diff --git a/.clang-format.example b/.clang-format.example new file mode 100644 index 00000000..fa3391b2 --- /dev/null +++ b/.clang-format.example @@ -0,0 +1,45 @@ +# This may be useful with https://clang.llvm.org/docs/ClangFormat.html#script-for-patch-reformatting +# but some of clang's reformattings may conflict with notes/c-style-guide.txt, which takes precedence. +# +# This is deliberately named .clang-format.example instead of .clang-format to avoid editors +# unexpectedly using this to reformatting entire files because of these shortcomings. +# It can be copied into .clang-format to use this for local development selectively. +# +# Note: this clang format file has many shortcomings. +# Attempting to apply this automatically to everything may make code less readable. +# However, this may be useful for spot checking new code, +# or if you're not certain of indentation style or general spacing/wrapping rules +# +# Known shortcomings: +# - Some places exceed the 80 column limit deliberately for readability, e.g. help strings or error messages or function prototypes. (BreakStringLiterals helps preserve some of those) +# - Some places deliberately put blocks on a single line when there are a lot of similar blocks. +# AllowShortBlocksOnASingleLine is not useful. +# - Some places deliberately put blocks on a single line when there are a lot of similar blocks. +# - No good way to eliminate space before and after PRIu64 and other macros for adjacent string literal concatenation +# - clang-format is not aware of macros, some of which have different styles from functions. +# - Function declarations are not typically aligned +# - Some variable declarations are aligned and others aren't on a case by case basis +# - The choice of function argument grouping should depends on which function arguments are semantically related, +# not just on fitting within 80 columns. + +AllowShortBlocksOnASingleLine: true +BasedOnStyle: LLVM +AlwaysBreakAfterDefinitionReturnType: All +UseTab: Never +IndentWidth: 4 +TabWidth: 4 +ColumnLimit: 80 +BreakBeforeBraces: Linux +SortIncludes: false + +BreakStringLiterals: false +# BitFieldColonSpacing is too new to work in clang-format 11 +# https://releases.llvm.org/11.0.0/tools/clang/docs/ClangFormatStyleOptions.html +# Latest: https://clang.llvm.org/docs/ClangFormatStyleOptions.html +# +# BitFieldColonSpacing: None +# +# XXX no way to treat the `*` indicating a value is a pointer as part of the aligned name for the declaration +# XXX function declarations are not typically aligned +AlignConsecutiveDeclarations: true +AlignConsecutiveMacros: true diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..2b1a09f5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,57 @@ +Dockerfile* +**/Dockerfile* +*.gz +src/nutredis +src/nutcracker +tests/_binaries/* + +### Entries copied from .gitignore +# pyc +*.pyc + +# Compiled Object files +*.lo +*.o + +# Compiled Dynamic libraries +*.so + +# Compiled Static libraries +*.la +*.a + +# Compiled misc +*.dep +*.gcda +*.gcno +*.gcov + +# Packages +*.tar.gz +*.tar.bz2 + +# Logs +*.log + +# Temporary +*.swp +*.~ +*.project +*.cproject + +# Core and executable +core* +nutcracker + +# extracted yaml +!/contrib/yaml-0.2.5.tar.gz + +# Autotools +.deps +.libs + +/aclocal.m4 +/autom4te.cache +/stamp-h1 +/autoscan.log +/libtool diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..73461bc7 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,19 @@ +# https://editorconfig.org/ + +root = true + +[*] +trim_trailing_whitespace = true +insert_final_newline = true +end_of_line = lf +charset = utf-8 + +# See twemproxy/notes/c-styleguide.txt +[*.c,*.h] +tab_width = 4 +indent_size = 4 +indent_style = space +# indent_brace_style depends on function vs conditional +max_line_length = 80 +spaces_around_brackets = none +spaces_around_operators = true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..4444674f --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,42 @@ +# This is a basic workflow to help you get started with Actions + +name: CI + +# Controls when the action will run. +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ master ] + pull_request: + branches: [ master ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # See https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-environment-variables-in-a-matrix + strategy: + # Run to completion even if one redis version has failures + fail-fast: false + matrix: + include: + - REDIS_VER: 3.0.7 + - REDIS_VER: 3.2.13 + - REDIS_VER: 4.0.14 + - REDIS_VER: 5.0.12 + - REDIS_VER: 6.2.4 + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + # Runs a single command using the runners shell + - name: Build and test in docker + run: bash ./test_in_docker.sh ${{ matrix.REDIS_VER }} diff --git a/.gitignore b/.gitignore index 2556202e..8e1230f5 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,7 @@ core* nutcracker # extracted yaml -!/contrib/yaml-0.1.4.tar.gz +!/contrib/yaml-0.2.5.tar.gz # Autotools .deps @@ -66,3 +66,11 @@ nutcracker Makefile Makefile.in + +# The .clang-format.example file may be copied here for use with -style=file +.clang-format + +test_all +*.trs +tags + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d9f33ed6..00000000 --- a/.travis.yml +++ /dev/null @@ -1,29 +0,0 @@ -language: python -python: - - "3.8" -cache: - directories: - - $HOME/.cache/pip - - $HOME/.cache/redis -before_install: - - sudo apt-get update - - sudo apt-get install -y git - - sudo apt-get install -y socat - - sudo apt-get install -y memcached - # build redis from source - - | - set -e; - ls $HOME/.cache/redis/src 1>/dev/null 2>&1 || git clone https://github.com/redis/redis.git -b 4.0.14 $HOME/.cache/redis - cd $HOME/.cache/redis - make -j $(nproc) - sudo make install - cd - - - pip install nose - - pip install python-memcached - - pip install redis -install: - - autoreconf -fvi - - CFLAGS="-ggdb3 -O3 -fno-strict-aliasing" ./configure --enable-debug=log - - make -j $(nproc) - - sudo make install -script: bash travis.sh diff --git a/ChangeLog b/ChangeLog index 439b9b91..d7e15161 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,46 @@ + 2021-13-07 Tyson Andre + * twemproxy: version 0.5.0 release + Same as 0.5.0-RC1 + + 2021-06-07 Tyson Andre + * twemproxy: version 0.5.0-RC1 release + Add 'tcpkeepalive' pool boolean config flag setting + to enable tcp keepalive (charsyam, manju) + Support redis bitpos command (clark kang) + Fix parsing of redis error response for error type with no space, + add tests (tyson, tom dalton) + Update integration tests, add C unit test suite for 'make check' (tyson) + Increase the maximum host length+port+identifier to 273 + in ketama_update (李广博) + Always initialize file permissions field when listening on a unix domain + socket (tyson) + Use number of servers instead of number of points on the continuum when + sharding requests to backend services to improve sharding performance + and fix potential invalid memory access when all hosts were ejected + from a pool. (tyson) + Optimize performance of deletion of single redis keys (vincentve) + Don't fragment memcache/redis get commands when they only have a single + key (improves performance and error handling of single key case) (tyson) + Don't let requests hang when there is a dns error when processing a + fragmented request (e.g. multiget with multiple keys) (tyson) + Allow extra parameters for redis spop (charsyam) + Update documentation and README (various) + Fix memory leak bug for redis mset (deep011) + Support arbitrarily deep nested redis multi-bulk + responses (nested arrays) (qingping209, tyson) + Upgrade from libyaml 0.1.4 to 0.2.5 (tyson) + Fix compiler warnings about wrong conversion specifiers in format + strings for logging (tyson) + Log the async backend used and any debug options in the + '--help'/'--version' output. + Add support for many more new redis commands and updates to existing + redis commands (tyson) + Optimization: Skip hashing and choosing server index when a pool has + exactly one server (tyson) + Support memcache 'version' requests by proxying the request to a single + backend memcache server to fetch the server version. (tyson) + Make error messages for creating the stats server during startup clearer. (tyson) + 2015-22-06 Manju Rajashekhar * twemproxy: version 0.4.1 release backend server hostnames are resolved lazily @@ -9,7 +52,6 @@ implemented support for parsing fine grained redis error response remove redundant conditional judgement in rbtree deletion (leo ma) fix bug mset has invalid pair (charsyam) - fix bug mset has invalid pair (charsyam) temp fix a core on kqueue (idning) support "touch" command for memcached (panmiaocai) fix redis parse rsp bug (charsyam) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..165dd9f2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM gcc + +COPY . /usr/src/twemproxy +WORKDIR /usr/src/twemproxy +RUN \ + autoreconf -h && \ + autoreconf -fvi && \ + ./configure && \ + make && \ + make install + +ENTRYPOINT [ "nutcracker" ] diff --git a/NOTICE b/NOTICE index efe6905d..257983f4 100644 --- a/NOTICE +++ b/NOTICE @@ -101,7 +101,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -The source also includes libyaml (yaml-0.1.4) in contrib/ directory +The source also includes libyaml (yaml-0.2.5) in contrib/ directory Copyright (c) 2006 Kirill Simonov diff --git a/README.md b/README.md index db2778a2..a2cb5cd1 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,17 @@ # twemproxy (nutcracker) [![Build Status](https://secure.travis-ci.org/colopl/twemproxy.png)](http://travis-ci.org/colopl/twemproxy) + **twemproxy** (pronounced "two-em-proxy"), aka **nutcracker** is a fast and lightweight proxy for [memcached](http://www.memcached.org/) and [redis](http://redis.io/) protocol. It was built primarily to reduce the number of connections to the caching servers on the backend. This, together with protocol pipelining and sharding enables you to horizontally scale your distributed caching architecture. ## Build -To build twemproxy from [distribution tarball](https://drive.google.com/open?id=0B6pVMMV5F5dfMUdJV25abllhUWM&authuser=0): +To build twemproxy 0.5.0+ from [distribution tarball](https://github.com/twitter/twemproxy/releases): $ ./configure $ make $ sudo make install -To build twemproxy from [distribution tarball](https://drive.google.com/open?id=0B6pVMMV5F5dfMUdJV25abllhUWM&authuser=0) in _debug mode_: +To build twemproxy 0.5.0+ from [distribution tarball](https://github.com/twitter/twemproxy/releases) in _debug mode_: $ CFLAGS="-ggdb3 -O0" ./configure --enable-debug=full $ make @@ -32,6 +33,13 @@ A quick checklist: + Use CFLAGS="-O3 -fno-strict-aliasing" ./configure && make + `autoreconf -fvi && ./configure` needs `automake` and `libtool` to be installed +`make check` will run unit tests. + +### Older Releases + +Distribution tarballs for older twemproxy releases (<= 0.4.1) can be found on [Google Drive](https://drive.google.com/open?id=0B6pVMMV5F5dfMUdJV25abllhUWM&authuser=0). +The build steps are the same (`./configure; make; sudo make install`). + ## Features + Fast. @@ -83,36 +91,37 @@ Twemproxy can be configured through a YAML file specified by the -c or --conf-fi + **listen**: The listening address and port (name:port or ip:port) or an absolute path to sock file (e.g. /var/run/nutcracker.sock) for this server pool. + **client_connections**: The maximum number of connections allowed from redis clients. Unlimited by default, though OS-imposed limitations will still apply. + **hash**: The name of the hash function. Possible values are: - + one_at_a_time - + md5 - + crc16 - + crc32 (crc32 implementation compatible with [libmemcached](http://libmemcached.org/)) - + crc32a (correct crc32 implementation as per the spec) - + fnv1_64 - + fnv1a_64 - + fnv1_32 - + fnv1a_32 - + hsieh - + murmur - + jenkins + + one_at_a_time + + md5 + + crc16 + + crc32 (crc32 implementation compatible with [libmemcached](http://libmemcached.org/)) + + crc32a (correct crc32 implementation as per the spec) + + fnv1_64 + + fnv1a_64 (default) + + fnv1_32 + + fnv1a_32 + + hsieh + + murmur + + jenkins + **hash_tag**: A two character string that specifies the part of the key used for hashing. Eg "{}" or "$$". [Hash tag](notes/recommendation.md#hash-tags) enable mapping different keys to the same server as long as the part of the key within the tag is the same. -+ **distribution**: The key distribution mode. Possible values are: - + ketama - + modula - + random ++ **distribution**: The key distribution mode for choosing backend servers based on the computed hash value. Possible values are: + + ketama (default, recommended. An implementation of https://en.wikipedia.org/wiki/Consistent_hashing) + + modula (use hash modulo number of servers to choose the backend) + + random (choose a random backend for each key of each request) + **timeout**: The timeout value in msec that we wait for to establish a connection to the server or receive a response from a server. By default, we wait indefinitely. + **abort_on_timeout**: A boolean value that controls if twemproxy abort a connection on timeout(ETIMEDOUT). Defaults to false. + **abort_on_refused**: A boolean value that controls if twemproxy abort a connection on connection refused(ECONNREFUSED). Defaults to false. + **abort_on_invalid**: A boolean value that controls if twemproxy abort a connection on invalid argument(EINVAL). Defaults to false. + **backlog**: The TCP backlog argument. Defaults to 512. ++ **tcpkeepalive**: A boolean value that controls if tcp keepalive is enabled for connections to servers. Defaults to false. + **preconnect**: A boolean value that controls if twemproxy should preconnect to all the servers in this pool on process start. Defaults to false. + **redis**: A boolean value that controls if a server pool speaks redis or memcached protocol. Defaults to false. + **redis_auth**: Authenticate to the Redis server on connect. + **redis_db**: The DB number to use on the pool servers. Defaults to 0. Note: Twemproxy will always present itself to clients as DB 0. + **server_connections**: The maximum number of connections that can be opened to each server. By default, we open at most 1 server connection. + **auto_eject_hosts**: A boolean value that controls if server should be ejected temporarily when it fails consecutively server_failure_limit times. See [liveness recommendations](notes/recommendation.md#liveness) for information. Defaults to false. -+ **server_retry_timeout**: The timeout value in msec to wait for before retrying on a temporarily ejected server, when auto_eject_host is set to true. Defaults to 30000 msec. -+ **server_failure_limit**: The number of consecutive failures on a server that would lead to it being temporarily ejected when auto_eject_host is set to true. Defaults to 2. ++ **server_retry_timeout**: The timeout value in msec to wait for before retrying on a temporarily ejected server, when auto_eject_hosts is set to true. Defaults to 30000 msec. ++ **server_failure_limit**: The number of consecutive failures on a server that would lead to it being temporarily ejected when auto_eject_hosts is set to true. Defaults to 2. + **servers**: A list of server address, port and weight (name:port:weight or ip:port:weight) for this server pool. @@ -186,13 +195,13 @@ For example, the configuration file in [conf/nutcracker.yml](conf/nutcracker.yml - 127.0.0.1:11214:100000 - 127.0.0.1:11215:1 -Finally, to make writing a syntactically correct configuration file easier, twemproxy provides a command-line argument -t or --test-conf that can be used to test the YAML configuration file for any syntax error. +Finally, to make writing a syntactically correct configuration file easier, twemproxy provides a command-line argument `-t` or `--test-conf` that can be used to test the YAML configuration file for any syntax error. ## Observability Observability in twemproxy is through logs and stats. -Twemproxy exposes stats at the granularity of server pool and servers per pool through the stats monitoring port. The stats are essentially JSON formatted key-value pairs, with the keys corresponding to counter names. By default stats are exposed on port 22222 and aggregated every 30 seconds. Both these values can be configured on program start using the -c or --conf-file and -i or --stats-interval command-line arguments respectively. You can print the description of all stats exported by using the -D or --describe-stats command-line argument. +Twemproxy exposes stats at the granularity of server pool and servers per pool through the stats monitoring port by responding with the raw data over TCP. The stats are essentially JSON formatted key-value pairs, with the keys corresponding to counter names. By default stats are exposed on port 22222 and aggregated every 30 seconds. Both these values can be configured on program start using the `-c` or `--conf-file` and `-i` or `--stats-interval` command-line arguments respectively. You can print the description of all stats exported by using the `-D` or `--describe-stats` command-line argument. $ nutcracker --describe-stats @@ -218,13 +227,15 @@ Twemproxy exposes stats at the granularity of server pool and servers per pool t out_queue "# requests in outgoing queue" out_queue_bytes "current request bytes in outgoing queue" -Logging in twemproxy is only available when twemproxy is built with logging enabled. By default logs are written to stderr. Twemproxy can also be configured to write logs to a specific file through the -o or --output command-line argument. On a running twemproxy, we can turn log levels up and down by sending it SIGTTIN and SIGTTOU signals respectively and reopen log files by sending it SIGHUP signal. +See [`notes/debug.txt`](notes/debug.txt) for examples of how to read the stats from the stats port. + +Logging in twemproxy is only available when twemproxy is built with logging enabled. By default logs are written to stderr. Twemproxy can also be configured to write logs to a specific file through the `-o` or `--output` command-line argument. On a running twemproxy, we can turn log levels up and down by sending it SIGTTIN and SIGTTOU signals respectively and reopen log files by sending it SIGHUP signal. ## Pipelining Twemproxy enables proxying multiple client connections onto one or few server connections. This architectural setup makes it ideal for pipelining requests and responses and hence saving on the round trip time. -For example, if twemproxy is proxying three client connections onto a single server and we get requests - 'get key\r\n', 'set key 0 0 3\r\nval\r\n' and 'delete key\r\n' on these three connections respectively, twemproxy would try to batch these requests and send them as a single message onto the server connection as 'get key\r\nset key 0 0 3\r\nval\r\ndelete key\r\n'. +For example, if twemproxy is proxying three client connections onto a single server and we get requests - `get key\r\n`, `set key 0 0 3\r\nval\r\n` and `delete key\r\n` on these three connections respectively, twemproxy would try to batch these requests and send them as a single message onto the server connection as `get key\r\nset key 0 0 3\r\nval\r\ndelete key\r\n`. Pipelining is the reason why twemproxy ends up doing better in terms of throughput even though it introduces an extra hop between the client and server. @@ -232,18 +243,6 @@ Pipelining is the reason why twemproxy ends up doing better in terms of throughp If you are deploying twemproxy in production, you might consider reading through the [recommendation document](notes/recommendation.md) to understand the parameters you could tune in twemproxy to run it efficiently in the production environment. -## Packages - -### Ubuntu - -#### PPA Stable - -https://launchpad.net/~twemproxy/+archive/ubuntu/stable - -#### PPA Daily - -https://launchpad.net/~twemproxy/+archive/ubuntu/daily - ## Utils + [collectd-plugin](https://github.com/bewie/collectd-twemproxy) + [munin-plugin](https://github.com/eveiga/contrib/tree/nutcracker/plugins/nutcracker) @@ -258,7 +257,7 @@ https://launchpad.net/~twemproxy/+archive/ubuntu/daily + [smitty for twemproxy failover](https://github.com/areina/smitty) + [Beholder, a Python agent for twemproxy failover](https://github.com/Serekh/beholder) + [chef cookbook](https://supermarket.getchef.com/cookbooks/twemproxy) -+ [twemsentinel] (https://github.com/yak0/twemsentinel) ++ [twemsentinel](https://github.com/yak0/twemsentinel) ## Companies using Twemproxy in Production + [Twitter](https://twitter.com/) diff --git a/ci/Dockerfile b/ci/Dockerfile new file mode 100644 index 00000000..6586490d --- /dev/null +++ b/ci/Dockerfile @@ -0,0 +1,76 @@ +# Dockerfile to create a slower debug build of nutcracker with assertions enabled +# for continuous integration checks. +# ARGS: REDIS_VER +# Also see test_in_docker.sh +FROM centos:7 + +ENV LAST_MODIFIED_DATE 2021-04-09 + +RUN yum install -y \ + https://repo.ius.io/ius-release-el7.rpm \ + https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + +# socat Allow tests to open log files +# which (used below) +# python-setuptools for pip +RUN yum install -y \ + tar git gcc make tcl \ + autoconf automake libtool wget \ + memcached \ + socat \ + which \ + python36u python36u-pip && \ + yum clean all + +# Install nosetest dependencies +RUN pip3.6 install nose && \ + pip3.6 install 'git+https://github.com/andymccurdy/redis-py.git@3.5.3' && \ + pip3.6 install 'git+https://github.com/linsomniac/python-memcached.git@1.58' +# I can't install redis or redis-server in centos:7 (didn't add package), adding to centos 6 +# Install redis and redis sentinel, needed for unit tests +# RUN yum install -y redis redis-sentinel + +ARG REDIS_VER=3.2.11 +RUN wget https://github.com/redis/redis/archive/$REDIS_VER.tar.gz && \ + tar zxvf $REDIS_VER.tar.gz && \ + pushd redis-$REDIS_VER && \ + make install && \ + popd && \ + rm -r redis-* + +# This will build twemproxy with compilation flags for running unit tests, integration tests. +# Annoyingly, this can't add multiple directories at once. +ADD conf /usr/src/twemproxy/conf +ADD contrib /usr/src/twemproxy/contrib +ADD man /usr/src/twemproxy/man +ADD m4 /usr/src/twemproxy/m4 +ADD notes /usr/src/twemproxy/notes +ADD scripts /usr/src/twemproxy/scripts +ADD src /usr/src/twemproxy/src +ADD ChangeLog configure.ac Makefile.am LICENSE NOTICE README.md /usr/src/twemproxy/ + +WORKDIR /usr/src/twemproxy + + +ADD ci/build-nutcracker.sh /usr/local/bin/build-nutcracker.sh +RUN /usr/local/bin/build-nutcracker.sh + +# Add the tests after adding source files, which makes it easy to quickly test changes to unit tests. +ADD tests /usr/src/twemproxy/tests + +# Not installing redis utilities, since we're not running those tests. +RUN mkdir tests/_binaries -p + +RUN ln -nsf $PWD/src/nutcracker tests/_binaries/nutcracker && \ + cp `which redis-server` tests/_binaries/redis-server && \ + cp `which redis-server` tests/_binaries/redis-sentinel && \ + cp `which memcached` tests/_binaries/memcached && \ + cp `which redis-cli` tests/_binaries/redis-cli + +WORKDIR /usr/src/twemproxy/tests + +# Allow tests to open log files +RUN chmod -R a+w log/ +RUN cat /etc/passwd +RUN chown -R daemon:daemon /usr/src/twemproxy +USER daemon diff --git a/ci/build-nutcracker.sh b/ci/build-nutcracker.sh new file mode 100755 index 00000000..a9bd8a28 --- /dev/null +++ b/ci/build-nutcracker.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -xeu +function cleanup { + rm -f *.gz +} +trap cleanup EXIT +trap cleanup INT + +cleanup + +export CFLAGS="-O3 -fno-strict-aliasing -I/usr/lib/x86_64-redhat-linux6E/include -B /usr/lib/x86_64-redhat-linux6E/lib64" +# TODO: Figure out how to make this apply only to the contrib/ directory. Maybe override the yaml directory's Makefile.am after extracting it. +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119 +CFLAGS+=" -Werror -Wall -Wno-pointer-sign -Wno-sign-conversion -Wno-missing-braces -Wno-unused-value -Wno-builtin-declaration-mismatch -Wno-maybe-uninitialized" +export LDFLAGS="-lc_nonshared" +cd /usr/src/twemproxy +autoreconf -fvi +./configure --enable-debug=yes --prefix=/usr/src/twemproxy/work/usr +make -j5 +make install diff --git a/configure.ac b/configure.ac index c311355a..0f96dbae 100644 --- a/configure.ac +++ b/configure.ac @@ -1,8 +1,8 @@ # Define the package version numbers and the bug reporting address m4_define([NC_MAJOR], 0) -m4_define([NC_MINOR], 4) -m4_define([NC_PATCH], 1) -m4_define([NC_BUGS], [manj@cs.stanford.edu]) +m4_define([NC_MINOR], 5) +m4_define([NC_PATCH], 0) +m4_define([NC_BUGS], [https://github.com/twitter/twemproxy/issues]) # Initialize autoconf AC_PREREQ([2.64]) @@ -84,6 +84,7 @@ AC_CHECK_FUNCS([dup2 gethostname gettimeofday strerror]) AC_CHECK_FUNCS([socket]) AC_CHECK_FUNCS([memchr memmove memset]) AC_CHECK_FUNCS([strchr strndup strtoul]) +AC_CHECK_FUNCS([strdup]) AC_CACHE_CHECK([if epoll works], [ac_cv_epoll_works], AC_TRY_RUN([ @@ -196,11 +197,11 @@ AS_IF([test "x$disable_stats" = xyes], [AC_DEFINE([HAVE_STATS], [1], [Define to 1 if stats is not disabled])]) AC_MSG_RESULT($disable_stats) -# Untar the yaml-0.1.4 in contrib/ before config.status is rerun -AC_CONFIG_COMMANDS_PRE([tar xvfz contrib/yaml-0.1.4.tar.gz -C contrib]) +# Untar the yaml-0.2.5 in contrib/ before config.status is rerun +AC_CONFIG_COMMANDS_PRE([tar xvfz contrib/yaml-0.2.5.tar.gz -C contrib]) -# Call yaml-0.1.4 ./configure recursively -AC_CONFIG_SUBDIRS([contrib/yaml-0.1.4]) +# Call yaml-0.2.5 ./configure recursively +AC_CONFIG_SUBDIRS([contrib/yaml-0.2.5]) # Define Makefiles AC_CONFIG_FILES([Makefile diff --git a/contrib/Makefile.am b/contrib/Makefile.am index a5c85a3a..d90e6080 100644 --- a/contrib/Makefile.am +++ b/contrib/Makefile.am @@ -1,3 +1,3 @@ -SUBDIRS = yaml-0.1.4 +SUBDIRS = yaml-0.2.5 -EXTRA_DIST = yaml-0.1.4.tar.gz +EXTRA_DIST = yaml-0.2.5.tar.gz diff --git a/contrib/yaml-0.1.4.tar.gz b/contrib/yaml-0.1.4.tar.gz deleted file mode 100644 index 87a7b8f7..00000000 Binary files a/contrib/yaml-0.1.4.tar.gz and /dev/null differ diff --git a/contrib/yaml-0.2.5.tar.gz b/contrib/yaml-0.2.5.tar.gz new file mode 100644 index 00000000..a78e0d99 Binary files /dev/null and b/contrib/yaml-0.2.5.tar.gz differ diff --git a/contrib/yaml-0.1.4/.gitignore b/contrib/yaml-0.2.5/.gitignore similarity index 100% rename from contrib/yaml-0.1.4/.gitignore rename to contrib/yaml-0.2.5/.gitignore diff --git a/notes/c-styleguide.txt b/notes/c-styleguide.txt index 30aa62f2..85fd16cc 100644 --- a/notes/c-styleguide.txt +++ b/notes/c-styleguide.txt @@ -6,7 +6,7 @@ - Make sure that your editor does not leave space at the end of the line. - snake_case for variable, function and file names. - Use your own judgment when naming variables and functions. Be as Spartan - as possible. Eg: Using name like this_variable_is_a_temporary_counter + as possible. E.g.: Using name like this_variable_is_a_temporary_counter will usually be frowned upon. - Don’t use local variables or parameters that shadow global identifiers. GCC’s ‘-Wshadow’ option can help you to detect this problem. @@ -17,19 +17,19 @@ you cannot get away from using int and char. - Use bool for boolean variables. You have to include - Avoid using a bool as type for struct member names. Instead use unsigned - 1-bit bit field. Eg: + 1-bit bit field. E.g.: struct foo { unsigned is_bar:1; }; - Always use size_t type when dealing with sizes of objects or memory ranges. - Your code should be 64-bit and 32-bit friendly. Bear in mind problems of printing, comparisons, and structure alignment. You have to include - to get generic format specifier macros for printing. + to get generic format specifier macros for printing. - 80 column line limit. - If you have to wrap a long statement (> 80 column), put the operator at the end of the line and indent the next line at the same column as the arguments - in the previous column. Eg: + in the previous column. E.g.: while (cnt < 20 && this_variable_name_is_too_long && ep != NULL) { z = a + really + long + statement + that + needs + three + lines + @@ -43,19 +43,19 @@ param_g, param_h, param_i, param_j, param_k, param_l); - Always use braces for all conditional blocks (if, switch, for, while, do). - This holds good even for single statement conditional blocks. Eg: + This holds good even for single statement conditional blocks. E.g.: if (cond) { stmt; } - Placement of braces for non-function statement blocks - put opening brace - last on the line and closing brace first. Eg: + last on the line and closing brace first. E.g.: if (x is true) { we do y } - Placement of brace for functions - put the opening brace at the beginning of the next line and closing brace first. This is useful because several tools look for opening brace in column one to find beginning of C - functions. Eg: + functions. E.g.: int function(int x) { @@ -79,7 +79,7 @@ function(int x) .... } -- Column align switch keyword and the corresponding case/default keyword. Eg: +- Column align switch keyword and the corresponding case/default keyword. E.g.: switch (alphabet) { case 'a': case 'b': @@ -90,7 +90,7 @@ function(int x) break; } -- Forever loops are done with for, and not while. Eg: +- Forever loops are done with for, and not while. E.g.: for (;;) { stmt; } @@ -102,14 +102,14 @@ function(int x) - Do not add spaces around (inside) parenthesized expressions. s = sizeof( sizeof(*p)) ); /* bad example */ s = sizeof(sizeof(*p)); /* good example */ -- Casts should not be followed by space. Eg: +- Casts should not be followed by space. E.g.: int q = *(int *)&p - There is no need to type cast when assigning a void pointer to a non-void pointer, or vice versa. - Avoid using goto statements. However there are some exceptions to this rule when a single goto label within a function and one or more goto statements come in handy when a function exits from multiple locations and some common - work such as cleanup has to be done. Eg: + work such as cleanup has to be done. E.g.: int fun(void) { @@ -135,7 +135,7 @@ out: return result; } - When declaring pointer data, use '*' adjacent to the data name and not - adjacent to the type name. Eg: + adjacent to the type name. E.g.: int function(int *p) { @@ -192,7 +192,7 @@ out: or by the header that uses it (which causes namespace pollution), or there must be a back-door mechanism for obtaining the typedef. - The only exception for using a typedef is when you are defining a type - for a function pointer or a type for an enum. Eg: + for a function pointer or a type for an enum. E.g.: typedef void (*foo_handler_t)(int, void *); @@ -205,7 +205,7 @@ out: - Use just one variable declaration per line when variables are part of a struct. This leaves you room for a small comment on each item, explaining - its use. Declarations should also be aligned. Eg, use: + its use. Declarations should also be aligned. E.g., use: struct foo { int *foo_a; /* comment for foo_a */ @@ -222,7 +222,7 @@ out: - For variable declaration outside a struct, either collect all the declarations of the same type on a single line, or use one variable - per line if the variables purpose needs to be commented. Eg: + per line if the variables purpose needs to be commented. E.g.: char *a, *b, c; or: @@ -235,7 +235,7 @@ out: - Function definitions should start the name of the function in column one. This is useful because it makes searching for function definitions - fairly trivial. Eg: + fairly trivial. E.g.: static char * concat(char *s1, char *s2) { @@ -244,7 +244,7 @@ concat(char *s1, char *s2) - Function and variables local to a file should be static. - Separate two successive functions with one blank line. -- Include parameter names with their datypes in function declaration. Eg: +- Include parameter names with their datatypes in function declaration. E.g.: void function(int param); - Functions should be short and sweet, and do just one thing. They should @@ -293,7 +293,7 @@ void function(int param); type char * which is really the address of the second character of a string, not the first), or any possible values that would not work the way one would expect (such as, that strings containing newlines are not - guaranteed to work), be sure to say so. Eg: + guaranteed to work), be sure to say so. E.g.: /* * Try to acquire a physical address lock while a pmap is locked. If we @@ -324,7 +324,7 @@ vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked) - Recommend using UPPERCASE for macro names. However, sometimes using lowercase for macro names makes sense when macros masquerade as well-known - function calls. Eg, it makes sense to write the wrapper for the + function calls. E.g., it makes sense to write the wrapper for the standard free() function in lowercase to keep the readability consistent: @@ -340,10 +340,10 @@ vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked) - For macros encapsulating compound statements, right justify the backslashes and enclose it in do { ... } while (0) - For parameterized macros, all the parameters used in the macro body must - be surrounded by parentheses. Eg: + be surrounded by parentheses. E.g.: #define ADD_1(_x) ((_x) + 1) -- Use sizeof(varname) instead of sizeof(type) whenever possible. Eg: +- Use sizeof(varname) instead of sizeof(type) whenever possible. E.g.: char *p; p = malloc(sizeof(*p)); /* good example */ p = malloc(sizeof(char)); /* bad example */ @@ -369,7 +369,7 @@ vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked) - Every header file in the source code must have preprocessor conditional to prevent the header file from being scanned multiple times and avoiding mutual dependency cycles. Alternatively you can use #pragma once directive, - as it avoids name clashes and increases the compile speed. Eg, for a + as it avoids name clashes and increases the compile speed. E.g., for a header file named foo.h, the entire contents of the header file must be between the guard macros as follows: @@ -398,7 +398,7 @@ Or, - Conditional compilation: when supporting configuration options already known when building your program we prefer using if (... ) over conditional compilation, as in the former case the compiler is able to perform more - extensive checking of all possible code paths. Eg, use: + extensive checking of all possible code paths. E.g., use: if (HAS_FOO) ... diff --git a/notes/debug.txt b/notes/debug.txt index 282bfc5c..176fa338 100644 --- a/notes/debug.txt +++ b/notes/debug.txt @@ -1,7 +1,7 @@ - strace strace -o strace.txt -ttT -s 1024 -p `pgrep nutcracker` -- libyaml (yaml-0.1.4) +- libyaml (yaml-0.2.5) - yaml tokens: @@ -109,9 +109,9 @@ EPOLLET = (1 << 31) /* opcodes */ - EPOLL_CTL_ADD = 1 /* add a file decriptor to the interface */ - EPOLL_CTL_DEL = 2 /* remove a file decriptor from the interface */ - EPOLL_CTL_MOD = 3 /* change file decriptor epoll_event structure */ + EPOLL_CTL_ADD = 1 /* add a file descriptor to the interface */ + EPOLL_CTL_DEL = 2 /* remove a file descriptor from the interface */ + EPOLL_CTL_MOD = 3 /* change file descriptor epoll_event structure */ - kqueue (bsd) diff --git a/notes/memcache.md b/notes/memcache.md index 5c42aaff..f25a1149 100644 --- a/notes/memcache.md +++ b/notes/memcache.md @@ -30,7 +30,7 @@ * - uint8_t[]: data block * - uint64_t -#### Ascii Retrival Command +#### Ascii Retrieval Command +-------------------+------------+--------------------------------------------------------------------------+ | Command | Supported? | Format | @@ -68,11 +68,15 @@ +-------------------+------------+--------------------------------------------------------------------------+ | touch | Yes | touch [noreply]\r\n | +-------------------+------------+--------------------------------------------------------------------------+ + | gat | Planned | gat +\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | gats | Planned | gats +\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ | quit | Yes | quit\r\n | +-------------------+------------+--------------------------------------------------------------------------+ | flush_all | No | flush_all [] [noreply]\r\n | +-------------------+------------+--------------------------------------------------------------------------+ - | version | No | version\r\n | + | version | Yes | version\r\n | +-------------------+------------+--------------------------------------------------------------------------+ | verbosity | No | verbosity [noreply]\r\n | +-------------------+------------+--------------------------------------------------------------------------+ @@ -112,7 +116,7 @@ NOT_FOUND\r\n DELETED\r\n -#### Retrival Responses +#### Retrieval Responses END\r\n VALUE []\r\n\r\nEND\r\n @@ -158,5 +162,7 @@ - ascii protocol is easier to debug - think using strace or tcpdump to see protocol on the wire, Or using telnet or netcat or socat to build memcache requests and responses - http://stackoverflow.com/questions/2525188/are-binary-protocols-dead - - http://news.ycombinator.com/item?id=1712788 + https://stackoverflow.com/questions/2525188/are-binary-protocols-dead + - nutcracker will support the more efficient meta-text protocol after the protocol + is marked as stable and memcached servers using it have had several releases. + - https://news.ycombinator.com/item?id=1712788 diff --git a/notes/recommendation.md b/notes/recommendation.md index 13289297..45782f9c 100644 --- a/notes/recommendation.md +++ b/notes/recommendation.md @@ -151,12 +151,6 @@ You can also graph the timestamp at which any given server was ejected by graphi ## server_connections: > 1 -By design, twemproxy multiplexes several client connections over few server connections. It is important to note that **"read my last write"** constraint doesn't necessarily hold true when twemproxy is configured with `server_connections: > 1`. +By design, twemproxy multiplexes several client connections over few server connections. It is important to note that **"read my last write"** constraint doesn't necessarily hold true when twemproxy is configured with `server_connections: > 1`. To illustrate this, consider a scenario where twemproxy is configured with `server_connections: 2`. If a client makes pipelined requests with the first request in pipeline being `set foo 0 0 3\r\nbar\r\n` (write) and the second request being `get foo\r\n` (read), the expectation is that the read of key `foo` would return the value `bar`. However, with configuration of two server connections it is possible that write and read request are sent on different server connections which would mean that their completion could race with one another. In summary, if the client expects "read my last write" constraint, you either configure twemproxy to use `server_connections:1` or use clients that only make synchronous requests to twemproxy. - -## twemproxy and python-memcached - -The implementation of delete command in [python-memcached](https://github.com/linsomniac/python-memcached) conflicts with the one in twemproxy. See [issue 283](https://github.com/twitter/twemproxy/pull/283) for details. The workaround for this issue is to call `delete_multi` in python-memcached as follows: - - mc.delete_multi([key1, key2, ... keyN], time=None) diff --git a/notes/redis.md b/notes/redis.md index 1f5fefea..641401a9 100644 --- a/notes/redis.md +++ b/notes/redis.md @@ -53,54 +53,62 @@ +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | Command | Supported? | Format | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | APPEND | Yes | APPEND key value | + | APPEND | Yes | APPEND key value | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | BITCOUNT | Yes | BITCOUNT key [start] [end] | + | BITCOUNT | Yes | BITCOUNT key [start end] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | BITPOS | Yes | BITPOS key bit [start] [end] | + | BITFIELD | Yes | BITFIELD key [GET] [SET] [INCRBY] [WRAP|SAT|FAIL] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | BITOP | No | BITOP operation destkey key [key ...] | + | BITOP | No | BITOP operation destkey key [key …] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | BITPOS | Yes | BITPOS key bit [start] [end] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | DECR | Yes | DECR key | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | DECRBY | Yes | DECRBY key decrement | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | GET | Yes | GET key | + | GET | Yes | GET key | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | GETBIT | Yes | GETBIT key offset | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | GETDEL | Yes | GETDEL key | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | GETEX | Yes | GETEX key [EX seconds|PX milliseconds|EXAT timestamp|PXAT milliseconds-timestamp|PERSIST] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | GETRANGE | Yes | GETRANGE key start end | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | GETSET | Yes | GETSET key value | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | INCR | Yes | INCR key | + | INCR | Yes | INCR key | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | INCRBY | Yes | INCRBY key increment | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | INCRBYFLOAT | Yes | INCRBYFLOAT key increment | + | INCRBYFLOAT | Yes | INCRBYFLOAT key increment | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | MGET | Yes | MGET key [key ...] | + | MGET | Yes | MGET key [key …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | MSET | Yes* | MSET key value [key value ...] | + | MSET | Yes | MSET key value [key value …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | MSETNX | No | MSETNX key value [key value ...] | + | MSETNX | No | MSETNX key value [key value …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | PSETEX | Yes | PSETEX key milliseconds value | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SET | Yes | SET key value [EX seconds] [PX milliseconds] [NX|XX] | + | SET | Yes | SET key value [EX seconds|PX milliseconds|EXAT timestamp|PXAT milliseconds-timestamp|KEEPTTL] [NX|XX] [GET] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | SETBIT | Yes | SETBIT key offset value | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SETEX | Yes | SETEX key seconds value | + | SETEX | Yes | SETEX key seconds value | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SETNX | Yes | SETNX key value | + | SETNX | Yes | SETNX key value | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SETRANGE | Yes | SETRANGE key offset value | + | SETRANGE | Yes | SETRANGE key offset value | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | STRALGO | No | STRALGO LCS algo-specific-argument [algo-specific-argument …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | STRLEN | Yes | STRLEN key | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ -* MSET support is not Atomic +* MSET support is not Atomic ### Hashes @@ -141,39 +149,45 @@ +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | Command | Supported? | Format | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | BLPOP | No | BLPOP key [key ...] timeout | + | BLMOVE | No | BLMOVE source destination LEFT|RIGHT LEFT|RIGHT timeout | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | BLPOP | No | BLPOP key [key …] timeout | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | BRPOP | No | BRPOP key [key ...] timeout | + | BRPOP | No | BRPOP key [key …] timeout | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | BRPOPLPUSH | No | BRPOPLPUSH source destination timeout | + | BRPOPLPUSH | No | BRPOPLPUSH source destination timeout | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | LINDEX | Yes | LINDEX key index | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | LINSERT | Yes | LINSERT key BEFORE|AFTER pivot value | + | LINSERT | Yes | LINSERT key BEFORE|AFTER pivot element | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | LLEN | Yes | LLEN key | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | LLEN | Yes | LLEN key | + | LMOVE | Yes | LMOVE source destination LEFT|RIGHT LEFT|RIGHT | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | LPOP | Yes | LPOP key | + | LPOP | Yes | LPOP key [count] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | LPUSH | Yes | LPUSH key value [value ...] | + | LPOS | Yes | LPOS key element [RANK] [COUNT] [MAXLEN] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | LPUSHX | Yes | LPUSHX key value | + | LPUSH | Yes | LPUSH key element [element …] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | LPUSHX | Yes | LPUSHX key element [element …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | LRANGE | Yes | LRANGE key start stop | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | LREM | Yes | LREM key count value | + | LREM | Yes | LREM key count element | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | LSET | Yes | LSET key index value | + | LSET | Yes | LSET key index element | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | LTRIM | Yes | LTRIM key start stop | + | LTRIM | Yes | LTRIM key start stop | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | RPOP | Yes | RPOP key | + | RPOP | Yes | RPOP key [count] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | RPOPLPUSH | Yes* | RPOPLPUSH source destination | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | RPUSH | Yes | RPUSH key value [value ...] | + | RPUSH | Yes | RPUSH key element [element …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | RPUSHX | Yes | RPUSHX key value | + | RPUSHX | Yes | RPUSHX key element [element …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ * RPOPLPUSH support requires that source and destination keys hash to the same server. You can ensure this by using the same [hashtag](recommendation.md#hash-tags) for source and destination key. Twemproxy does no checking on its end to verify that source and destination key hash to the same server, and the RPOPLPUSH command is forwarded to the server that the source key hashes to @@ -183,35 +197,37 @@ +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | Command | Supported? | Format | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SADD | Yes | SADD key member [member ...] | + | SADD | Yes | SADD key member [member …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SCARD | Yes | SCARD key | + | SCARD | Yes | SCARD key | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SDIFF | Yes* | SDIFF key [key ...] | + | SDIFF | Yes* | SDIFF key [key …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SDIFFSTORE | Yes* | SDIFFSTORE destination key [key ...] | + | SDIFFSTORE | Yes* | SDIFFSTORE destination key [key …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SINTER | Yes* | SINTER key [key ...] | + | SINTER | Yes* | SINTER key [key …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SINTERSTORE | Yes* | SINTERSTORE destination key [key ...] | + | SINTERSTORE | Yes* | SINTERSTORE destination key [key …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | SISMEMBER | Yes | SISMEMBER key member | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | SMEMBERS | Yes | SMEMBERS key | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SMOVE | Yes* | SMOVE source destination member | + | SMISMEMBER | Yes | SMISMEMBER key member [member …] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | SMOVE | Yes* | SMOVE source destination member | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SPOP | Yes | SPOP key | + | SPOP | Yes | SPOP key [count] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | SRANDMEMBER | Yes | SRANDMEMBER key [count] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SREM | Yes | SREM key member [member ...] | + | SREM | Yes | SREM key member [member …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SUNION | Yes* | SUNION key [key ...] | + | SSCAN | Yes | SSCAN key cursor [MATCH] [COUNT] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SUNIONSTORE | Yes* | SUNIONSTORE destination key [key ...] | + | SUNION | Yes* | SUNION key [key …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SSCAN | Yes | SSCAN key cursor [MATCH pattern] [COUNT count] | + | SUNIONSTORE | Yes* | SUNIONSTORE destination key [key …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ * SIDFF, SDIFFSTORE, SINTER, SINTERSTORE, SMOVE, SUNION and SUNIONSTORE support requires that the supplied keys hash to the same server. You can ensure this by using the same [hashtag](recommendation.md#hash-tags) for all keys in the command. Twemproxy does no checking on its end to verify that all the keys hash to the same server, and the given command is forwarded to the server that the first key hashes to. @@ -222,45 +238,69 @@ +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | Command | Supported? | Format | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZADD | Yes | ZADD key score member [score] [member] | + | BZPOPMAX | No | BZPOPMAX key [key …] timeout | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | BZPOPMIN | No | BZPOPMIN key [key …] timeout | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZCARD | Yes | ZCARD key | + | ZADD | Yes | ZADD key [NX|XX] [GT|LT] [CH] [INCR] score member [score member …] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | ZCARD | Yes | ZCARD key | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | ZCOUNT | Yes | ZCOUNT key min max | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | ZDIFF | Yes* | ZDIFF numkeys key [key …] [WITHSCORES] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | ZDIFFSTORE | Yes* | ZDIFFSTORE destination numkeys key [key …] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | ZINCRBY | Yes | ZINCRBY key increment member | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZINTERSTORE | Yes* | ZINTERSTORE destination numkeys key [key ...] [WEIGHTS weight [weight ...]] [AGGREGATE SUM|MIN|MAX] | - +------------------------------------------------------------------------------------------------------------------------------------------------------+ - | ZLEXCOUNT | Yes | ZLEXCOUNT key min max | + | ZINTER | Yes* | ZINTER numkeys key [key …] [WEIGHTS] [SUM|MIN|MAX] [WITHSCORES] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZRANGE | Yes | ZRANGE key start stop [WITHSCORES] | - +------------------------------------------------------------------------------------------------------------------------------------------------------+ - | ZRANGEBYLEX | Yes | ZRANGEBYLEX key min max [LIMIT offset count] | + | ZINTERSTORE | Yes* | ZINTERSTORE destination numkeys key [key …] [WEIGHTS] [SUM|MIN|MAX] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZRANGEBYSCORE | Yes | ZRANGEBYSCORE key min max [WITHSCORES] [LIMIT offset count] | + | ZLEXCOUNT | Yes | ZLEXCOUNT key min max | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZRANK | Yes | ZRANK key member | + | ZMSCORE | Yes | ZMSCORE key member [member …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZREM | Yes | ZREM key member [member ...] | + | ZPOPMAX | Yes | ZPOPMAX key [count] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZREMRANGEBYLEX | Yes | ZREMRANGEBYLEX key min max | + | ZPOPMIN | Yes | ZPOPMIN key [count] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZREMRANGEBYRANK | Yes | ZREMRANGEBYRANK key start stop | + | ZRANDMEMBER | Yes | ZRANDMEMBER key [options] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZREMRANGEBYSCORE | Yes | ZREMRANGEBYSCORE key min max | + | ZRANGE | Yes | ZRANGE key min max [BYSCORE|BYLEX] [REV] [LIMIT] [WITHSCORES] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZREVRANGE | Yes | ZREVRANGE key start stop [WITHSCORES] | + | ZRANGEBYLEX | Yes | ZRANGEBYLEX key min max [LIMIT] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZREVRANGEBYSCORE | Yes | ZREVRANGEBYSCORE key max min [WITHSCORES] [LIMIT offset count] | + | ZRANGEBYSCORE | Yes | ZRANGEBYSCORE key min max [WITHSCORES] [LIMIT] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | ZRANGESTORE | Yes | ZRANGESTORE dst src min max [BYSCORE|BYLEX] [REV] [LIMIT] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | ZRANK | Yes | ZRANK key member | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | ZREM | Yes | ZREM key member [member …] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | ZREMRANGEBYLEX | Yes | ZREMRANGEBYLEX key min max | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | ZREMRANGEBYRANK | Yes | ZREMRANGEBYRANK key start stop | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | ZREMRANGEBYSCORE | Yes | ZREMRANGEBYSCORE key min max | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | ZREVRANGE | Yes | ZREVRANGE key start stop [WITHSCORES] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | ZREVRANGEBYLEX | Yes | ZREVRANGEBYLEX key max min [LIMIT] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | ZREVRANGEBYSCORE | Yes | ZREVRANGEBYSCORE key max min [WITHSCORES] [LIMIT] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | ZREVRANK | Yes | ZREVRANK key member | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZSCORE | Yes | ZSCORE key member | + | ZSCAN | Yes | ZSCAN key cursor [MATCH] [COUNT] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZUNIONSTORE | Yes* | ZUNIONSTORE destination numkeys key [key ...] [WEIGHTS weight [weight ...]] [AGGREGATE SUM|MIN|MAX] | + | ZSCORE | Yes | ZSCORE key member | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | ZSCAN | Yes | ZSCAN key cursor [MATCH pattern] [COUNT count] | + | ZUNION | Yes* | ZUNION numkeys key [key …] [WEIGHTS] [SUM|MIN|MAX] [WITHSCORES] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | ZUNIONSTORE | Yes* | ZUNIONSTORE destination numkeys key [key …] [WEIGHTS] [SUM|MIN|MAX] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ * ZINTERSTORE and ZUNIONSTORE support requires that the supplied keys hash to the same server. You can ensure this by using the same [hashtag](recommendation.md#hash-tags) for all keys in the command. Twemproxy does no checking on its end to verify that all the keys hash to the same server, and the given command is forwarded to the server that the first key hashes to. @@ -270,30 +310,53 @@ +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | Command | Supported? | Format | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | PFADD | Yes | PFADD key element [element ...] | + | PFADD | Yes | PFADD key element [element …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | PFCOUNT | Yes | PFCOUNT key [key ...] | + | PFCOUNT | Yes | PFCOUNT key [key …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | PFMERGE | Yes* | PFMERGE destkey sourcekey [sourcekey ...] | + | PFMERGE | Yes* | PFMERGE destkey sourcekey [sourcekey …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ * PFMERGE support requires that the supplied keys hash to the same server. You can ensure this by using the same [hashtag](recommendation.md#hash-tags) for all keys in the command. Twemproxy does no checking on its end to verify that all the keys hash to the same server, and the given command is forwarded to the server that the first key hashes to. +### Geo Command + + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | Command | Supported? | Format | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | GEOADD | Yes | GEOADD key [NX|XX] [CH] longitude latitude member [longitude latitude member …] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | GEODIST | Yes | GEODIST key member1 member2 [m|km|ft|mi] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | GEOHASH | Yes | GEOHASH key member [member …] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | GEOPOS | Yes | GEOPOS key member [member …] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | GEORADIUS | Yes | GEORADIUS key longitude latitude radius m|km|ft|mi [WITHCOORD] [WITHDIST] [WITHHASH] [count] [ASC|DESC] [STORE] [STOREDIST]| + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | GEORADIUSBYMEMBER | Yes | GEORADIUSBYMEMBER key member radius m|km|ft|mi [WITHCOORD] [WITHDIST] [WITHHASH] [count] [ASC|DESC] [STORE] [STOREDIST]| + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | GEOSEARCH | Yes | GEOSEARCH key [FROMMEMBER] [FROMLONLAT] [circle] [box] [ASC|DESC] [count] [WITHCOORD] [WITHDIST] [WITHHASH] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | GEOSEARCHSTORE | Yes | GEOSEARCHSTORE destination source [FROMMEMBER] [FROMLONLAT] [circle] [box] [ASC|DESC] [count] [WITHCOORD] [WITHDIST] [WITHHASH] [STOREDIST]| + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ ### Pub/Sub +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | Command | Supported? | Format | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | PSUBSCRIBE | No | PSUBSCRIBE pattern [pattern ...] | + | PSUBSCRIBE | No | PSUBSCRIBE pattern [pattern …] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | PUBLISH | No | PUBLISH channel message | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | PUBLISH | No | PUBLISH channel message | + | PUBSUB | No | PUBSUB subcommand [argument [argument …]] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | PUNSUBSCRIBE | No | PUNSUBSCRIBE [pattern [pattern ...]] | + | PUNSUBSCRIBE | No | PUNSUBSCRIBE [pattern [pattern …]] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SUBSCRIBE | No | SUBSCRIBE channel [channel ...] | + | SUBSCRIBE | No | SUBSCRIBE channel [channel …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | UNSUBSCRIBE | No | UNSUBSCRIBE [channel [channel ...]] | + | UNSUBSCRIBE | No | UNSUBSCRIBE [channel [channel …]] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ ### Transactions @@ -309,7 +372,7 @@ +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | UNWATCH | No | UNWATCH | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | WATCH | No | WATCH key [key ...] | + | WATCH | No | WATCH key [key …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ ### Scripting @@ -317,13 +380,15 @@ +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | Command | Supported? | Format | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | EVAL | Yes* | EVAL script numkeys key [key ...] arg [arg ...] | + | EVAL | Yes* | EVAL script numkeys key [key …] arg [arg …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | EVALSHA | Yes* | EVALSHA sha1 numkeys key [key ...] arg [arg ...] | + | EVALSHA | Yes* | EVALSHA sha1 numkeys key [key …] arg [arg …] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SCRIPT EXISTS | No | SCRIPT EXISTS script [script ...] | + | SCRIPT DEBUG | No | SCRIPT DEBUG YES|SYNC|NO | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SCRIPT FLUSH | No | SCRIPT FLUSH | + | SCRIPT EXISTS | No | SCRIPT EXISTS sha1 [sha1 …] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | SCRIPT FLUSH | No | SCRIPT FLUSH [ASYNC|SYNC] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | SCRIPT KILL | No | SCRIPT KILL | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ @@ -395,11 +460,22 @@ +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | TIME | No | TIME | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | COMMAND | Yes | COMMAND | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + | LOLWUT | Yes | LOLWUT [ argument ...] | + +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ + +* COMMAND is forwarded to an arbitrarily chosen redis backend for the benefit of tools such as `redis-cli`. + It assumes that all proxied redis servers support the same backend. + There is no post-processing; Commands that are not supported by twemproxy are not filtered out. +* LOLWUT is also forwarded to an arbitrarily chosen redis backend because it does not depend on the state of the database and the presence of the command would not affect production applications. + It can be used as a reference for adding future commands with no keys and a variable-length argument list. + Other commands such as `TIME` continue to be unsupported. They may vary based on state of the database (e.g. out of sync clocks). Using EVAL with a key known to be on the specific backend server can be done instead. ## Note - redis commands are not case sensitive -- only vectored commands 'MGET key [key ...]', 'MSET key value [key value ...]', 'DEL key [key ...]' needs to be fragmented +- only vectored commands 'MGET key [key ...]', 'MSET key value [key value ...]', 'DEL key [key ...]', 'UNLINK key [key ...]', 'EXISTS key [key ...]' needs to be fragmented ## Performance @@ -458,6 +534,30 @@ + notice: + *MUST* set all redis with a same passwd, and all twemproxy with the same passwd - + Length of password should less than 256 + + Length of password should be less than 256 + +## redis-sentinel feature + ++ You can configure sentinel for a pool with 'sentinels' to let twemproxy works with sentinel: + sigma: + listen: 127.0.0.1:22125 + hash: fnv1a_64 + distribution: ketama + auto_eject_hosts: false + redis: true + server_retry_timeout: 2000 + server_failure_limit: 1 + servers: + - 127.0.0.1:6379:1 server1 + - 127.0.0.1:6380:1 server2 + sentinels: + - 127.0.0.1:26379:1 + - 127.0.0.1:26380:1 + - 127.0.0.1:26381:1 + ++ notice: + + You should configure all the sentinels you used. Twemproxy will connect to the alive sentinels when some are down + + Weight of sentinel is not used. Twemproxy keeps it because of the server load code reuse +See [sentinel.md](./sentinel.md) for more details. diff --git a/scripts/extract_redis_commands_argcounts.php b/scripts/extract_redis_commands_argcounts.php new file mode 100755 index 00000000..fbeeeeec --- /dev/null +++ b/scripts/extract_redis_commands_argcounts.php @@ -0,0 +1,359 @@ +#!/usr/bin/env php + version_compare($b['since'], $a['since'])); + +const INFINITE_ARGS = 100000; + +function categorize_arg(array $arg, string $commandName): array { + $min = 1; + $max = 1; + if ($arg['multiple']) { + $min = 0; + $max = INFINITE_ARGS; + } + if ($arg['optional']) { + $min = 0; + } + if ($arg['type'] === 'key') { + return ['min_key' => $min, 'max_key' => $max]; + } + return ['min_arg' => $min, 'max_arg' => $max]; +} + +function categorize(array $command, string $commandName): string { + $minKeyCount = 0; + $maxKeyCount = 0; + $minArgCount = 0; + $maxArgCount = 0; + $arguments = $command['arguments'] ?? []; + foreach ($arguments as $arg) { + $data = categorize_arg($arg, $commandName); + $minKeyCount += ($data['min_key'] ?? 0); + $maxKeyCount += ($data['max_key'] ?? 0); + $minArgCount += ($data['min_arg'] ?? 0); + $maxArgCount += ($data['max_arg'] ?? 0); + } + if (in_array($commandName, ['DEL', 'MGET', 'MSET', 'TOUCH', 'UNLINK'])) { + return "keyn"; + } + if ($maxKeyCount > $minKeyCount || $maxArgCount > $minArgCount) { + // return "key${minKeyCount}_argx"; + return "key1_argx"; + } + // Assume that + // min=max for arg and key + if ($minArgCount > 0 && $minKeyCount >= 2) { + return "key1_arg" . ($minArgCount + $maxKeyCount - 1); + } + return "key${minKeyCount}_arg" . $minArgCount; +} + +const KEY1 = [ + 'PERSIST', + 'PTTL', + 'TTL', + 'TYPE', + 'DUMP', + + 'DECR', + 'GET', + 'GETDEL', + 'INCR', + 'STRLEN', + + 'HGETALL', + 'HKEYS', + 'HLEN', + 'HVALS', + + 'LLEN', + + 'SCARD', + 'SMEMBERS', + + 'ZCARD', + // 'AUTH', +]; + +const KEY1_ARG1 = [ + 'EXPIRE', + 'EXPIREAT', + 'PEXPIRE', + 'PEXPIREAT', + 'MOVE', + + 'APPEND', + 'DECRBY', + 'GETBIT', + 'GETSET', + 'INCRBY', + 'INCRBYFLOAT', + 'SETNX', + + 'HEXISTS', + 'HGET', + 'HSTRLEN', + + 'LINDEX', + 'RPOPLPUSH', + + 'SISMEMBER', + + 'ZRANK', + 'ZREVRANK', + 'ZSCORE', +]; + +const KEY1_ARG2 = [ + 'GETRANGE', + 'PSETEX', + 'SETBIT', + 'SETEX', + 'SETRANGE', + + 'HINCRBY', + 'HINCRBYFLOAT', + 'HSETNX', + + 'LRANGE', + 'LREM', + 'LSET', + 'LTRIM', + + 'SMOVE', + + 'ZCOUNT', + 'ZLEXCOUNT', + 'ZINCRBY', + 'ZREMRANGEBYLEX', + 'ZREMRANGEBYRANK', + 'ZREMRANGEBYSCORE', +]; + +const KEY1_ARG3 = [ + 'LINSERT', + 'LMOVE', +]; + +const KEY1_ARGN = [ + 'SORT', + + 'BITCOUNT', + 'BITPOS', + 'BITFIELD', + 'BITOP', + + 'EXISTS', + 'GETEX', + 'SET', + + 'HDEL', + 'HMGET', + 'HMSET', + 'HSCAN', + 'HSET', + 'HRANDFIELD', + + 'LPUSH', + 'LPUSHX', + 'RPUSH', + 'RPUSHX', + 'LPOP', + 'RPOP', + 'LPOS', + + 'SADD', + 'SDIFF', + 'SDIFFSTORE', + 'SINTER', + 'SINTERSTORE', + 'SREM', + 'SUNION', + 'SUNIONSTORE', + 'SRANDMEMBER', + 'SSCAN', + 'SPOP', + 'SMISMEMBER', + + 'PFADD', + 'PFMERGE', + 'PFCOUNT', + + 'ZADD', + 'ZDIFF', + 'ZDIFFSTORE', + 'ZINTER', + 'ZINTERSTORE', + 'ZMSCORE', + 'ZPOPMAX', + 'ZPOPMIN', + 'ZRANDMEMBER', + 'ZRANGE', + 'ZRANGEBYLEX', + 'ZRANGEBYSCORE', + 'ZRANGESTORE', + 'ZREM', + 'ZREVRANGE', + 'ZREVRANGEBYLEX', + 'ZREVRANGEBYSCORE', + 'ZSCAN', + 'ZUNION', + 'ZUNIONSTORE', + + 'GEODIST', + 'GEOPOS', + 'GEOHASH', + 'GEOADD', + 'GEOSEARCH', + + 'RESTORE', +]; + +const EXPECTED_MAPS = [ + 'key1_arg0' => KEY1, + 'key1_arg1' => KEY1_ARG1, + 'key1_arg2' => KEY1_ARG2, + 'key1_arg3' => KEY1_ARG3, + 'key1_argx' => KEY1_ARGN, +]; + +function compute_types(): array { + global $commands; + $types = []; + foreach ($commands as $name => $cmd) { + // printf("%s: %s\n", $name, json_encode($cmd, JSON_PRETTY_PRINT)); + try { + $type = categorize($cmd, $name); + } catch (Exception $e) { + $type = "unknown: {$e->getMessage()} " . json_encode($cmd); + } + $types[$name] = $type; + } + return $types; +} + +function dump_mismatched_argument_types(array $types, array $commands): void { + foreach (EXPECTED_MAPS as $expected => $maps) { + foreach ($maps as $key) { + $actual = $types[$key]; + if ($actual !== $expected) { + echo "Unexpected type for $key: got $actual, want $expected: " . json_encode($commands[$key]['arguments']) . "\n"; + } + } + foreach ($types as $other_name => $type) { + if ($type === $expected && !in_array($other_name, $maps)) { + $command = $commands[$other_name]; + echo "Expected $other_name in $expected: " . json_encode($command['arguments']) . "\n"; + echo "> " . $command['group'] . ": " . $command['summary'] . "\n\n"; + } + } + } +} + +function render_arg(array $argument): string { + if ($argument['optional'] ?? false) { + unset($argument['optional']); + return '[' . render_arg($argument) . ']'; + } + if ($argument['enum'] ?? null) { + return implode('|', $argument['enum']); + } + if ($argument['command'] ?? null) { + return $argument['command']; + } + $name = $argument['name']; + + $repr = is_array($name) ? implode(' ', $name) : $name; + if ($argument['multiple'] ?? false) { + return "$repr [$repr …]"; + } + return $repr; +} + +function render_command(string $name, array $command): string { + $repr = $name; + foreach ($command['arguments'] ?? [] as $argument) { + $repr .= ' ' . render_arg($argument); + } + return $repr; +} + +function center_pad(string $name, int $len) { + if (mb_strlen($name) >= $len) { + return $name; + } + $name = str_repeat(' ', ($len - mb_strlen($name)) >> 1) . $name; + $name .= str_repeat(' ', $len - mb_strlen($name)); + return $name; +} + +function right_pad(string $name, int $len) { + if (mb_strlen($name) >= $len) { + return $name; + } + $name .= str_repeat(' ', $len - mb_strlen($name)); + return $name; +} + +function dump_table(array $commands) { + $header = << $command) { + $key = center_pad($name, 19); + $commandRepr = render_command($name, $command); + $supports = 'Yes'; + printf(" |%s|%s|%s|\n", center_pad($name, $nameLen), center_pad($supports, $supportsLen), right_pad(' ' . $commandRepr, $commandLen)); + echo $rowLine . "\n"; + + } + echo "\n"; +} + +function dump_table_groups(array $commands): void { + $groups = []; + foreach ($commands as $name => $command) { + $groups[$command['group']][$name] = $command; + } + foreach ($groups as $groupName => $group) { + printf("### %s Command\n\n", $groupName); + + dump_table($group); + } +} + +$types = compute_types(); +foreach ($types as $name => $type) { + printf("%s: %s\n", $name, $type); +} + +dump_mismatched_argument_types($types, $commands); +dump_table_groups($commands); diff --git a/scripts/nutcracker.spec b/scripts/nutcracker.spec index a8b529ae..d109c007 100644 --- a/scripts/nutcracker.spec +++ b/scripts/nutcracker.spec @@ -1,6 +1,6 @@ Summary: Twitter's nutcracker redis and memcached proxy Name: nutcracker -Version: 0.4.1 +Version: 0.5.0 Release: 1 URL: https://github.com/twitter/twemproxy/ @@ -66,6 +66,34 @@ fi %config(noreplace)%{_sysconfdir}/%{name}/%{name}.yml %changelog +* Tue Jul 06 2021 Tyson Andre +- twemproxy: version 0.5.0 release +- Same as 0.5.0-RC1 + +* Tue Jul 06 2021 Tyson Andre +- twemproxy: version 0.5.0-RC1 release +- Add 'tcpkeepalive' pool boolean config flag setting to enable tcp keepalive (charsyam, manju) +- Support redis bitpos command (clark kang) +- Fix parsing of redis error response for error type with no space, add tests (tyson, tom dalton) +- Update integration tests, add C unit test suite for 'make check' (tyson) +- Increase the maximum host length+port+identifier to 273 in ketama_update (李广博) +- Always initialize file permissions field when listening on a unix domain socket (tyson) +- Use number of servers instead of number of points on the continuum when sharding requests to backend services to improve sharding performance and fix potential invalid memory access when all hosts were ejected from a pool. (tyson) +- Optimize performance of deletion of single redis keys (vincentve) +- Don't fragment memcache/redis get commands when they only have a single key (improves performance and error handling of single key case) (tyson) +- Don't let requests hang when there is a dns error when processing a fragmented request (e.g. multiget with multiple keys) (tyson) +- Allow extra parameters for redis spop (charsyam) +- Update documentation and README (various) +- Fix memory leak bug for redis mset (deep011) +- Support arbitrarily deep nested redis multi-bulk responses (nested arrays) (qingping209, tyson) +- Upgrade from libyaml 0.1.4 to 0.2.5 (tyson) +- Fix compiler warnings about wrong conversion specifiers in format strings for logging (tyson) +- Log the async backend used and any debug options in the '--help'/'--version' output. +- Add support for many more new redis commands and updates to existing redis commands (tyson) +- Optimization: Skip hashing and choosing server index when a pool has exactly one server (tyson) +- Support memcache 'version' requests by proxying the request to a single backend memcache server to fetch the server version. (tyson) +- Make error messages for creating the stats server during startup clearer. (tyson) + * Mon Jun 22 2015 Manju Rajashekhar - twemproxy: version 0.4.1 release - backend server hostnames are resolved lazily @@ -109,7 +137,7 @@ fi - support for set ex/px/nx/xx for redis 2.6.12 and up (ypocat) - kqueue (bsd) support (ferenyx) - fix parsing redis response to accept integer reply (charsyam) - + * Tue Jul 30 2013 Tait Clarridge - Rebuild SPEC to work with CentOS - Added buildrequires if building with mock/koji diff --git a/src/Makefile.am b/src/Makefile.am index 3d400c5e..dd07a25a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,9 +7,9 @@ endif AM_CPPFLAGS += -I $(top_srcdir)/src/hashkit AM_CPPFLAGS += -I $(top_srcdir)/src/proto AM_CPPFLAGS += -I $(top_srcdir)/src/event -AM_CPPFLAGS += -I $(top_srcdir)/contrib/yaml-0.1.4/include +AM_CPPFLAGS += -I $(top_srcdir)/contrib/yaml-0.2.5/include -AM_CFLAGS = +AM_CFLAGS = # about -fno-strict-aliasing: https://github.com/twitter/twemproxy/issues/276 AM_CFLAGS += -fno-strict-aliasing AM_CFLAGS += -Wall -Wshadow @@ -19,6 +19,7 @@ AM_CFLAGS += -Wunused-function -Wunused-variable -Wunused-value AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value AM_CFLAGS += -Wconversion -Wsign-compare AM_CFLAGS += -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Wmissing-declarations +AM_CFLAGS += -Wno-format-zero-length AM_LDFLAGS = AM_LDFLAGS += -lm -lpthread -rdynamic @@ -57,4 +58,32 @@ nutcracker_SOURCES = \ nutcracker_LDADD = $(top_builddir)/src/hashkit/libhashkit.a nutcracker_LDADD += $(top_builddir)/src/proto/libproto.a nutcracker_LDADD += $(top_builddir)/src/event/libevent.a -nutcracker_LDADD += $(top_builddir)/contrib/yaml-0.1.4/src/.libs/libyaml.a +nutcracker_LDADD += $(top_builddir)/contrib/yaml-0.2.5/src/.libs/libyaml.a + +TESTS = test_all +bin_PROGRAMS = test_all + +test_all_SOURCES = test_all.c \ + nc_core.c nc_core.h \ + nc_connection.c nc_connection.h \ + nc_client.c nc_client.h \ + nc_server.c nc_server.h \ + nc_proxy.c nc_proxy.h \ + nc_message.c nc_message.h \ + nc_request.c \ + nc_response.c \ + nc_mbuf.c nc_mbuf.h \ + nc_conf.c nc_conf.h \ + nc_stats.c nc_stats.h \ + nc_signal.c nc_signal.h \ + nc_rbtree.c nc_rbtree.h \ + nc_log.c nc_log.h \ + nc_string.c nc_string.h \ + nc_array.c nc_array.h \ + nc_util.c nc_util.h \ + nc_queue.h + +test_all_LDADD = $(top_builddir)/src/hashkit/libhashkit.a +test_all_LDADD += $(top_builddir)/src/proto/libproto.a +test_all_LDADD += $(top_builddir)/src/event/libevent.a +test_all_LDADD += $(top_builddir)/contrib/yaml-0.2.5/src/.libs/libyaml.a diff --git a/src/hashkit/nc_crc32.c b/src/hashkit/nc_crc32.c index f5223285..7a7eafdb 100644 --- a/src/hashkit/nc_crc32.c +++ b/src/hashkit/nc_crc32.c @@ -111,7 +111,7 @@ hash_crc32(const char *key, size_t key_length) uint32_t hash_crc32a(const char *key, size_t key_length) { - const uint8_t *p = key; + const uint8_t *p = (const uint8_t *)key; uint32_t crc; crc = ~0U; diff --git a/src/hashkit/nc_fnv.c b/src/hashkit/nc_fnv.c index 7ea3c450..3fcf5ef5 100644 --- a/src/hashkit/nc_fnv.c +++ b/src/hashkit/nc_fnv.c @@ -17,10 +17,10 @@ #include -static uint64_t FNV_64_INIT = UINT64_C(0xcbf29ce484222325); -static uint64_t FNV_64_PRIME = UINT64_C(0x100000001b3); -static uint32_t FNV_32_INIT = 2166136261UL; -static uint32_t FNV_32_PRIME = 16777619; +static const uint64_t FNV_64_INIT = UINT64_C(0xcbf29ce484222325); +static const uint64_t FNV_64_PRIME = UINT64_C(0x100000001b3); +static const uint32_t FNV_32_INIT = 2166136261UL; +static const uint32_t FNV_32_PRIME = 16777619; uint32_t hash_fnv1_64(const char *key, size_t key_length) diff --git a/src/hashkit/nc_hashkit.h b/src/hashkit/nc_hashkit.h index 544e38b5..5956edf4 100644 --- a/src/hashkit/nc_hashkit.h +++ b/src/hashkit/nc_hashkit.h @@ -69,10 +69,11 @@ uint32_t hash_jenkins(const char *key, size_t length); uint32_t hash_murmur(const char *key, size_t length); rstatus_t ketama_update(struct server_pool *pool); -uint32_t ketama_dispatch(struct continuum *continuum, uint32_t ncontinuum, uint32_t hash); +uint32_t ketama_dispatch(const struct continuum *continuum, uint32_t ncontinuum, uint32_t hash); rstatus_t modula_update(struct server_pool *pool); -uint32_t modula_dispatch(struct continuum *continuum, uint32_t ncontinuum, uint32_t hash); +uint32_t modula_dispatch(const struct continuum *continuum, uint32_t ncontinuum, uint32_t hash); rstatus_t random_update(struct server_pool *pool); -uint32_t random_dispatch(struct continuum *continuum, uint32_t ncontinuum, uint32_t hash); +uint32_t random_dispatch(const struct continuum *continuum, uint32_t ncontinuum, uint32_t hash); +uint32_t ketama_hash(const char *key, size_t key_length, uint32_t alignment); #endif diff --git a/src/hashkit/nc_ketama.c b/src/hashkit/nc_ketama.c index 9d43f2b2..87e05faf 100644 --- a/src/hashkit/nc_ketama.c +++ b/src/hashkit/nc_ketama.c @@ -25,9 +25,9 @@ #define KETAMA_CONTINUUM_ADDITION 10 /* # extra slots to build into continuum */ #define KETAMA_POINTS_PER_SERVER 160 /* 40 points per hash */ -#define KETAMA_MAX_HOSTLEN 86 +#define KETAMA_MAX_HOSTLEN 273 /* 273 is 255(domain or ip)+1(:)+5(port)+1(-)+10(uint32)+1(\0) */ -static uint32_t +uint32_t ketama_hash(const char *key, size_t key_length, uint32_t alignment) { unsigned char results[16]; @@ -178,6 +178,12 @@ ketama_update(struct server_pool *pool) hostlen = snprintf(host, KETAMA_MAX_HOSTLEN, "%.*s-%u", server->name.len, server->name.data, pointer_index - 1); + if (hostlen >= KETAMA_MAX_HOSTLEN) { + // > The generated string has a length of at most n-1, leaving space for the additional terminating null character. + // Not really important since this should never get hit in practice according to https://devblogs.microsoft.com/oldnewthing/20120412-00/?p=7873 + hostlen = KETAMA_MAX_HOSTLEN - 1; + log_error("Unexpectedly forced to truncate a hostname in ketama pool to %d characters for %.*s", KETAMA_MAX_HOSTLEN - 1, KETAMA_MAX_HOSTLEN - 1, host); + } for (x = 0; x < pointer_per_hash; x++) { value = ketama_hash(host, hostlen, x); @@ -213,9 +219,9 @@ ketama_update(struct server_pool *pool) } uint32_t -ketama_dispatch(struct continuum *continuum, uint32_t ncontinuum, uint32_t hash) +ketama_dispatch(const struct continuum *continuum, uint32_t ncontinuum, uint32_t hash) { - struct continuum *begin, *end, *left, *right, *middle; + const struct continuum *begin, *end, *left, *right, *middle; ASSERT(continuum != NULL); ASSERT(ncontinuum != 0); diff --git a/src/hashkit/nc_md5.c b/src/hashkit/nc_md5.c index 94faa169..7bd73a8b 100644 --- a/src/hashkit/nc_md5.c +++ b/src/hashkit/nc_md5.c @@ -85,10 +85,10 @@ typedef struct { * This processes one or more 64-byte data blocks, but does NOT update * the bit counters. There are no alignment requirements. */ -static void * -body(MD5_CTX *ctx, void *data, unsigned long size) +static const void * +body(MD5_CTX *ctx, const void *data, unsigned long size) { - unsigned char *ptr; + const unsigned char *ptr; MD5_u32plus a, b, c, d; MD5_u32plus saved_a, saved_b, saved_c, saved_d; @@ -206,7 +206,7 @@ MD5_Init(MD5_CTX *ctx) } void -MD5_Update(MD5_CTX *ctx, void *data, unsigned long size) +MD5_Update(MD5_CTX *ctx, const void *data, unsigned long size) { MD5_u32plus saved_lo; unsigned long used, free; @@ -228,7 +228,7 @@ MD5_Update(MD5_CTX *ctx, void *data, unsigned long size) } memcpy(&ctx->buffer[used], data, free); - data = (unsigned char *)data + free; + data = (const unsigned char *)data + free; size -= free; body(ctx, ctx->buffer, 64); } @@ -298,7 +298,7 @@ MD5_Final(unsigned char *result, MD5_CTX *ctx) * result must be == 16 */ void -md5_signature(unsigned char *key, unsigned long length, unsigned char *result) +md5_signature(const unsigned char *key, unsigned long length, unsigned char *result) { MD5_CTX my_md5; @@ -312,7 +312,7 @@ hash_md5(const char *key, size_t key_length) { unsigned char results[16]; - md5_signature((unsigned char*)key, (unsigned long)key_length, results); + md5_signature((const unsigned char*)key, (unsigned long)key_length, results); return ((uint32_t) (results[3] & 0xFF) << 24) | ((uint32_t) (results[2] & 0xFF) << 16) | diff --git a/src/hashkit/nc_modula.c b/src/hashkit/nc_modula.c index 083f89a9..9b973cf5 100644 --- a/src/hashkit/nc_modula.c +++ b/src/hashkit/nc_modula.c @@ -143,9 +143,9 @@ modula_update(struct server_pool *pool) } uint32_t -modula_dispatch(struct continuum *continuum, uint32_t ncontinuum, uint32_t hash) +modula_dispatch(const struct continuum *continuum, uint32_t ncontinuum, uint32_t hash) { - struct continuum *c; + const struct continuum *c; ASSERT(continuum != NULL); ASSERT(ncontinuum != 0); diff --git a/src/hashkit/nc_random.c b/src/hashkit/nc_random.c index 8c6261c7..375fd020 100644 --- a/src/hashkit/nc_random.c +++ b/src/hashkit/nc_random.c @@ -133,9 +133,9 @@ random_update(struct server_pool *pool) } uint32_t -random_dispatch(struct continuum *continuum, uint32_t ncontinuum, uint32_t hash) +random_dispatch(const struct continuum *continuum, uint32_t ncontinuum, uint32_t hash) { - struct continuum *c; + const struct continuum *c; ASSERT(continuum != NULL); ASSERT(ncontinuum != 0); diff --git a/src/nc.c b/src/nc.c index bde044c2..0d606bad 100644 --- a/src/nc.c +++ b/src/nc.c @@ -51,7 +51,7 @@ static int test_conf; static int daemonize; static int describe_stats; -static struct option long_options[] = { +static const struct option long_options[] = { { "help", no_argument, NULL, 'h' }, { "version", no_argument, NULL, 'V' }, { "test-conf", no_argument, NULL, 't' }, @@ -68,7 +68,7 @@ static struct option long_options[] = { { NULL, 0, NULL, 0 } }; -static char short_options[] = "hVtdDv:o:c:s:i:a:p:m:"; +static const char short_options[] = "hVtdDv:o:c:s:i:a:p:m:"; static rstatus_t nc_daemonize(int dump_core) @@ -173,7 +173,7 @@ nc_daemonize(int dump_core) } static void -nc_print_run(struct instance *nci) +nc_print_run(const struct instance *nci) { int status; struct utsname name; @@ -397,8 +397,8 @@ nc_get_options(int argc, char **argv, struct instance *nci) } if (value < NC_MBUF_MIN_SIZE || value > NC_MBUF_MAX_SIZE) { - log_stderr("nutcracker: mbuf chunk size must be between %zu and" - " %zu bytes", NC_MBUF_MIN_SIZE, NC_MBUF_MAX_SIZE); + log_stderr("nutcracker: mbuf chunk size must be between %d and" + " %d bytes", NC_MBUF_MIN_SIZE, NC_MBUF_MAX_SIZE); return NC_ERROR; } @@ -446,7 +446,7 @@ nc_get_options(int argc, char **argv, struct instance *nci) * returns false */ static bool -nc_test_conf(struct instance *nci) +nc_test_conf(const struct instance *nci) { struct conf *cf; @@ -551,7 +551,22 @@ main(int argc, char **argv) } if (show_version) { - log_stderr("This is nutcracker-%s" CRLF, NC_VERSION_STRING); + log_stderr("This is nutcracker-%s", NC_VERSION_STRING); +#if NC_HAVE_EPOLL + log_stderr("async event backend: epoll"); +#elif NC_HAVE_KQUEUE + log_stderr("async event backend: kqueue"); +#elif NC_HAVE_EVENT_PORTS + log_stderr("async event backend: event_ports"); +#else + log_stderr("async event backend: unknown"); +#endif +#if HAVE_ASSERT_PANIC || HAVE_ASSERT_LOG + log_stderr("debugging assertions are enabled (--enable-debug=yes|full), nutcracker may be less efficient"); +#endif + // Log a blank line after the version + log_stderr(""); + if (show_help) { nc_show_usage(); } diff --git a/src/nc_array.c b/src/nc_array.c index 8a389827..efc8c08c 100644 --- a/src/nc_array.c +++ b/src/nc_array.c @@ -79,9 +79,9 @@ array_deinit(struct array *a) } uint32_t -array_idx(struct array *a, void *elem) +array_idx(const struct array *a, const void *elem) { - uint8_t *p, *q; + const uint8_t *p, *q; uint32_t off, idx; ASSERT(elem >= a->elem); @@ -136,7 +136,7 @@ array_pop(struct array *a) } void * -array_get(struct array *a, uint32_t idx) +array_get(const struct array *a, uint32_t idx) { void *elem; @@ -149,7 +149,7 @@ array_get(struct array *a, uint32_t idx) } void * -array_top(struct array *a) +array_top(const struct array *a) { ASSERT(a->nelem != 0); @@ -183,7 +183,7 @@ array_sort(struct array *a, array_compare_t compare) * success. On failure short-circuits and returns the error status. */ rstatus_t -array_each(struct array *a, array_each_t func, void *data) +array_each(const struct array *a, array_each_t func, void *data) { uint32_t i, nelem; diff --git a/src/nc_array.h b/src/nc_array.h index 6e79453c..61669457 100644 --- a/src/nc_array.h +++ b/src/nc_array.h @@ -61,13 +61,13 @@ void array_destroy(struct array *a); rstatus_t array_init(struct array *a, uint32_t n, size_t size); void array_deinit(struct array *a); -uint32_t array_idx(struct array *a, void *elem); +uint32_t array_idx(const struct array *a, const void *elem); void *array_push(struct array *a); void *array_pop(struct array *a); -void *array_get(struct array *a, uint32_t idx); -void *array_top(struct array *a); +void *array_get(const struct array *a, uint32_t idx); +void *array_top(const struct array *a); void array_swap(struct array *a, struct array *b); void array_sort(struct array *a, array_compare_t compare); -rstatus_t array_each(struct array *a, array_each_t func, void *data); +rstatus_t array_each(const struct array *a, array_each_t func, void *data); #endif diff --git a/src/nc_client.c b/src/nc_client.c index b904f61e..8558056e 100644 --- a/src/nc_client.c +++ b/src/nc_client.c @@ -66,7 +66,7 @@ client_unref(struct conn *conn) } bool -client_active(struct conn *conn) +client_active(const struct conn *conn) { ASSERT(conn->client && !conn->proxy); diff --git a/src/nc_client.h b/src/nc_client.h index 2becb0cb..347ca280 100644 --- a/src/nc_client.h +++ b/src/nc_client.h @@ -20,7 +20,7 @@ #include -bool client_active(struct conn *conn); +bool client_active(const struct conn *conn); void client_ref(struct conn *conn, void *owner); void client_unref(struct conn *conn); void client_close(struct context *ctx, struct conn *conn); diff --git a/src/nc_conf.c b/src/nc_conf.c index bd71b0e7..e785754a 100644 --- a/src/nc_conf.c +++ b/src/nc_conf.c @@ -21,27 +21,27 @@ #include #define DEFINE_ACTION(_hash, _name) string(#_name), -static struct string hash_strings[] = { +static const struct string hash_strings[] = { HASH_CODEC( DEFINE_ACTION ) null_string }; #undef DEFINE_ACTION #define DEFINE_ACTION(_hash, _name) hash_##_name, -static hash_t hash_algos[] = { +static const hash_t hash_algos[] = { HASH_CODEC( DEFINE_ACTION ) NULL }; #undef DEFINE_ACTION #define DEFINE_ACTION(_dist, _name) string(#_name), -static struct string dist_strings[] = { +static const struct string dist_strings[] = { DIST_CODEC( DEFINE_ACTION ) null_string }; #undef DEFINE_ACTION -static struct command conf_commands[] = { +static const struct command conf_commands[] = { { string("listen"), conf_set_listen, offsetof(struct conf_pool, listen) }, @@ -125,6 +125,9 @@ static struct command conf_commands[] = { null_command }; +static const struct string true_str = string("true"); +static const struct string false_str = string("false"); + static void conf_server_init(struct conf_server *cs) { @@ -187,7 +190,7 @@ conf_server_each_transform(void *elem, void *data) } static rstatus_t -conf_pool_init(struct conf_pool *cp, struct string *name) +conf_pool_init(struct conf_pool *cp, const struct string *name) { rstatus_t status; @@ -305,9 +308,9 @@ conf_pool_each_transform(void *elem, void *data) sp->redis = cp->redis ? 1 : 0; sp->timeout = cp->timeout; - sp->abort_on_timeout = cp->abort_on_timeout; - sp->abort_on_refused = cp->abort_on_refused; - sp->abort_on_invalid = cp->abort_on_invalid; + sp->abort_on_timeout = cp->abort_on_timeout ? 1 : 0; + sp->abort_on_refused = cp->abort_on_refused ? 1 : 0; + sp->abort_on_invalid = cp->abort_on_invalid ? 1 : 0; sp->backlog = cp->backlog; sp->redis_db = cp->redis_db; @@ -333,7 +336,7 @@ conf_pool_each_transform(void *elem, void *data) } static void -conf_dump(struct conf *cf) +conf_dump(const struct conf *cf) { uint32_t i, j, npool, nserver; struct conf_pool *cp; @@ -518,7 +521,7 @@ conf_pop_scalar(struct conf *cf) static rstatus_t conf_handler(struct conf *cf, void *data) { - struct command *cmd; + const struct command *cmd; struct string *key, *value; uint32_t narg; @@ -536,7 +539,7 @@ conf_handler(struct conf *cf, void *data) value->len, value->data); for (cmd = conf_commands; cmd->name.len != 0; cmd++) { - char *rv; + const char *rv; if (string_compare(key, &cmd->name) != 0) { continue; @@ -771,7 +774,7 @@ conf_parse(struct conf *cf) } static struct conf * -conf_open(char *filename) +conf_open(const char *filename) { rstatus_t status; struct conf *cf; @@ -1329,7 +1332,7 @@ conf_post_validate(struct conf *cf) npool = array_n(&cf->pool); if (npool == 0) { - log_error("conf: '%.*s' has no pools", cf->fname); + log_error("conf: '%s' has no pools", cf->fname); return NC_ERROR; } @@ -1387,7 +1390,7 @@ conf_post_validate(struct conf *cf) } struct conf * -conf_create(char *filename) +conf_create(const char *filename) { rstatus_t status; struct conf *cf; @@ -1447,12 +1450,13 @@ conf_destroy(struct conf *cf) nc_free(cf); } -char * -conf_set_string(struct conf *cf, struct command *cmd, void *conf) +const char * +conf_set_string(struct conf *cf, const struct command *cmd, void *conf) { rstatus_t status; uint8_t *p; - struct string *field, *value; + struct string *field; + const struct string *value; p = conf; field = (struct string *)(p + cmd->offset); @@ -1471,8 +1475,8 @@ conf_set_string(struct conf *cf, struct command *cmd, void *conf) return CONF_OK; } -char * -conf_set_listen(struct conf *cf, struct command *cmd, void *conf) +const char * +conf_set_listen(struct conf *cf, const struct command *cmd, void *conf) { rstatus_t status; struct string *value; @@ -1496,8 +1500,6 @@ conf_set_listen(struct conf *cf, struct command *cmd, void *conf) if (value->data[0] == '/') { uint8_t *q, *start, *perm; - uint32_t permlen; - /* parse "socket_path permissions" from the end */ p = value->data + value->len -1; @@ -1507,9 +1509,9 @@ conf_set_listen(struct conf *cf, struct command *cmd, void *conf) /* no permissions field, so use defaults */ name = value->data; namelen = value->len; + field->perm = (mode_t)0; } else { perm = q + 1; - permlen = (uint32_t)(p - perm + 1); p = q - 1; name = start; @@ -1562,8 +1564,8 @@ conf_set_listen(struct conf *cf, struct command *cmd, void *conf) return CONF_OK; } -char * -conf_add_server(struct conf *cf, struct command *cmd, void *conf) +const char * +conf_add_server(struct conf *cf, const struct command *cmd, void *conf) { rstatus_t status; struct array *a; @@ -1572,7 +1574,7 @@ conf_add_server(struct conf *cf, struct command *cmd, void *conf) uint8_t *p, *q, *start; uint8_t *pname, *addr, *port, *weight, *name; uint32_t k, delimlen, pnamelen, addrlen, portlen, weightlen, namelen; - char delim[] = " ::"; + const char *const delim = " ::"; p = conf; a = (struct array *)(p + cmd->offset); @@ -1702,12 +1704,12 @@ conf_add_server(struct conf *cf, struct command *cmd, void *conf) return CONF_OK; } -char * -conf_set_num(struct conf *cf, struct command *cmd, void *conf) +const char * +conf_set_num(struct conf *cf, const struct command *cmd, void *conf) { uint8_t *p; int num, *np; - struct string *value; + const struct string *value; p = conf; np = (int *)(p + cmd->offset); @@ -1728,12 +1730,12 @@ conf_set_num(struct conf *cf, struct command *cmd, void *conf) return CONF_OK; } -char * -conf_set_bool(struct conf *cf, struct command *cmd, void *conf) +const char * +conf_set_bool(struct conf *cf, const struct command *cmd, void *conf) { uint8_t *p; int *bp; - struct string *value, true_str, false_str; + const struct string *value; p = conf; bp = (int *)(p + cmd->offset); @@ -1743,8 +1745,6 @@ conf_set_bool(struct conf *cf, struct command *cmd, void *conf) } value = array_top(&cf->arg); - string_set_text(&true_str, "true"); - string_set_text(&false_str, "false"); if (string_compare(value, &true_str) == 0) { *bp = 1; @@ -1757,12 +1757,12 @@ conf_set_bool(struct conf *cf, struct command *cmd, void *conf) return CONF_OK; } -char * -conf_set_hash(struct conf *cf, struct command *cmd, void *conf) +const char * +conf_set_hash(struct conf *cf, const struct command *cmd, void *conf) { uint8_t *p; hash_type_t *hp; - struct string *value, *hash; + const struct string *value, *hash; p = conf; hp = (hash_type_t *)(p + cmd->offset); @@ -1778,7 +1778,7 @@ conf_set_hash(struct conf *cf, struct command *cmd, void *conf) continue; } - *hp = hash - hash_strings; + *hp = (hash_type_t)(hash - hash_strings); return CONF_OK; } @@ -1786,12 +1786,12 @@ conf_set_hash(struct conf *cf, struct command *cmd, void *conf) return "is not a valid hash"; } -char * -conf_set_distribution(struct conf *cf, struct command *cmd, void *conf) +const char * +conf_set_distribution(struct conf *cf, const struct command *cmd, void *conf) { uint8_t *p; dist_type_t *dp; - struct string *value, *dist; + const struct string *value, *dist; p = conf; dp = (dist_type_t *)(p + cmd->offset); @@ -1807,7 +1807,7 @@ conf_set_distribution(struct conf *cf, struct command *cmd, void *conf) continue; } - *dp = dist - dist_strings; + *dp = (dist_type_t)(dist - dist_strings); return CONF_OK; } @@ -1815,12 +1815,13 @@ conf_set_distribution(struct conf *cf, struct command *cmd, void *conf) return "is not a valid distribution"; } -char * -conf_set_hashtag(struct conf *cf, struct command *cmd, void *conf) +const char * +conf_set_hashtag(struct conf *cf, const struct command *cmd, void *conf) { rstatus_t status; uint8_t *p; - struct string *field, *value; + struct string *field; + const struct string *value; p = conf; field = (struct string *)(p + cmd->offset); diff --git a/src/nc_conf.h b/src/nc_conf.h index f8020afa..7dd8384f 100644 --- a/src/nc_conf.h +++ b/src/nc_conf.h @@ -104,7 +104,7 @@ struct conf_pool { }; struct conf { - char *fname; /* file name (ref in argv[]) */ + const char *fname; /* file name (ref in argv[]) */ FILE *fh; /* file handle */ struct array arg; /* string[] (parsed {key, value} pairs) */ struct array pool; /* conf_pool[] (parsed pools) */ @@ -123,25 +123,25 @@ struct conf { struct command { struct string name; - char *(*set)(struct conf *cf, struct command *cmd, void *data); + const char *(*set)(struct conf *cf, const struct command *cmd, void *data); int offset; }; #define null_command { null_string, NULL, 0 } -char *conf_set_string(struct conf *cf, struct command *cmd, void *conf); -char *conf_set_listen(struct conf *cf, struct command *cmd, void *conf); -char *conf_add_server(struct conf *cf, struct command *cmd, void *conf); -char *conf_set_num(struct conf *cf, struct command *cmd, void *conf); -char *conf_set_bool(struct conf *cf, struct command *cmd, void *conf); -char *conf_set_hash(struct conf *cf, struct command *cmd, void *conf); -char *conf_set_distribution(struct conf *cf, struct command *cmd, void *conf); -char *conf_set_hashtag(struct conf *cf, struct command *cmd, void *conf); +const char *conf_set_string(struct conf *cf, const struct command *cmd, void *conf); +const char *conf_set_listen(struct conf *cf, const struct command *cmd, void *conf); +const char *conf_add_server(struct conf *cf, const struct command *cmd, void *conf); +const char *conf_set_num(struct conf *cf, const struct command *cmd, void *conf); +const char *conf_set_bool(struct conf *cf, const struct command *cmd, void *conf); +const char *conf_set_hash(struct conf *cf, const struct command *cmd, void *conf); +const char *conf_set_distribution(struct conf *cf, const struct command *cmd, void *conf); +const char *conf_set_hashtag(struct conf *cf, const struct command *cmd, void *conf); rstatus_t conf_server_each_transform(void *elem, void *data); rstatus_t conf_pool_each_transform(void *elem, void *data); -struct conf *conf_create(char *filename); +struct conf *conf_create(const char *filename); void conf_destroy(struct conf *cf); #endif diff --git a/src/nc_connection.c b/src/nc_connection.c index a0d1d9da..d37ac8bb 100644 --- a/src/nc_connection.c +++ b/src/nc_connection.c @@ -91,7 +91,7 @@ static uint32_t ncurr_cconn; /* current # client connections */ * Return the context associated with this connection. */ struct context * -conn_to_ctx(struct conn *conn) +conn_to_ctx(const struct conn *conn) { struct server_pool *pool; @@ -245,9 +245,8 @@ conn_get(void *owner, bool client, bool redis) } struct conn * -conn_get_proxy(void *owner) +conn_get_proxy(struct server_pool *pool) { - struct server_pool *pool = owner; struct conn *conn; conn = _conn_get(); @@ -278,7 +277,7 @@ conn_get_proxy(void *owner) conn->enqueue_outq = NULL; conn->dequeue_outq = NULL; - conn->ref(conn, owner); + conn->ref(conn, pool); log_debug(LOG_VVERB, "get conn %p proxy %d", conn, conn->proxy); @@ -312,7 +311,7 @@ conn_put(struct conn *conn) void conn_init(void) { - log_debug(LOG_DEBUG, "conn size %d", sizeof(struct conn)); + log_debug(LOG_DEBUG, "conn size %d", (int)sizeof(struct conn)); nfree_connq = 0; TAILQ_INIT(&free_connq); } @@ -382,7 +381,7 @@ conn_recv(struct conn *conn, void *buf, size_t size) } ssize_t -conn_sendv(struct conn *conn, struct array *sendv, size_t nsend) +conn_sendv(struct conn *conn, const struct array *sendv, size_t nsend) { ssize_t n; @@ -453,7 +452,7 @@ conn_ncurr_cconn(void) * authentication, otherwise return false */ bool -conn_authenticated(struct conn *conn) +conn_authenticated(const struct conn *conn) { struct server_pool *pool; diff --git a/src/nc_connection.h b/src/nc_connection.h index 7ece25ef..2fe83d42 100644 --- a/src/nc_connection.h +++ b/src/nc_connection.h @@ -29,7 +29,7 @@ typedef struct msg* (*conn_send_next_t)(struct context *, struct conn *); typedef void (*conn_send_done_t)(struct context *, struct conn *, struct msg *); typedef void (*conn_close_t)(struct context *, struct conn *); -typedef bool (*conn_active_t)(struct conn *); +typedef bool (*conn_active_t)(const struct conn *); typedef void (*conn_ref_t)(struct conn *, void *); typedef void (*conn_unref_t)(struct conn *); @@ -93,17 +93,17 @@ struct conn { TAILQ_HEAD(conn_tqh, conn); -struct context *conn_to_ctx(struct conn *conn); +struct context *conn_to_ctx(const struct conn *conn); struct conn *conn_get(void *owner, bool client, bool redis); -struct conn *conn_get_proxy(void *owner); +struct conn *conn_get_proxy(struct server_pool *pool); void conn_put(struct conn *conn); ssize_t conn_recv(struct conn *conn, void *buf, size_t size); -ssize_t conn_sendv(struct conn *conn, struct array *sendv, size_t nsend); +ssize_t conn_sendv(struct conn *conn, const struct array *sendv, size_t nsend); void conn_init(void); void conn_deinit(void); uint32_t conn_ncurr_conn(void); uint64_t conn_ntotal_conn(void); uint32_t conn_ncurr_cconn(void); -bool conn_authenticated(struct conn *conn); +bool conn_authenticated(const struct conn *conn); #endif diff --git a/src/nc_core.c b/src/nc_core.c index c39a8057..430bba7c 100644 --- a/src/nc_core.c +++ b/src/nc_core.c @@ -220,7 +220,8 @@ static void core_close(struct context *ctx, struct conn *conn) { rstatus_t status; - char type, *addrstr; + char type; + const char *addrstr; ASSERT(conn->sd > 0); diff --git a/src/nc_core.h b/src/nc_core.h index 3166ec9b..3ac6dce6 100644 --- a/src/nc_core.h +++ b/src/nc_core.h @@ -137,15 +137,15 @@ struct context { struct instance { struct context *ctx; /* active context */ int log_level; /* log level */ - char *log_filename; /* log filename */ - char *conf_filename; /* configuration filename */ + const char *log_filename; /* log filename */ + const char *conf_filename; /* configuration filename */ uint16_t stats_port; /* stats monitoring port */ int stats_interval; /* stats aggregation interval */ - char *stats_addr; /* stats monitoring addr */ + const char *stats_addr; /* stats monitoring addr */ char hostname[NC_MAXHOSTNAMELEN]; /* hostname */ size_t mbuf_chunk_size; /* mbuf chunk size */ pid_t pid; /* process id */ - char *pid_filename; /* pid filename */ + const char *pid_filename; /* pid filename */ unsigned pidfile:1; /* pid file created? */ }; diff --git a/src/nc_log.c b/src/nc_log.c index 3d451ffc..7beca678 100644 --- a/src/nc_log.c +++ b/src/nc_log.c @@ -27,7 +27,7 @@ static struct logger logger; int -log_init(int level, char *name) +log_init(int level, const char *name) { struct logger *l = &logger; @@ -202,7 +202,7 @@ _log_stderr(const char *fmt, ...) * See -C option in man hexdump */ void -_log_hexdump(const char *file, int line, char *data, int datalen, +_log_hexdump(const char *file, int line, const char *data, int datalen, const char *fmt, ...) { struct logger *l = &logger; @@ -221,7 +221,7 @@ _log_hexdump(const char *file, int line, char *data, int datalen, size = 8 * LOG_MAX_LEN; /* size of output buffer */ while (datalen != 0 && (len < size - 1)) { - char *save, *str; + const char *save, *str; unsigned char c; int savelen; diff --git a/src/nc_log.h b/src/nc_log.h index e19e91f5..df55d307 100644 --- a/src/nc_log.h +++ b/src/nc_log.h @@ -18,11 +18,13 @@ #ifndef _NC_LOG_H_ #define _NC_LOG_H_ +#include + struct logger { - char *name; /* log file name */ - int level; /* log level */ - int fd; /* log file descriptor */ - int nerror; /* # log error */ + const char *name; /* log file name */ + int level; /* log level */ + int fd; /* log file descriptor */ + int nerror; /* # log error */ }; #define LOG_EMERG 0 /* system in unusable */ @@ -114,7 +116,8 @@ struct logger { } \ } while (0) -int log_init(int level, char *filename); + +int log_init(int level, const char *filename); void log_deinit(void); void log_level_up(void); void log_level_down(void); @@ -122,10 +125,10 @@ void log_level_set(int level); void log_stacktrace(void); void log_reopen(void); int log_loggable(int level); -void _log(const char *file, int line, int panic, const char *fmt, ...); -void _log_stderr(const char *fmt, ...); -void _log_safe(const char *fmt, ...); -void _log_stderr_safe(const char *fmt, ...); -void _log_hexdump(const char *file, int line, char *data, int datalen, const char *fmt, ...); +void _log(const char *file, int line, int panic, const char *fmt, ...) NC_ATTRIBUTE_FORMAT(printf, 4, 5); +void _log_stderr(const char *fmt, ...) NC_ATTRIBUTE_FORMAT(printf, 1, 2); +void _log_safe(const char *fmt, ...) NC_ATTRIBUTE_FORMAT(printf, 1, 2); +void _log_stderr_safe(const char *fmt, ...) NC_ATTRIBUTE_FORMAT(printf, 1, 2); +void _log_hexdump(const char *file, int line, const char *data, int datalen, const char *fmt, ...); #endif diff --git a/src/nc_mbuf.c b/src/nc_mbuf.c index 50aa59af..50fd742f 100644 --- a/src/nc_mbuf.c +++ b/src/nc_mbuf.c @@ -106,7 +106,7 @@ mbuf_free(struct mbuf *mbuf) { uint8_t *buf; - log_debug(LOG_VVERB, "put mbuf %p len %d", mbuf, mbuf->last - mbuf->pos); + log_debug(LOG_VVERB, "put mbuf %p len %d", mbuf, (int)(mbuf->last - mbuf->pos)); ASSERT(STAILQ_NEXT(mbuf, next) == NULL); ASSERT(mbuf->magic == MBUF_MAGIC); @@ -118,7 +118,7 @@ mbuf_free(struct mbuf *mbuf) void mbuf_put(struct mbuf *mbuf) { - log_debug(LOG_VVERB, "put mbuf %p len %d", mbuf, mbuf->last - mbuf->pos); + log_debug(LOG_VVERB, "put mbuf %p len %d", mbuf, (int)(mbuf->last - mbuf->pos)); ASSERT(STAILQ_NEXT(mbuf, next) == NULL); ASSERT(mbuf->magic == MBUF_MAGIC); @@ -143,7 +143,7 @@ mbuf_rewind(struct mbuf *mbuf) * 2^32 bytes (4G). */ uint32_t -mbuf_length(struct mbuf *mbuf) +mbuf_length(const struct mbuf *mbuf) { ASSERT(mbuf->last >= mbuf->pos); @@ -155,7 +155,7 @@ mbuf_length(struct mbuf *mbuf) * contain more than 2^32 bytes (4G). */ uint32_t -mbuf_size(struct mbuf *mbuf) +mbuf_size(const struct mbuf *mbuf) { ASSERT(mbuf->end >= mbuf->last); @@ -179,7 +179,8 @@ void mbuf_insert(struct mhdr *mhdr, struct mbuf *mbuf) { STAILQ_INSERT_TAIL(mhdr, mbuf, next); - log_debug(LOG_VVERB, "insert mbuf %p len %d", mbuf, mbuf->last - mbuf->pos); + log_debug(LOG_VVERB, "insert mbuf %p len %d", mbuf, + (int)(mbuf->last - mbuf->pos)); } /* @@ -188,7 +189,8 @@ mbuf_insert(struct mhdr *mhdr, struct mbuf *mbuf) void mbuf_remove(struct mhdr *mhdr, struct mbuf *mbuf) { - log_debug(LOG_VVERB, "remove mbuf %p len %d", mbuf, mbuf->last - mbuf->pos); + log_debug(LOG_VVERB, "remove mbuf %p len %d", mbuf, + (int)(mbuf->last - mbuf->pos)); STAILQ_REMOVE(mhdr, mbuf, mbuf, next); STAILQ_NEXT(mbuf, next) = NULL; @@ -201,7 +203,7 @@ mbuf_remove(struct mhdr *mhdr, struct mbuf *mbuf) * enough space for n bytes. */ void -mbuf_copy(struct mbuf *mbuf, uint8_t *pos, size_t n) +mbuf_copy(struct mbuf *mbuf, const uint8_t *pos, size_t n) { if (n == 0) { return; @@ -260,7 +262,7 @@ mbuf_split(struct mhdr *h, uint8_t *pos, mbuf_copy_t cb, void *cbarg) } void -mbuf_init(struct instance *nci) +mbuf_init(const struct instance *nci) { nfree_mbufq = 0; STAILQ_INIT(&free_mbufq); @@ -269,7 +271,7 @@ mbuf_init(struct instance *nci) mbuf_offset = mbuf_chunk_size - MBUF_HSIZE; log_debug(LOG_DEBUG, "mbuf hsize %d chunk size %zu offset %zu length %zu", - MBUF_HSIZE, mbuf_chunk_size, mbuf_offset, mbuf_offset); + (int)MBUF_HSIZE, mbuf_chunk_size, mbuf_offset, mbuf_offset); } void diff --git a/src/nc_mbuf.h b/src/nc_mbuf.h index 21d67704..c4b5dfd9 100644 --- a/src/nc_mbuf.h +++ b/src/nc_mbuf.h @@ -40,28 +40,28 @@ STAILQ_HEAD(mhdr, mbuf); #define MBUF_HSIZE sizeof(struct mbuf) static inline bool -mbuf_empty(struct mbuf *mbuf) +mbuf_empty(const struct mbuf *mbuf) { - return mbuf->pos == mbuf->last ? true : false; + return mbuf->pos == mbuf->last; } static inline bool -mbuf_full(struct mbuf *mbuf) +mbuf_full(const struct mbuf *mbuf) { - return mbuf->last == mbuf->end ? true : false; + return mbuf->last == mbuf->end; } -void mbuf_init(struct instance *nci); +void mbuf_init(const struct instance *nci); void mbuf_deinit(void); struct mbuf *mbuf_get(void); void mbuf_put(struct mbuf *mbuf); void mbuf_rewind(struct mbuf *mbuf); -uint32_t mbuf_length(struct mbuf *mbuf); -uint32_t mbuf_size(struct mbuf *mbuf); +uint32_t mbuf_length(const struct mbuf *mbuf); +uint32_t mbuf_size(const struct mbuf *mbuf); size_t mbuf_data_size(void); void mbuf_insert(struct mhdr *mhdr, struct mbuf *mbuf); void mbuf_remove(struct mhdr *mhdr, struct mbuf *mbuf); -void mbuf_copy(struct mbuf *mbuf, uint8_t *pos, size_t n); +void mbuf_copy(struct mbuf *mbuf, const uint8_t *pos, size_t n); struct mbuf *mbuf_split(struct mhdr *h, uint8_t *pos, mbuf_copy_t cb, void *cbarg); #endif diff --git a/src/nc_message.c b/src/nc_message.c index db98a82a..d76a98e1 100644 --- a/src/nc_message.c +++ b/src/nc_message.c @@ -117,7 +117,7 @@ static struct rbtree tmo_rbt; /* timeout rbtree */ static struct rbnode tmo_rbs; /* timeout rbtree sentinel */ #define DEFINE_ACTION(_name) string(#_name), -static struct string msg_type_strings[] = { +static const struct string msg_type_strings[] = { MSG_TYPE_CODEC( DEFINE_ACTION ) null_string }; @@ -256,6 +256,10 @@ _msg_get(void) msg->narg = 0; msg->rnarg = 0; msg->rlen = 0; + /* + * This is used for both parsing redis responses + * and as a counter for coalescing responses such as DEL + */ msg->integer = 0; msg->err = 0; @@ -328,8 +332,8 @@ msg_get_error(bool redis, err_t err) struct msg *msg; struct mbuf *mbuf; int n; - char *errstr = err ? strerror(err) : "unknown"; - char *protstr = redis ? "-ERR" : "SERVER_ERROR"; + const char *errstr = err ? strerror(err) : "unknown"; + const char *protstr = redis ? "-ERR" : "SERVER_ERROR"; msg = _msg_get(); if (msg == NULL) { @@ -393,9 +397,9 @@ msg_put(struct msg *msg) } void -msg_dump(struct msg *msg, int level) +msg_dump(const struct msg *msg, int level) { - struct mbuf *mbuf; + const struct mbuf *mbuf; if (log_loggable(level) == 0) { return; @@ -420,7 +424,7 @@ msg_dump(struct msg *msg, int level) void msg_init(void) { - log_debug(LOG_DEBUG, "msg size %d", sizeof(struct msg)); + log_debug(LOG_DEBUG, "msg size %d", (int)sizeof(struct msg)); msg_id = 0; frag_id = 0; nfree_msgq = 0; @@ -442,20 +446,20 @@ msg_deinit(void) ASSERT(nfree_msgq == 0); } -struct string * +const struct string * msg_type_string(msg_type_t type) { return &msg_type_strings[type]; } bool -msg_empty(struct msg *msg) +msg_empty(const struct msg *msg) { - return msg->mlen == 0 ? true : false; + return msg->mlen == 0; } uint32_t -msg_backend_idx(struct msg *msg, uint8_t *key, uint32_t keylen) +msg_backend_idx(const struct msg *msg, const uint8_t *key, uint32_t keylen) { struct conn *conn = msg->owner; struct server_pool *pool = conn->owner; @@ -487,7 +491,7 @@ msg_ensure_mbuf(struct msg *msg, size_t len) * into mbuf */ rstatus_t -msg_append(struct msg *msg, uint8_t *pos, size_t n) +msg_append(struct msg *msg, const uint8_t *pos, size_t n) { struct mbuf *mbuf; @@ -511,7 +515,7 @@ msg_append(struct msg *msg, uint8_t *pos, size_t n) * into mbuf */ rstatus_t -msg_prepend(struct msg *msg, uint8_t *pos, size_t n) +msg_prepend(struct msg *msg, const uint8_t *pos, size_t n) { struct mbuf *mbuf; @@ -942,3 +946,21 @@ msg_send(struct context *ctx, struct conn *conn) return NC_OK; } + +/* + * Set a placeholder key for a command with no key that is forwarded to an + * arbitrary backend. + */ +bool msg_set_placeholder_key(struct msg *r) +{ + struct keypos *kpos; + ASSERT(array_n(r->keys) == 0); + kpos = array_push(r->keys); + if (kpos == NULL) { + return false; + } + kpos->start = (uint8_t *)"placeholder"; + kpos->end = kpos->start + sizeof("placeholder") - 1; + return true; +} + diff --git a/src/nc_message.h b/src/nc_message.h index 2844c824..26a063bc 100644 --- a/src/nc_message.h +++ b/src/nc_message.h @@ -19,13 +19,14 @@ #define _NC_MESSAGE_H_ #include +#include typedef void (*msg_parse_t)(struct msg *); typedef rstatus_t (*msg_add_auth_t)(struct context *ctx, struct conn *c_conn, struct conn *s_conn); typedef rstatus_t (*msg_fragment_t)(struct msg *, uint32_t, struct msg_tqh *); typedef void (*msg_coalesce_t)(struct msg *r); typedef rstatus_t (*msg_reply_t)(struct msg *r); -typedef bool (*msg_failure_t)(struct msg *r); +typedef bool (*msg_failure_t)(const struct msg *r); typedef enum msg_parse_result { MSG_PARSE_OK, /* parsing ok */ @@ -49,6 +50,7 @@ typedef enum msg_parse_result { ACTION( REQ_MC_DECR ) \ ACTION( REQ_MC_TOUCH ) /* memcache touch request */ \ ACTION( REQ_MC_QUIT ) /* memcache quit request */ \ + ACTION( REQ_MC_VERSION ) /* memcache version request */ \ ACTION( RSP_MC_NUM ) /* memcache arithmetic response */ \ ACTION( RSP_MC_STORED ) /* memcache cas and storage response */ \ ACTION( RSP_MC_NOT_STORED ) \ @@ -58,28 +60,36 @@ typedef enum msg_parse_result { ACTION( RSP_MC_VALUE ) \ ACTION( RSP_MC_DELETED ) /* memcache delete response */ \ ACTION( RSP_MC_TOUCHED ) /* memcache touch response */ \ + ACTION( RSP_MC_VERSION ) /* memcache version response */ \ ACTION( RSP_MC_ERROR ) /* memcache error responses */ \ ACTION( RSP_MC_CLIENT_ERROR ) \ ACTION( RSP_MC_SERVER_ERROR ) \ - ACTION( REQ_REDIS_DEL ) /* redis commands - keys */ \ + ACTION( REQ_REDIS_COPY ) /* redis commands - keys */ \ + ACTION( REQ_REDIS_DEL ) \ ACTION( REQ_REDIS_EXISTS ) \ ACTION( REQ_REDIS_EXPIRE ) \ ACTION( REQ_REDIS_EXPIREAT ) \ + ACTION( REQ_REDIS_MOVE ) \ ACTION( REQ_REDIS_PEXPIRE ) \ ACTION( REQ_REDIS_PEXPIREAT ) \ ACTION( REQ_REDIS_PERSIST ) \ ACTION( REQ_REDIS_PTTL ) \ ACTION( REQ_REDIS_SORT ) \ + ACTION( REQ_REDIS_TOUCH ) \ ACTION( REQ_REDIS_TTL ) \ ACTION( REQ_REDIS_TYPE ) \ + ACTION( REQ_REDIS_UNLINK ) \ ACTION( REQ_REDIS_APPEND ) /* redis requests - string */ \ ACTION( REQ_REDIS_BITCOUNT ) \ - ACTION( REQ_REDIS_BITPOS ) \ + ACTION( REQ_REDIS_BITFIELD ) \ + ACTION( REQ_REDIS_BITPOS ) \ ACTION( REQ_REDIS_DECR ) \ ACTION( REQ_REDIS_DECRBY ) \ ACTION( REQ_REDIS_DUMP ) \ ACTION( REQ_REDIS_GET ) \ ACTION( REQ_REDIS_GETBIT ) \ + ACTION( REQ_REDIS_GETDEL ) \ + ACTION( REQ_REDIS_GETEX ) \ ACTION( REQ_REDIS_GETRANGE ) \ ACTION( REQ_REDIS_GETSET ) \ ACTION( REQ_REDIS_INCR ) \ @@ -105,14 +115,18 @@ typedef enum msg_parse_result { ACTION( REQ_REDIS_HLEN ) \ ACTION( REQ_REDIS_HMGET ) \ ACTION( REQ_REDIS_HMSET ) \ + ACTION( REQ_REDIS_HRANDFIELD ) \ ACTION( REQ_REDIS_HSET ) \ ACTION( REQ_REDIS_HSETNX ) \ ACTION( REQ_REDIS_HSCAN) \ + ACTION( REQ_REDIS_HSTRLEN) \ ACTION( REQ_REDIS_HVALS ) \ ACTION( REQ_REDIS_LINDEX ) /* redis requests - lists */ \ ACTION( REQ_REDIS_LINSERT ) \ ACTION( REQ_REDIS_LLEN ) \ + ACTION( REQ_REDIS_LMOVE ) \ ACTION( REQ_REDIS_LPOP ) \ + ACTION( REQ_REDIS_LPOS ) \ ACTION( REQ_REDIS_LPUSH ) \ ACTION( REQ_REDIS_LPUSHX ) \ ACTION( REQ_REDIS_LRANGE ) \ @@ -133,6 +147,7 @@ typedef enum msg_parse_result { ACTION( REQ_REDIS_SINTER ) \ ACTION( REQ_REDIS_SINTERSTORE ) \ ACTION( REQ_REDIS_SISMEMBER ) \ + ACTION( REQ_REDIS_SMISMEMBER ) \ ACTION( REQ_REDIS_SMEMBERS ) \ ACTION( REQ_REDIS_SMOVE ) \ ACTION( REQ_REDIS_SPOP ) \ @@ -144,29 +159,49 @@ typedef enum msg_parse_result { ACTION( REQ_REDIS_ZADD ) /* redis requests - sorted sets */ \ ACTION( REQ_REDIS_ZCARD ) \ ACTION( REQ_REDIS_ZCOUNT ) \ + ACTION( REQ_REDIS_ZDIFF ) \ + ACTION( REQ_REDIS_ZDIFFSTORE ) \ ACTION( REQ_REDIS_ZINCRBY ) \ + ACTION( REQ_REDIS_ZINTER ) \ ACTION( REQ_REDIS_ZINTERSTORE ) \ ACTION( REQ_REDIS_ZLEXCOUNT ) \ + ACTION( REQ_REDIS_ZMSCORE ) \ + ACTION( REQ_REDIS_ZPOPMIN ) \ + ACTION( REQ_REDIS_ZPOPMAX ) \ + ACTION( REQ_REDIS_ZRANDMEMBER ) \ ACTION( REQ_REDIS_ZRANGE ) \ ACTION( REQ_REDIS_ZRANGEBYLEX ) \ ACTION( REQ_REDIS_ZRANGEBYSCORE ) \ + ACTION( REQ_REDIS_ZRANGESTORE ) \ ACTION( REQ_REDIS_ZRANK ) \ ACTION( REQ_REDIS_ZREM ) \ ACTION( REQ_REDIS_ZREMRANGEBYRANK ) \ ACTION( REQ_REDIS_ZREMRANGEBYLEX ) \ ACTION( REQ_REDIS_ZREMRANGEBYSCORE ) \ ACTION( REQ_REDIS_ZREVRANGE ) \ + ACTION( REQ_REDIS_ZREVRANGEBYLEX ) \ ACTION( REQ_REDIS_ZREVRANGEBYSCORE ) \ ACTION( REQ_REDIS_ZREVRANK ) \ + ACTION( REQ_REDIS_ZUNION ) \ + ACTION( REQ_REDIS_ZSCAN ) \ ACTION( REQ_REDIS_ZSCORE ) \ ACTION( REQ_REDIS_ZUNIONSTORE ) \ - ACTION( REQ_REDIS_ZSCAN) \ + ACTION( REQ_REDIS_GEOADD ) /* redis requests - geo */ \ + ACTION( REQ_REDIS_GEODIST ) \ + ACTION( REQ_REDIS_GEOHASH ) \ + ACTION( REQ_REDIS_GEORADIUS ) \ + ACTION( REQ_REDIS_GEORADIUSBYMEMBER ) \ + ACTION( REQ_REDIS_GEOPOS ) \ + ACTION( REQ_REDIS_GEOSEARCH) \ + ACTION( REQ_REDIS_GEOSEARCHSTORE) \ ACTION( REQ_REDIS_EVAL ) /* redis requests - eval */ \ ACTION( REQ_REDIS_EVALSHA ) \ ACTION( REQ_REDIS_PING ) /* redis requests - ping/quit */ \ ACTION( REQ_REDIS_QUIT) \ ACTION( REQ_REDIS_AUTH) \ ACTION( REQ_REDIS_SELECT) /* only during init */ \ + ACTION( REQ_REDIS_COMMAND) /* Sent to random server for redis-cli completions*/ \ + ACTION( REQ_REDIS_LOLWUT) /* Vitally important */ \ ACTION( RSP_REDIS_STATUS ) /* redis response */ \ ACTION( RSP_REDIS_ERROR ) \ ACTION( RSP_REDIS_ERROR_ERR ) \ @@ -195,10 +230,14 @@ typedef enum msg_type { #undef DEFINE_ACTION struct keypos { - uint8_t *start; /* key start pos */ - uint8_t *end; /* key end pos */ + uint8_t *start; /* key start pos */ + uint8_t *end; /* key end pos */ }; +/* + * This represents a message with a list of mbufs + * that can be a redis/memcache request/response/error response. + */ struct msg { TAILQ_ENTRY(msg) c_tqe; /* link in client q */ TAILQ_ENTRY(msg) s_tqe; /* link in server q */ @@ -238,10 +277,11 @@ struct msg { uint8_t *narg_start; /* narg start (redis) */ uint8_t *narg_end; /* narg end (redis) */ - uint32_t narg; /* # arguments (redis) */ + uint32_t narg; /* # arguments (redis, memcache) */ uint32_t rnarg; /* running # arg used by parsing fsa (redis) */ uint32_t rlen; /* running length in parsing fsa (redis) */ uint32_t integer; /* integer reply value (redis) */ + uint8_t is_top_level; /* is this top level (redis) */ struct msg *frag_owner; /* owner of fragment message */ uint32_t nfrag; /* # fragment */ @@ -270,25 +310,26 @@ void msg_tmo_delete(struct msg *msg); void msg_init(void); void msg_deinit(void); -struct string *msg_type_string(msg_type_t type); +const struct string *msg_type_string(msg_type_t type); struct msg *msg_get(struct conn *conn, bool request, bool redis); void msg_put(struct msg *msg); struct msg *msg_get_error(bool redis, err_t err); -void msg_dump(struct msg *msg, int level); -bool msg_empty(struct msg *msg); +void msg_dump(const struct msg *msg, int level); +bool msg_empty(const struct msg *msg); rstatus_t msg_recv(struct context *ctx, struct conn *conn); rstatus_t msg_send(struct context *ctx, struct conn *conn); uint64_t msg_gen_frag_id(void); -uint32_t msg_backend_idx(struct msg *msg, uint8_t *key, uint32_t keylen); +uint32_t msg_backend_idx(const struct msg *msg, const uint8_t *key, uint32_t keylen); struct mbuf *msg_ensure_mbuf(struct msg *msg, size_t len); -rstatus_t msg_append(struct msg *msg, uint8_t *pos, size_t n); -rstatus_t msg_prepend(struct msg *msg, uint8_t *pos, size_t n); +rstatus_t msg_append(struct msg *msg, const uint8_t *pos, size_t n); +rstatus_t msg_prepend(struct msg *msg, const uint8_t *pos, size_t n); rstatus_t msg_prepend_format(struct msg *msg, const char *fmt, ...); +bool msg_set_placeholder_key(struct msg *r); struct msg *req_get(struct conn *conn); void req_put(struct msg *msg); -bool req_done(struct conn *conn, struct msg *msg); -bool req_error(struct conn *conn, struct msg *msg); +bool req_done(const struct conn *conn, struct msg *msg); +bool req_error(const struct conn *conn, struct msg *msg); void req_server_enqueue_imsgq(struct context *ctx, struct conn *conn, struct msg *msg); void req_server_enqueue_imsgq_head(struct context *ctx, struct conn *conn, struct msg *msg); void req_server_dequeue_imsgq(struct context *ctx, struct conn *conn, struct msg *msg); @@ -298,6 +339,7 @@ void req_client_dequeue_omsgq(struct context *ctx, struct conn *conn, struct msg void req_server_dequeue_omsgq(struct context *ctx, struct conn *conn, struct msg *msg); struct msg *req_recv_next(struct context *ctx, struct conn *conn, bool alloc); void req_recv_done(struct context *ctx, struct conn *conn, struct msg *msg, struct msg *nmsg); +struct msg *req_fake(struct context *ctx, struct conn *conn); struct msg *req_send_next(struct context *ctx, struct conn *conn); void req_send_done(struct context *ctx, struct conn *conn, struct msg *msg); diff --git a/src/nc_queue.h b/src/nc_queue.h index 5ff70cf9..98f4ec02 100644 --- a/src/nc_queue.h +++ b/src/nc_queue.h @@ -168,6 +168,13 @@ struct qm_trace { (head)->trace.lastfile = __FILE__; \ } while (0) +#define QMD_TRACE_HEAD_INIT(head) do { \ + (head)->trace.prevline = __LINE__; \ + (head)->trace.prevfile = __FILE__; \ + (head)->trace.lastline = __LINE__; \ + (head)->trace.lastfile = __FILE__; \ +} while (0) + #define QMD_TRACE_ELEM(elem) do { \ (elem)->trace.prevline = (elem)->trace.lastline; \ (elem)->trace.prevfile = (elem)->trace.lastfile; \ @@ -179,6 +186,7 @@ struct qm_trace { #define QMD_TRACE_ELEM(elem) #define QMD_TRACE_HEAD(head) +#define QMD_TRACE_HEAD_INIT(head) #define TRACEBUF #endif /* QUEUE_MACRO_TRACE */ @@ -595,7 +603,7 @@ struct { \ #define TAILQ_INIT(head) do { \ TAILQ_FIRST((head)) = NULL; \ (head)->tqh_last = &TAILQ_FIRST((head)); \ - QMD_TRACE_HEAD(head); \ + QMD_TRACE_HEAD_INIT(head); \ } while (0) #define TAILQ_INSERT_AFTER(head, listelm, elm, field) do { \ diff --git a/src/nc_rbtree.c b/src/nc_rbtree.c index 404789fe..d76ad861 100644 --- a/src/nc_rbtree.c +++ b/src/nc_rbtree.c @@ -38,7 +38,7 @@ rbtree_init(struct rbtree *tree, struct rbnode *node) } static struct rbnode * -rbtree_node_min(struct rbnode *node, struct rbnode *sentinel) +rbtree_node_min(struct rbnode *node, const struct rbnode *sentinel) { /* traverse left links */ @@ -50,10 +50,10 @@ rbtree_node_min(struct rbnode *node, struct rbnode *sentinel) } struct rbnode * -rbtree_min(struct rbtree *tree) +rbtree_min(const struct rbtree *tree) { struct rbnode *node = tree->root; - struct rbnode *sentinel = tree->sentinel; + const struct rbnode *sentinel = tree->sentinel; /* empty tree */ diff --git a/src/nc_rbtree.h b/src/nc_rbtree.h index bdf19525..4b6137ac 100644 --- a/src/nc_rbtree.h +++ b/src/nc_rbtree.h @@ -40,7 +40,7 @@ struct rbtree { void rbtree_node_init(struct rbnode *node); void rbtree_init(struct rbtree *tree, struct rbnode *node); -struct rbnode *rbtree_min(struct rbtree *tree); +struct rbnode *rbtree_min(const struct rbtree *tree); void rbtree_insert(struct rbtree *tree, struct rbnode *node); void rbtree_delete(struct rbtree *tree, struct rbnode *node); diff --git a/src/nc_request.c b/src/nc_request.c index 3190416f..df6a2501 100644 --- a/src/nc_request.c +++ b/src/nc_request.c @@ -33,14 +33,14 @@ req_get(struct conn *conn) } static void -req_log(struct msg *req) +req_log(const struct msg *req) { - struct msg *rsp; /* peer message (response) */ + const struct msg *rsp; /* peer message (response) */ int64_t req_time; /* time cost for this request */ - char *peer_str; /* peer client ip:port */ + const char *peer_str; /* peer client ip:port */ uint32_t req_len, rsp_len; /* request and response length */ - struct string *req_type; /* request type string */ - struct keypos *kpos; + const struct string *req_type; /* request type string */ + const struct keypos *kpos; if (log_loggable(LOG_NOTICE) == 0) { return; @@ -75,9 +75,6 @@ req_log(struct msg *req) } kpos = array_get(req->keys, 0); - if (kpos->end != NULL) { - *(kpos->end) = '\0'; - } /* * FIXME: add backend addr here @@ -90,10 +87,11 @@ req_log(struct msg *req) log_debug(LOG_NOTICE, "req %"PRIu64" done on c %d req_time %"PRIi64".%03"PRIi64 " msec type %.*s narg %"PRIu32" req_len %"PRIu32" rsp_len %"PRIu32 - " key0 '%s' peer '%s' done %d error %d", + " key0 '%.*s' peer '%s' done %d error %d", req->id, req->owner->sd, req_time / 1000, req_time % 1000, req_type->len, req_type->data, req->narg, req_len, rsp_len, - kpos->start, peer_str, req->done, req->error); + (int)(kpos->end ? kpos->end - kpos->start : 0), kpos->start, + peer_str, req->done, req->error); } void @@ -124,11 +122,13 @@ req_put(struct msg *msg) * A request is done, if we received response for the given request. * A request vector is done if we received responses for all its * fragments. + * + * msg->fdone is modified to cache whether this request was done. */ bool -req_done(struct conn *conn, struct msg *msg) +req_done(const struct conn *conn, struct msg *msg) { - struct msg *cmsg, *pmsg; /* current and previous message */ + struct msg *cmsg; /* current message */ uint64_t id; /* fragment id */ uint32_t nfragment; /* # fragment */ @@ -155,18 +155,18 @@ req_done(struct conn *conn, struct msg *msg) /* check all fragments of the given request vector are done */ - for (pmsg = msg, cmsg = TAILQ_PREV(msg, msg_tqh, c_tqe); + for (cmsg = TAILQ_PREV(msg, msg_tqh, c_tqe); cmsg != NULL && cmsg->frag_id == id; - pmsg = cmsg, cmsg = TAILQ_PREV(cmsg, msg_tqh, c_tqe)) { + cmsg = TAILQ_PREV(cmsg, msg_tqh, c_tqe)) { if (!cmsg->done) { return false; } } - for (pmsg = msg, cmsg = TAILQ_NEXT(msg, c_tqe); + for (cmsg = TAILQ_NEXT(msg, c_tqe); cmsg != NULL && cmsg->frag_id == id; - pmsg = cmsg, cmsg = TAILQ_NEXT(cmsg, c_tqe)) { + cmsg = TAILQ_NEXT(cmsg, c_tqe)) { if (!cmsg->done) { return false; @@ -184,16 +184,16 @@ req_done(struct conn *conn, struct msg *msg) msg->fdone = 1; nfragment = 0; - for (pmsg = msg, cmsg = TAILQ_PREV(msg, msg_tqh, c_tqe); + for (cmsg = TAILQ_PREV(msg, msg_tqh, c_tqe); cmsg != NULL && cmsg->frag_id == id; - pmsg = cmsg, cmsg = TAILQ_PREV(cmsg, msg_tqh, c_tqe)) { + cmsg = TAILQ_PREV(cmsg, msg_tqh, c_tqe)) { cmsg->fdone = 1; nfragment++; } - for (pmsg = msg, cmsg = TAILQ_NEXT(msg, c_tqe); + for (cmsg = TAILQ_NEXT(msg, c_tqe); cmsg != NULL && cmsg->frag_id == id; - pmsg = cmsg, cmsg = TAILQ_NEXT(cmsg, c_tqe)) { + cmsg = TAILQ_NEXT(cmsg, c_tqe)) { cmsg->fdone = 1; nfragment++; } @@ -217,7 +217,7 @@ req_done(struct conn *conn, struct msg *msg) * receiving response for any its fragments. */ bool -req_error(struct conn *conn, struct msg *msg) +req_error(const struct conn *conn, struct msg *msg) { struct msg *cmsg; /* current message */ uint64_t id; @@ -472,7 +472,7 @@ req_make_reply(struct context *ctx, struct conn *conn, struct msg *req) } static bool -req_filter(struct context *ctx, struct conn *conn, struct msg *msg) +req_filter(struct conn *conn, struct msg *msg) { ASSERT(conn->client && !conn->proxy); @@ -574,7 +574,6 @@ req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg) { rstatus_t status; struct conn *s_conn; - struct server_pool *pool; uint8_t *key; uint32_t keylen; struct keypos *kpos; @@ -586,8 +585,6 @@ req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg) c_conn->enqueue_outq(ctx, c_conn, msg); } - pool = c_conn->owner; - ASSERT(array_n(msg->keys) > 0); kpos = array_get(msg->keys, 0); key = kpos->start; @@ -595,6 +592,21 @@ req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg) s_conn = server_pool_conn(ctx, c_conn->owner, key, keylen); if (s_conn == NULL) { + /* + * Handle a failure to establish a new connection to a server, + * e.g. due to dns resolution errors. + * + * If this is a fragmented request sent to multiple servers such as + * a memcache get(multiget), + * mark the fragment for this request to the server as done. + * + * Normally, this would be done when the request was forwarded to the + * server, but due to failing to connect to the server this check is + * repeated here. + */ + if (msg->frag_owner != NULL) { + msg->frag_owner->nfrag_done++; + } req_forward_error(ctx, c_conn, msg); return; } @@ -647,7 +659,7 @@ req_recv_done(struct context *ctx, struct conn *conn, struct msg *msg, /* enqueue next message (request), if any */ conn->rmsg = nmsg; - if (req_filter(ctx, conn, msg)) { + if (req_filter(conn, msg)) { return; } @@ -675,7 +687,7 @@ req_recv_done(struct context *ctx, struct conn *conn, struct msg *msg, /* do fragment */ pool = conn->owner; TAILQ_INIT(&frag_msgq); - status = msg->fragment(msg, pool->ncontinuum, &frag_msgq); + status = msg->fragment(msg, array_n(&pool->server), &frag_msgq); if (status != NC_OK) { if (!msg->noreply) { conn->enqueue_outq(ctx, conn, msg); diff --git a/src/nc_server.c b/src/nc_server.c index 7b1d5e4f..dab6a79b 100644 --- a/src/nc_server.c +++ b/src/nc_server.c @@ -92,7 +92,7 @@ server_timeout(struct conn *conn) } bool -server_active(struct conn *conn) +server_active(const struct conn *conn) { ASSERT(!conn->client && !conn->proxy); @@ -293,7 +293,7 @@ server_failure(struct context *ctx, struct server *server) next = now + pool->server_retry_timeout; log_debug(LOG_INFO, "update pool %"PRIu32" '%.*s' to delete server '%.*s' " - "for next %"PRIu32" secs", pool->idx, pool->name.len, + "for next %"PRId64" secs", pool->idx, pool->name.len, pool->name.data, server->pname.len, server->pname.data, pool->server_retry_timeout / 1000 / 1000); @@ -627,7 +627,7 @@ server_pool_update(struct server_pool *pool) } static uint32_t -server_pool_hash(struct server_pool *pool, uint8_t *key, uint32_t keylen) +server_pool_hash(const struct server_pool *pool, const uint8_t *key, uint32_t keylen) { ASSERT(array_n(&pool->server) != 0); ASSERT(key != NULL); @@ -640,25 +640,31 @@ server_pool_hash(struct server_pool *pool, uint8_t *key, uint32_t keylen) return 0; } - return pool->key_hash((char *)key, keylen); + return pool->key_hash((const char *)key, keylen); } uint32_t -server_pool_idx(struct server_pool *pool, uint8_t *key, uint32_t keylen) +server_pool_idx(const struct server_pool *pool, const uint8_t *key, uint32_t keylen) { uint32_t hash, idx; + uint32_t nserver = array_n(&pool->server); - ASSERT(array_n(&pool->server) != 0); + ASSERT(nserver != 0); ASSERT(key != NULL); + if (nserver == 1) { + /* Optimization: Skip hashing and dispatching for pools with only one server */ + return 0; + } + /* * If hash_tag: is configured for this server pool, we use the part of * the key within the hash tag as an input to the distributor. Otherwise * we use the full key */ if (!string_empty(&pool->hash_tag)) { - struct string *tag = &pool->hash_tag; - uint8_t *tag_start, *tag_end; + const struct string *tag = &pool->hash_tag; + const uint8_t *tag_start, *tag_end; tag_start = nc_strchr(key, key + keylen, tag->data[0]); if (tag_start != NULL) { @@ -694,7 +700,7 @@ server_pool_idx(struct server_pool *pool, uint8_t *key, uint32_t keylen) } static struct server * -server_pool_server(struct server_pool *pool, uint8_t *key, uint32_t keylen) +server_pool_server(struct server_pool *pool, const uint8_t *key, uint32_t keylen) { struct server *server; uint32_t idx; @@ -709,7 +715,7 @@ server_pool_server(struct server_pool *pool, uint8_t *key, uint32_t keylen) } struct conn * -server_pool_conn(struct context *ctx, struct server_pool *pool, uint8_t *key, +server_pool_conn(struct context *ctx, struct server_pool *pool, const uint8_t *key, uint32_t keylen) { rstatus_t status; diff --git a/src/nc_server.h b/src/nc_server.h index ee76deb8..6a0c038a 100644 --- a/src/nc_server.h +++ b/src/nc_server.h @@ -129,7 +129,7 @@ struct server_pool { void server_ref(struct conn *conn, void *owner); void server_unref(struct conn *conn); int server_timeout(struct conn *conn); -bool server_active(struct conn *conn); +bool server_active(const struct conn *conn); rstatus_t server_init(struct array *server, struct array *conf_server, struct server_pool *sp); void server_deinit(struct array *server); struct conn *server_conn(struct server *server); @@ -138,8 +138,8 @@ void server_close(struct context *ctx, struct conn *conn); void server_connected(struct context *ctx, struct conn *conn); void server_ok(struct context *ctx, struct conn *conn); -uint32_t server_pool_idx(struct server_pool *pool, uint8_t *key, uint32_t keylen); -struct conn *server_pool_conn(struct context *ctx, struct server_pool *pool, uint8_t *key, uint32_t keylen); +uint32_t server_pool_idx(const struct server_pool *pool, const uint8_t *key, uint32_t keylen); +struct conn *server_pool_conn(struct context *ctx, struct server_pool *pool, const uint8_t *key, uint32_t keylen); rstatus_t server_pool_run(struct server_pool *pool); rstatus_t server_pool_preconnect(struct context *ctx); void server_pool_disconnect(struct context *ctx); diff --git a/src/nc_signal.c b/src/nc_signal.c index cd5cdaa6..b669009e 100644 --- a/src/nc_signal.c +++ b/src/nc_signal.c @@ -21,23 +21,23 @@ #include #include -static struct signal signals[] = { - { SIGUSR1, "SIGUSR1", 0, signal_handler }, - { SIGUSR2, "SIGUSR2", 0, signal_handler }, - { SIGTTIN, "SIGTTIN", 0, signal_handler }, - { SIGTTOU, "SIGTTOU", 0, signal_handler }, - { SIGHUP, "SIGHUP", 0, signal_handler }, - { SIGINT, "SIGINT", 0, signal_handler }, - { SIGTERM, "SIGTERM", 0, signal_handler }, - { SIGSEGV, "SIGSEGV", (int)SA_RESETHAND, signal_handler }, - { SIGPIPE, "SIGPIPE", 0, SIG_IGN }, - { 0, NULL, 0, NULL } +static const struct signal signals[] = { + { SIGUSR1, "SIGUSR1", 0, signal_handler }, + { SIGUSR2, "SIGUSR2", 0, signal_handler }, + { SIGTTIN, "SIGTTIN", 0, signal_handler }, + { SIGTTOU, "SIGTTOU", 0, signal_handler }, + { SIGHUP, "SIGHUP", 0, signal_handler }, + { SIGINT, "SIGINT", 0, signal_handler }, + { SIGTERM, "SIGTERM", 0, signal_handler }, + { SIGSEGV, "SIGSEGV", (int)SA_RESETHAND, signal_handler }, + { SIGPIPE, "SIGPIPE", 0, SIG_IGN }, + { 0, NULL, 0, NULL } }; rstatus_t signal_init(void) { - struct signal *sig; + const struct signal *sig; for (sig = signals; sig->signo != 0; sig++) { rstatus_t status; @@ -67,7 +67,7 @@ signal_deinit(void) void signal_handler(int signo) { - struct signal *sig; + const struct signal *sig; void (*action)(void); char *actionstr; bool done; diff --git a/src/nc_signal.h b/src/nc_signal.h index e87804df..1b367568 100644 --- a/src/nc_signal.h +++ b/src/nc_signal.h @@ -20,6 +20,8 @@ #include +#define EXIT_MSG_MAX_LEN 256 /* max length of exit message */ + struct signal { int signo; char *signame; diff --git a/src/nc_stats.c b/src/nc_stats.c index 6777e77c..b53b3c74 100644 --- a/src/nc_stats.c +++ b/src/nc_stats.c @@ -42,11 +42,11 @@ static struct stats_metric stats_server_codec[] = { #undef DEFINE_ACTION #define DEFINE_ACTION(_name, _type, _desc) { .name = #_name, .desc = _desc }, -static struct stats_desc stats_pool_desc[] = { +static const struct stats_desc stats_pool_desc[] = { STATS_POOL_CODEC( DEFINE_ACTION ) }; -static struct stats_desc stats_server_desc[] = { +static const struct stats_desc stats_server_desc[] = { STATS_SERVER_CODEC( DEFINE_ACTION ) }; #undef DEFINE_ACTION @@ -188,7 +188,7 @@ stats_server_init(struct stats_server *sts, struct server *s) } static rstatus_t -stats_server_map(struct array *stats_server, struct array *server) +stats_server_map(struct array *stats_server, const struct array *server) { rstatus_t status; uint32_t i, nserver; @@ -233,7 +233,7 @@ stats_server_unmap(struct array *stats_server) } static rstatus_t -stats_pool_init(struct stats_pool *stp, struct server_pool *sp) +stats_pool_init(struct stats_pool *stp, const struct server_pool *sp) { rstatus_t status; @@ -281,7 +281,7 @@ stats_pool_reset(struct array *stats_pool) } static rstatus_t -stats_pool_map(struct array *stats_pool, struct array *server_pool) +stats_pool_map(struct array *stats_pool, const struct array *server_pool) { rstatus_t status; uint32_t i, npool; @@ -295,7 +295,7 @@ stats_pool_map(struct array *stats_pool, struct array *server_pool) } for (i = 0; i < npool; i++) { - struct server_pool *sp = array_get(server_pool, i); + const struct server_pool *sp = array_get(server_pool, i); struct stats_pool *stp = array_push(stats_pool); status = stats_pool_init(stp, sp); @@ -327,7 +327,7 @@ stats_pool_unmap(struct array *stats_pool) } static rstatus_t -stats_pool_redis_db_init(struct array *stats_pool, struct array *server_pool) +stats_pool_redis_db_init(struct array *stats_pool, const struct array *server_pool) { uint32_t i, npool; @@ -453,7 +453,7 @@ stats_destroy_buf(struct stats *st) } static rstatus_t -stats_add_string(struct stats *st, struct string *key, struct string *val) +stats_add_string(struct stats *st, const struct string *key, const struct string *val) { struct stats_buffer *buf; uint8_t *pos; @@ -476,7 +476,7 @@ stats_add_string(struct stats *st, struct string *key, struct string *val) } static rstatus_t -stats_add_num(struct stats *st, struct string *key, int64_t val) +stats_add_num(struct stats *st, const struct string *key, int64_t val) { struct stats_buffer *buf; uint8_t *pos; @@ -572,7 +572,7 @@ stats_add_footer(struct stats *st) } static rstatus_t -stats_begin_nesting(struct stats *st, struct string *key) +stats_begin_nesting(struct stats *st, const struct string *key) { struct stats_buffer *buf; uint8_t *pos; @@ -649,12 +649,13 @@ stats_copy_metric(struct stats *st, struct array *metric) } static void -stats_aggregate_metric(struct array *dst, struct array *src) +stats_aggregate_metric(struct array *dst, const struct array *src) { uint32_t i; for (i = 0; i < array_n(src); i++) { - struct stats_metric *stm1, *stm2; + const struct stats_metric *stm1; + struct stats_metric *stm2; stm1 = array_get(src, i); stm2 = array_get(dst, i); @@ -794,7 +795,7 @@ stats_send_rsp(struct stats *st) return NC_ERROR; } - log_debug(LOG_VERB, "send stats on sd %d %d bytes", sd, st->buf.len); + log_debug(LOG_VERB, "send stats on sd %d %zu bytes", sd, st->buf.len); n = nc_sendn(sd, st->buf.data, st->buf.len); if (n < 0) { @@ -851,24 +852,25 @@ stats_listen(struct stats *st) status = nc_set_reuseaddr(st->sd); if (status < 0) { - log_error("set reuseaddr on m %d failed: %s", st->sd, strerror(errno)); + log_error("set reuseaddr on m %d failed for stats server: %s", st->sd, strerror(errno)); return NC_ERROR; } status = bind(st->sd, (struct sockaddr *)&si.addr, si.addrlen); if (status < 0) { - log_error("bind on m %d to addr '%.*s:%u' failed: %s", st->sd, + log_error("bind on m %d to stats server addr '%.*s:%u' failed: %s", st->sd, st->addr.len, st->addr.data, st->port, strerror(errno)); return NC_ERROR; } status = listen(st->sd, SOMAXCONN); if (status < 0) { - log_error("listen on m %d failed: %s", st->sd, strerror(errno)); + log_error("listen on m %d for stats server '%.*s:%u' failed: %s", st->sd, + st->addr.len, st->addr.data, st->port, strerror(errno)); return NC_ERROR; } - log_debug(LOG_NOTICE, "m %d listening on '%.*s:%u'", st->sd, + log_debug(LOG_NOTICE, "m %d listening on stats server '%.*s:%u'", st->sd, st->addr.len, st->addr.data, st->port); return NC_OK; @@ -908,8 +910,8 @@ stats_stop_aggregator(struct stats *st) } struct stats * -stats_create(uint16_t stats_port, char *stats_ip, int stats_interval, - char *source, struct array *server_pool) +stats_create(uint16_t stats_port, const char *stats_ip, int stats_interval, + const char *source, const struct array *server_pool) { rstatus_t status; struct stats *st; @@ -1039,7 +1041,7 @@ stats_swap(struct stats *st) } static struct stats_metric * -stats_pool_to_metric(struct context *ctx, struct server_pool *pool, +stats_pool_to_metric(struct context *ctx, const struct server_pool *pool, stats_pool_field_t fidx) { struct stats *st; @@ -1062,7 +1064,7 @@ stats_pool_to_metric(struct context *ctx, struct server_pool *pool, } void -_stats_pool_incr(struct context *ctx, struct server_pool *pool, +_stats_pool_incr(struct context *ctx, const struct server_pool *pool, stats_pool_field_t fidx) { struct stats_metric *stm; @@ -1077,7 +1079,7 @@ _stats_pool_incr(struct context *ctx, struct server_pool *pool, } void -_stats_pool_decr(struct context *ctx, struct server_pool *pool, +_stats_pool_decr(struct context *ctx, const struct server_pool *pool, stats_pool_field_t fidx) { struct stats_metric *stm; @@ -1092,7 +1094,7 @@ _stats_pool_decr(struct context *ctx, struct server_pool *pool, } void -_stats_pool_incr_by(struct context *ctx, struct server_pool *pool, +_stats_pool_incr_by(struct context *ctx, const struct server_pool *pool, stats_pool_field_t fidx, int64_t val) { struct stats_metric *stm; @@ -1107,7 +1109,7 @@ _stats_pool_incr_by(struct context *ctx, struct server_pool *pool, } void -_stats_pool_decr_by(struct context *ctx, struct server_pool *pool, +_stats_pool_decr_by(struct context *ctx, const struct server_pool *pool, stats_pool_field_t fidx, int64_t val) { struct stats_metric *stm; @@ -1122,7 +1124,7 @@ _stats_pool_decr_by(struct context *ctx, struct server_pool *pool, } void -_stats_pool_set_ts(struct context *ctx, struct server_pool *pool, +_stats_pool_set_ts(struct context *ctx, const struct server_pool *pool, stats_pool_field_t fidx, int64_t val) { struct stats_metric *stm; @@ -1137,7 +1139,7 @@ _stats_pool_set_ts(struct context *ctx, struct server_pool *pool, } static struct stats_metric * -stats_server_to_metric(struct context *ctx, struct server *server, +stats_server_to_metric(struct context *ctx, const struct server *server, stats_server_field_t fidx) { struct stats *st; @@ -1163,7 +1165,7 @@ stats_server_to_metric(struct context *ctx, struct server *server, } void -_stats_server_incr(struct context *ctx, struct server *server, +_stats_server_incr(struct context *ctx, const struct server *server, stats_server_field_t fidx) { struct stats_metric *stm; @@ -1178,7 +1180,7 @@ _stats_server_incr(struct context *ctx, struct server *server, } void -_stats_server_decr(struct context *ctx, struct server *server, +_stats_server_decr(struct context *ctx, const struct server *server, stats_server_field_t fidx) { struct stats_metric *stm; @@ -1193,7 +1195,7 @@ _stats_server_decr(struct context *ctx, struct server *server, } void -_stats_server_incr_by(struct context *ctx, struct server *server, +_stats_server_incr_by(struct context *ctx, const struct server *server, stats_server_field_t fidx, int64_t val) { struct stats_metric *stm; @@ -1208,7 +1210,7 @@ _stats_server_incr_by(struct context *ctx, struct server *server, } void -_stats_server_decr_by(struct context *ctx, struct server *server, +_stats_server_decr_by(struct context *ctx, const struct server *server, stats_server_field_t fidx, int64_t val) { struct stats_metric *stm; @@ -1223,7 +1225,7 @@ _stats_server_decr_by(struct context *ctx, struct server *server, } void -_stats_server_set_ts(struct context *ctx, struct server *server, +_stats_server_set_ts(struct context *ctx, const struct server *server, stats_server_field_t fidx, int64_t val) { struct stats_metric *stm; diff --git a/src/nc_stats.h b/src/nc_stats.h index 1703b0a8..ee769fbd 100644 --- a/src/nc_stats.h +++ b/src/nc_stats.h @@ -197,19 +197,19 @@ typedef enum stats_server_field { void stats_describe(void); -void _stats_pool_incr(struct context *ctx, struct server_pool *pool, stats_pool_field_t fidx); -void _stats_pool_decr(struct context *ctx, struct server_pool *pool, stats_pool_field_t fidx); -void _stats_pool_incr_by(struct context *ctx, struct server_pool *pool, stats_pool_field_t fidx, int64_t val); -void _stats_pool_decr_by(struct context *ctx, struct server_pool *pool, stats_pool_field_t fidx, int64_t val); -void _stats_pool_set_ts(struct context *ctx, struct server_pool *pool, stats_pool_field_t fidx, int64_t val); - -void _stats_server_incr(struct context *ctx, struct server *server, stats_server_field_t fidx); -void _stats_server_decr(struct context *ctx, struct server *server, stats_server_field_t fidx); -void _stats_server_incr_by(struct context *ctx, struct server *server, stats_server_field_t fidx, int64_t val); -void _stats_server_decr_by(struct context *ctx, struct server *server, stats_server_field_t fidx, int64_t val); -void _stats_server_set_ts(struct context *ctx, struct server *server, stats_server_field_t fidx, int64_t val); - -struct stats *stats_create(uint16_t stats_port, char *stats_ip, int stats_interval, char *source, struct array *server_pool); +void _stats_pool_incr(struct context *ctx, const struct server_pool *pool, stats_pool_field_t fidx); +void _stats_pool_decr(struct context *ctx, const struct server_pool *pool, stats_pool_field_t fidx); +void _stats_pool_incr_by(struct context *ctx, const struct server_pool *pool, stats_pool_field_t fidx, int64_t val); +void _stats_pool_decr_by(struct context *ctx, const struct server_pool *pool, stats_pool_field_t fidx, int64_t val); +void _stats_pool_set_ts(struct context *ctx, const struct server_pool *pool, stats_pool_field_t fidx, int64_t val); + +void _stats_server_incr(struct context *ctx, const struct server *server, stats_server_field_t fidx); +void _stats_server_decr(struct context *ctx, const struct server *server, stats_server_field_t fidx); +void _stats_server_incr_by(struct context *ctx, const struct server *server, stats_server_field_t fidx, int64_t val); +void _stats_server_decr_by(struct context *ctx, const struct server *server, stats_server_field_t fidx, int64_t val); +void _stats_server_set_ts(struct context *ctx, const struct server *server, stats_server_field_t fidx, int64_t val); + +struct stats *stats_create(uint16_t stats_port, const char *stats_ip, int stats_interval, const char *source, const struct array *server_pool); void stats_destroy(struct stats *stats); void stats_swap(struct stats *stats); diff --git a/src/nc_string.c b/src/nc_string.c index 31637c8e..dffaeafd 100644 --- a/src/nc_string.c +++ b/src/nc_string.c @@ -61,7 +61,7 @@ string_empty(const struct string *str) { ASSERT((str->len == 0 && str->data == NULL) || (str->len != 0 && str->data != NULL)); - return str->len == 0 ? true : false; + return str->len == 0; } rstatus_t @@ -108,10 +108,11 @@ string_compare(const struct string *s1, const struct string *s2) return nc_strncmp(s1->data, s2->data, s1->len); } +static const char *const hex = "0123456789abcdef"; + static char * _safe_utoa(int _base, uint64_t val, char *buf) { - char hex[] = "0123456789abcdef"; uint32_t base = (uint32_t) _base; *buf-- = 0; do { @@ -123,7 +124,6 @@ _safe_utoa(int _base, uint64_t val, char *buf) static char * _safe_itoa(int base, int64_t val, char *buf) { - char hex[] = "0123456789abcdef"; char *orig_buf = buf; const int32_t is_neg = (val < 0); *buf-- = 0; diff --git a/src/nc_util.c b/src/nc_util.c index e2768089..a8842958 100644 --- a/src/nc_util.c +++ b/src/nc_util.c @@ -188,7 +188,7 @@ nc_get_rcvbuf(int sd) } int -_nc_atoi(uint8_t *line, size_t n) +_nc_atoi(const uint8_t *line, size_t n) { int value; @@ -462,7 +462,7 @@ nc_msec_now(void) } static int -nc_resolve_inet(struct string *name, int port, struct sockinfo *si) +nc_resolve_inet(const struct string *name, int port, struct sockinfo *si) { int status; struct addrinfo *ai, *cai; /* head and current addrinfo */ @@ -532,7 +532,7 @@ nc_resolve_inet(struct string *name, int port, struct sockinfo *si) } static int -nc_resolve_unix(struct string *name, struct sockinfo *si) +nc_resolve_unix(const struct string *name, struct sockinfo *si) { struct sockaddr_un *un; @@ -560,7 +560,7 @@ nc_resolve_unix(struct string *name, struct sockinfo *si) * This routine is reentrant */ int -nc_resolve(struct string *name, int port, struct sockinfo *si) +nc_resolve(const struct string *name, int port, struct sockinfo *si) { if (name != NULL && name->data[0] == '/') { return nc_resolve_unix(name, si); @@ -575,7 +575,7 @@ nc_resolve(struct string *name, int port, struct sockinfo *si) * * This routine is not reentrant */ -char * +const char * nc_unresolve_addr(struct sockaddr *addr, socklen_t addrlen) { static char unresolve[NI_MAXHOST + NI_MAXSERV]; @@ -600,7 +600,7 @@ nc_unresolve_addr(struct sockaddr *addr, socklen_t addrlen) * * This routine is not reentrant */ -char * +const char * nc_unresolve_peer_desc(int sd) { static struct sockinfo si; @@ -626,7 +626,7 @@ nc_unresolve_peer_desc(int sd) * * This routine is not reentrant */ -char * +const char * nc_unresolve_desc(int sd) { static struct sockinfo si; diff --git a/src/nc_util.h b/src/nc_util.h index 986b7aef..da13aedc 100644 --- a/src/nc_util.h +++ b/src/nc_util.h @@ -20,6 +20,18 @@ #include +#ifdef __GNUC__ +# define NC_GCC_VERSION (__GNUC__ * 1000 + __GNUC_MINOR__) +#else +# define NC_GCC_VERSION 0 +#endif +#if NC_GCC_VERSION >= 2007 +#define NC_ATTRIBUTE_FORMAT(type, idx, first) __attribute__ ((format(type, idx, first))) +#else +#define NC_ATTRIBUTE_FORMAT(type, idx, first) +#endif + + #define LF (uint8_t) 10 #define CR (uint8_t) 13 #define CRLF "\x0d\x0a" @@ -90,7 +102,7 @@ int nc_get_soerror(int sd); int nc_get_sndbuf(int sd); int nc_get_rcvbuf(int sd); -int _nc_atoi(uint8_t *line, size_t n); +int _nc_atoi(const uint8_t *line, size_t n); bool nc_valid_port(int n); /* @@ -188,7 +200,7 @@ void nc_assert(const char *cond, const char *file, int line, int panic); void nc_stacktrace(int skip_count); void nc_stacktrace_fd(int fd); -int _scnprintf(char *buf, size_t size, const char *fmt, ...); +int _scnprintf(char *buf, size_t size, const char *fmt, ...) NC_ATTRIBUTE_FORMAT(printf, 3, 4); int _vscnprintf(char *buf, size_t size, const char *fmt, va_list args); int64_t nc_usec_now(void); int64_t nc_msec_now(void); @@ -208,9 +220,9 @@ struct sockinfo { } addr; }; -int nc_resolve(struct string *name, int port, struct sockinfo *si); -char *nc_unresolve_addr(struct sockaddr *addr, socklen_t addrlen); -char *nc_unresolve_peer_desc(int sd); -char *nc_unresolve_desc(int sd); +int nc_resolve(const struct string *name, int port, struct sockinfo *si); +const char *nc_unresolve_addr(struct sockaddr *addr, socklen_t addrlen); +const char *nc_unresolve_peer_desc(int sd); +const char *nc_unresolve_desc(int sd); #endif diff --git a/src/proto/nc_memcache.c b/src/proto/nc_memcache.c index be3c5f54..79545976 100644 --- a/src/proto/nc_memcache.c +++ b/src/proto/nc_memcache.c @@ -37,7 +37,7 @@ * return false */ static bool -memcache_storage(struct msg *r) +memcache_storage(const struct msg *r) { switch (r->type) { case MSG_REQ_MC_SET: @@ -60,7 +60,7 @@ memcache_storage(struct msg *r) * return false */ static bool -memcache_cas(struct msg *r) +memcache_cas(const struct msg *r) { if (r->type == MSG_REQ_MC_CAS) { return true; @@ -74,7 +74,7 @@ memcache_cas(struct msg *r) * return false */ static bool -memcache_retrieval(struct msg *r) +memcache_retrieval(const struct msg *r) { switch (r->type) { case MSG_REQ_MC_GET: @@ -88,12 +88,43 @@ memcache_retrieval(struct msg *r) return false; } +/* + * Return true, if the memcache command should be fragmented, + * otherwise return false. + * + * The only supported memcache commands that can have multiple keys + * are get/gets. Both are multigets, and the latter returns CAS token with the + * value. + * + * Fragmented requests are assumed to be slower due to the fact that they need + * to allocate an array to track which key went to which server, + * so avoid them when possible. + */ +static bool +memcache_should_fragment(const struct msg *r) +{ + switch (r->type) { + case MSG_REQ_MC_GET: + case MSG_REQ_MC_GETS: + /* + * A memcache get for a single key is only sent to one server. + * Fragmenting it would work but be less efficient. + */ + return array_n(r->keys) != 1; + + default: + break; + } + + return false; +} + /* * Return true, if the memcache command is a arithmetic command, otherwise * return false */ static bool -memcache_arithmetic(struct msg *r) +memcache_arithmetic(const struct msg *r) { switch (r->type) { case MSG_REQ_MC_INCR: @@ -112,7 +143,7 @@ memcache_arithmetic(struct msg *r) * return false */ static bool -memcache_delete(struct msg *r) +memcache_delete(const struct msg *r) { if (r->type == MSG_REQ_MC_DELETE) { return true; @@ -126,7 +157,7 @@ memcache_delete(struct msg *r) * return false */ static bool -memcache_touch(struct msg *r) +memcache_touch(const struct msg *r) { if (r->type == MSG_REQ_MC_TOUCH) { return true; @@ -289,6 +320,14 @@ memcache_parse_req(struct msg *r) break; } + if (str7cmp(m, 'v', 'e', 'r', 's', 'i', 'o', 'n')) { + r->type = MSG_REQ_MC_VERSION; + if (!msg_set_placeholder_key(r)) { + goto enomem; + } + break; + } + break; } @@ -311,6 +350,7 @@ memcache_parse_req(struct msg *r) state = SW_SPACES_BEFORE_KEY; break; + case MSG_REQ_MC_VERSION: case MSG_REQ_MC_QUIT: p = p - 1; /* go back by 1 byte */ state = SW_CRLF; @@ -348,7 +388,7 @@ memcache_parse_req(struct msg *r) log_error("parsed bad req %"PRIu64" of type %d with key " "prefix '%.*s...' and length %d that exceeds " "maximum key length", r->id, r->type, 16, - r->token, p - r->token); + r->token, (int)(p - r->token)); goto error; } else if (keylen == 0) { log_error("parsed bad req %"PRIu64" of type %d with an " @@ -371,11 +411,10 @@ memcache_parse_req(struct msg *r) state = SW_SPACES_BEFORE_FLAGS; } else if (memcache_arithmetic(r) || memcache_touch(r) ) { state = SW_SPACES_BEFORE_NUM; - } else if (memcache_delete(r)) { - state = SW_RUNTO_CRLF; } else if (memcache_retrieval(r)) { state = SW_SPACES_BEFORE_KEYS; } else { + /* delete, etc. */ state = SW_RUNTO_CRLF; } @@ -714,7 +753,7 @@ memcache_parse_req(struct msg *r) log_hexdump(LOG_VERB, b->pos, mbuf_length(b), "parsed req %"PRIu64" res %d " "type %d state %d rpos %d of %d", r->id, r->result, r->type, - r->state, r->pos - b->pos, b->last - b->pos); + r->state, (int)(r->pos - b->pos), (int)(b->last - b->pos)); return; done: @@ -726,7 +765,7 @@ memcache_parse_req(struct msg *r) log_hexdump(LOG_VERB, b->pos, mbuf_length(b), "parsed req %"PRIu64" res %d " "type %d state %d rpos %d of %d", r->id, r->result, r->type, - r->state, r->pos - b->pos, b->last - b->pos); + r->state, (int)(r->pos - b->pos), (int)(b->last - b->pos)); return; enomem: @@ -886,6 +925,11 @@ memcache_parse_rsp(struct msg *r) break; } + if (str7cmp(m, 'V', 'E', 'R', 'S', 'I', 'O', 'N')) { + r->type = MSG_RSP_MC_VERSION; + break; + } + break; case 9: @@ -945,6 +989,7 @@ memcache_parse_rsp(struct msg *r) case MSG_RSP_MC_CLIENT_ERROR: case MSG_RSP_MC_SERVER_ERROR: + case MSG_RSP_MC_VERSION: state = SW_RUNTO_CRLF; break; @@ -1174,7 +1219,7 @@ memcache_parse_rsp(struct msg *r) log_hexdump(LOG_VERB, b->pos, mbuf_length(b), "parsed rsp %"PRIu64" res %d " "type %d state %d rpos %d of %d", r->id, r->result, r->type, - r->state, r->pos - b->pos, b->last - b->pos); + r->state, (int)(r->pos - b->pos), (int)(b->last - b->pos)); return; done: @@ -1187,7 +1232,7 @@ memcache_parse_rsp(struct msg *r) log_hexdump(LOG_VERB, b->pos, mbuf_length(b), "parsed rsp %"PRIu64" res %d " "type %d state %d rpos %d of %d", r->id, r->result, r->type, - r->state, r->pos - b->pos, b->last - b->pos); + r->state, (int)(r->pos - b->pos), (int)(b->last - b->pos)); return; error: @@ -1201,13 +1246,13 @@ memcache_parse_rsp(struct msg *r) } bool -memcache_failure(struct msg *r) +memcache_failure(const struct msg *r) { return false; } static rstatus_t -memcache_append_key(struct msg *r, uint8_t *key, uint32_t keylen) +memcache_append_key(struct msg *r, const uint8_t *key, uint32_t keylen) { struct mbuf *mbuf; struct keypos *kpos; @@ -1227,7 +1272,7 @@ memcache_append_key(struct msg *r, uint8_t *key, uint32_t keylen) mbuf_copy(mbuf, key, keylen); r->mlen += keylen; - mbuf_copy(mbuf, (uint8_t *)" ", 1); + mbuf_copy(mbuf, (const uint8_t *)" ", 1); r->mlen += 1; return NC_OK; } @@ -1236,7 +1281,7 @@ memcache_append_key(struct msg *r, uint8_t *key, uint32_t keylen) * read the comment in proto/nc_redis.c */ static rstatus_t -memcache_fragment_retrieval(struct msg *r, uint32_t ncontinuum, +memcache_fragment_retrieval(struct msg *r, uint32_t nserver, struct msg_tqh *frag_msgq, uint32_t key_step) { @@ -1245,7 +1290,7 @@ memcache_fragment_retrieval(struct msg *r, uint32_t ncontinuum, uint32_t i; rstatus_t status; - sub_msgs = nc_zalloc(ncontinuum * sizeof(*sub_msgs)); + sub_msgs = nc_zalloc(nserver * sizeof(*sub_msgs)); if (sub_msgs == NULL) { return NC_ENOMEM; } @@ -1275,10 +1320,12 @@ memcache_fragment_retrieval(struct msg *r, uint32_t ncontinuum, r->nfrag = 0; r->frag_owner = r; + /* Build up the key1 key2 ... to be sent to a given server at index idx */ for (i = 0; i < array_n(r->keys); i++) { /* for each key */ struct msg *sub_msg; struct keypos *kpos = array_get(r->keys, i); uint32_t idx = msg_backend_idx(r, kpos->start, kpos->end - kpos->start); + ASSERT(idx < nserver); if (sub_msgs[idx] == NULL) { sub_msgs[idx] = msg_get(r->owner, r->request, r->redis); @@ -1296,8 +1343,11 @@ memcache_fragment_retrieval(struct msg *r, uint32_t ncontinuum, return status; } } - - for (i = 0; i < ncontinuum; i++) { /* prepend mget header, and forward it */ + /* + * prepend mget header, and forward the get[s] key1 key2\r\n + * to the corresponding server(s) + */ + for (i = 0; i < nserver; i++) { struct msg *sub_msg = sub_msgs[i]; if (sub_msg == NULL) { continue; @@ -1305,9 +1355,9 @@ memcache_fragment_retrieval(struct msg *r, uint32_t ncontinuum, /* prepend get/gets */ if (r->type == MSG_REQ_MC_GET) { - status = msg_prepend(sub_msg, (uint8_t *)"get ", 4); + status = msg_prepend(sub_msg, (const uint8_t *)"get ", 4); } else if (r->type == MSG_REQ_MC_GETS) { - status = msg_prepend(sub_msg, (uint8_t *)"gets ", 5); + status = msg_prepend(sub_msg, (const uint8_t *)"gets ", 5); } if (status != NC_OK) { nc_free(sub_msgs); @@ -1315,7 +1365,7 @@ memcache_fragment_retrieval(struct msg *r, uint32_t ncontinuum, } /* append \r\n */ - status = msg_append(sub_msg, (uint8_t *)CRLF, CRLF_LEN); + status = msg_append(sub_msg, (const uint8_t *)CRLF, CRLF_LEN); if (status != NC_OK) { nc_free(sub_msgs); return status; @@ -1334,10 +1384,10 @@ memcache_fragment_retrieval(struct msg *r, uint32_t ncontinuum, } rstatus_t -memcache_fragment(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msgq) +memcache_fragment(struct msg *r, uint32_t nserver, struct msg_tqh *frag_msgq) { - if (memcache_retrieval(r)) { - return memcache_fragment_retrieval(r, ncontinuum, frag_msgq, 1); + if (memcache_should_fragment(r)) { + return memcache_fragment_retrieval(r, nserver, frag_msgq, 1); } return NC_OK; } @@ -1422,7 +1472,8 @@ static rstatus_t memcache_copy_bulk(struct msg *dst, struct msg *src) { struct mbuf *mbuf, *nbuf; - uint8_t *p; + const uint8_t *p; + const uint8_t *last; uint32_t len = 0; uint32_t bytes = 0; uint32_t i = 0; @@ -1440,31 +1491,46 @@ memcache_copy_bulk(struct msg *dst, struct msg *src) return NC_OK; /* key not exists */ } p = mbuf->pos; + last = mbuf->last; /* - * get : VALUE key 0 len\r\nval\r\n - * gets: VALUE key 0 len cas\r\nval\r\n + * get : VALUE key flags len\r\nval\r\n + * gets: VALUE key flags len cas\r\nval\r\n */ ASSERT(*p == 'V'); - for (i = 0; i < 3; i++) { /* eat 'VALUE key 0 ' */ - for (; *p != ' ';) { - p++; + i = 0; + while (p < last) { /* eat 'VALUE key flags ' */ + if (*p == ' ') { + i++; + if (i >= 3) { + p++; + break; + } } p++; } len = 0; - for (; p < mbuf->last && isdigit(*p); p++) { + for (; p < last && isdigit(*p); p++) { len = len * 10 + (uint32_t)(*p - '0'); } - for (; p < mbuf->last && ('\r' != *p); p++) { /* eat cas for gets */ + for (; p < last && ('\r' != *p); p++) { /* eat cas for gets */ ; } + /* "*p" should be pointing to '\r' */ len += CRLF_LEN * 2; len += (p - mbuf->pos); + if (p >= last) { + log_error("Saw memcache value response where header was not " + "parsed or header length %d unexpectedly exceeded mbuf size limit", + (int)(p - mbuf->pos)); + return NC_ERROR; + } + + bytes = len; /* copy len bytes to dst */ @@ -1514,8 +1580,8 @@ memcache_post_coalesce(struct msg *request) return; } - for (i = 0; i < array_n(request->keys); i++) { /* for each key */ - sub_msg = request->frag_seq[i]->peer; /* get it's peer response */ + for (i = 0; i < array_n(request->keys); i++) { /* for each key */ + sub_msg = request->frag_seq[i]->peer; /* get its peer response */ if (sub_msg == NULL) { response->owner->err = 1; return; @@ -1528,7 +1594,7 @@ memcache_post_coalesce(struct msg *request) } /* append END\r\n */ - status = msg_append(response, (uint8_t *)"END\r\n", 5); + status = msg_append(response, (const uint8_t *)"END\r\n", 5); if (status != NC_OK) { response->owner->err = 1; return; diff --git a/src/proto/nc_proto.h b/src/proto/nc_proto.h index 62370290..6470f7b7 100644 --- a/src/proto/nc_proto.h +++ b/src/proto/nc_proto.h @@ -138,24 +138,28 @@ (str15icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14) && \ (m[15] == c15 || m[15] == (c15 ^ 0x20))) +#define str17icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16) \ + (str16icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15) && \ + (m[16] == c16 || m[16] == (c16 ^ 0x20))) + void memcache_parse_req(struct msg *r); void memcache_parse_rsp(struct msg *r); -bool memcache_failure(struct msg *r); +bool memcache_failure(const struct msg *r); void memcache_pre_coalesce(struct msg *r); void memcache_post_coalesce(struct msg *r); rstatus_t memcache_add_auth(struct context *ctx, struct conn *c_conn, struct conn *s_conn); -rstatus_t memcache_fragment(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msgq); +rstatus_t memcache_fragment(struct msg *r, uint32_t nserver, struct msg_tqh *frag_msgq); rstatus_t memcache_reply(struct msg *r); void memcache_post_connect(struct context *ctx, struct conn *conn, struct server *server); void memcache_swallow_msg(struct conn *conn, struct msg *pmsg, struct msg *msg); void redis_parse_req(struct msg *r); void redis_parse_rsp(struct msg *r); -bool redis_failure(struct msg *r); +bool redis_failure(const struct msg *r); void redis_pre_coalesce(struct msg *r); void redis_post_coalesce(struct msg *r); rstatus_t redis_add_auth(struct context *ctx, struct conn *c_conn, struct conn *s_conn); -rstatus_t redis_fragment(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msgq); +rstatus_t redis_fragment(struct msg *r, uint32_t nserver, struct msg_tqh *frag_msgq); rstatus_t redis_reply(struct msg *r); void redis_post_connect(struct context *ctx, struct conn *conn, struct server *server); void redis_swallow_msg(struct conn *conn, struct msg *pmsg, struct msg *msg); diff --git a/src/proto/nc_redis.c b/src/proto/nc_redis.c index 7a91993d..a5ea210f 100644 --- a/src/proto/nc_redis.c +++ b/src/proto/nc_redis.c @@ -40,11 +40,13 @@ static rstatus_t redis_handle_auth_req(struct msg *request, struct msg *response * return false */ static bool -redis_argz(struct msg *r) +redis_argz(const struct msg *r) { switch (r->type) { + /* TODO: PING has an optional argument, emulate that? */ case MSG_REQ_REDIS_PING: case MSG_REQ_REDIS_QUIT: + case MSG_REQ_REDIS_COMMAND: return true; default: @@ -59,10 +61,9 @@ redis_argz(struct msg *r) * return false */ static bool -redis_arg0(struct msg *r) +redis_arg0(const struct msg *r) { switch (r->type) { - case MSG_REQ_REDIS_EXISTS: case MSG_REQ_REDIS_PERSIST: case MSG_REQ_REDIS_PTTL: case MSG_REQ_REDIS_TTL: @@ -71,6 +72,7 @@ redis_arg0(struct msg *r) case MSG_REQ_REDIS_DECR: case MSG_REQ_REDIS_GET: + case MSG_REQ_REDIS_GETDEL: case MSG_REQ_REDIS_INCR: case MSG_REQ_REDIS_STRLEN: @@ -80,14 +82,12 @@ redis_arg0(struct msg *r) case MSG_REQ_REDIS_HVALS: case MSG_REQ_REDIS_LLEN: - case MSG_REQ_REDIS_LPOP: - case MSG_REQ_REDIS_RPOP: case MSG_REQ_REDIS_SCARD: case MSG_REQ_REDIS_SMEMBERS: case MSG_REQ_REDIS_ZCARD: - case MSG_REQ_REDIS_PFCOUNT: + /* TODO: Support emulating 2-arg username+password auth by just checking password? */ case MSG_REQ_REDIS_AUTH: return true; @@ -103,13 +103,14 @@ redis_arg0(struct msg *r) * return false */ static bool -redis_arg1(struct msg *r) +redis_arg1(const struct msg *r) { switch (r->type) { case MSG_REQ_REDIS_EXPIRE: case MSG_REQ_REDIS_EXPIREAT: case MSG_REQ_REDIS_PEXPIRE: case MSG_REQ_REDIS_PEXPIREAT: + case MSG_REQ_REDIS_MOVE: case MSG_REQ_REDIS_APPEND: case MSG_REQ_REDIS_DECRBY: @@ -121,11 +122,10 @@ redis_arg1(struct msg *r) case MSG_REQ_REDIS_HEXISTS: case MSG_REQ_REDIS_HGET: + case MSG_REQ_REDIS_HSTRLEN: case MSG_REQ_REDIS_LINDEX: - case MSG_REQ_REDIS_LPUSHX: case MSG_REQ_REDIS_RPOPLPUSH: - case MSG_REQ_REDIS_RPUSHX: case MSG_REQ_REDIS_SISMEMBER: @@ -146,7 +146,7 @@ redis_arg1(struct msg *r) * return false */ static bool -redis_arg2(struct msg *r) +redis_arg2(const struct msg *r) { switch (r->type) { case MSG_REQ_REDIS_GETRANGE: @@ -157,7 +157,6 @@ redis_arg2(struct msg *r) case MSG_REQ_REDIS_HINCRBY: case MSG_REQ_REDIS_HINCRBYFLOAT: - case MSG_REQ_REDIS_HSET: case MSG_REQ_REDIS_HSETNX: case MSG_REQ_REDIS_LRANGE: @@ -173,8 +172,6 @@ redis_arg2(struct msg *r) case MSG_REQ_REDIS_ZREMRANGEBYLEX: case MSG_REQ_REDIS_ZREMRANGEBYRANK: case MSG_REQ_REDIS_ZREMRANGEBYSCORE: - - case MSG_REQ_REDIS_RESTORE: return true; default: @@ -189,10 +186,11 @@ redis_arg2(struct msg *r) * return false */ static bool -redis_arg3(struct msg *r) +redis_arg3(const struct msg *r) { switch (r->type) { case MSG_REQ_REDIS_LINSERT: + case MSG_REQ_REDIS_LMOVE: return true; default: @@ -203,26 +201,39 @@ redis_arg3(struct msg *r) } /* - * Return true, if the redis command accepts 0 or more arguments, otherwise + * Return true, if the redis command operates on one key and accepts 0 or more arguments, otherwise * return false */ static bool -redis_argn(struct msg *r) +redis_argn(const struct msg *r) { switch (r->type) { case MSG_REQ_REDIS_SORT: + case MSG_REQ_REDIS_COPY: case MSG_REQ_REDIS_BITCOUNT: case MSG_REQ_REDIS_BITPOS: + case MSG_REQ_REDIS_BITFIELD: + /* TODO: Support REDIS_BITOP operation destkey key ... and add tests - this requires handling key in a position other than the first one */ + case MSG_REQ_REDIS_EXISTS: + case MSG_REQ_REDIS_GETEX: case MSG_REQ_REDIS_SET: + case MSG_REQ_REDIS_HDEL: case MSG_REQ_REDIS_HMGET: case MSG_REQ_REDIS_HMSET: case MSG_REQ_REDIS_HSCAN: + case MSG_REQ_REDIS_HSET: + case MSG_REQ_REDIS_HRANDFIELD: case MSG_REQ_REDIS_LPUSH: + case MSG_REQ_REDIS_LPUSHX: case MSG_REQ_REDIS_RPUSH: + case MSG_REQ_REDIS_RPUSHX: + case MSG_REQ_REDIS_LPOP: + case MSG_REQ_REDIS_RPOP: + case MSG_REQ_REDIS_LPOS: case MSG_REQ_REDIS_SADD: case MSG_REQ_REDIS_SDIFF: @@ -235,20 +246,43 @@ redis_argn(struct msg *r) case MSG_REQ_REDIS_SRANDMEMBER: case MSG_REQ_REDIS_SSCAN: case MSG_REQ_REDIS_SPOP: + case MSG_REQ_REDIS_SMISMEMBER: case MSG_REQ_REDIS_PFADD: case MSG_REQ_REDIS_PFMERGE: + case MSG_REQ_REDIS_PFCOUNT: case MSG_REQ_REDIS_ZADD: + case MSG_REQ_REDIS_ZDIFF: + case MSG_REQ_REDIS_ZDIFFSTORE: + case MSG_REQ_REDIS_ZINTER: case MSG_REQ_REDIS_ZINTERSTORE: + case MSG_REQ_REDIS_ZMSCORE: + case MSG_REQ_REDIS_ZPOPMAX: + case MSG_REQ_REDIS_ZPOPMIN: + case MSG_REQ_REDIS_ZRANDMEMBER: case MSG_REQ_REDIS_ZRANGE: + case MSG_REQ_REDIS_ZRANGEBYLEX: case MSG_REQ_REDIS_ZRANGEBYSCORE: + case MSG_REQ_REDIS_ZRANGESTORE: case MSG_REQ_REDIS_ZREM: case MSG_REQ_REDIS_ZREVRANGE: - case MSG_REQ_REDIS_ZRANGEBYLEX: + case MSG_REQ_REDIS_ZREVRANGEBYLEX: case MSG_REQ_REDIS_ZREVRANGEBYSCORE: - case MSG_REQ_REDIS_ZUNIONSTORE: case MSG_REQ_REDIS_ZSCAN: + case MSG_REQ_REDIS_ZUNION: + case MSG_REQ_REDIS_ZUNIONSTORE: + + case MSG_REQ_REDIS_GEODIST: + case MSG_REQ_REDIS_GEOPOS: + case MSG_REQ_REDIS_GEOHASH: + case MSG_REQ_REDIS_GEOADD: + case MSG_REQ_REDIS_GEORADIUS: + case MSG_REQ_REDIS_GEORADIUSBYMEMBER: + case MSG_REQ_REDIS_GEOSEARCH: + case MSG_REQ_REDIS_GEOSEARCHSTORE: + + case MSG_REQ_REDIS_RESTORE: return true; default: @@ -263,11 +297,13 @@ redis_argn(struct msg *r) * more keys, otherwise return false */ static bool -redis_argx(struct msg *r) +redis_argx(const struct msg *r) { switch (r->type) { case MSG_REQ_REDIS_MGET: case MSG_REQ_REDIS_DEL: + case MSG_REQ_REDIS_UNLINK: + case MSG_REQ_REDIS_TOUCH: return true; default: @@ -282,7 +318,7 @@ redis_argx(struct msg *r) * more key-value pairs, otherwise return false */ static bool -redis_argkvx(struct msg *r) +redis_argkvx(const struct msg *r) { switch (r->type) { case MSG_REQ_REDIS_MSET: @@ -302,7 +338,7 @@ redis_argkvx(struct msg *r) * that at least one argument is required, but that shouldn't be the case). */ static bool -redis_argeval(struct msg *r) +redis_argeval(const struct msg *r) { switch (r->type) { case MSG_REQ_REDIS_EVAL: @@ -316,12 +352,26 @@ redis_argeval(struct msg *r) return false; } +static bool +redis_nokey(const struct msg *r) +{ + switch (r->type) { + case MSG_REQ_REDIS_LOLWUT: + return true; + + default: + break; + } + + return false; +} + /* * Return true, if the redis response is an error response i.e. a simple * string whose first character is '-', otherwise return false. */ static bool -redis_error(struct msg *r) +redis_error(const struct msg *r) { switch (r->type) { case MSG_RSP_REDIS_ERROR: @@ -426,17 +476,23 @@ redis_parse_req(struct msg *r) switch (state) { case SW_START: + ASSERT(r->token == NULL); + if (ch != '*') { + /* redis commands are always arrays */ + goto error; + } + r->token = p; + /* req_start <- p */ + r->narg_start = p; + r->rnarg = 0; + state = SW_NARG; + + break; + case SW_NARG: - if (r->token == NULL) { - if (ch != '*') { - goto error; - } - r->token = p; - /* req_start <- p */ - r->narg_start = p; - r->rnarg = 0; - state = SW_NARG; - } else if (isdigit(ch)) { + /* SW_NARG: The number of arguments in the redis command array */ + ASSERT(r->token != NULL); + if (isdigit(ch)) { r->rnarg = r->rnarg * 10 + (uint32_t)(ch - '0'); } else if (ch == CR) { if (r->rnarg == 0) { @@ -596,6 +652,11 @@ redis_parse_req(struct msg *r) break; } + if (str4icmp(m, 'l', 'p', 'o', 's')) { + r->type = MSG_REQ_REDIS_LPOS; + break; + } + if (str4icmp(m, 'l', 'r', 'e', 'm')) { r->type = MSG_REQ_REDIS_LREM; break; @@ -678,6 +739,17 @@ redis_parse_req(struct msg *r) break; } + if (str4icmp(m, 'm', 'o', 'v', 'e')) { + r->type = MSG_REQ_REDIS_MOVE; + r->noforward = 1; + break; + } + + if (str4icmp(m, 'c', 'o', 'p', 'y')) { + r->type = MSG_REQ_REDIS_COPY; + break; + } + break; case 5: @@ -756,6 +828,11 @@ redis_parse_req(struct msg *r) break; } + if (str5icmp(m, 'z', 'd', 'i', 'f', 'f')) { + r->type = MSG_REQ_REDIS_ZDIFF; + break; + } + if (str5icmp(m, 'z', 'r', 'a', 'n', 'k')) { r->type = MSG_REQ_REDIS_ZRANK; break; @@ -771,6 +848,21 @@ redis_parse_req(struct msg *r) break; } + if (str5icmp(m, 'g', 'e', 't', 'e', 'x')) { + r->type = MSG_REQ_REDIS_GETEX; + break; + } + + if (str5icmp(m, 't', 'o', 'u', 'c', 'h')) { + r->type = MSG_REQ_REDIS_TOUCH; + break; + } + + if (str5icmp(m, 'l', 'm', 'o', 'v', 'e')) { + r->type = MSG_REQ_REDIS_LMOVE; + break; + } + break; case 6: @@ -879,6 +971,44 @@ redis_parse_req(struct msg *r) break; } + if (str6icmp(m, 'g', 'e', 'o', 'p', 'o', 's')) { + r->type = MSG_REQ_REDIS_GEOPOS; + break; + } + + if (str6icmp(m, 'g', 'e', 'o', 'a', 'd', 'd')) { + r->type = MSG_REQ_REDIS_GEOADD; + break; + } + + if (str6icmp(m, 'g', 'e', 't', 'd', 'e', 'l')) { + r->type = MSG_REQ_REDIS_GETDEL; + break; + } + + if (str6icmp(m, 'z', 'u', 'n', 'i', 'o', 'n')) { + r->type = MSG_REQ_REDIS_ZUNION; + break; + } + + if (str6icmp(m, 'z', 'i', 'n', 't', 'e', 'r')) { + r->type = MSG_REQ_REDIS_ZINTER; + break; + } + + if (str6icmp(m, 'u', 'n', 'l', 'i', 'n', 'k')) { + r->type = MSG_REQ_REDIS_UNLINK; + break; + } + + if (str6icmp(m, 'l', 'o', 'l', 'w', 'u', 't')) { + r->type = MSG_REQ_REDIS_LOLWUT; + if (!msg_set_placeholder_key(r)) { + goto enomem; + } + break; + } + break; case 7: @@ -937,6 +1067,44 @@ redis_parse_req(struct msg *r) break; } + if (str7icmp(m, 'z', 'm', 's', 'c', 'o', 'r', 'e')) { + r->type = MSG_REQ_REDIS_ZMSCORE; + break; + } + + if (str7icmp(m, 'z', 'p', 'o', 'p', 'm', 'i', 'n')) { + r->type = MSG_REQ_REDIS_ZPOPMIN; + break; + } + + if (str7icmp(m, 'z', 'p', 'o', 'p', 'm', 'a', 'x')) { + r->type = MSG_REQ_REDIS_ZPOPMAX; + break; + } + + if (str7icmp(m, 'g', 'e', 'o', 'd', 'i', 's', 't')) { + r->type = MSG_REQ_REDIS_GEODIST; + break; + } + + if (str7icmp(m, 'g', 'e', 'o', 'h', 'a', 's', 'h')) { + r->type = MSG_REQ_REDIS_GEOHASH; + break; + } + + if (str7icmp(m, 'h', 's', 't', 'r', 'l', 'e', 'n')) { + r->type = MSG_REQ_REDIS_HSTRLEN; + break; + } + + if (str7icmp(m, 'c', 'o', 'm', 'm', 'a', 'n', 'd')) { + r->type = MSG_REQ_REDIS_COMMAND; + if (!msg_set_placeholder_key(r)) { + goto enomem; + } + break; + } + break; case 8: @@ -970,6 +1138,11 @@ redis_parse_req(struct msg *r) break; } + if (str8icmp(m, 'b', 'i', 't', 'f', 'i', 'e', 'l', 'd')) { + r->type = MSG_REQ_REDIS_BITFIELD; + break; + } + break; case 9: @@ -998,6 +1171,16 @@ redis_parse_req(struct msg *r) break; } + if (str9icmp(m, 'g', 'e', 'o', 's', 'e', 'a', 'r', 'c', 'h')) { + r->type = MSG_REQ_REDIS_GEOSEARCH; + break; + } + + if (str9icmp(m, 'g', 'e', 'o', 'r', 'a', 'd', 'i', 'u', 's')) { + r->type = MSG_REQ_REDIS_GEORADIUS; + break; + } + break; case 10: @@ -1006,6 +1189,24 @@ redis_parse_req(struct msg *r) break; } + if (str10icmp(m, 'h', 'r', 'a', 'n', 'd', 'f', 'i', 'e', 'l', 'd')) { + r->type = MSG_REQ_REDIS_HRANDFIELD; + break; + } + + if (str10icmp(m, 's', 'm', 'i', 's', 'm', 'e', 'm', 'b', 'e', 'r')) { + r->type = MSG_REQ_REDIS_SMISMEMBER; + break; + } + + if (str10icmp(m, 'z', 'd', 'i', 'f', 'f', 's', 't', 'o', 'r', 'e')) { + r->type = MSG_REQ_REDIS_ZDIFFSTORE; + break; + } + + + break; + case 11: if (str11icmp(m, 'i', 'n', 'c', 'r', 'b', 'y', 'f', 'l', 'o', 'a', 't')) { r->type = MSG_REQ_REDIS_INCRBYFLOAT; @@ -1042,6 +1243,16 @@ redis_parse_req(struct msg *r) break; } + if (str11icmp(m, 'z', 'r', 'a', 'n', 'd', 'm', 'e', 'm', 'b', 'e', 'r')) { + r->type = MSG_REQ_REDIS_ZRANDMEMBER; + break; + } + + if (str11icmp(m, 'z', 'r', 'a', 'n', 'g', 'e', 's', 't', 'o', 'r', 'e')) { + r->type = MSG_REQ_REDIS_ZRANGESTORE; + break; + } + break; case 12: @@ -1067,6 +1278,16 @@ redis_parse_req(struct msg *r) break; } + if (str14icmp(m, 'z', 'r', 'e', 'v', 'r', 'a', 'n', 'g', 'e', 'b', 'y', 'l', 'e', 'x')) { + r->type = MSG_REQ_REDIS_ZREVRANGEBYLEX; + break; + } + if (str14icmp(m, 'g', 'e', 'o', 's', 'e', 'a', 'r', 'c', 'h', 's', 't', 'o', 'r', 'e')) { + r->type = MSG_REQ_REDIS_GEOSEARCHSTORE; + break; + } + + break; case 15: @@ -1087,19 +1308,24 @@ redis_parse_req(struct msg *r) r->type = MSG_REQ_REDIS_ZREVRANGEBYSCORE; break; } - break; + case 17: + if (str17icmp(m, 'g', 'e', 'o', 'r', 'a', 'd', 'i', 'u', 's', 'b', 'y', 'm', 'e', 'm', 'b', 'e', 'r')) { + r->type = MSG_REQ_REDIS_GEORADIUSBYMEMBER; + break; + } + default: break; } if (r->type == MSG_UNKNOWN) { - log_error("parsed unsupported command '%.*s'", p - m, m); + log_error("parsed unsupported command '%.*s'", (int)(p - m), m); goto error; } - log_debug(LOG_VERB, "parsed command '%.*s'", p - m, m); + log_debug(LOG_VERB, "parsed command '%.*s'", (int)(p - m), m); state = SW_REQ_TYPE_LF; break; @@ -1108,7 +1334,16 @@ redis_parse_req(struct msg *r) switch (ch) { case LF: if (redis_argz(r)) { + if (r->narg != 1) { + /* It's an error to provide more than one argument. */ + goto error; + } goto done; + } else if (redis_nokey(r)) { + if (r->narg == 1) { + goto done; + } + state = SW_ARGN_LEN; } else if (r->narg == 1) { goto error; } else if (redis_argeval(r)) { @@ -1137,7 +1372,7 @@ redis_parse_req(struct msg *r) if (r->rlen >= mbuf_data_size()) { log_error("parsed bad req %"PRIu64" of type %d with key " "length %d that greater than or equal to maximum" - " redis key length of %d", r->id, r->type, + " redis key length of %zu", r->id, r->type, r->rlen, mbuf_data_size()); goto error; } @@ -1613,7 +1848,7 @@ redis_parse_req(struct msg *r) case SW_ARGN_LF: switch (ch) { case LF: - if (redis_argn(r) || redis_argeval(r)) { + if (redis_argn(r) || redis_argeval(r) || redis_nokey(r)) { if (r->rnarg == 0) { goto done; } @@ -1651,7 +1886,7 @@ redis_parse_req(struct msg *r) log_hexdump(LOG_VERB, b->pos, mbuf_length(b), "parsed req %"PRIu64" res %d " "type %d state %d rpos %d of %d", r->id, r->result, r->type, - r->state, r->pos - b->pos, b->last - b->pos); + r->state, (int)(r->pos - b->pos), (int)(b->last - b->pos)); return; done: @@ -1664,7 +1899,7 @@ redis_parse_req(struct msg *r) log_hexdump(LOG_VERB, b->pos, mbuf_length(b), "parsed req %"PRIu64" res %d " "type %d state %d rpos %d of %d", r->id, r->result, r->type, - r->state, r->pos - b->pos, b->last - b->pos); + r->state, (int)(r->pos - b->pos), (int)(b->last - b->pos)); return; enomem: @@ -1725,7 +1960,6 @@ redis_parse_rsp(struct msg *r) SW_START, SW_STATUS, SW_ERROR, - SW_INTEGER, SW_INTEGER_START, SW_SIMPLE, SW_BULK, @@ -1761,6 +1995,9 @@ redis_parse_rsp(struct msg *r) switch (state) { case SW_START: r->type = MSG_UNKNOWN; + r->rnarg = 1; + r->is_top_level = 1; + switch (ch) { case '+': p = p - 1; /* go back by 1 byte */ @@ -1776,8 +2013,8 @@ redis_parse_rsp(struct msg *r) case ':': r->type = MSG_RSP_REDIS_INTEGER; - p = p - 1; /* go back by 1 byte */ - state = SW_INTEGER; + r->integer = 0; + state = SW_INTEGER_START; break; case '$': @@ -1920,21 +2157,21 @@ redis_parse_rsp(struct msg *r) break; } - state = SW_RUNTO_CRLF; + if (ch == '\r') { + state = SW_ALMOST_DONE; + } else { + /* Read remaining characters until '\r' */ + state = SW_RUNTO_CRLF; + } } break; - case SW_INTEGER: - /* rsp_start <- p */ - state = SW_INTEGER_START; - r->integer = 0; - break; - case SW_SIMPLE: if (ch == CR) { - state = SW_MULTIBULK_ARGN_LF; - r->rnarg--; + ASSERT(r->rnarg > 0); + r->rnarg--; + state = SW_MULTIBULK_ARGN_LF; } break; @@ -1975,6 +2212,11 @@ redis_parse_rsp(struct msg *r) break; case SW_BULK: + /* + * SW_BULK is used for top-level bulk string replies. + * Within an array, SW_MULTIBULK_ARG... helpers are used + * to parse bulk strings instead. + */ if (r->token == NULL) { if (ch != '$') { goto error; @@ -2047,22 +2289,42 @@ redis_parse_rsp(struct msg *r) if (ch != '*') { goto error; } + r->vlen = 0; r->token = p; /* rsp_start <- p */ - r->narg_start = p; - r->rnarg = 0; + if (r->is_top_level) { + r->narg_start = p; + } } else if (ch == '-') { - state = SW_RUNTO_CRLF; + p = p-1; + r->token = NULL; + /* + * This is a null array (e.g. from BLPOP). Don't increment rnarg + * https://redis.io/topics/protocol + */ + r->vlen = 1; + state = SW_MULTIBULK_ARGN_LEN; } else if (isdigit(ch)) { - r->rnarg = r->rnarg * 10 + (uint32_t)(ch - '0'); + r->vlen = r->vlen * 10 + (uint32_t)(ch - '0'); } else if (ch == CR) { if ((p - r->token) <= 1) { goto error; } - r->narg = r->rnarg; - r->narg_end = p; + if (r->is_top_level) { + /* For multiget responses, we may need to know the number of responses to combine them. */ + r->narg = r->vlen; + r->narg_end = p; + } + r->is_top_level = 0; + ASSERT(r->rnarg > 0); + r->rnarg += r->vlen - 1; r->token = NULL; + + /* + * The stack is always initialized before transitioning + * to another state. + */ state = SW_MULTIBULK_NARG_LF; } else { goto error; @@ -2101,9 +2363,9 @@ redis_parse_rsp(struct msg *r) * there is a special case for sscan/hscan/zscan, these command * replay a nested multi-bulk with a number and a multi bulk like this: * - * - mulit-bulk + * - multi-bulk * - cursor - * - mulit-bulk + * - multi-bulk * - val1 * - val2 * - val3 @@ -2147,6 +2409,7 @@ redis_parse_rsp(struct msg *r) } else { state = SW_MULTIBULK_ARGN_LEN_LF; } + ASSERT(r->rnarg > 0); r->rnarg--; r->token = NULL; } else { @@ -2224,7 +2487,7 @@ redis_parse_rsp(struct msg *r) log_hexdump(LOG_VERB, b->pos, mbuf_length(b), "parsed rsp %"PRIu64" res %d " "type %d state %d rpos %d of %d", r->id, r->result, r->type, - r->state, r->pos - b->pos, b->last - b->pos); + r->state, (int)(r->pos - b->pos), (int)(b->last - b->pos)); return; done: @@ -2237,7 +2500,7 @@ redis_parse_rsp(struct msg *r) log_hexdump(LOG_VERB, b->pos, mbuf_length(b), "parsed rsp %"PRIu64" res %d " "type %d state %d rpos %d of %d", r->id, r->result, r->type, - r->state, r->pos - b->pos, b->last - b->pos); + r->state, (int)(r->pos - b->pos), (int)(b->last - b->pos)); return; error: @@ -2264,7 +2527,7 @@ redis_parse_rsp(struct msg *r) * See issue: https://github.com/twitter/twemproxy/issues/369 */ bool -redis_failure(struct msg *r) +redis_failure(const struct msg *r) { ASSERT(!r->request); @@ -2381,7 +2644,7 @@ redis_pre_coalesce(struct msg *r) switch (r->type) { case MSG_RSP_REDIS_INTEGER: /* only redis 'del' fragmented request sends back integer reply */ - ASSERT(pr->type == MSG_REQ_REDIS_DEL); + ASSERT(pr->type == MSG_REQ_REDIS_DEL || pr->type == MSG_REQ_REDIS_TOUCH || pr->type == MSG_REQ_REDIS_UNLINK); mbuf = STAILQ_FIRST(&r->mhdr); /* @@ -2408,7 +2671,7 @@ redis_pre_coalesce(struct msg *r) /* * Muti-bulk reply can span over multiple mbufs and in each reply * we should skip over the narg token. Our response parser - * guarantees thaat the narg token and the immediately following + * guarantees that the narg token and the immediately following * '\r\n' will exist in a contiguous region in the first mbuf */ ASSERT(r->narg_start == mbuf->pos); @@ -2444,7 +2707,7 @@ redis_pre_coalesce(struct msg *r) } static rstatus_t -redis_append_key(struct msg *r, uint8_t *key, uint32_t keylen) +redis_append_key(struct msg *r, const uint8_t *key, uint32_t keylen) { uint32_t len; struct mbuf *mbuf; @@ -2489,9 +2752,9 @@ redis_append_key(struct msg *r, uint8_t *key, uint32_t keylen) /* * input a msg, return a msg chain. - * ncontinuum is the number of backend redis/memcache server + * nserver is the number of backend redis/memcache server * - * the original msg will be fragment into at most ncontinuum fragments. + * the original msg will be fragmented into at most nserver fragments. * all the keys map to the same backend will group into one fragment. * * frag_id: @@ -2540,23 +2803,24 @@ redis_append_key(struct msg *r, uint8_t *key, uint32_t keylen) * */ static rstatus_t -redis_fragment_argx(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msgq, +redis_fragment_argx(struct msg *r, uint32_t nserver, struct msg_tqh *frag_msgq, uint32_t key_step) { struct mbuf *mbuf; struct msg **sub_msgs; uint32_t i; rstatus_t status; + struct array *keys = r->keys; - ASSERT(array_n(r->keys) == (r->narg - 1) / key_step); + ASSERT(array_n(keys) == (r->narg - 1) / key_step); - sub_msgs = nc_zalloc(ncontinuum * sizeof(*sub_msgs)); + sub_msgs = nc_zalloc(nserver * sizeof(*sub_msgs)); if (sub_msgs == NULL) { return NC_ENOMEM; } ASSERT(r->frag_seq == NULL); - r->frag_seq = nc_alloc(array_n(r->keys) * sizeof(*r->frag_seq)); + r->frag_seq = nc_alloc(array_n(keys) * sizeof(*r->frag_seq)); if (r->frag_seq == NULL) { nc_free(sub_msgs); return NC_ENOMEM; @@ -2582,10 +2846,12 @@ redis_fragment_argx(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msg r->nfrag = 0; r->frag_owner = r; - for (i = 0; i < array_n(r->keys); i++) { /* for each key */ + /* Build up the key1 key2 ... to be sent to a given server at index idx */ + for (i = 0; i < array_n(keys); i++) { /* for each key */ struct msg *sub_msg; - struct keypos *kpos = array_get(r->keys, i); + struct keypos *kpos = array_get(keys, i); uint32_t idx = msg_backend_idx(r, kpos->start, kpos->end - kpos->start); + ASSERT(idx < nserver); if (sub_msgs[idx] == NULL) { sub_msgs[idx] = msg_get(r->owner, r->request, r->redis); @@ -2622,7 +2888,11 @@ redis_fragment_argx(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msg } } - for (i = 0; i < ncontinuum; i++) { /* prepend mget header, and forward it */ + /* + * prepend mget header, and forward the command (command type+key(s)+suffix) + * to the corresponding server(s) + */ + for (i = 0; i < nserver; i++) { struct msg *sub_msg = sub_msgs[i]; if (sub_msg == NULL) { continue; @@ -2637,6 +2907,12 @@ redis_fragment_argx(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msg } else if (r->type == MSG_REQ_REDIS_MSET) { status = msg_prepend_format(sub_msg, "*%d\r\n$4\r\nmset\r\n", sub_msg->narg + 1); + } else if (r->type == MSG_REQ_REDIS_TOUCH) { + status = msg_prepend_format(sub_msg, "*%d\r\n$5\r\ntouch\r\n", + sub_msg->narg + 1); + } else if (r->type == MSG_REQ_REDIS_UNLINK) { + status = msg_prepend_format(sub_msg, "*%d\r\n$6\r\nunlink\r\n", + sub_msg->narg + 1); } else { NOT_REACHED(); } @@ -2658,7 +2934,7 @@ redis_fragment_argx(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msg } rstatus_t -redis_fragment(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msgq) +redis_fragment(struct msg *r, uint32_t nserver, struct msg_tqh *frag_msgq) { if (1 == array_n(r->keys)){ return NC_OK; @@ -2667,10 +2943,13 @@ redis_fragment(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msgq) switch (r->type) { case MSG_REQ_REDIS_MGET: case MSG_REQ_REDIS_DEL: - return redis_fragment_argx(r, ncontinuum, frag_msgq, 1); + case MSG_REQ_REDIS_TOUCH: + case MSG_REQ_REDIS_UNLINK: + return redis_fragment_argx(r, nserver, frag_msgq, 1); + /* TODO: MSETNX - instead of responding with OK, respond with 1 if all fragments respond with 1 */ case MSG_REQ_REDIS_MSET: - return redis_fragment_argx(r, ncontinuum, frag_msgq, 2); + return redis_fragment_argx(r, nserver, frag_msgq, 2); default: return NC_OK; @@ -2718,7 +2997,7 @@ redis_post_coalesce_mset(struct msg *request) } void -redis_post_coalesce_del(struct msg *request) +redis_post_coalesce_del_or_touch(struct msg *request) { struct msg *response = request->peer; rstatus_t status; @@ -2785,7 +3064,9 @@ redis_post_coalesce(struct msg *r) return redis_post_coalesce_mget(r); case MSG_REQ_REDIS_DEL: - return redis_post_coalesce_del(r); + case MSG_REQ_REDIS_TOUCH: + case MSG_REQ_REDIS_UNLINK: + return redis_post_coalesce_del_or_touch(r); case MSG_REQ_REDIS_MSET: return redis_post_coalesce_mset(r); @@ -2799,15 +3080,15 @@ static rstatus_t redis_handle_auth_req(struct msg *req, struct msg *rsp) { struct conn *conn = (struct conn *)rsp->owner; - struct server_pool *pool; - struct keypos *kpos; - uint8_t *key; + const struct server_pool *pool; + const struct keypos *kpos; + const uint8_t *key; uint32_t keylen; bool valid; ASSERT(conn->client && !conn->proxy); - pool = (struct server_pool *)conn->owner; + pool = (const struct server_pool *)conn->owner; if (!pool->require_auth) { /* @@ -2821,7 +3102,7 @@ redis_handle_auth_req(struct msg *req, struct msg *rsp) key = kpos->start; keylen = (uint32_t)(kpos->end - kpos->start); valid = (keylen == pool->redis_auth.len) && - (memcmp(pool->redis_auth.data, key, keylen) == 0) ? true : false; + (memcmp(pool->redis_auth.data, key, keylen) == 0); if (valid) { conn->authenticated = 1; return msg_append(rsp, rsp_ok.data, rsp_ok.len); diff --git a/src/test_all.c b/src/test_all.c new file mode 100644 index 00000000..c22ec7c6 --- /dev/null +++ b/src/test_all.c @@ -0,0 +1,607 @@ +#include +#include +#include +#include +#include + +static int failures = 0; +static int successes = 0; + +static void expect_same_int(int expected, int actual, const char* message) { + if (expected != actual) { + printf("FAIL Expected %d, got %d (%s)\n", expected, actual, message); + failures++; + } else { + /* printf("PASS (%s)\n", message); */ + successes++; + } +} + +static void expect_same_uint32_t(uint32_t expected, uint32_t actual, const char* message) { + if (expected != actual) { + printf("FAIL Expected %u, got %u (%s)\n", (unsigned int) expected, + (unsigned int) actual, message); + failures++; + } else { + /* printf("PASS (%s)\n", message); */ + successes++; + } +} + +static void expect_same_ptr(const void *expected, const void *actual, const char* message) { + if (expected != actual) { + printf("FAIL Expected %p, got %p (%s)\n", expected, actual, message); + failures++; + } else { + /* printf("PASS (%s)\n", message); */ + successes++; + } +} + +static void test_hash_algorithms(void) { + /* refer to libmemcached tests/hash_results.h */ + expect_same_uint32_t(2297466611U, hash_one_at_a_time("apple", 5), "should have expected one_at_a_time hash for key \"apple\""); + expect_same_uint32_t(3195025439U, hash_md5("apple", 5), "should have expected md5 hash for key \"apple\""); + + expect_same_uint32_t(3662830516U, hash_crc16("apple", 5), "should have expected crc16 hash for key \"apple\""); + expect_same_uint32_t(10542U, hash_crc32("apple", 5), "should have expected crc32 hash for key \"apple\""); + expect_same_uint32_t(2838417488U, hash_crc32a("apple", 5), "should have expected crc32a hash for key \"apple\""); + expect_same_uint32_t(67176023U, hash_fnv1_32("apple", 5), "should have expected fnv1_32 hash for key \"apple\""); + expect_same_uint32_t(280767167U, hash_fnv1a_32("apple", 5), "should have expected fnv1a_32 hash for key \"apple\""); + expect_same_uint32_t(473199127U, hash_fnv1_64("apple", 5), "should have expected fnv1_64 hash for key \"apple\""); + expect_same_uint32_t(1488911807U, hash_fnv1a_64("apple", 5), "should have expected fnv1a_64 hash for key \"apple\""); + expect_same_uint32_t(3738850110U, hash_hsieh("apple", 5), "should have expected hsieh hash for key \"apple\""); + expect_same_uint32_t(1442444624U, hash_jenkins("apple", 5), "should have expected jenkins hash for key \"apple\""); + expect_same_uint32_t(4142305122U, hash_murmur("apple", 5), "should have expected murmur hash for key \"apple\""); + /* The above have exactly the same result as libmemcached/tests/hash_results.h */ + + expect_same_uint32_t(3853726576U, ketama_hash("server1-8", strlen("server1-8"), 0), "should have expected ketama_hash for server1-8 index 0"); + expect_same_uint32_t(2667054752U, ketama_hash("server1-8", strlen("server1-8"), 3), "should have expected ketama_hash for server1-8 index 3"); +} + +static void test_config_parsing(void) { + const char* conf_file = "../conf/nutcracker.yml"; + struct conf * conf = conf_create(conf_file); + if (conf == NULL) { + printf("FAIL could not parse %s (this test should be run within src/ folder)\n", conf_file); + failures++; + } else { + printf("PASS parsed %s\n", conf_file); + + conf_destroy(conf); + successes++; + } +} + +static void test_redis_parse_req_success_case(const char* data, int expected_type) { + const int original_failures = failures; + struct conn fake_client = {0}; + struct mbuf *m = mbuf_get(); + const int SW_START = 0; /* Same as SW_START in redis_parse_req */ + + struct msg *req = msg_get(&fake_client, 1, 1); + req->state = SW_START; + req->token = NULL; + const size_t datalen = strlen(data); + + /* Copy data into the message */ + mbuf_copy(m, (const uint8_t*)data, datalen); + /* Insert a single buffer into the message mbuf header */ + STAILQ_INIT(&req->mhdr); + ASSERT(STAILQ_EMPTY(&req->mhdr)); + mbuf_insert(&req->mhdr, m); + req->pos = m->start; + + redis_parse_req(req); + expect_same_ptr(m->last, req->pos, "redis_parse_req: expected req->pos to be m->last"); + expect_same_int(SW_START, req->state, "redis_parse_req: expected full buffer to be parsed"); + expect_same_int(expected_type, req->type, "redis_parse_req: expected request type to be parsed"); + expect_same_int(0, fake_client.err, "redis_parse_req: expected no connection error"); + + msg_put(req); + /* mbuf_put(m); */ + if (failures > original_failures) { + fprintf(stderr, "test_redis_parse_req_success_case failed for (%s)", data); + } +} + +/* Test support for https://redis.io/topics/protocol */ +static void test_redis_parse_req_success(void) { + /* Redis requests from clients are serialized as arrays before sending them (* is array length, $ is string length) */ + + test_redis_parse_req_success_case("*4\r\n$4\r\neval\r\n$10\r\nreturn 123\r\n$1\r\n1\r\n$1\r\n1\r\n", MSG_REQ_REDIS_EVAL); + test_redis_parse_req_success_case("*7\r\n$4\r\neval\r\n$40\r\nreturn {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}\r\n$1\r\n2\r\n$9\r\nkey1{tag}\r\n$4\r\narg1\r\n$9\r\nkey2{tag}\r\n$4\r\narg2\r\n", MSG_REQ_REDIS_EVAL); + + test_redis_parse_req_success_case("*3\r\n$6\r\nappend\r\n$3\r\n999\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_APPEND); + test_redis_parse_req_success_case("*2\r\n$8\r\nbitcount\r\n$3\r\nfoo\r\n", MSG_REQ_REDIS_BITCOUNT); + test_redis_parse_req_success_case("*4\r\n$8\r\nbitcount\r\n$3\r\nfoo\r\n$1\r\n1\r\n$1\r\n1\r\n", MSG_REQ_REDIS_BITCOUNT); + test_redis_parse_req_success_case("*1\r\n$7\r\nCOMMAND\r\n", MSG_REQ_REDIS_COMMAND); + test_redis_parse_req_success_case("*2\r\n$4\r\ndecr\r\n$7\r\ncounter\r\n", MSG_REQ_REDIS_DECR); + test_redis_parse_req_success_case("*3\r\n$6\r\ndecrby\r\n$7\r\ncounter\r\n$3\r\n100\r\n", MSG_REQ_REDIS_DECRBY); + test_redis_parse_req_success_case("*2\r\n$3\r\ndel\r\n$3\r\nfoo\r\n", MSG_REQ_REDIS_DEL); + test_redis_parse_req_success_case("*3\r\n$3\r\ndel\r\n$3\r\nfoo\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_DEL); + test_redis_parse_req_success_case("*2\r\n$4\r\ndump\r\n$3\r\nfoo\r\n", MSG_REQ_REDIS_DUMP); + test_redis_parse_req_success_case("*2\r\n$6\r\nexists\r\n$3\r\nfoo\r\n", MSG_REQ_REDIS_EXISTS); + test_redis_parse_req_success_case("*3\r\n$6\r\nexpire\r\n$3\r\nfoo\r\n$1\r\n0\r\n", MSG_REQ_REDIS_EXPIRE); + test_redis_parse_req_success_case("*3\r\n$8\r\nexpireat\r\n$3\r\nfoo\r\n$10\r\n1282463464\r\n", MSG_REQ_REDIS_EXPIREAT); + test_redis_parse_req_success_case("*2\r\n$3\r\nGET\r\n$3\r\nkey\r\n", MSG_REQ_REDIS_GET); + test_redis_parse_req_success_case("*3\r\n$6\r\ngetbit\r\n$3\r\nfoo\r\n$1\r\n1\r\n", MSG_REQ_REDIS_GETBIT); + test_redis_parse_req_success_case("*4\r\n$8\r\ngetrange\r\n$3\r\nfoo\r\n$1\r\n1\r\n$1\r\n2\r\n", MSG_REQ_REDIS_GETRANGE); + test_redis_parse_req_success_case("*3\r\n$6\r\ngetset\r\n$3\r\nfoo\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_GETSET); + test_redis_parse_req_success_case("*4\r\n$4\r\nhdel\r\n$4\r\nhfoo\r\n$6\r\nfield1\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_HDEL); + test_redis_parse_req_success_case("*3\r\n$7\r\nhexists\r\n$4\r\nhfoo\r\n$6\r\nfield1\r\n", MSG_REQ_REDIS_HEXISTS); + test_redis_parse_req_success_case("*3\r\n$4\r\nhget\r\n$4\r\nhfoo\r\n$6\r\nfield1\r\n", MSG_REQ_REDIS_HGET); + test_redis_parse_req_success_case("*2\r\n$7\r\nhgetall\r\n$4\r\nhfoo\r\n", MSG_REQ_REDIS_HGETALL); + test_redis_parse_req_success_case("*4\r\n$7\r\nhincrby\r\n$4\r\nhfoo\r\n$6\r\nfield1\r\n$3\r\n100\r\n", MSG_REQ_REDIS_HINCRBY); + test_redis_parse_req_success_case("*4\r\n$12\r\nhincrbyfloat\r\n$4\r\nhfoo\r\n$6\r\nfield1\r\n$6\r\n100.12\r\n", MSG_REQ_REDIS_HINCRBYFLOAT); + test_redis_parse_req_success_case("*2\r\n$5\r\nhkeys\r\n$4\r\nhfoo\r\n", MSG_REQ_REDIS_HKEYS); + test_redis_parse_req_success_case("*2\r\n$4\r\nhlen\r\n$4\r\nhfoo\r\n", MSG_REQ_REDIS_HLEN); + test_redis_parse_req_success_case("*3\r\n$5\r\nhmget\r\n$4\r\nhfoo\r\n$6\r\nfield1\r\n", MSG_REQ_REDIS_HMGET); + test_redis_parse_req_success_case("*4\r\n$5\r\nhmget\r\n$4\r\nhfoo\r\n$6\r\nfield1\r\n$6\r\n1dleif\r\n", MSG_REQ_REDIS_HMGET); + test_redis_parse_req_success_case("*6\r\n$5\r\nhmset\r\n$4\r\nhfoo\r\n$6\r\nfield1\r\n$3\r\nbar\r\n$6\r\nfield2\r\n$3\r\nbas\r\n", MSG_REQ_REDIS_HMSET); + test_redis_parse_req_success_case("*4\r\n$4\r\nhset\r\n$4\r\nhfoo\r\n$6\r\n1dleif\r\n$3\r\nrab\r\n", MSG_REQ_REDIS_HSET); + test_redis_parse_req_success_case("*4\r\n$6\r\nhsetnx\r\n$4\r\nhfoo\r\n$6\r\nfield1\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_HSETNX); + test_redis_parse_req_success_case("*2\r\n$5\r\nhvals\r\n$4\r\nhfoo\r\n", MSG_REQ_REDIS_HVALS); + test_redis_parse_req_success_case("*2\r\n$4\r\nincr\r\n$7\r\ncounter\r\n", MSG_REQ_REDIS_INCR); + test_redis_parse_req_success_case("*3\r\n$6\r\nincrby\r\n$7\r\ncounter\r\n$3\r\n100\r\n", MSG_REQ_REDIS_INCRBY); + test_redis_parse_req_success_case("*3\r\n$11\r\nincrbyfloat\r\n$7\r\ncounter\r\n$5\r\n10.10\r\n", MSG_REQ_REDIS_INCRBYFLOAT); + test_redis_parse_req_success_case("*3\r\n$6\r\nlindex\r\n$4\r\nlfoo\r\n$1\r\n0\r\n", MSG_REQ_REDIS_LINDEX); + test_redis_parse_req_success_case("*5\r\n$7\r\nlinsert\r\n$4\r\nlfoo\r\n$6\r\nBEFORE\r\n$3\r\nbar\r\n$3\r\nbaq\r\n", MSG_REQ_REDIS_LINSERT); + test_redis_parse_req_success_case("*2\r\n$4\r\nllen\r\n$4\r\nlfoo\r\n", MSG_REQ_REDIS_LLEN); + test_redis_parse_req_success_case("*2\r\n$4\r\nLLEN\r\n$6\r\nmylist\r\n", MSG_REQ_REDIS_LLEN); /* LLEN command */ + test_redis_parse_req_success_case("*1\r\n$6\r\nLOLWUT\r\n", MSG_REQ_REDIS_LOLWUT); + test_redis_parse_req_success_case("*2\r\n$6\r\nLOLWUT\r\n$2\r\n40\r\n", MSG_REQ_REDIS_LOLWUT); + test_redis_parse_req_success_case("*2\r\n$4\r\nlpop\r\n$4\r\nlfoo\r\n", MSG_REQ_REDIS_LPOP); + test_redis_parse_req_success_case("*3\r\n$5\r\nlpush\r\n$4\r\nlfoo\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_LPUSH); + test_redis_parse_req_success_case("*4\r\n$5\r\nlpush\r\n$4\r\nlfoo\r\n$3\r\nbaq\r\n$3\r\nbap\r\n", MSG_REQ_REDIS_LPUSH); + test_redis_parse_req_success_case("*6\r\n$5\r\nlpush\r\n$4\r\nlfoo\r\n$3\r\nbar\r\n$3\r\nbar\r\n$3\r\nbar\r\n$3\r\nbau\r\n", MSG_REQ_REDIS_LPUSH); + test_redis_parse_req_success_case("*3\r\n$6\r\nlpushx\r\n$4\r\nlfoo\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_LPUSHX); + test_redis_parse_req_success_case("*4\r\n$6\r\nlrange\r\n$4\r\nlfoo\r\n$1\r\n0\r\n$1\r\n2\r\n", MSG_REQ_REDIS_LRANGE); + test_redis_parse_req_success_case("*4\r\n$6\r\nlrange\r\n$4\r\nlfoo\r\n$1\r\n0\r\n$1\r\n3\r\n", MSG_REQ_REDIS_LRANGE); + test_redis_parse_req_success_case("*4\r\n$4\r\nlrem\r\n$4\r\nlfoo\r\n$1\r\n2\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_LREM); + test_redis_parse_req_success_case("*4\r\n$4\r\nlset\r\n$4\r\nlfoo\r\n$1\r\n0\r\n$3\r\nbaq\r\n", MSG_REQ_REDIS_LSET); + test_redis_parse_req_success_case("*4\r\n$5\r\nltrim\r\n$4\r\nlfoo\r\n$1\r\n0\r\n$1\r\n2\r\n", MSG_REQ_REDIS_LTRIM); + test_redis_parse_req_success_case("*13\r\n$4\r\nmget\r\n$3\r\nfoo\r\n$3\r\nbar\r\n$3\r\nbar\r\n$3\r\nbar\r\n$3\r\nbar\r\n$3\r\nbar\r\n$3\r\nbar\r\n$3\r\nbar\r\n$3\r\nbar\r\n$3\r\nbar\r\n$3\r\nbar\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_MGET); + test_redis_parse_req_success_case("*2\r\n$4\r\nMGET\r\n$1\r\nx\r\n", MSG_REQ_REDIS_MGET); + test_redis_parse_req_success_case("*3\r\n$4\r\nMGET\r\n$1\r\nx\r\n$10\r\nabcdefghij\r\n", MSG_REQ_REDIS_MGET); + test_redis_parse_req_success_case("*3\r\n$4\r\nmget\r\n$3\r\nfoo\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_MGET); + test_redis_parse_req_success_case("*3\r\n$4\r\nmget\r\n$3\r\nfoo\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_MGET); + test_redis_parse_req_success_case("*2\r\n$7\r\npersist\r\n$3\r\nfoo\r\n", MSG_REQ_REDIS_PERSIST); + test_redis_parse_req_success_case("*3\r\n$7\r\npexpire\r\n$3\r\nfoo\r\n$1\r\n0\r\n", MSG_REQ_REDIS_PEXPIRE); + test_redis_parse_req_success_case("*3\r\n$5\r\npfadd\r\n$7\r\n{pfoo}2\r\n$3\r\nbas\r\n", MSG_REQ_REDIS_PFADD); + test_redis_parse_req_success_case("*4\r\n$5\r\npfadd\r\n$4\r\npfoo\r\n$3\r\nbar\r\n$3\r\nbas\r\n", MSG_REQ_REDIS_PFADD); + test_redis_parse_req_success_case("*2\r\n$7\r\npfcount\r\n$4\r\npfoo\r\n", MSG_REQ_REDIS_PFCOUNT); + test_redis_parse_req_success_case("*5\r\n$7\r\npfmerge\r\n$7\r\n{pfoo}3\r\n$1\r\n2\r\n$6\r\n{pfoo}\r\n$7\r\n{pfoo}2\r\n", MSG_REQ_REDIS_PFMERGE); + test_redis_parse_req_success_case("*1\r\n$4\r\nPING\r\n", MSG_REQ_REDIS_PING); + test_redis_parse_req_success_case("*4\r\n$6\r\npsetex\r\n$3\r\nfoo\r\n$4\r\n1000\r\n$3\r\noof\r\n", MSG_REQ_REDIS_PSETEX); + test_redis_parse_req_success_case("*2\r\n$4\r\npttl\r\n$3\r\nfoo\r\n", MSG_REQ_REDIS_PTTL); + test_redis_parse_req_success_case("*4\r\n$7\r\nrestore\r\n$3\r\nfoo\r\n$1\r\n0\r\n$3\r\noof\r\n", MSG_REQ_REDIS_RESTORE); + test_redis_parse_req_success_case("*2\r\n$4\r\nrpop\r\n$4\r\nlfoo\r\n", MSG_REQ_REDIS_RPOP); + test_redis_parse_req_success_case("*3\r\n$9\r\nrpoplpush\r\n$6\r\n{lfoo}\r\n$7\r\n{lfoo}2\r\n", MSG_REQ_REDIS_RPOPLPUSH); + test_redis_parse_req_success_case("*3\r\n$5\r\nrpush\r\n$4\r\nlfoo\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_RPUSH); + test_redis_parse_req_success_case("*4\r\n$5\r\nrpush\r\n$4\r\nlfoo\r\n$3\r\nbat\r\n$3\r\nbau\r\n", MSG_REQ_REDIS_RPUSH); + test_redis_parse_req_success_case("*3\r\n$6\r\nrpushx\r\n$4\r\nlfoo\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_RPUSHX); + test_redis_parse_req_success_case("*3\r\n$4\r\nsadd\r\n$7\r\n{sfoo}2\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_SADD); + test_redis_parse_req_success_case("*4\r\n$4\r\nsadd\r\n$4\r\nsfoo\r\n$3\r\nbar\r\n$3\r\nbas\r\n", MSG_REQ_REDIS_SADD); + test_redis_parse_req_success_case("*5\r\n$4\r\nsadd\r\n$4\r\nsfoo\r\n$3\r\nbar\r\n$3\r\nbas\r\n$3\r\nbat\r\n", MSG_REQ_REDIS_SADD); + test_redis_parse_req_success_case("*2\r\n$5\r\nscard\r\n$4\r\nsfoo\r\n", MSG_REQ_REDIS_SCARD); + test_redis_parse_req_success_case("*3\r\n$5\r\nsdiff\r\n$6\r\n{sfoo}\r\n$7\r\n{sfoo}2\r\n", MSG_REQ_REDIS_SDIFF); + test_redis_parse_req_success_case("*4\r\n$10\r\nsdiffstore\r\n$7\r\n{sfoo}3\r\n$6\r\n{sfoo}\r\n$7\r\n{sfoo}2\r\n", MSG_REQ_REDIS_SDIFFSTORE); + test_redis_parse_req_success_case("*3\r\n$3\r\nSET\r\n$10\r\nkey4567890\r\n$5\r\nVALUE\r\n", MSG_REQ_REDIS_SET); + test_redis_parse_req_success_case("*3\r\n$3\r\nset\r\n$3\r\nbar\r\n$3\r\nrab\r\n", MSG_REQ_REDIS_SET); + test_redis_parse_req_success_case("*4\r\n$6\r\nsetbit\r\n$3\r\nfoo\r\n$1\r\n1\r\n$1\r\n1\r\n", MSG_REQ_REDIS_SETBIT); + test_redis_parse_req_success_case("*4\r\n$5\r\nsetex\r\n$3\r\nfoo\r\n$4\r\n1000\r\n$3\r\noof\r\n", MSG_REQ_REDIS_SETEX); + test_redis_parse_req_success_case("*3\r\n$5\r\nsetnx\r\n$3\r\nfoo\r\n$3\r\noof\r\n", MSG_REQ_REDIS_SETNX); + test_redis_parse_req_success_case("*4\r\n$8\r\nsetrange\r\n$3\r\nfoo\r\n$1\r\n1\r\n$3\r\noof\r\n", MSG_REQ_REDIS_SETRANGE); + test_redis_parse_req_success_case("*3\r\n$6\r\nsinter\r\n$6\r\n{sfoo}\r\n$7\r\n{sfoo}2\r\n", MSG_REQ_REDIS_SINTER); + test_redis_parse_req_success_case("*4\r\n$11\r\nsinterstore\r\n$7\r\n{sfoo}3\r\n$6\r\n{sfoo}\r\n$7\r\n{sfoo}2\r\n", MSG_REQ_REDIS_SINTERSTORE); + test_redis_parse_req_success_case("*3\r\n$9\r\nsismember\r\n$4\r\nsfoo\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_SISMEMBER); + test_redis_parse_req_success_case("*2\r\n$8\r\nsmembers\r\n$4\r\nsfoo\r\n", MSG_REQ_REDIS_SMEMBERS); + test_redis_parse_req_success_case("*4\r\n$5\r\nsmove\r\n$6\r\n{sfoo}\r\n$7\r\n{sfoo}2\r\n$3\r\nbas\r\n", MSG_REQ_REDIS_SMOVE); + test_redis_parse_req_success_case("*2\r\n$11\r\nsrandmember\r\n$4\r\nsfoo\r\n", MSG_REQ_REDIS_SRANDMEMBER); + test_redis_parse_req_success_case("*3\r\n$11\r\nsrandmember\r\n$4\r\nsfoo\r\n$1\r\n2\r\n", MSG_REQ_REDIS_SRANDMEMBER); + test_redis_parse_req_success_case("*3\r\n$4\r\nsrem\r\n$4\r\nsfoo\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_SREM); + test_redis_parse_req_success_case("*5\r\n$4\r\nsrem\r\n$4\r\nsfoo\r\n$3\r\nbas\r\n$3\r\nbat\r\n$3\r\nrab\r\n", MSG_REQ_REDIS_SREM); + test_redis_parse_req_success_case("*3\r\n$6\r\nsunion\r\n$6\r\n{sfoo}\r\n$7\r\n{sfoo}2\r\n", MSG_REQ_REDIS_SUNION); + test_redis_parse_req_success_case("*4\r\n$11\r\nsunionstore\r\n$7\r\n{sfoo}3\r\n$6\r\n{sfoo}\r\n$7\r\n{sfoo}2\r\n", MSG_REQ_REDIS_SUNIONSTORE); + test_redis_parse_req_success_case("*2\r\n$3\r\nttl\r\n$3\r\nfoo\r\n", MSG_REQ_REDIS_TTL); + test_redis_parse_req_success_case("*2\r\n$4\r\ntype\r\n$3\r\nfoo\r\n", MSG_REQ_REDIS_TYPE); + test_redis_parse_req_success_case("*4\r\n$4\r\nzadd\r\n$4\r\nzfoo\r\n$3\r\n100\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_ZADD); + test_redis_parse_req_success_case("*6\r\n$4\r\nzadd\r\n$4\r\nzfoo\r\n$3\r\n101\r\n$3\r\nbat\r\n$3\r\n102\r\n$3\r\nbau\r\n", MSG_REQ_REDIS_ZADD); + test_redis_parse_req_success_case("*8\r\n$4\r\nzadd\r\n$4\r\nzfoo\r\n$3\r\n100\r\n$3\r\nbar\r\n$3\r\n101\r\n$3\r\nbat\r\n$3\r\n102\r\n$3\r\nbau\r\n", MSG_REQ_REDIS_ZADD); + test_redis_parse_req_success_case("*2\r\n$5\r\nzcard\r\n$4\r\nzfoo\r\n", MSG_REQ_REDIS_ZCARD); + test_redis_parse_req_success_case("*4\r\n$6\r\nzcount\r\n$4\r\nzfoo\r\n$3\r\n100\r\n$3\r\n101\r\n", MSG_REQ_REDIS_ZCOUNT); + test_redis_parse_req_success_case("*4\r\n$7\r\nzincrby\r\n$4\r\nzfoo\r\n$3\r\n100\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_ZINCRBY); + test_redis_parse_req_success_case("*5\r\n$11\r\nzinterstore\r\n$7\r\n{zfoo}3\r\n$1\r\n2\r\n$6\r\n{zfoo}\r\n$7\r\n{zfoo}2\r\n", MSG_REQ_REDIS_ZINTERSTORE); + test_redis_parse_req_success_case("*4\r\n$9\r\nzlexcount\r\n$4\r\nzfoo\r\n$1\r\n-\r\n$1\r\n+\r\n", MSG_REQ_REDIS_ZLEXCOUNT); + test_redis_parse_req_success_case("*4\r\n$6\r\nzrange\r\n$4\r\nzfoo\r\n$1\r\n0\r\n$1\r\n3\r\n", MSG_REQ_REDIS_ZRANGE); + test_redis_parse_req_success_case("*5\r\n$6\r\nzrange\r\n$4\r\nzfoo\r\n$1\r\n0\r\n$1\r\n3\r\n$10\r\nWITHSCORES\r\n", MSG_REQ_REDIS_ZRANGE); + test_redis_parse_req_success_case("*4\r\n$11\r\nzrangebylex\r\n$4\r\nzfoo\r\n$1\r\n-\r\n$1\r\n+\r\n", MSG_REQ_REDIS_ZRANGEBYLEX); + test_redis_parse_req_success_case("*4\r\n$13\r\nzrangebyscore\r\n$4\r\nzfoo\r\n$3\r\n100\r\n$3\r\n101\r\n", MSG_REQ_REDIS_ZRANGEBYSCORE); + test_redis_parse_req_success_case("*3\r\n$5\r\nzrank\r\n$4\r\nzfoo\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_ZRANK); + test_redis_parse_req_success_case("*8\r\n$4\r\nzrem\r\n$4\r\nzfoo\r\n$3\r\n100\r\n$3\r\nbar\r\n$3\r\n101\r\n$3\r\nbat\r\n$3\r\n102\r\n$3\r\nbau\r\n", MSG_REQ_REDIS_ZREM); + test_redis_parse_req_success_case("*4\r\n$14\r\nzremrangebylex\r\n$4\r\nzfoo\r\n$1\r\n-\r\n$1\r\n+\r\n", MSG_REQ_REDIS_ZREMRANGEBYLEX); + test_redis_parse_req_success_case("*4\r\n$15\r\nzremrangebyrank\r\n$4\r\nzfoo\r\n$1\r\n0\r\n$1\r\n1\r\n", MSG_REQ_REDIS_ZREMRANGEBYRANK); + test_redis_parse_req_success_case("*4\r\n$16\r\nzremrangebyscore\r\n$4\r\nzfoo\r\n$3\r\n100\r\n$3\r\n101\r\n", MSG_REQ_REDIS_ZREMRANGEBYSCORE); + test_redis_parse_req_success_case("*4\r\n$9\r\nzrevrange\r\n$4\r\nzfoo\r\n$1\r\n0\r\n$1\r\n2\r\n", MSG_REQ_REDIS_ZREVRANGE); + test_redis_parse_req_success_case("*3\r\n$8\r\nzrevrank\r\n$4\r\nzfoo\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_ZREVRANK); + test_redis_parse_req_success_case("*3\r\n$6\r\nzscore\r\n$4\r\nzfoo\r\n$3\r\nbar\r\n", MSG_REQ_REDIS_ZSCORE); + test_redis_parse_req_success_case("*5\r\n$11\r\nzunionstore\r\n$7\r\n{zfoo}3\r\n$1\r\n2\r\n$6\r\n{zfoo}\r\n$7\r\n{zfoo}2\r\n", MSG_REQ_REDIS_ZUNIONSTORE); +} + +static void test_redis_parse_rsp_success_case(const char* data, int expected) { + int original_failures = failures; + struct conn fake_client = {0}; + struct mbuf *m = mbuf_get(); + const int SW_START = 0; /* Same as SW_START in redis_parse_rsp */ + + struct msg *rsp = msg_get(&fake_client, 0, 1); + rsp->state = SW_START; + rsp->token = NULL; + const size_t datalen = strlen(data); + + /* Copy data into the message */ + mbuf_copy(m, (const uint8_t*)data, datalen); + /* Insert a single buffer into the message mbuf header */ + STAILQ_INIT(&rsp->mhdr); + ASSERT(STAILQ_EMPTY(&rsp->mhdr)); + mbuf_insert(&rsp->mhdr, m); + rsp->pos = m->start; + errno = 0; + + redis_parse_rsp(rsp); + expect_same_ptr(m->last, rsp->pos, "redis_parse_rsp: expected rsp->pos to be m->last"); + expect_same_int(SW_START, rsp->state, "redis_parse_rsp: expected full buffer to be parsed"); + expect_same_int(expected, rsp->type, "redis_parse_rsp: expected response type to be parsed"); + expect_same_int(0, errno, "redis_parse_rsp: expected errno=0"); + expect_same_uint32_t(1, rsp->rnarg ? rsp->rnarg : 1, "expected remaining args to be 0 or 1"); + + msg_put(rsp); + /* mbuf_put(m); */ + if (failures > original_failures) { + fprintf(stderr, "test_redis_parse_rsp_success_case failed for (%s)", data); + } +} + +/* Test support for https://redis.io/topics/protocol */ +static void test_redis_parse_rsp_success(void) { + /* Error message without a space */ + test_redis_parse_rsp_success_case("-CUSTOMERR\r\n", MSG_RSP_REDIS_ERROR); + /* Error message */ + test_redis_parse_rsp_success_case("-Error message\r\n", MSG_RSP_REDIS_ERROR); + /* Error message without a space */ + test_redis_parse_rsp_success_case("+OK\r\n", MSG_RSP_REDIS_STATUS); + /* bulk string */ + test_redis_parse_rsp_success_case("$6\r\nfoobar\r\n", MSG_RSP_REDIS_BULK); + /* empty bulk string */ + test_redis_parse_rsp_success_case("$0\r\n\r\n", MSG_RSP_REDIS_BULK); + /* null value */ + test_redis_parse_rsp_success_case("$-1\r\n", MSG_RSP_REDIS_BULK); + /* empty array */ + test_redis_parse_rsp_success_case("*0\r\n", MSG_RSP_REDIS_MULTIBULK); + /* array with 2 bulk strings */ + test_redis_parse_rsp_success_case("*2\r\n$3\r\nfoo\r\n$3\r\nbar\r\n", + MSG_RSP_REDIS_MULTIBULK); + /* array with 3 integers */ + test_redis_parse_rsp_success_case("*3\r\n:1\r\n:2\r\n:3\r\n", + MSG_RSP_REDIS_MULTIBULK); + /* null array for BLPOP */ + test_redis_parse_rsp_success_case("*-1\r\n", MSG_RSP_REDIS_MULTIBULK); + /* + * Test support for parsing arrays of arrays. + * They can be returned by COMMAND, EVAL, etc. + */ + test_redis_parse_rsp_success_case("*2\r\n" + "*3\r\n" + ":1\r\n" + ":2\r\n" + ":3\r\n" + "*2\r\n" + "+Foo\r\n" + "-Bar\r\n", MSG_RSP_REDIS_MULTIBULK); /* array of 2 arrays */ +} + +static void test_redis_parse_rsp_failure_case(const char* data) { + int original_failures = failures; + struct conn fake_client = {0}; + struct mbuf *m = mbuf_get(); + const int SW_START = 0; /* Same as SW_START in redis_parse_rsp */ + + struct msg *rsp = msg_get(&fake_client, 0, 1); + rsp->state = SW_START; + rsp->token = NULL; + const size_t datalen = strlen(data); + + /* Copy data into the message */ + mbuf_copy(m, (const uint8_t*)data, datalen); + /* Insert a single buffer into the message mbuf header */ + STAILQ_INIT(&rsp->mhdr); + ASSERT(STAILQ_EMPTY(&rsp->mhdr)); + mbuf_insert(&rsp->mhdr, m); + rsp->pos = m->start; + errno = 0; + + redis_parse_rsp(rsp); + expect_same_ptr(m->start, rsp->pos, "redis_parse_rsp: expected rsp->pos to be m->start"); + expect_same_int(MSG_PARSE_ERROR, rsp->result, "redis_parse_rsp: expected MSG_PARSE_ERROR"); + expect_same_int(EINVAL, errno, "redis_parse_rsp: expected errno=EINVAL"); + + msg_put(rsp); + /* mbuf_put(m); */ + if (failures > original_failures) { + fprintf(stderr, "test_redis_parse_rsp_failure_case failed for (%s)", data); + } +} + + +/* Test support for https://redis.io/topics/protocol */ +static void test_redis_parse_rsp_failure(void) { + test_redis_parse_rsp_failure_case("*\r\n"); + test_redis_parse_rsp_failure_case(":x\r\n"); + test_redis_parse_rsp_failure_case("$6\r\nfoobarr\r\n"); + test_redis_parse_rsp_failure_case("$6\r\nfoobar\n\n"); + test_redis_parse_rsp_failure_case("$0\r\nx\r\n"); + test_redis_parse_rsp_failure_case("$0\n"); + test_redis_parse_rsp_failure_case("*2\r\n" + "*3\r\n" + ":1\r\n" + ":2\r\n" + ":3\r\n" + "*2\r\n" + "\r\n"); +} + +static void test_memcache_parse_rsp_success_case(const char* data, int expected) { + struct conn fake_client = {0}; + struct mbuf *m = mbuf_get(); + const int SW_START = 0; /* Same as SW_START in memcache_parse_rsp */ + const int original_failures = failures; + + struct msg *rsp = msg_get(&fake_client, 0, 0); + rsp->state = SW_START; + rsp->token = NULL; + const size_t datalen = strlen(data); + + /* Copy data into the message */ + mbuf_copy(m, (const uint8_t*)data, datalen); + /* Insert a single buffer into the message mbuf header */ + STAILQ_INIT(&rsp->mhdr); + ASSERT(STAILQ_EMPTY(&rsp->mhdr)); + mbuf_insert(&rsp->mhdr, m); + rsp->pos = m->start; + errno=0; + + memcache_parse_rsp(rsp); + expect_same_ptr(m->last, rsp->pos, "memcache_parse_rsp: expected rsp->pos to be m->last"); + expect_same_int(SW_START, rsp->state, "memcache_parse_rsp: expected state to be SW_START after parsing full buffer"); + expect_same_int(expected, rsp->type, "memcache_parse_rsp: expected response type to be parsed"); + expect_same_int(0, fake_client.err, "memcache_parse_rsp: expected no connection error"); + expect_same_int(0, rsp->request, "memcache_parse_rsp: expected response"); + expect_same_int(0, rsp->error, "memcache_parse_rsp: expected no error"); + expect_same_int(0, rsp->swallow, "memcache_parse_rsp: expected swallow=0"); + expect_same_int(0, errno, "memcache_parse_rsp: expected errno=0"); + + msg_put(rsp); + /* mbuf_put(m); */ + if (original_failures != failures) { + printf("Saw test failures for test_memcache_parse_rsp_success_case (%s)\n", + data); + } +} + +static void test_memcache_parse_rsp_success(void) { + test_memcache_parse_rsp_success_case("0\r\n", MSG_RSP_MC_NUM); + /* The number returned by the server may be space-padded at the end */ + test_memcache_parse_rsp_success_case("0 \r\n", MSG_RSP_MC_NUM); + test_memcache_parse_rsp_success_case("9223372036854775807\r\n", MSG_RSP_MC_NUM); + test_memcache_parse_rsp_success_case("DELETED\r\n", MSG_RSP_MC_DELETED); + test_memcache_parse_rsp_success_case("END\r\n", MSG_RSP_MC_END); + test_memcache_parse_rsp_success_case("ERROR\r\n", MSG_RSP_MC_ERROR); + test_memcache_parse_rsp_success_case("EXISTS\r\n", MSG_RSP_MC_EXISTS); + test_memcache_parse_rsp_success_case("NOT_FOUND\r\n", MSG_RSP_MC_NOT_FOUND); + test_memcache_parse_rsp_success_case("STORED\r\n", MSG_RSP_MC_STORED); + test_memcache_parse_rsp_success_case("TOUCHED\r\n", MSG_RSP_MC_TOUCHED); + test_memcache_parse_rsp_success_case("VALUE key 0 2\r\nab\r\nEND\r\n", MSG_RSP_MC_END); + test_memcache_parse_rsp_success_case("VALUE key 0 2\r\nab\r\nVALUE key2 0 2\r\ncd\r\nEND\r\n", MSG_RSP_MC_END); + test_memcache_parse_rsp_success_case("VERSION 1.5.22\r\n", MSG_RSP_MC_VERSION); +} + +static void test_memcache_parse_rsp_failure_case(const char* data) { + struct conn fake_client = {0}; + struct mbuf *m = mbuf_get(); + const int SW_START = 0; /* Same as SW_START in memcache_parse_rsp */ + const int original_failures = failures; + + struct msg *rsp = msg_get(&fake_client, 0, 0); + rsp->state = SW_START; + rsp->token = NULL; + const size_t datalen = strlen(data); + + /* Copy data into the message */ + mbuf_copy(m, (const uint8_t*)data, datalen); + /* Insert a single buffer into the message mbuf header */ + STAILQ_INIT(&rsp->mhdr); + ASSERT(STAILQ_EMPTY(&rsp->mhdr)); + mbuf_insert(&rsp->mhdr, m); + rsp->pos = m->start; + errno = 0; + + memcache_parse_rsp(rsp); + expect_same_ptr(m->start, rsp->pos, "memcache_parse_rsp: expected rsp->pos to be m->start"); + expect_same_int(0, rsp->type, "memcache_parse_rsp: expected response type to be parsed"); + expect_same_int(MSG_PARSE_ERROR, rsp->result, "memcache_parse_rsp: expected MSG_PARSE_ERROR"); + expect_same_int(EINVAL, errno, "memcache_parse_rsp: expected EINVAL"); + + msg_put(rsp); + /* mbuf_put(m); */ + if (original_failures != failures) { + printf("Saw test failures for test_memcache_parse_rsp_success_case (%s)\n", + data); + } +} + + +static void test_memcache_parse_rsp_failure(void) { + test_memcache_parse_rsp_failure_case("\r\n"); + test_memcache_parse_rsp_failure_case("ENDD\r\n"); + test_memcache_parse_rsp_failure_case("\r"); + test_memcache_parse_rsp_failure_case("-1\r\n"); +} + +static void test_memcache_parse_req_success_case(const char* data, int expected) { + const int original_failures = failures; + struct conn fake_client = {0}; + struct mbuf *m = mbuf_get(); + const int SW_START = 0; /* Same as SW_START in memcache_parse_req */ + /* in the test cases, the substring noreply only occurs for valid noreply requests */ + const int expected_noreply = strstr(data, " noreply") != NULL; + + struct msg *req = msg_get(&fake_client, 1, 0); + req->state = SW_START; + req->token = NULL; + const size_t datalen = strlen(data); + + /* Copy data into the message */ + mbuf_copy(m, (const uint8_t*)data, datalen); + /* Insert a single buffer into the message mbuf header */ + STAILQ_INIT(&req->mhdr); + ASSERT(STAILQ_EMPTY(&req->mhdr)); + mbuf_insert(&req->mhdr, m); + req->pos = m->start; + + memcache_parse_req(req); + expect_same_ptr(m->last, req->pos, "memcache_parse_req: expected req->pos to be m->last"); + expect_same_int(SW_START, req->state, "memcache_parse_req: expected state to be SW_START after parsing full buffer"); + expect_same_int(expected, req->type, "memcache_parse_req: expected response type to be parsed"); + expect_same_int(expected_noreply, req->noreply, "memcache_parse_req: unexpected noreply value"); + expect_same_int(0, req->noforward, "memcache_parse_req: unexpected noforward value"); + expect_same_int(1, req->request, "memcache_parse_req: expected request"); + expect_same_int(0, req->error, "memcache_parse_req: expected no error"); + expect_same_int(strstr(data, "quit\r\n") != NULL ? 1 : 0, req->quit, + "memcache_parse_req: unexpected quit value"); + expect_same_int(0, fake_client.err, "memcache_parse_req: expected no connection error"); + + msg_put(req); + /* mbuf_put(m); */ + if (original_failures != failures) { + printf("Saw test failures for test_memcache_parse_req_success_case (%s)\n", + data); + } +} + +static void test_memcache_parse_req_success(void) { + test_memcache_parse_req_success_case("add key 0 600 5\r\nvalue\r\n", MSG_REQ_MC_ADD); + /* Can add any binary data such as '\n' */ + test_memcache_parse_req_success_case("add key 0 0 1 noreply\r\n\n\r\n", MSG_REQ_MC_ADD); + test_memcache_parse_req_success_case("append key 0 600 5\r\nvalue\r\n", MSG_REQ_MC_APPEND); + test_memcache_parse_req_success_case("append key 0 1 0 noreply\r\n\r\n", MSG_REQ_MC_APPEND); + test_memcache_parse_req_success_case("cas key 0 600 5 123456\r\nvalue\r\n", MSG_REQ_MC_CAS); + test_memcache_parse_req_success_case("cas key 0 1 1 1 noreply\r\nx\r\n", MSG_REQ_MC_CAS); + test_memcache_parse_req_success_case("decr key 0\r\n", MSG_REQ_MC_DECR); + test_memcache_parse_req_success_case("decr key 0 noreply\r\n", MSG_REQ_MC_DECR); + test_memcache_parse_req_success_case("delete a noreply\r\n", MSG_REQ_MC_DELETE); + test_memcache_parse_req_success_case("delete key\r\n", MSG_REQ_MC_DELETE); + /* TODO https://github.com/twitter/twemproxy/issues/631 gat/gats */ + /* test_memcache_parse_req_success_case("gat 3600 key\r\n", MSG_REQ_MC_GAT); */ + test_memcache_parse_req_success_case("get a b xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n", MSG_REQ_MC_GET); + test_memcache_parse_req_success_case("get key\r\n", MSG_REQ_MC_GET); + test_memcache_parse_req_success_case("gets u\r\n", MSG_REQ_MC_GETS); + test_memcache_parse_req_success_case("incr key 1\r\n", MSG_REQ_MC_INCR); + test_memcache_parse_req_success_case("incr key 9223372036854775807 noreply\r\n", MSG_REQ_MC_INCR); + test_memcache_parse_req_success_case("prepend key 0 600 5\r\nvalue\r\n", MSG_REQ_MC_PREPEND); + test_memcache_parse_req_success_case("prepend key 0 600 0 noreply\r\n\r\n", MSG_REQ_MC_PREPEND); + test_memcache_parse_req_success_case("quit\r\n", MSG_REQ_MC_QUIT); + test_memcache_parse_req_success_case("replace key 0 600 5\r\nvalue\r\n", MSG_REQ_MC_REPLACE); + test_memcache_parse_req_success_case("replace key 0 9 0 noreply\r\n\r\n", MSG_REQ_MC_REPLACE); + test_memcache_parse_req_success_case("set key 0 5 10 noreply\r\nvalue12345\r\n", MSG_REQ_MC_SET); + test_memcache_parse_req_success_case("set key 0 600 5\r\nvalue\r\n", MSG_REQ_MC_SET); + test_memcache_parse_req_success_case("touch key 12345\r\n", MSG_REQ_MC_TOUCH); + test_memcache_parse_req_success_case("touch key 12345 noreply\r\n", MSG_REQ_MC_TOUCH); + test_memcache_parse_req_success_case("version\r\n", MSG_REQ_MC_VERSION); +} + +static void test_memcache_parse_req_failure_case(const char* data) { + const int original_failures = failures; + struct conn fake_client = {0}; + struct mbuf *m = mbuf_get(); + const int SW_START = 0; /* Same as SW_START in memcache_parse_req */ + + struct msg *req = msg_get(&fake_client, 1, 0); + req->state = SW_START; + req->token = NULL; + const size_t datalen = strlen(data); + + /* Copy data into the message */ + mbuf_copy(m, (const uint8_t*)data, datalen); + /* Insert a single buffer into the message mbuf header */ + STAILQ_INIT(&req->mhdr); + ASSERT(STAILQ_EMPTY(&req->mhdr)); + mbuf_insert(&req->mhdr, m); + req->pos = m->start; + errno = 0; + + memcache_parse_req(req); + + expect_same_ptr(m->start, req->pos, "memcache_parse_rsp: expected rsp->pos to be m->start"); + expect_same_int(MSG_PARSE_ERROR, req->result, "memcache_parse_rsp: expected MSG_PARSE_ERROR"); + expect_same_int(EINVAL, errno, "memcache_parse_rsp: expected EINVAL"); + + msg_put(req); + /* mbuf_put(m); */ + if (original_failures != failures) { + printf("Saw test failures for test_memcache_parse_req_success_case (%s)\n", + data); + } +} + +static void test_memcache_parse_req_failure(void) { + /* key length exceeds 250 */ + test_memcache_parse_req_failure_case("add xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 0 600 5\r\nvalue\r\n"); + test_memcache_parse_req_failure_case("add\r\n"); + test_memcache_parse_req_failure_case("get\r\n"); + test_memcache_parse_req_failure_case("get \r\n"); + test_memcache_parse_req_failure_case("get key\r\r"); + test_memcache_parse_req_failure_case("append key 0 600\r\n"); + test_memcache_parse_req_failure_case("cas key 0 600 5 \r\n"); + test_memcache_parse_req_failure_case("decr key 0 extra\r\n"); + test_memcache_parse_req_failure_case("decr key\r\n"); + test_memcache_parse_req_failure_case("delete \r\n"); + test_memcache_parse_req_failure_case("DELETE key\r\n"); + test_memcache_parse_req_failure_case("gets\r\n"); + test_memcache_parse_req_failure_case("incr\r\n"); + test_memcache_parse_req_failure_case("incr key 0notanint\r\n"); + test_memcache_parse_req_failure_case("prepend key 0 600 5\r\nvalueextra\r\n"); + test_memcache_parse_req_failure_case("prepend key 0 600 0 noreply\r\r"); + /* test_memcache_parse_req_failure_case("quit unknownarg\r\n"); */ + test_memcache_parse_req_failure_case("replace key 0 9 ?\r\n\r\n"); + test_memcache_parse_req_failure_case("set key 0 5 10 noreply\r\nvalue12345\r\r"); + test_memcache_parse_req_failure_case("set key 0 600 5\r\nvaluee\r\n"); + test_memcache_parse_req_failure_case("touch missingarg\r\n"); + test_memcache_parse_req_failure_case("version extra\r\n"); +} + +int main(int argc, char **argv) { + struct instance nci = {0}; + nci.mbuf_chunk_size = MBUF_SIZE; + mbuf_init(&nci); + msg_init(); + log_init(7, NULL); + + test_hash_algorithms(); + test_config_parsing(); + test_redis_parse_rsp_success(); + test_redis_parse_req_success(); + test_memcache_parse_rsp_success(); + test_memcache_parse_req_success(); + printf("Starting tests of request/response parsing failures\n"); + test_memcache_parse_rsp_failure(); + test_memcache_parse_req_failure(); + test_redis_parse_rsp_failure(); + printf("%d successes, %d failures\n", successes, failures); + + msg_deinit(); + mbuf_deinit(); + log_deinit(); + + return failures > 0 ? 1 : 0; +} diff --git a/test_in_docker.sh b/test_in_docker.sh new file mode 100755 index 00000000..e9ec214f --- /dev/null +++ b/test_in_docker.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Main ci script for nutcracker tests +set -xeu + +function print_usage() { + echo "Usage: $0 [REDIS_VER]" 1>&2 + echo "e.g. $0 3.2.12" 1>&2 + exit 1 +} + +REDIS_VER=3.2.11 +if [[ "$#" > 1 ]]; then + echo "Too many arguments" 1>&2 + print_usage +elif [[ "$#" > 0 ]]; then + REDIS_VER="$1" +fi + +PACKAGE_NAME="nutcrackerci" + +TAG=$( git describe --always ) +DOCKER_IMG_NAME=twemproxy-build-$PACKAGE_NAME-$REDIS_VER-$TAG + +rm -rf twemproxy + +DOCKER_TAG=twemproxy-$PACKAGE_NAME-$REDIS_VER:$TAG + +docker build -f ci/Dockerfile \ + --tag $DOCKER_TAG \ + --build-arg=REDIS_VER=$REDIS_VER \ + . + +# Run c unit tests +UNIT_TEST_FAIL=no +if ! docker run \ + --rm \ + -e REDIS_VER=$REDIS_VER \ + --workdir=/usr/src/twemproxy/src \ + --name=$DOCKER_IMG_NAME \ + --entrypoint=/bin/sh \ + $DOCKER_TAG \ + -c 'make test_all && ./test_all'; then + + UNIT_TEST_FAIL=yes +fi + +# Run nose tests +docker run \ + --rm \ + -e REDIS_VER=$REDIS_VER \ + --name=$DOCKER_IMG_NAME \ + $DOCKER_TAG \ + nosetests -v test_redis test_memcache test_system + +if [ $UNIT_TEST_FAIL = yes ]; then + echo "See earlier output, unit tests failed" 1>&2 + exit 1 +fi diff --git a/tests/README.rst b/tests/README.rst index acceea41..0ae657ea 100644 --- a/tests/README.rst +++ b/tests/README.rst @@ -1,19 +1,25 @@ Python testing facilities for twemproxy, this test suite is based on https://github.com/idning/redis-mgr -already add to https://travis-ci.org/idning/twemproxy as travis-ci +Testing in docker +================= -see https://github.com/idning/twemproxy/blob/travis-ci/travis.sh +The script `test_in_docker.sh` can be run to run all of twemproxy's compiler checks, C unit tests, and this folder's integration tests in docker. + + REDIS_VERSION=6.2.4 + ./test_in_docker.sh $REDIS_VERSION usage ===== -1. install dependency:: +Information on setting up integration tests, running integration tests, and creating new integration tests is below. + +1. install dependencies (redis-py must be 3.0 or newer):: pip install nose - pip install git+https://github.com/andymccurdy/redis-py.git@2.10.3 - pip install git+https://github.com/idning/python-memcached.git#egg=memcache + pip install git+https://github.com/andymccurdy/redis-py.git@3.5.3 + pip install git+https://github.com/linsomniac/python-memcached.git#egg=memcache -2. copy binarys to _binaries/:: +2. copy binaries to _binaries/:: _binaries/ |-- nutcracker @@ -25,9 +31,9 @@ usage |-- redis-server |-- memcached -3. run:: +3. run with nosetests (or ./nosetests_verbose.sh):: - $ nosetests -v + $ python3 -m nose -v test_del.test_multi_delete_on_readonly ... ok test_mget.test_mget ... ok @@ -36,7 +42,7 @@ usage OK -4. add A case:: +4. add a case:: cp tests/test_del.py tests/test_xxx.py vim tests/test_xxx.py @@ -69,6 +75,13 @@ T_LOGFILE: notes ===== -- After all the tests. you may got a core because we have a case in test_signal which will send SEGV to nutcracker +- After all of the integration tests, you may get a core dump because we have a case in test_signal which will send SEGV to nutcracker + +- If tests are failing, you may have to `pkill` redis-server, redis-sentinel, or nutcracker + +Unit tests +========== +See src/test_all.c - unit tests are separate from these integration tests and do not require python. They can be compiled and run with `make check`. +To view the output of the failing tests, run `cd src; make test_all; ./test_all`. diff --git a/tests/conf/conf.py b/tests/conf/conf.py index c02f72e3..003247c1 100644 --- a/tests/conf/conf.py +++ b/tests/conf/conf.py @@ -1,5 +1,3 @@ -#coding: utf-8 - import os import sys diff --git a/tests/conf/redis.conf b/tests/conf/redis.conf index d25a3f08..301b6a2d 100644 --- a/tests/conf/redis.conf +++ b/tests/conf/redis.conf @@ -16,10 +16,6 @@ # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. daemonize yes -#whitelist configure -#whitelist yes -#whitelist-file ./whitelist - # When running daemonized, Redis writes a pid file in /var/run/redis.pid by # default. You can specify a custom pid file location here. pidfile ${pidfile} diff --git a/tests/conf/sentinel.conf b/tests/conf/sentinel.conf new file mode 100644 index 00000000..7acc8ccb --- /dev/null +++ b/tests/conf/sentinel.conf @@ -0,0 +1,48 @@ +# Example sentinel.conf + +# port +# The port that this sentinel instance will run on +port ${port} + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +daemonize yes + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile ${pidfile} + +# Specify the log file name. Also 'stdout' can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile ${logfile} + +# sentinel announce-ip +# sentinel announce-port +# +# The above two configuration directives are useful in environments where, +# because of NAT, Sentinel is reachable from outside via a non-local address. +# +# When announce-ip is provided, the Sentinel will claim the specified IP address +# in HELLO messages used to gossip its presence, instead of auto-detecting the +# local address as it usually does. +# +# Similarly when announce-port is provided and is valid and non-zero, Sentinel +# will announce the specified TCP port. +# +# The two options don't need to be used together, if only announce-ip is +# provided, the Sentinel will announce the specified IP and the server port +# as specified by the "port" option. If only announce-port is provided, the +# Sentinel will announce the auto-detected local IP and the specified port. +# +# Example: +# +# sentinel announce-ip 1.2.3.4 + +# dir +# Every long running process should have a well-defined working directory. +# For Redis Sentinel to chdir to /tmp at startup is the simplest thing +# for the process to don't interfere with administrative tasks such as +# unmounting filesystems. +dir ${dir} + diff --git a/tests/lib/server_modules.py b/tests/lib/server_modules.py index e002186c..a92b88bd 100644 --- a/tests/lib/server_modules.py +++ b/tests/lib/server_modules.py @@ -10,6 +10,11 @@ from utils import * import conf +if sys.version_info[0] < 3: + # Give a clear error message instead of a confusing one. + sys.stderr.write("Error: must use python 3 to run these nosetests, e.g. python3 -m nose [options] test_modules\n") + sys.exit(2) + class Base: ''' Sub class should implement: @@ -39,13 +44,13 @@ def deploy(self): mkdir -p $path/conf && \ mkdir -p $path/log && \ mkdir -p $path/data', - self.args)) + self.args)) self._pre_deploy() self._gen_control_script() def _gen_control_script(self): - content = open(os.path.join(WORKDIR, 'conf/control.sh')).read() + content = open(os.path.join(WORKDIR, 'conf/control.sh'), 'r').read() content = TT(content, self.args) control_filename = TT('${path}/${name}_control', self.args) @@ -57,7 +62,7 @@ def _gen_control_script(self): def start(self): if self._alive(): - logging.warn('%s already running' %(self) ) + logging.warning('%s already running' % (self)) return logging.debug('starting %s' % self) @@ -72,15 +77,15 @@ def start(self): if sleeptime < 5: sleeptime *= 2 else: - sleeptime = 5 - logging.warn('%s still not alive' % self) + sleeptime = 5.0 + logging.warning('%s still not alive' % self) t2 = time.time() - logging.info('%s start ok in %.2f seconds' %(self, t2-t1) ) + logging.info('%s start ok in %.2f seconds' %(self, t2-t1)) def stop(self): if not self._alive(): - logging.warn('%s already stop' %(self) ) + logging.warning('%s already stop' %(self)) return cmd = TT("cd $path && ./${name}_control stop", self.args) @@ -90,21 +95,21 @@ def stop(self): while self._alive(): lets_sleep() t2 = time.time() - logging.info('%s stop ok in %.2f seconds' %(self, t2-t1) ) + logging.info('%s stop ok in %.2f seconds' %(self, t2-t1)) def pid(self): cmd = TT("pgrep -f '^$runcmd'", self.args) return self._run(cmd) def status(self): - logging.warn("status: not implement") + logging.warning("status: not implement") def _alive(self): - logging.warn("_alive: not implement") + logging.warning("_alive: not implement") def _run(self, raw_cmd): ret = system(raw_cmd, logging.debug) - logging.debug('return : [%d] [%s] ' % (len(ret), shorten(ret)) ) + logging.debug('return : [%d] [%s] ' % (len(ret), shorten(ret))) return ret def clean(self): @@ -121,8 +126,8 @@ class RedisServer(Base): def __init__(self, host, port, path, cluster_name, server_name, auth = None): Base.__init__(self, 'redis', host, port, path) - self.args['startcmd'] = TT('bin/redis-server conf/redis.conf --port $port', self.args) - self.args['runcmd'] = TT('redis-server.*$port', self.args) + self.args['startcmd'] = TT('bin/redis-server conf/redis.conf', self.args) + self.args['runcmd'] = TT('redis-server \\*:$port', self.args) self.args['conf'] = TT('$path/conf/redis.conf', self.args) self.args['pidfile'] = TT('$path/log/redis.pid', self.args) self.args['logfile'] = TT('$path/log/redis.log', self.args) @@ -154,7 +159,7 @@ def _alive(self): return strstr(self._ping(), 'PONG') def _gen_conf(self): - content = open(os.path.join(WORKDIR, 'conf/redis.conf')).read() + content = open(os.path.join(WORKDIR, 'conf/redis.conf'), 'r').read() content = TT(content, self.args) if self.args['auth']: content += '\r\nrequirepass %s' % self.args['auth'] @@ -193,6 +198,49 @@ def rediscmd(self, cmd): logging.info('%s %s' % (self, cmd)) return self._run(cmd) +class RedisSentinel(RedisServer): + def __init__(self, host, port, path, cluster_name, server_name, masters, quorum, down_time, auth = None): + RedisServer.__init__(self, host, port, path, cluster_name, server_name, auth) + + self.masters = masters + self.quorum = quorum + self.down_time = down_time + + self.args['startcmd'] = TT('bin/redis-sentinel conf/sentinel.conf', self.args) + self.args['runcmd'] = TT('redis-sentinel \\*:$port', self.args) + self.args['conf'] = TT('$path/conf/sentinel.conf', self.args) + self.args['pidfile'] = TT('$path/log/sentinel.pid', self.args) + self.args['logfile'] = TT('$path/log/sentinel.log', self.args) + + def _gen_conf_section(self): + template = ''' +sentinel monitor $server_name $host $port %d +sentinel down-after-milliseconds $server_name %d +sentinel parallel-syncs $server_name 1 +sentinel failover-timeout $server_name 180000 +''' % (self.quorum, self.down_time) + cfg = '\n'.join([TT(template, master.args) for master in self.masters]) + return cfg + + def _gen_conf(self): + content = open(os.path.join(WORKDIR, 'conf/sentinel.conf'), 'r').read() + content = TT(content, self.args) + if self.args['auth']: + content += '\r\nrequirepass %s' % self.args['auth'] + return content + self._gen_conf_section() + + def _pre_deploy(self): + self.args['BINS'] = conf.BINARYS['REDIS_SERVER_BINS'] + self._run(TT('cp $BINS $path/bin/', self.args)) + + fout = open(TT('$path/conf/sentinel.conf', self.args), 'w+') + fout.write(self._gen_conf()) + fout.close() + + def failover(self, server_name): + cmd = 'SENTINEL FAILOVER %s' % server_name + return self.rediscmd(cmd) + class Memcached(Base): def __init__(self, host, port, path, cluster_name, server_name): Base.__init__(self, 'memcached', host, port, path) @@ -214,10 +262,12 @@ def _pre_deploy(self): class NutCracker(Base): def __init__(self, host, port, path, cluster_name, masters, mbuf=512, - verbose=5, is_redis=True, redis_auth=None, redis_db=0, timeout=400): + verbose=5, is_redis=True, redis_auth=None, sentinels=None, redis_db=0, timeout=400): + Base.__init__(self, 'nutcracker', host, port, path) self.masters = masters + self.sentinels = sentinels self.args['mbuf'] = mbuf self.args['verbose'] = verbose @@ -241,9 +291,9 @@ def __init__(self, host, port, path, cluster_name, masters, mbuf=512, def _alive(self): return self._info_dict() - def _gen_conf_section(self): + def _gen_conf_section(self, servers): template = ' - $host:$port:1 $server_name' - cfg = '\n'.join([TT(template, master.args) for master in self.masters]) + cfg = '\n'.join([TT(template, server.args) for server in servers]) return cfg def _gen_conf(self): @@ -268,7 +318,13 @@ def _gen_conf(self): content = content.replace('redis: $is_redis', 'redis: $is_redis\r\n redis_auth: $redis_auth') content = TT(content, self.args) - return content + self._gen_conf_section() + content += self._gen_conf_section(self.masters) + if self.args['is_redis'] and self.sentinels: + content += ''' + sentinels: +''' + content += self._gen_conf_section(self.sentinels) + return content def _pre_deploy(self): self.args['BINS'] = conf.BINARYS['NUTCRACKER_BINS'] @@ -293,8 +349,10 @@ def _info_dict(self): [Exception: %s]' % (e, )) return None - def reconfig(self, masters): + def reconfig(self, masters, sentinels=None): self.masters = masters + if sentinels: + self.sentinels = sentinels self.stop() self.deploy() self.start() diff --git a/tests/lib/utils.py b/tests/lib/utils.py index 694b0149..b27dd2e3 100644 --- a/tests/lib/utils.py +++ b/tests/lib/utils.py @@ -65,7 +65,7 @@ def f_retry(*args, **kwargs): if logger: logger.info(e) else: - print((str(e))) + print(str(e)) return f_retry # true decorator return deco_retry @@ -77,9 +77,12 @@ def json_encode(j): return json.dumps(j, indent=4, cls=MyEncoder) def json_decode(j): + if isinstance(j, bytes): + j = str(j, encoding="utf-8") + return json.loads(j) -#commands dose not work on windows.. +#commands does not work on windows.. def system(cmd, log_fun=logging.info): if log_fun: log_fun(cmd) r = subprocess.getoutput(cmd) diff --git a/tests/nosetests_verbose.sh b/tests/nosetests_verbose.sh new file mode 100755 index 00000000..b2e022e5 --- /dev/null +++ b/tests/nosetests_verbose.sh @@ -0,0 +1,12 @@ +#!/bin/bash -xeu +# A simple utility script to run tests with extremely verbose output. + +if [[ $# == 0 ]]; then + echo "Usage: $0 test_a [test_b, ...]" 1>&2 + exit 1 +fi + +# Print test logging to stderr +export T_LOGFILE=- + +python3 -m nose -v --nologcapture --nocapture "$@" diff --git a/tests/test_memcache/test_gets.py b/tests/test_memcache/test_gets.py index 9bbe6f6e..e38cc420 100644 --- a/tests/test_memcache/test_gets.py +++ b/tests/test_memcache/test_gets.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -#coding: utf-8 +#!/usr/bin/env python3 import os import sys @@ -67,7 +66,6 @@ def test_mget_mset(kv=default_kv): conn.set_multi(kv) keys = sorted(kv.keys()) - assert(conn.get_multi(keys) == kv) assert(conn.get_multi(keys) == kv) #del diff --git a/tests/test_redis/common.py b/tests/test_redis/common.py index ed8af7a4..7c59c9dc 100644 --- a/tests/test_redis/common.py +++ b/tests/test_redis/common.py @@ -1,9 +1,9 @@ -#!/usr/bin/env python -#coding: utf-8 +#!/usr/bin/env python3 import os import sys import redis +import time PWD = os.path.dirname(os.path.realpath(__file__)) WORKDIR = os.path.join(PWD,'../') @@ -29,7 +29,7 @@ all_redis, mbuf=mbuf, verbose=nc_verbose) def setup(): - print(('setup(mbuf=%s, verbose=%s)' %(mbuf, nc_verbose))) + print('setup(mbuf=%s, verbose=%s)' %(mbuf, nc_verbose)) for r in all_redis + [nc]: r.clean() r.deploy() @@ -37,25 +37,19 @@ def setup(): r.start() def teardown(): + all_alive = True for r in all_redis + [nc]: - assert(r._alive()) + if not r._alive(): + all_alive = False r.stop() + assert(all_alive) -default_kv = {'kkk-%s' % i : 'vvv-%s' % i for i in range(10)} +default_kv = {b'kkk-%d' % i : b'vvv-%d' % i for i in range(10)} def getconn(): - for r in all_redis: - c = redis.Redis(r.host(), r.port(), decode_responses=True) - c.flushdb() - - r = redis.Redis(nc.host(), nc.port(), decode_responses=True) - return r - -def getconn_no_decode(): for r in all_redis: c = redis.Redis(r.host(), r.port()) c.flushdb() r = redis.Redis(nc.host(), nc.port()) return r - diff --git a/tests/test_redis/test_auth.py b/tests/test_redis/test_auth.py index 3ca761a8..c3f188a9 100644 --- a/tests/test_redis/test_auth.py +++ b/tests/test_redis/test_auth.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -#coding: utf-8 +#!/usr/bin/env python3 from .common import * @@ -21,7 +20,7 @@ all_redis, mbuf=mbuf, verbose=nc_verbose) def setup(): - print(('setup(mbuf=%s, verbose=%s)' %(mbuf, nc_verbose))) + print('setup(mbuf=%s, verbose=%s)' %(mbuf, nc_verbose)) for r in all_redis + [nc, nc_badpass, nc_nopass]: r.clean() r.deploy() @@ -33,10 +32,10 @@ def teardown(): assert(r._alive()) r.stop() -default_kv = {'kkk-%s' % i : 'vvv-%s' % i for i in range(10)} +default_kv = {bytes('kkk-%s' % i, encoding='utf-8') : bytes('vvv-%s' % i, encoding='utf-8') for i in range(10)} def getconn(): - r = redis.Redis(nc.host(), nc.port(), decode_responses=True) + r = redis.Redis(nc.host(), nc.port()) return r ''' @@ -56,56 +55,63 @@ def getconn(): def test_auth_basic(): # we hope to have same behavior when the server is redis or twemproxy conns = [ - redis.Redis(all_redis[0].host(), all_redis[0].port(), decode_responses=True), - redis.Redis(nc.host(), nc.port(), decode_responses=True), + redis.Redis(all_redis[0].host(), all_redis[0].port()), + redis.Redis(nc.host(), nc.port()), ] for r in conns: - assert_fail('NOAUTH|operation not permitted|Authentication required', r.ping) - assert_fail('NOAUTH|operation not permitted|Authentication required', r.set, 'k', 'v') - assert_fail('NOAUTH|operation not permitted|Authentication required', r.get, 'k') + assert_fail('Authentication required', r.ping) + assert_fail('Authentication required', r.set, 'k', 'v') + assert_fail('Authentication required', r.get, 'k') # bad passwd - assert_fail('invalid password', r.execute_command, 'AUTH', 'badpasswd') + assert_fail('invalid password|WRONGPASS', r.execute_command, 'AUTH', 'badpasswd') # everything is ok after auth r.execute_command('AUTH', 'hellopasswd') r.set('k', 'v') assert(r.ping() == True) - assert(r.get('k') == 'v') + assert_equal(b'v', r.get('k')) # auth fail here, should we return ok or not => we will mark the conn state as not authed - assert_fail('invalid password', r.execute_command, 'AUTH', 'badpasswd') - - assert_fail('NOAUTH|operation not permitted|Authentication required', r.ping) - assert_fail('NOAUTH|operation not permitted|Authentication required', r.get, 'k') + assert_fail('invalid password|WRONGPASS', r.execute_command, 'AUTH', 'badpasswd') + # https://redis.io/commands/auth changed in redis 6.0.0 and auth now appears to be additive for valid credentials? + # We can get the redis version by invoking a shell command, but not going to bother. Just assert that it if it throws, it's for the expected reason. + try: + r.ping() + except Exception as e: + assert re.search('Authentication required', str(e)) + try: + r.get('k') + except Exception as e: + assert re.search('Authentication required', str(e)) def test_nopass_on_proxy(): - r = redis.Redis(nc_nopass.host(), nc_nopass.port(), decode_responses=True) + r = redis.Redis(nc_nopass.host(), nc_nopass.port()) # if you config pass on redis but not on twemproxy, # twemproxy will reply ok for ping, but once you do get/set, you will get errmsg from redis assert(r.ping() == True) - assert_fail('NOAUTH|operation not permitted|Authentication required', r.set, 'k', 'v') - assert_fail('NOAUTH|operation not permitted|Authentication required', r.get, 'k') + assert_fail('Authentication required', r.set, 'k', 'v') + assert_fail('Authentication required', r.get, 'k') # proxy has no pass, when we try to auth assert_fail('Client sent AUTH, but no password is set', r.execute_command, 'AUTH', 'anypasswd') pass def test_badpass_on_proxy(): - r = redis.Redis(nc_badpass.host(), nc_badpass.port(), decode_responses=True) + r = redis.Redis(nc_badpass.host(), nc_badpass.port()) - assert_fail('NOAUTH|operation not permitted|Authentication required', r.ping) - assert_fail('NOAUTH|operation not permitted|Authentication required', r.set, 'k', 'v') - assert_fail('NOAUTH|operation not permitted|Authentication required', r.get, 'k') + assert_fail('Authentication required', r.ping) + assert_fail('Authentication required', r.set, 'k', 'v') + assert_fail('Authentication required', r.get, 'k') # we can auth with bad pass (twemproxy will say ok for this) r.execute_command('AUTH', 'badpasswd') # after that, we still got NOAUTH for get/set (return from redis-server) assert(r.ping() == True) - assert_fail('NOAUTH|operation not permitted|Authentication required', r.set, 'k', 'v') - assert_fail('NOAUTH|operation not permitted|Authentication required', r.get, 'k') + assert_fail('Authentication required', r.set, 'k', 'v') + assert_fail('Authentication required', r.get, 'k') def setup_and_wait(): time.sleep(60*60) diff --git a/tests/test_redis/test_basic.py b/tests/test_redis/test_basic.py index 770dd106..f20d13af 100644 --- a/tests/test_redis/test_basic.py +++ b/tests/test_redis/test_basic.py @@ -1,32 +1,35 @@ #!/usr/bin/env python #coding: utf-8 +from nose.tools import nottest from .common import * def test_setget(): r = getconn() rst = r.set('k', 'v') - assert(r.get('k') == 'v') + assert_equal(True, rst) + assert_equal(b'v', r.get('k')) def test_msetnx(): r = getconn() - #not supported - keys = default_kv.keys() - assert_fail('Socket closed|Connection closed', r.msetnx, default_kv) + # https://redis.io/commands/msetnx + # MSETNX is not supported because the keys can get sent to different backends, which is not supported. + normalized_kv = {str(key, encoding='utf-8'): val for key, val in default_kv.items()} + assert_fail('Socket closed|Connection closed', r.msetnx, normalized_kv) def test_null_key(): r = getconn() rst = r.set('', 'v') - assert(r.get('') == 'v') + assert_equal(b'v', r.get('')) rst = r.set('', '') - assert(r.get('') == '') + assert_equal(b'', r.get('')) kv = {'' : 'val', 'k': 'v'} ret = r.mset(kv) - assert(r.get('') == 'val') + assert_equal(b'val', r.get('')) def test_ping_quit(): r = getconn() @@ -34,11 +37,25 @@ def test_ping_quit(): #get set rst = r.set('k', 'v') - assert(r.get('k') == 'v') + assert_equal(b'v', r.get('k')) assert_fail('Socket closed|Connection closed', r.execute_command, 'QUIT') -def test_slow_req(): +def test_slow_req_lua(): + r = getconn() + pipe = r.pipeline(transaction=False) + pipe.eval("local x=0;for i = 1,300000000,1 do x = x+ i; end; return x", 1, 'tmp') + assert_fail('timed out', pipe.execute) + +def test_fast_req_lua(): + r = getconn() + pipe = r.pipeline(transaction=False) + pipe.eval("local x=0;for i = 1,100,1 do x = x+ i; end; return x", 1, 'tmp') + assert_equal([5050], pipe.execute()) + +# Disabled because this uses a lot of memory and would sometimes complete before the timeout. +@nottest +def disabled_test_slow_req(): r = getconn() kv = {'mkkk-%s' % i : 'mvvv-%s' % i for i in range(500000)} @@ -55,6 +72,7 @@ def test_slow_req(): def test_signal(): #init nc.cleanlog() + time.sleep(.1) nc.signal('HUP') nc.signal('HUP') @@ -63,7 +81,7 @@ def test_signal(): nc.signal('SEGV') time.sleep(.3) - log = open(nc.logfile()).read() + log = open(nc.logfile(), 'r').read() assert(strstr(log, 'HUP')) assert(strstr(log, 'TTIN')) @@ -92,7 +110,7 @@ def test_nc_stats(): nc.start() r = getconn() kv = {'kkk-%s' % i :'vvv-%s' % i for i in range(10)} - for k, v in kv.items(): + for k, v in list(kv.items()): r.set(k, v) r.get(k) @@ -115,7 +133,7 @@ def get_stat(name): assert(get_stat('responses') == 20) ##### mget - keys = kv.keys() + keys = list(kv.keys()) r.mget(keys) #for version<=0.3.0 @@ -144,12 +162,15 @@ def get_stat(name): def test_issue_323(): # do on redis r = all_redis[0] - c = redis.Redis(r.host(), r.port(), decode_responses=True) - assert([1, 'OK'] == c.eval(b"return {1, redis.call('set', 'x', '1')}", 1, 'tmp')) + c = redis.Redis(r.host(), r.port()) + assert_equal([1, b'OK'], c.eval("return {1, redis.call('set', 'x', '1')}", 1, 'tmp')) # do on twemproxy c = getconn() - assert([1, 'OK'] == c.eval(b"return {1, redis.call('set', 'x', '1')}", 1, 'tmp')) + assert_equal([1, b'OK'], c.eval("return {1, redis.call('set', 'x', '1')}", 1, 'tmp')) + + # Test processing deeply nested multibulk responses + assert_equal([[[[[[[[[[[[[[[[[[[[b'value']]]]]]]]]]]]]]]]]]], b'other'], c.eval("return {{{{{{{{{{{{{{{{{{{{'value'}}}}}}}}}}}}}}}}}}}, 'other'}", 1, 'tmp')) def setup_and_wait(): time.sleep(60*60) diff --git a/tests/test_redis/test_commands.py b/tests/test_redis/test_commands.py index 242845f6..f4ed3a73 100644 --- a/tests/test_redis/test_commands.py +++ b/tests/test_redis/test_commands.py @@ -11,12 +11,19 @@ def test_linsert(): r.linsert('mylist', 'BEFORE', 'World', 'There') rst = r.lrange('mylist', 0, -1) - assert(rst == ['Hello', 'There', 'World']) + assert_equal([b'Hello', b'There', b'World'], rst) + +def test_exists(): + r = getconn() + + r.set('exists1', 'foo') + assert_equal(1, r.exists('exists1')) + assert_equal(0, r.exists('doesnotexist')) def test_lpush_lrange(): r = getconn() - vals = ['vvv-%s' % i for i in range(10) ] + vals = [b'vvv-%d' % i for i in range(10) ] assert([] == r.lrange('mylist', 0, -1)) r.lpush('mylist', *vals) @@ -27,7 +34,7 @@ def test_lpush_lrange(): def test_hscan(): r = getconn() - kv = {'kkk-%s' % i : 'vvv-%s' % i for i in range(10)} + kv = {b'kkk-%d' % i : b'vvv-%d' % i for i in range(10)} r.hmset('a', kv) cursor, dic = r.hscan('a') @@ -36,12 +43,13 @@ def test_hscan(): cursor, dic = r.hscan('a', match='kkk-5') assert(str(cursor) == '0') - assert(dic == {'kkk-5': 'vvv-5'}) + assert(dic == {b'kkk-5': b'vvv-5'}) def test_hscan_large(): r = getconn() - kv = {'x'* 100 + 'kkk-%s' % i : 'vvv-%s' % i for i in range(1000)} + kv = {b'x'* 100 + (b'kkk-%d' % i) : (b'vvv-%d' % i) + for i in range(1000)} r.hmset('a', kv) cursor = '0' @@ -70,15 +78,15 @@ def test_hscan_large(): def test_zscan(): r = getconn() - r.zadd('a', {'a':1, 'b':2, 'c':3}) + r.zadd('a', {'a': 1, 'b': 2, 'c': 3}) cursor, pairs = r.zscan('a') - assert(str(cursor) == '0') - assert(set(pairs) == set([('a', 1), ('b', 2), ('c', 3)])) + assert_equal(0, cursor) + assert_equal({(b'a', 1), (b'b', 2), (b'c', 3)}, set(pairs)) cursor, pairs = r.zscan('a', match='a') - assert(str(cursor) == '0') - assert(set(pairs) == set([('a', 1)])) + assert_equal(0, cursor) + assert_equal({(b'a', 1)}, set(pairs)) def test_sscan(): r = getconn() @@ -86,10 +94,10 @@ def test_sscan(): r.sadd('a', 1, 2, 3) cursor, members = r.sscan('a') - assert(str(cursor) == '0') - assert(set(members) == set(['1', '2', '3'])) + assert_equal(0, cursor) + assert_equal({b'1', b'2', b'3'}, set(members)) cursor, members = r.sscan('a', match='1') - assert(str(cursor) == '0') - assert(set(members) == set(['1'])) + assert_equal('0', str(cursor)) + assert_equal({b'1'}, set(members)) diff --git a/tests/test_redis/test_mget_large_binary.py b/tests/test_redis/test_mget_large_binary.py index 24442774..5d7f6b82 100644 --- a/tests/test_redis/test_mget_large_binary.py +++ b/tests/test_redis/test_mget_large_binary.py @@ -12,7 +12,7 @@ all_redis, mbuf=mbuf, verbose=nc_verbose, timeout=2000) def setup(): - print(('special setup(mbuf=%s, verbose=%s)' %(mbuf, nc_verbose))) + print('special setup(mbuf=%s, verbose=%s)' %(mbuf, nc_verbose)) for r in all_redis + [nc]: r.deploy() r.stop() @@ -27,8 +27,8 @@ def teardown(): def test_mget_binary_value(cnt=5): kv = {} for i in range(cnt): - kv['kkk-%s' % i] = os.urandom(1024*1024*16+1024) #16M + kv[bytes('kkk-%s' % i, encoding='utf-8')] = os.urandom(1024*1024*16+1024) #16M for i in range(cnt): - kv['kkk2-%s' % i] = b'' - _mget_mset(kv, no_decode=True) + kv[bytes('kkk2-%s' % i, encoding='utf-8')] = b'' + _mget_mset(kv) diff --git a/tests/test_redis/test_mget_mset.py b/tests/test_redis/test_mget_mset.py index 0d0d7cdc..99d61b36 100644 --- a/tests/test_redis/test_mget_mset.py +++ b/tests/test_redis/test_mget_mset.py @@ -1,17 +1,13 @@ -#!/usr/bin/env python -#coding: utf-8 +#!/usr/bin/env python3 from .common import * -def test_mget_mset(kv=default_kv, no_decode=False): - if no_decode: - r = getconn_no_decode() - else: - r = getconn() +def test_mget_mset(kv=default_kv): + r = getconn() def insert_by_pipeline(): pipe = r.pipeline(transaction=False) - for k, v in kv.items(): + for k, v in list(kv.items()): pipe.set(k, v) pipe.execute() @@ -25,12 +21,12 @@ def insert_by_mset(): except: insert_by_pipeline() - keys = kv.keys() + keys = list(kv.keys()) #mget to check vals = r.mget(keys) for i, k in enumerate(keys): - assert(kv[k] == vals[i]) + assert_equal(kv[k], vals[i]) #del assert (len(keys) == r.delete(*keys) ) @@ -46,7 +42,7 @@ def test_mget_mset_on_key_not_exist(kv=default_kv): def insert_by_pipeline(): pipe = r.pipeline(transaction=False) - for k, v in kv.items(): + for k, v in list(kv.items()): pipe.set(k, v) pipe.execute() @@ -59,7 +55,7 @@ def insert_by_mset(): insert_by_pipeline() keys = list(kv.keys()) - keys2 = ['x-'+k for k in keys] + keys2 = [b'x-'+k for k in keys] keys = keys + keys2 random.shuffle(keys) @@ -67,9 +63,9 @@ def insert_by_mset(): vals = r.mget(keys) for i, k in enumerate(keys): if k in kv: - assert(kv[k] == vals[i]) + assert_equal(kv[k], vals[i]) else: - assert(vals[i] == None) + assert_equal(None, vals[i]) #del assert (len(kv) == r.delete(*keys) ) @@ -82,16 +78,16 @@ def insert_by_mset(): def test_mget_mset_large(): for cnt in range(171, large, 171): - kv = {'kkk-%s' % i :'vvv-%s' % i for i in range(cnt)} + kv = {b'kkk-%d' % i : b'vvv-%d' % i for i in range(cnt)} test_mget_mset(kv) def test_mget_special_key(cnt=5): #key length = 512-48-1 kv = {} for i in range(cnt): - k = 'kkk-%s' % i - k = k + 'x'*(512-48-1-len(k)) - kv[k] = 'vvv' + k = b'kkk-%d' % i + k = k + b'x'*(512-48-1-len(k)) + kv[k] = b'vvv' test_mget_mset(kv) @@ -99,16 +95,16 @@ def test_mget_special_key_2(cnt=5): #key length = 512-48-2 kv = {} for i in range(cnt): - k = 'kkk-%s' % i - k = k + 'x'*(512-48-2-len(k)) - kv[k] = 'vvv'*9 + k = b'kkk-%d' % i + k = k + b'x'*(512-48-2-len(k)) + kv[k] = b'vvv'*9 test_mget_mset(kv) def test_mget_on_backend_down(): #one backend down - r = redis.Redis(nc.host(), nc.port(), decode_responses=True) + r = redis.Redis(nc.host(), nc.port()) assert_equal(None, r.get('key-2')) assert_equal(None, r.get('key-1')) @@ -123,7 +119,7 @@ def test_mget_on_backend_down(): #all backend down all_redis[1].stop() - r = redis.Redis(nc.host(), nc.port(), decode_responses=True) + r = redis.Redis(nc.host(), nc.port()) assert_fail('Connection refused|reset by peer|Broken pipe', r.mget, 'key-1') assert_fail('Connection refused|reset by peer|Broken pipe', r.mget, 'key-2') @@ -136,7 +132,7 @@ def test_mget_on_backend_down(): def test_mset_on_backend_down(): all_redis[0].stop() - r = redis.Redis(nc.host(),nc.port(), decode_responses=True) + r = redis.Redis(nc.host(),nc.port()) assert_fail('Connection refused|Broken pipe',r.mset,default_kv) @@ -147,25 +143,25 @@ def test_mset_on_backend_down(): r.start() def test_mget_pipeline(): - r = getconn_no_decode() + r = getconn() pipe = r.pipeline(transaction=False) - for k,v in default_kv.items(): + for k,v in list(default_kv.items()): pipe.set(k,v) - keys = default_kv.keys() + keys = list(default_kv.keys()) pipe.mget(keys) kv = {} for i in range(large): - kv['kkk-%s' % i] = os.urandom(100) - for k,v in kv.items(): + kv[b'kkk-%d' % i] = os.urandom(100) + for k,v in list(kv.items()): pipe.set(k,v) - for k in kv.keys(): + for k in list(kv.keys()): pipe.get(k) rst = pipe.execute() #print rst #check the result - keys = default_kv.keys() + keys = list(default_kv.keys()) #mget to check vals = r.mget(keys) @@ -186,8 +182,9 @@ def test_multi_delete_normal(): for i in range(100): r.set('key-%s'%i, 'val-%s'%i) + for i in range(100): - assert_equal('val-%s'%i, r.get('key-%s'%i) ) + assert_equal(bytes('val-%s'%i, encoding='utf-8'), r.get('key-%s'%i) ) keys = ['key-%s'%i for i in range(100)] assert_equal(100, r.delete(*keys)) @@ -198,39 +195,45 @@ def test_multi_delete_normal(): def test_multi_delete_on_readonly(): all_redis[0].slaveof(all_redis[1].args['host'], all_redis[1].args['port']) - r = redis.Redis(nc.host(), nc.port(), decode_responses=True) + r = redis.Redis(nc.host(), nc.port()) - # got "You can't write against a read only slave" - assert_fail("You can't write against a read only (slave|replica).", r.delete, 'key-1') + # got "You can't write against a read only (replica)" + assert_fail('READONLY|Invalid|You can\'t write against a read only', r.delete, 'key-1') assert_equal(0, r.delete('key-2')) - assert_fail("You can't write against a read only (slave|replica).", r.delete, 'key-3') + assert_fail('READONLY|Invalid|You can\'t write against a read only', r.delete, 'key-3') keys = ['key-1', 'key-2', 'kkk-3'] assert_fail('Invalid argument', r.delete, *keys) # got "Invalid argument" def test_multi_delete_on_backend_down(): - #one backend down - all_redis[0].stop() - r = redis.Redis(nc.host(), nc.port(), decode_responses=True) - - assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, 'key-1') + r = redis.Redis(nc.host(), nc.port()) assert_equal(None, r.get('key-2')) - keys = ['key-1', 'key-2', 'kkk-3'] - assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, *keys) - - #all backend down - all_redis[1].stop() - r = redis.Redis(nc.host(), nc.port(), decode_responses=True) - - assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, 'key-1') - assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, 'key-2') - - keys = ['key-1', 'key-2', 'kkk-3'] - assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, *keys) + # one backend down + all_redis[0].stop() - for r in all_redis: - r.start() + try: + # Saw this fail in redis 6.2.2 spuriously in GitHub actions with a timeout. + # Continue to assert that subsequent commands will recover. + assert_fail('Connection refused|reset by peer|Broken pipe|Connection timed out', r.delete, 'key-1') + assert_equal(None, r.get('key-2')) + + keys = ['key-1', 'key-2', 'kkk-3'] + assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, *keys) + + #all backend down + all_redis[1].stop() + r = redis.Redis(nc.host(), nc.port()) + + assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, 'key-1') + assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, 'key-2') + + keys = ['key-1', 'key-2', 'kkk-3'] + assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, *keys) + finally: + # Start is idempotent. + for r in all_redis: + r.start() def test_multi_delete_20140525(): diff --git a/tests/test_redis/test_pipeline.py b/tests/test_redis/test_pipeline.py index 2f1afca1..8982934f 100644 --- a/tests/test_redis/test_pipeline.py +++ b/tests/test_redis/test_pipeline.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -#coding: utf-8 +#!/usr/bin/env python3 from .common import * @@ -8,18 +7,20 @@ def test_pipeline(): pipe = r.pipeline(transaction = False) - pipe.set('a', 'a1').get('a').zadd('z', {'z1':1}).zadd('z', {'z2':4}) + pipe.set('a', 'a1').get('a').zadd('z', {'z1': 1}).zadd('z', {'z2': 4}) pipe.zincrby('z', 1, 'z1').zrange('z', 0, 5, withscores=True) - assert pipe.execute() == \ + assert_equal( [ True, - 'a1', + b'a1', True, True, 2.0, - [('z1', 2.0), ('z2', 4)], - ] + [(b'z1', 2.0), (b'z2', 4)], + ], + pipe.execute() + ) def test_invalid_pipeline(): r = getconn() @@ -29,8 +30,8 @@ def test_invalid_pipeline(): pipe.set('a', 1).set('b', 2).lpush('a', 3).set('d', 4).get('a') result = pipe.execute(raise_on_error = False) - assert result[0] - assert result[1] + assert_equal(True, result[0]) + assert_equal(True, result[1]) # we can't lpush to a key that's a string value, so this should # be a ResponseError exception @@ -38,8 +39,8 @@ def test_invalid_pipeline(): # since this isn't a transaction, the other commands after the # error are still executed - assert result[3] - assert result[4] == '1' + assert_equal(True, result[3]) + assert_equal(b'1', result[4]) # make sure the pipe was restored to a working state assert pipe.set('z', 'zzz').execute() == [True] diff --git a/tests/test_redis/test_protocol.py b/tests/test_redis/test_protocol.py index c7d2064d..b5aa44e0 100644 --- a/tests/test_redis/test_protocol.py +++ b/tests/test_redis/test_protocol.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from .common import * from pprint import pprint @@ -11,17 +11,15 @@ def get_conn(): def _test(req, resp, sleep=0): s = get_conn() - if isinstance(req, bytes): - req = [req] - + # Send a single byte at a time for i in req: - s.sendall(i) + s.sendall(bytes([i])) time.sleep(sleep) s.settimeout(.3) data = s.recv(10000) - assert(data == resp) + assert_equal(resp, data) def test_slow(): req = b'*1\r\n$4\r\nPING\r\n' @@ -38,35 +36,38 @@ def test_pingpong(): req = b'*1\r\n$4\r\nPING\r\n' resp = b'+PONG\r\n' _test(req, resp) + # Sanity check there's no error + info = nc._info_dict() + assert_equal(0, info['ntest']['client_err']) +# twemproxy for redis doesn't appear to have any code to send +OK\r\n, it just disconnects. def test_quit(): - if nc.version() < '0.4.2': - return req = b'*1\r\n$4\r\nQUIT\r\n' - resp = b'+OK\r\n' + # NOTE: Nutcracker doesn't appear to have any code to send +OK\r\n, it just disconnects. + # +OK\r\n would also be valid. + resp = b'' _test(req, resp) +# twemproxy for redis doesn't appear to have any code to send +OK\r\n, it just disconnects. +# If it doesn't try to send anything, there's no client_err. def test_quit_without_recv(): - if nc.version() < '0.4.2': - return req = b'*1\r\n$4\r\nQUIT\r\n' resp = b'+OK\r\n' s = get_conn() s.sendall(req) s.close() + time.sleep(0.1) info = nc._info_dict() - #pprint(info) - assert(info['ntest']['client_err'] == 1) + assert_equal(0, info['ntest']['client_err']) def _test_bad(req): s = get_conn() s.sendall(req) data = s.recv(10000) - print(data) - assert(b'' == s.recv(1000)) # peer is closed + assert_equal(b'', s.recv(1000)) # peer is closed def test_badreq(): reqs = [ @@ -90,4 +91,4 @@ def test_wrong_argc(): s = get_conn() s.sendall(b'*1\r\n$3\r\nGET\r\n') - assert(b'' == s.recv(1000)) # peer is closed + assert_equal(b'', s.recv(1000)) # peer is closed diff --git a/tests/test_system/test_reload.py b/tests/test_system/test_reload.py index 590f2597..072f0b8c 100644 --- a/tests/test_system/test_reload.py +++ b/tests/test_system/test_reload.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -#coding: utf-8 +#!/usr/bin/env python3 #file : test_reload.py #author : ning #date : 2014-09-03 12:28:16 @@ -19,6 +18,8 @@ from utils import * from nose import with_setup +# Reload support with SIGUSR1 was at one planned to be added in 0.4.2 of twitter/twemproxy but has not actually been added. +VERSION_SUPPORTING_RELOAD = '99.99.99' CLUSTER_NAME = 'ntest' nc_verbose = int(getenv('T_VERBOSE', 5)) mbuf = int(getenv('T_MBUF', 512)) @@ -35,7 +36,7 @@ all_redis, mbuf=mbuf, verbose=nc_verbose) def _setup(): - print(('setup(mbuf=%s, verbose=%s)' %(mbuf, nc_verbose))) + print('setup(mbuf=%s, verbose=%s)' %(mbuf, nc_verbose)) for r in all_redis + [nc]: r.deploy() r.stop() @@ -59,15 +60,15 @@ def send_cmd(s, req, resp): @with_setup(_setup, _teardown) def test_reload_with_old_conf(): - if nc.version() < '0.4.2': - print(('Ignore test_reload for version %s' % nc.version())) + if nc.version() < VERSION_SUPPORTING_RELOAD: + print('Ignore test_reload for version %s' % nc.version()) return pid = nc.pid() # print 'old pid:', pid - r = redis.Redis(nc.host(), nc.port(), decode_responses=True) + r = redis.Redis(nc.host(), nc.port()) r.set('k', 'v') - conn = get_tcp_conn(nc.host(), nc.port(), decode_responses=True) + conn = get_tcp_conn(nc.host(), nc.port()) send_cmd(conn, '*2\r\n$3\r\nGET\r\n$1\r\nk\r\n', '$1\r\nv\r\n') # nc.reload() is same as nc.stop() and nc.start() @@ -78,7 +79,7 @@ def test_reload_with_old_conf(): send_cmd(conn, '*2\r\n$3\r\nGET\r\n$1\r\nk\r\n', '$1\r\nv\r\n') # conn2 should connect to new instance - conn2 = get_tcp_conn(nc.host(), nc.port(), decode_responses=True) + conn2 = get_tcp_conn(nc.host(), nc.port()) send_cmd(conn2, '*2\r\n$3\r\nGET\r\n$1\r\nk\r\n', '$1\r\nv\r\n') # the old connection is still ok in T_RELOAD_DELAY seconds @@ -93,16 +94,16 @@ def test_reload_with_old_conf(): # conn2 should survive send_cmd(conn2, '*2\r\n$3\r\nGET\r\n$1\r\nk\r\n', '$1\r\nv\r\n') - r = redis.Redis(nc.host(), nc.port(), decode_responses=True) + r = redis.Redis(nc.host(), nc.port()) rst = r.set('k', 'v') assert(r.get('k') == 'v') @with_setup(_setup, _teardown) def test_new_port(): - if nc.version() < '0.4.2': - print(('Ignore test_reload for version %s' % nc.version())) + if nc.version() < VERSION_SUPPORTING_RELOAD: + print('Ignore test_reload for version %s' % nc.version()) return - r = redis.Redis(nc.host(), nc.port(), decode_responses=True) + r = redis.Redis(nc.host(), nc.port()) r.set('k', 'v') content = ''' @@ -120,19 +121,19 @@ def test_new_port(): nc.set_config(content) time.sleep(T_RELOAD_DELAY) - r1 = redis.Redis(nc.host(), nc.port(), decode_responses=True) - r2 = redis.Redis(nc.host(), 4101, decode_responses=True) + r1 = redis.Redis(nc.host(), nc.port()) + r2 = redis.Redis(nc.host(), 4101) assert_fail('Connection refused', r1.get, 'k') assert(r2.get('k') == 'v') @with_setup(_setup, _teardown) def test_pool_add_del(): - if nc.version() < '0.4.2': - print(('Ignore test_reload for version %s' % nc.version())) + if nc.version() < VERSION_SUPPORTING_RELOAD: + print('Ignore test_reload for version %s' % nc.version()) return - r = redis.Redis(nc.host(), nc.port(), decode_responses=True) + r = redis.Redis(nc.host(), nc.port()) r.set('k', 'v') content = ''' @@ -158,8 +159,8 @@ def test_pool_add_del(): nc.set_config(content) time.sleep(T_RELOAD_DELAY) - r1 = redis.Redis(nc.host(), nc.port(), decode_responses=True) - r2 = redis.Redis(nc.host(), 4101, decode_responses=True) + r1 = redis.Redis(nc.host(), nc.port()) + r2 = redis.Redis(nc.host(), 4101) assert(r1.get('k') == 'v') assert(r2.get('k') == 'v') @@ -178,9 +179,9 @@ def test_pool_add_del(): nc.set_config(content) time.sleep(T_RELOAD_DELAY) pid = nc.pid() - print((system('ls -l /proc/%s/fd/' % pid))) + print(system('ls -l /proc/%s/fd/' % pid)) - r3 = redis.Redis(nc.host(), 4102, decode_responses=True) + r3 = redis.Redis(nc.host(), 4102) assert_fail('Connection refused', r1.get, 'k') assert_fail('Connection refused', r2.get, 'k') diff --git a/travis.sh b/travis.sh deleted file mode 100644 index 211410d8..00000000 --- a/travis.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -#file : travis.sh -#author : ning -#date : 2014-05-10 16:54:43 - -cp `which nutcracker` tests/_binaries/ -cp `which redis-server` tests/_binaries/ -cp `which redis-cli` tests/_binaries/ -cp `which memcached` tests/_binaries/ - -#run test -# export T_LOGFILE=- -# export T_VERBOSE=10 -redis-server --version -nosetests -w tests/ --nologcapture -x -v \ No newline at end of file