diff --git a/AUTHORS b/AUTHORS index addec92..72afd8f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -4,7 +4,7 @@ many contributions from the community. Below follows a list of people, who contributed their code. Vasily Soshnikov, Andrew Drozdow, E. Blih, Konstantin Osipov, Valery Kholodkov, -Yichun Zhang (eval module, valgrind suppess, debug.h), Alex Turenko +Yichun Zhang (valgrind suppess, debug.h), Alex Turenko NOTE: If you can commit a change to this list, please do not hesitate to add your name to it. diff --git a/Makefile b/Makefile index a7a3913..ca804cd 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,8 @@ configure-debug: --prefix=$(PREFIX_PATH) \ --with-http_addition_module \ --add-module=$(MODULE_PATH) \ + --add-module=$(MODULE_PATH)/../ngx_devel_kit \ + --add-module=$(MODULE_PATH)/../lua-nginx-module \ --with-debug configure-for-testing: configure-debug diff --git a/README.md b/README.md index 550d7d3..78f231d 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,7 @@ Tarantool - https://hub.docker.com/r/tarantool/tarantool * [JSON](#json) * [Directives](#directives) * [tnt_pass](#tnt_pass) - * [tnt_eval](#tnt_eval) - * [tnt_eval_buffer_size](#tnt_eval_buffer_size) + * [HTTP headers and status](#HTTP headers and status) * [tnt_http_methods](#tnt_http_methods) * [tnt_http_rest_methods](#tnt_http_rest_methods) * [tnt_pass_http_request](#tnt_pass_http_request) @@ -211,7 +210,7 @@ end [ { "result": JSON_RESULT_OBJECT, "id":UINT, "error": { "message": STR, "code": INT } }, ...N ] - "result" - DEPRECATED in 2.4.0+ + "result" Version 2.4.0+ output a raw result, i.e. "JSON_RESULT_OBJECT". @@ -253,11 +252,11 @@ end rpc call 1: --> { "method": "echo", "params": [42, 23], "id": 1 } - <-- [42, 23] + <-- { "id": 1, "result": [42, 23] rpc call 2: --> { "method": "echo", "params": [ [ {"hello": "world"} ], "!" ], "id": 2 } - <-- [ {"hello": "world"} ], "!" ] + <-- { "id": 2, "result": [ {"hello": "world"} ], "!" ]} rpc call of a non-existent method: --> { "method": "echo_2", "id": 1 } @@ -273,8 +272,8 @@ end { "method": "echo", "params": [ [ {"hello": "world"} ], "!" ], "id": 2 } ] <-- [ - [42, 23], - [{"hello": "world"} ], "!" ], + { "id": 1, "result": [42, 23]}, + { "id": 2, "result" : [{"hello": "world"} ], "!" ]}, ] rpc call Batch of a non-existent method: @@ -284,7 +283,7 @@ end ] <-- [ { "error": {"code": -32601, "message": "Method not found"}, "id": 1 }, - [ {"hello": "world"} ], "!" ] + {"id": 2, "result": [ {"hello": "world"} ], "!" ]} ] rpc call Batch with invalid JSON: @@ -327,27 +326,18 @@ Specify the Tarantool server backend. [Back to content](#content) -tnt_eval --------- -**syntax:** *tnt_eval $HTTP_STATUS_VAR_NAME $HTTP_BODY_VAR_NAME* +HTTP headers and status +----------------------- -**default:** *no* - -**context:** *location* - -This directive put execution of tnt_pass into the nginx REWRITE PHASE. -That exactly this mean? That means that you can have a access to the body (in for of -JSON), http codes and http headers which have been passed from the Tarantool -to the nginx inside nginx config. This very useful for setting custom HTTP -statuses, headers and for post-processing of the original body. +Sometimes you have to set status or headers which came from the Tarantool. +For this you have to use something like [ngx_lua](https://github.com/openresty/lua-nginx-module) +or [ngx_perl](http://nginx.org/en/docs/http/ngx_http_perl_module.html) and so on. -Even more, you can use this for using this module with OpenResty, Nginx Script, -Nginx Perl and so on. +Also using the methods you can transform result from the `Tarantool` into +something else -NOTICE! +Here is an example with `ngx_lua`: -1) This directive expects that tarantool returns special object with meta -information about an HTTP status and an HTTP headers. Example @@ -358,7 +348,7 @@ Example -- First arg. if __ngx exists and tnt_eval is used, then it will be -- readed by nginx { - __ngx = { + ngx = { 200, -- set status HTTP 200 { ["X-Tarantool"] = "FROM_TNT" } -- set headers } @@ -373,60 +363,67 @@ Example upstream tnt_upstream { 127.0.0.1:9999; + keepalive 10000; } - location = /tnt { - - tnt_eval_buffer_size 1m; - - tnt_eval $tnt_http_status $tnt_body { - tnt_method foo; - tnt_pass 127.0.0.1:9999; - } - - if ($tnt_http_status = 404) { - return 404 $tnt_body; - } - - if ($tnt_body ~= 'Tarantool') { - return 200 '

Found Tarantool!

'; - } - - return 200 $tnt_body; + location /tnt_proxy { + tnt_method tnt_proxy; + tnt_buffer_size 100k; + tnt_pass_http_request on parse_args; + tnt_pass tnt_upstream; } - location = /tnt/with_echo_module { - - tnt_eval_buffer_size 1m; - - tnt_eval $tnt_http_status $tnt_body { - tnt_method foo; - tnt_pass 127.0.0.1:9999; + location /api { + + lua_need_request_body on; + + rewrite_by_lua ' + + local cjson = require("cjson") + + local map = { + GET = ngx.HTTP_GET, + POST = ngx.HTTP_POST, + PUT = ngx.HTTP_PUT, + -- ... + } + + local res = ngx.location.capture("/tnt_proxy", { + args = ngx.var.args, + method = map[ngx.var.request_method], + body = ngx.body + }) + + if res.status == ngx.HTTP_OK then + local answ = cjson.decode(res.body) + + -- Read reply + local result = answ["result"] + + if result ~= nil then + ngx.status = result[1]["ngx"][1] + for k, v in pairs(result[1]["ngx"][2]) do + ngx.header[k] = v + end + + table.remove(result, 1) + ngx.say(cjson.encode(result)) + else + ngx.status = 502 + ngx.say(res.body) + end + + -- Finalize execution + ngx.exit(ngx.OK) + else + ngx.status = 502 + ngx.say("Tarantool does not work") + end + '; } - echo $tnt_body; - } - - # ... - # Also those variables are available in any nginx's languages; ``` -2) '$'-prefix is required, means that tnt_eval http_code body { ... } will rise an error, - it should be tnt_eval $http_status $body { ... }. - -[Back to content](#content) - -tnt_eval_buffer_size --------------------- - -**syntax:** *tnt_eval_buffer_size size* - -**default:** *PAGE_SIZE x 16* - -**context:** *main, server, location* - -Specify the size of the buffer used for `tnt_eval`. - [Back to content](#content) tnt_http_methods @@ -703,7 +700,7 @@ The 0 value turns off this limitation. [Back to content](#content) -tnt_pure_result - DEPRECATED in 2.4.0+ +tnt_pure_result -------------------------------------- **syntax:** *tnt_pure_result [on|off]* @@ -714,7 +711,7 @@ tnt_pure_result - DEPRECATED in 2.4.0+ Whether to wrap tnt response or not. When this option is off: ``` -{"id":0, "result": [[ 1 ]]} +{"id":0, "result": [ 1 ]} ``` When this option is on: ``` diff --git a/config b/config index f5efe7b..0ae3c11 100644 --- a/config +++ b/config @@ -1,42 +1,47 @@ ngx_addon_name="ngx_http_tnt_module" -__libs=" \ - $ngx_addon_dir/third_party/yajl/build/yajl-2.1.0/lib/libyajl_s.a \ - $ngx_addon_dir/third_party/msgpuck/libmsgpuck.a \ - " +libs="-lyajl -lmsgpuck" -__module_src_dir="$ngx_addon_dir/src" +test -f $ngx_addon_dir/third_party/yajl/build/yajl-2.1.0/lib/libyajl_s.a && +test -f $ngx_addon_dir/third_party/msgpuck/libmsgpuck.a && { + libs=" \ + $ngx_addon_dir/third_party/yajl/build/yajl-2.1.0/lib/libyajl_s.a \ + $ngx_addon_dir/third_party/msgpuck/libmsgpuck.a \ + " +} -__include_paths=" \ +module_src_dir="$ngx_addon_dir/src" + +include_paths=" \ $ngx_addon_dir/src \ $ngx_addon_dir/third_party \ $ngx_addon_dir/third_party/msgpuck \ $ngx_addon_dir/third_party/yajl/build/yajl-2.1.0/include \ " -__sources=" \ - $__module_src_dir/json_encoders.c \ - $__module_src_dir/tp_transcode.c \ - $__module_src_dir/ngx_http_tnt_module.c \ - $__module_src_dir/ngx_http_tnt_handlers.c \ +sources=" \ + $module_src_dir/json_encoders.c \ + $module_src_dir/tp_transcode.c \ + $module_src_dir/ngx_http_tnt_module.c \ + $module_src_dir/ngx_http_tnt_handlers.c \ " -__headers=" \ - $__module_src_dir/debug.h \ - $__module_src_dir/tp_ext.h \ - $__module_src_dir/json_encoders.h \ - $__module_src_dir/tp_transcode.h \ - $__module_src_dir/ngx_http_tnt_handlers.h \ +headers=" \ + $module_src_dir/debug.h \ + $module_src_dir/tp_ext.h \ + $module_src_dir/json_encoders.h \ + $module_src_dir/tp_transcode.h \ + $module_src_dir/ngx_http_tnt_handlers.h \ " -__old_style_build=yes +old_style_build=yes if test -n "$ngx_module_link"; then - __old_style_build=no + old_style_build=no fi # # Old-style build [[ -if test "$__old_style_build" = "yes"; then +if test "$old_style_build" = "yes"; then CORE_INCS=" \ $CORE_INCS \ @@ -46,7 +51,7 @@ if test "$__old_style_build" = "yes"; then CORE_LIBS=" \ $CORE_LIBS \ $ngx_feature_libs \ - $__libs \ + $libs \ " HTTP_MODULES=" \ @@ -56,10 +61,10 @@ if test "$__old_style_build" = "yes"; then NGX_ADDON_SRCS=" \ $NGX_ADDON_SRCS \ - $__sources \ + $sources \ " - for path in $__include_paths; do + for path in $include_paths; do CFLAGS="$CFLAGS -I$path" done # ]] @@ -69,18 +74,10 @@ else ngx_module_type=HTTP ngx_module_name=$ngx_addon_name - ngx_module_incs=$__include_paths - ngx_module_deps=$__headers - ngx_module_srcs=$__sources - ngx_module_libs=$__libs - . auto/module - - ngx_module_type=HTTP_AUX_FILTER - ngx_module_name=ngx_http_tnt_eval_module - ngx_module_incs=$__include_paths - ngx_module_deps=$__headers - ngx_module_srcs=$__module_src_dir/ngx_http_tnt_eval_module.c - ngx_module_libs= + ngx_module_incs=$include_paths + ngx_module_deps=$headers + ngx_module_srcs=$sources + ngx_module_libs=$libs . auto/module # ]] diff --git a/src/ngx_http_tnt_eval_module.c b/src/ngx_http_tnt_eval_module.c deleted file mode 100644 index a748103..0000000 --- a/src/ngx_http_tnt_eval_module.c +++ /dev/null @@ -1,1024 +0,0 @@ -/* - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * 1. Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF - * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * Copyright (C) 2017 Tarantool AUTHORS: - * please see AUTHORS file. - */ - -#include "debug.h" - -#include - -#include -#include -#include -#include - - -typedef struct { - ngx_http_variable_t *variable; - ngx_uint_t index; -} ngx_http_tnt_eval_variable_t; - - -typedef struct { - ngx_array_t *variables; - ngx_str_t eval_location; - size_t buffer_size; -} ngx_http_tnt_eval_loc_conf_t; - - -typedef struct { - ngx_http_tnt_eval_loc_conf_t *base_conf; - ngx_http_variable_value_t **values; - ngx_int_t status; - ngx_int_t done:1; - ngx_int_t in_progress:1; - - /* A reference to the main request */ - ngx_http_request_t *r; - - /* A body passed from the Tarantool */ - ngx_buf_t buffer; - -} ngx_http_tnt_eval_ctx_t; - - -static ngx_int_t -ngx_http_tnt_eval_init_variables(ngx_http_request_t *r, - ngx_http_tnt_eval_ctx_t *ctx, ngx_http_tnt_eval_loc_conf_t *ecf); - -static ngx_int_t ngx_http_tnt_eval_post_subrequest_handler(ngx_http_request_t *r, - void *data, ngx_int_t rc); - -static void *ngx_http_tnt_eval_create_loc_conf(ngx_conf_t *cf); -static char *ngx_http_tnt_eval_merge_loc_conf(ngx_conf_t *cf, void *parent, - void *child); - -static char *ngx_http_tnt_eval_block(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf); - -static ngx_int_t ngx_http_tnt_eval_header_filter(ngx_http_request_t *r); -static ngx_int_t ngx_http_tnt_eval_body_filter(ngx_http_request_t *r, - ngx_chain_t *in); - -static ngx_int_t ngx_http_tnt_eval_discard_bufs(ngx_pool_t *pool, ngx_chain_t *in); - -static ngx_int_t ngx_http_tnt_eval_init(ngx_conf_t *cf); - -static ngx_int_t ngx_http_tnt_eval_parse_meta(ngx_http_request_t *r, - ngx_http_tnt_eval_ctx_t *ctx, ngx_http_variable_value_t *v); -static ngx_int_t ngx_http_tnt_eval_output(ngx_http_request_t *r, - ngx_http_tnt_eval_ctx_t *ctx); -static ngx_int_t ngx_http_tnt_subrequest(ngx_http_request_t *r, - ngx_str_t *uri, ngx_str_t *args, ngx_http_request_t **psr, - ngx_http_post_subrequest_t *ps, ngx_uint_t flags); -static void ngx_http_tnt_eval_finalize_request(ngx_http_request_t *r); - - -static ngx_http_output_header_filter_pt ngx_http_next_header_filter; -static ngx_http_output_body_filter_pt ngx_http_next_body_filter; - -/** if tarantool wants return some headers, status and so on, then - * nginx expectes that tarantool will output: - * [{__nginx = [STATUS, {HEADERS}]}, {H1:V1 ..}] - */ -static const char *tarantool_ngx_path[] = { "__ngx", (const char *) 0 }; - - -static ngx_command_t ngx_http_tnt_eval_commands[] = { - - { ngx_string("tnt_eval"), - NGX_HTTP_LOC_CONF|NGX_CONF_2MORE|NGX_CONF_BLOCK, - ngx_http_tnt_eval_block, - NGX_HTTP_LOC_CONF_OFFSET, - 0, - NULL }, - - { ngx_string("tnt_eval_buffer_size"), - NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, - ngx_conf_set_size_slot, - NGX_HTTP_LOC_CONF_OFFSET, - offsetof(ngx_http_tnt_eval_loc_conf_t, buffer_size), - NULL }, - - ngx_null_command -}; - - -static ngx_http_module_t ngx_http_tnt_eval_module_ctx = { - NULL, /* preconfiguration */ - ngx_http_tnt_eval_init, /* postconfiguration */ - - NULL, /* create main configuration */ - NULL, /* init main configuration */ - - NULL, /* create server configuration */ - NULL, /* merge server configuration */ - - ngx_http_tnt_eval_create_loc_conf, /* create location configuration */ - ngx_http_tnt_eval_merge_loc_conf /* merge location configuration */ -}; - - -ngx_module_t ngx_http_tnt_eval_module = { - NGX_MODULE_V1, - &ngx_http_tnt_eval_module_ctx, /* module context */ - ngx_http_tnt_eval_commands, /* module directives */ - NGX_HTTP_MODULE, /* module type */ - NULL, /* init master */ - NULL, /* init module */ - NULL, /* init process */ - NULL, /* init thread */ - NULL, /* exit thread */ - NULL, /* exit process */ - NULL, /* exit master */ - NGX_MODULE_V1_PADDING -}; - - -static ngx_int_t -ngx_http_tnt_eval_handler(ngx_http_request_t *r) -{ - ngx_str_t args; - ngx_str_t subrequest_uri; - ngx_uint_t flags; - ngx_http_tnt_eval_loc_conf_t *ecf; - ngx_http_tnt_eval_ctx_t *ctx; - ngx_http_tnt_eval_ctx_t *sr_ctx; - ngx_http_request_t *sr; - ngx_int_t rc; - ngx_http_post_subrequest_t *psr; - u_char *p; - - ecf = ngx_http_get_module_loc_conf(r, ngx_http_tnt_eval_module); - - /* Modules is not on */ - if (ecf->variables == NULL || !ecf->variables->nelts) { - return NGX_DECLINED; - } - - rc = ngx_http_read_client_request_body(r, - ngx_http_tnt_eval_finalize_request); - if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { - return rc; - } - - ctx = ngx_http_get_module_ctx(r, ngx_http_tnt_eval_module); - if (ctx == NULL) { - - ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_tnt_eval_ctx_t)); - if (ctx == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } - - dd("A new context has been created, r = %p, ctx = %p", r, ctx); - - ctx->base_conf = ecf; - ctx->r = r; - - ngx_http_set_ctx(r, ctx, ngx_http_tnt_eval_module); - } - - if (ctx->done) { - dd("subrequest done, r = %p, ctx = %p", r, ctx); - - /** If tnt_pass is down, then status should be passed to the client. - * 0 meas that upstream works - */ - if (ctx->status != 0) { - return ctx->status; - } - - return NGX_DECLINED; - } - - if (ctx->in_progress) { - dd("still in progress, r = %p, ctx = %p", r, ctx); - return NGX_DONE; - } - - psr = ngx_palloc(r->pool, sizeof(ngx_http_post_subrequest_t)); - if (psr == NULL) { - return NGX_ERROR; - } - - if (ngx_http_tnt_eval_init_variables(r, ctx, ecf) != NGX_OK) { - return NGX_ERROR; - } - - args = r->args; - flags = 0; - - subrequest_uri.len = ecf->eval_location.len + r->uri.len; - - p = ngx_palloc(r->pool, subrequest_uri.len); - if (p == NULL) { - return NGX_ERROR; - } - - subrequest_uri.data = p; - - p = ngx_copy(p, ecf->eval_location.data, ecf->eval_location.len); - ngx_memcpy(p, r->uri.data, r->uri.len); - - if (ngx_http_parse_unsafe_uri(r, &subrequest_uri, &args, &flags) - != NGX_OK) - { - return NGX_ERROR; - } - - psr->handler = ngx_http_tnt_eval_post_subrequest_handler; - psr->data = ctx; - - dd("issue subrequest"); - - flags |= NGX_HTTP_SUBREQUEST_WAITED; - - rc = ngx_http_tnt_subrequest(r, &subrequest_uri, &args, &sr, psr, flags); - if (rc == NGX_ERROR || rc == NGX_DONE) { - return rc; - } - - ctx->in_progress = 1; - - /** We don't allow eval in subrequests - */ - sr_ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_tnt_eval_ctx_t)); - if (sr_ctx == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } - - ngx_http_set_ctx(sr, sr_ctx, ngx_http_tnt_eval_module); - - dd("wait for subrequest to complete"); - - return NGX_DONE; -} - - -static ngx_int_t -ngx_http_tnt_eval_init_variables(ngx_http_request_t *r, - ngx_http_tnt_eval_ctx_t *ctx, ngx_http_tnt_eval_loc_conf_t *ecf) -{ - ngx_uint_t i; - ngx_http_tnt_eval_variable_t *variable; - - ctx->values = ngx_pcalloc(r->pool, - ecf->variables->nelts - * sizeof(ngx_http_variable_value_t *)); - - if (ctx->values == NULL) { - return NGX_ERROR; - } - - variable = ecf->variables->elts; - - for (i = 0; i < ecf->variables->nelts; i++) { - ctx->values[i] = r->variables + variable[i].index; - ctx->values[i]->valid = 0; - ctx->values[i]->not_found = 1; - } - - return NGX_OK; -} - - -static ngx_int_t -ngx_http_tnt_eval_post_subrequest_handler(ngx_http_request_t *r, void *data, - ngx_int_t rc) -{ - ngx_http_tnt_eval_ctx_t *ctx = data; - - ngx_http_tnt_eval_output(r, ctx); - - ctx->done = 1; - - ctx->status = rc; - -#if 0 - /* work-around a bug in the nginx core (ngx_http_named_location) - */ - r->parent->write_event_handler = ngx_http_core_run_phases; -#endif - - return NGX_OK; -} - - -/** Parse meta data iff it was sent by Tarantool - */ -static ngx_int_t -ngx_http_tnt_eval_parse_meta(ngx_http_request_t *r, - ngx_http_tnt_eval_ctx_t *ctx, ngx_http_variable_value_t *v /* body */) -{ - char errbuf[1024]; - const char *y_header; - const char *tmp; - yajl_val y_root; - yajl_val y_node; - yajl_val y_headers; - yajl_val y_header_value; - ngx_uint_t i; - ngx_table_elt_t *h; - ngx_http_variable_value_t *status; - ngx_int_t len; - - /* A conf var was'nt setted, so just call next filter */ - if (v->data == NULL || v->len <= 0) { - return NGX_DECLINED; - } - - /* TODO This should be fixed! {{{ */ - u_char *buf = ngx_pcalloc(ctx->r->pool, sizeof(u_char) * v->len + 1); - if (buf == NULL) { - return NGX_ERROR; - } - memcpy(buf, v->data, v->len); - buf[v->len] = 0; - /* }}} */ - y_root = yajl_tree_parse((const char *) buf, - errbuf, sizeof(errbuf)); - if (y_root == NULL || !(YAJL_IS_ARRAY(y_root) && - y_root->u.array.len > 0)) { - - ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "parse reply failed, message = \"%s\", body = \"%s\"", - errbuf, v->data); - return NGX_ERROR; - } - - y_node = yajl_tree_get(y_root->u.array.values[0], tarantool_ngx_path, - yajl_t_array); - if (y_node != NULL) { - - /** Set HTTP status - */ - if (y_node->u.array.len >= 1 && - YAJL_GET_INTEGER(y_node->u.array.values[0])) - { - status = ctx->values[0]; - - len = sizeof(u_char) + sizeof("2147483647") /* int32_t max */; - - status->data = (u_char *) ngx_palloc(ctx->r->pool, len); - if (status->data == NULL) { - return NGX_ERROR; - } - - status->len = snprintf((char *) status->data, len, "%i", - (int) YAJL_GET_INTEGER(y_node->u.array.values[0])); - status->valid = 1; - status->not_found = 0; - - dd("read a new HTTP status, r = %p, ctx = %p, ctx.status = %s", - r, ctx, status->data); - } - - /** Set HTTP headers - */ - if (ctx->r && - y_node->u.array.len >= 2 && - YAJL_IS_OBJECT(y_node->u.array.values[1])) - { - y_headers = y_node->u.array.values[1]; - - for (i = 0; i < y_headers->u.object.len; ++i) { - - y_header = y_headers->u.object.keys[i]; - y_header_value = y_headers->u.object.values[i]; - - if (!YAJL_IS_STRING(y_header_value)) { - continue; - } - - h = ngx_list_push(&ctx->r->headers_out.headers); - if (h == NULL) { - goto ngx_error; - } - - /** Here is insertion of headers passed from the Tarantool {{{ - */ - h->hash = 1; - - h->key.len = ngx_strlen(y_header); - h->key.data = ngx_pnalloc(r->pool, h->key.len * sizeof(u_char)); - if (h->key.data == NULL) { - goto ngx_error; - } - memcpy(h->key.data, y_header, h->key.len); - - tmp = YAJL_GET_STRING(y_header_value); - h->value.len = strlen(tmp); - h->value.data = ngx_pnalloc(r->pool, - h->value.len * sizeof(u_char)); - if (h->value.data == NULL) { - goto ngx_error; - } - tmp = YAJL_GET_STRING(y_header_value); - memcpy(h->value.data, tmp, h->value.len); - /** }}} - */ - } - } - } - - if (y_node != NULL) { - yajl_tree_free(y_node); - y_node = NULL; - } - - return NGX_OK; - -ngx_error: - if (y_node != NULL) { - yajl_tree_free(y_node); - y_node = NULL; - } - - return NGX_ERROR; -} - - -/* - * Evaluate tarantool output into the variable. - * - * This evaluation method assume, that we have at least one varible. - * ngx_http_tnt_eval_handler must guarantee this. * - */ -static ngx_int_t -ngx_http_tnt_eval_output(ngx_http_request_t *r, - ngx_http_tnt_eval_ctx_t *ctx) -{ - ngx_http_variable_value_t *body_value; - ngx_http_tnt_eval_ctx_t *sr_ctx; - - dd("Output stream for the tarantool into the variable"); - - body_value = ctx->values[1]; - sr_ctx = ngx_http_get_module_ctx(r, ngx_http_tnt_eval_module); - - if (sr_ctx && sr_ctx->buffer.start) { - - body_value->len = sr_ctx->buffer.last - sr_ctx->buffer.pos; - body_value->data = sr_ctx->buffer.pos; - body_value->valid = 1; - body_value->not_found = 0; - - } else if (r->upstream) { - - body_value->len = r->upstream->buffer.last - r->upstream->buffer.pos; - body_value->data = r->upstream->buffer.pos; - body_value->valid = 1; - body_value->not_found = 0; - - dd("found upstream buffer %d: %.*s, no cacheable = %d", - (int) body_value->len, (int) body_value->len, - body_value->data, (int) body_value->no_cacheable); - } - - if (ngx_http_tnt_eval_parse_meta(r, ctx, body_value) == NGX_ERROR) { - return NGX_ERROR; - } - - return NGX_OK; -} - - -static ngx_int_t -ngx_http_tnt_subrequest(ngx_http_request_t *r, - ngx_str_t *uri, ngx_str_t *args, ngx_http_request_t **psr, - ngx_http_post_subrequest_t *ps, ngx_uint_t flags) -{ - ngx_time_t *tp; - ngx_connection_t *c; - ngx_http_request_t *sr; - ngx_http_core_srv_conf_t *cscf; - ngx_http_postponed_request_t *pr, *p; - - if (r->subrequests == 0) { - ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "subrequests cycle while processing \"%V\"", uri); - return NGX_ERROR; - } - - /* - * 1000 is reserved for other purposes. - */ - if (r->main->count >= 65535 - 1000) { - ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0, - "request reference counter overflow " - "while processing \"%V\"", uri); - return NGX_ERROR; - } - - sr = ngx_pcalloc(r->pool, sizeof(ngx_http_request_t)); - if (sr == NULL) { - return NGX_ERROR; - } - - sr->signature = NGX_HTTP_MODULE; - - c = r->connection; - sr->connection = c; - - sr->ctx = ngx_pcalloc(r->pool, sizeof(void *) * ngx_http_max_module); - if (sr->ctx == NULL) { - return NGX_ERROR; - } - - if (ngx_list_init(&sr->headers_out.headers, r->pool, 20, - sizeof(ngx_table_elt_t)) - != NGX_OK) - { - return NGX_ERROR; - } - - cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); - sr->main_conf = cscf->ctx->main_conf; - sr->srv_conf = cscf->ctx->srv_conf; - sr->loc_conf = cscf->ctx->loc_conf; - - sr->pool = r->pool; - - sr->method = r->method; - sr->method_name = r->method_name; - sr->http_version = r->http_version; - - sr->request_line = r->request_line; - sr->uri = *uri; - - sr->headers_in = r->headers_in; - sr->request_body = r->request_body; - - if (args) { - sr->args = *args; - } - - ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, - "http tnt subrequest \"%V?%V\"", uri, &sr->args); - - sr->subrequest_in_memory = (flags & NGX_HTTP_SUBREQUEST_IN_MEMORY) != 0; - sr->waited = (flags & NGX_HTTP_SUBREQUEST_WAITED) != 0; - - sr->unparsed_uri = r->unparsed_uri; - sr->http_protocol = r->http_protocol; - - ngx_http_set_exten(sr); - - sr->main = r->main; - sr->parent = r; - sr->post_subrequest = ps; - sr->read_event_handler = ngx_http_request_empty_handler; - sr->write_event_handler = ngx_http_handler; - - if (c->data == r && r->postponed == NULL) { - c->data = sr; - } - - sr->variables = r->variables; - - sr->log_handler = r->log_handler; - - pr = ngx_palloc(r->pool, sizeof(ngx_http_postponed_request_t)); - if (pr == NULL) { - return NGX_ERROR; - } - - pr->request = sr; - pr->out = NULL; - pr->next = NULL; - - if (r->postponed) { - for (p = r->postponed; p->next; p = p->next) { /* void */ } - p->next = pr; - - } else { - r->postponed = pr; - } - - sr->internal = 1; - sr->discard_body = r->discard_body; - sr->expect_tested = 1; - sr->main_filter_need_in_memory = r->main_filter_need_in_memory; - - sr->uri_changes = NGX_HTTP_MAX_URI_CHANGES + 1; - sr->subrequests = r->subrequests - 1; - - tp = ngx_timeofday(); - sr->start_sec = tp->sec; - sr->start_msec = tp->msec; - - r->main->count++; - *psr = sr; - - return ngx_http_post_request(sr, NULL); -} - - -static void -ngx_http_tnt_eval_finalize_request(ngx_http_request_t *r) -{ - dd("finalize request"); - ngx_http_finalize_request(r, NGX_DONE); -} - - -static void * -ngx_http_tnt_eval_create_loc_conf(ngx_conf_t *cf) -{ - ngx_http_tnt_eval_loc_conf_t *conf; - - conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_tnt_eval_loc_conf_t)); - - if (conf == NULL) { - return NULL; - } - - conf->buffer_size = NGX_CONF_UNSET_SIZE; - - return conf; -} - - -static char * -ngx_http_tnt_eval_merge_loc_conf(ngx_conf_t *cf, - void *parent, void *child) -{ - ngx_http_tnt_eval_loc_conf_t *prev = parent; - ngx_http_tnt_eval_loc_conf_t *conf = child; - - ngx_conf_merge_size_value(conf->buffer_size, prev->buffer_size, - (size_t) ngx_pagesize * 16); - - return NGX_CONF_OK; -} - - -static ngx_int_t -ngx_http_tnt_eval_variable(ngx_http_request_t *r, - ngx_http_variable_value_t *v, uintptr_t data) -{ - (void) r; - (void) data; - - dd("XXX variable get_handle"); - - v->valid = 1; - v->no_cacheable = 0; - v->not_found = 0; - - v->len = 0; - v->data = (u_char*) ""; - - return NGX_OK; -} - - -static char * -ngx_http_tnt_eval_add_variables(ngx_conf_t *cf, - ngx_command_t *cmd, void *conf) -{ - ngx_http_tnt_eval_loc_conf_t *ecf = conf; - - ngx_uint_t i; - ngx_int_t index; - ngx_str_t *value; - ngx_http_variable_t *v; - ngx_http_tnt_eval_variable_t *variable; - - value = cf->args->elts; - - ecf->variables = ngx_array_create(cf->pool, - cf->args->nelts, sizeof(ngx_http_tnt_eval_variable_t)); - - if (ecf->variables == NULL) { - return NGX_CONF_ERROR; - } - - if (cf->args->nelts != 3) { - - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "invalid number of args, it should be tnt_eval " - "$http_stat $http_body { BLOCK }"); - return NGX_CONF_ERROR; - } - - /* Skip the first one, which is tnt_eval - */ - for (i = 1; i < cf->args->nelts; i++) { - - if (value[i].data[0] != '$') { - - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "invalid variable name \"%V\", it should be \"$VAR_NAME\"", - &value[i]); - return NGX_CONF_ERROR; - } - - variable = ngx_array_push(ecf->variables); - if (variable == NULL) { - return NGX_CONF_ERROR; - } - - value[i].len--; - value[i].data++; - - v = ngx_http_add_variable(cf, &value[i], NGX_HTTP_VAR_CHANGEABLE); - if (v == NULL) { - return NGX_CONF_ERROR; - } - - index = ngx_http_get_variable_index(cf, &value[i]); - if (index == NGX_ERROR) { - return NGX_CONF_ERROR; - } - - if (v->get_handler == NULL) { - v->get_handler = ngx_http_tnt_eval_variable; - v->data = index; - } - - variable->variable = v; - variable->index = index; - } - - return NGX_CONF_OK; -} - - -static char * -ngx_http_tnt_eval_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) -{ - ngx_http_tnt_eval_loc_conf_t *pecf = conf; - - char *rv; - void *mconf; - ngx_str_t name; - ngx_uint_t i; - ngx_conf_t save; - ngx_module_t **modules; - ngx_http_module_t *module; - ngx_http_conf_ctx_t *ctx, *pctx; - ngx_http_core_loc_conf_t *clcf, *rclcf; - ngx_http_core_srv_conf_t *cscf; - -#if defined(nginx_version) && nginx_version >= 8042 && nginx_version <= 8053 - return "does not work with " NGINX_VER; -#endif - - if (ngx_http_tnt_eval_add_variables(cf, cmd, conf) != NGX_CONF_OK) { - return NGX_CONF_ERROR; - } - - ctx = ngx_pcalloc(cf->pool, sizeof(ngx_http_conf_ctx_t)); - if (ctx == NULL) { - return NGX_CONF_ERROR; - } - - pctx = cf->ctx; - ctx->main_conf = pctx->main_conf; - ctx->srv_conf = pctx->srv_conf; - - ctx->loc_conf = ngx_pcalloc(cf->pool, sizeof(void *) * ngx_http_max_module); - if (ctx->loc_conf == NULL) { - return NGX_CONF_ERROR; - } - -#if defined(nginx_version) && nginx_version >= 1009011 - modules = cf->cycle->modules; -#else - modules = ngx_modules; -#endif - - for (i = 0; modules[i]; i++) { - - if (modules[i]->type != NGX_HTTP_MODULE) { - continue; - } - - module = modules[i]->ctx; - - if (module->create_loc_conf) { - - mconf = module->create_loc_conf(cf); - if (mconf == NULL) { - return NGX_CONF_ERROR; - } - - ctx->loc_conf[modules[i]->ctx_index] = mconf; - } - } - - clcf = ctx->loc_conf[ngx_http_core_module.ctx_index]; - - name.len = sizeof("/tnt_eval_") - 1 + NGX_OFF_T_LEN; - - name.data = ngx_palloc(cf->pool, name.len); - - if (name.data == NULL) { - return NGX_CONF_ERROR; - } - - name.len = ngx_sprintf(name.data, "/tnt_eval_%O", (off_t)(uintptr_t) clcf) - - name.data; - - clcf->loc_conf = ctx->loc_conf; - clcf->name = name; - clcf->exact_match = 0; - clcf->noname = 0; - clcf->internal = 1; - clcf->noregex = 1; - - cscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_core_module); - if (cscf == NULL || cscf->ctx == NULL) { - return NGX_CONF_ERROR; - } - - rclcf = cscf->ctx->loc_conf[ngx_http_core_module.ctx_index]; - if (rclcf == NULL) { - return NGX_CONF_ERROR; - } - - if (ngx_http_add_location(cf, &rclcf->locations, clcf) != NGX_OK) { - return NGX_CONF_ERROR; - } - - pecf->eval_location = clcf->name; - - save = *cf; - cf->ctx = ctx; - cf->cmd_type = NGX_HTTP_LOC_CONF; - - rv = ngx_conf_parse(cf, NULL); - - *cf = save; - - return rv; -} - - -static ngx_int_t -ngx_http_tnt_eval_init(ngx_conf_t *cf) -{ - ngx_http_handler_pt *h; - ngx_http_core_main_conf_t *cmcf; - - cmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_core_module); - - h = ngx_array_push(&cmcf->phases[NGX_HTTP_REWRITE_PHASE].handlers); - if (h == NULL) { - return NGX_ERROR; - } - - *h = ngx_http_tnt_eval_handler; - - ngx_http_next_header_filter = ngx_http_top_header_filter; - ngx_http_top_header_filter = ngx_http_tnt_eval_header_filter; - - ngx_http_next_body_filter = ngx_http_top_body_filter; - ngx_http_top_body_filter = ngx_http_tnt_eval_body_filter; - - return NGX_OK; -} - - -static ngx_int_t -ngx_http_tnt_eval_header_filter(ngx_http_request_t *r) -{ - ngx_http_tnt_eval_ctx_t *ctx; - - dd("Processing in a header filter"); - - /* If this module does not active, just goes to a next filter */ - ctx = ngx_http_get_module_ctx(r, ngx_http_tnt_eval_module); - if (ctx == NULL) { - return ngx_http_next_header_filter(r); - } - - /* Comeback to the main request */ - if (r == r->main) { - - dd("Comeback! Goes to a next header filter, r = %p, ctx = %p", - r, ctx); - return ngx_http_next_header_filter(r); - } - - r->filter_need_in_memory = 1; - - return NGX_OK; -} - - -static ngx_int_t -ngx_http_tnt_eval_body_filter(ngx_http_request_t *r, ngx_chain_t *in) -{ - ngx_http_tnt_eval_ctx_t *ctx; - ngx_chain_t *cl; - ngx_buf_t *b; - ngx_http_tnt_eval_loc_conf_t *conf; - size_t len; - ssize_t rest; - - if (r == r->main) { - return ngx_http_next_body_filter(r, in); - } - - ctx = ngx_http_get_module_ctx(r, ngx_http_tnt_eval_module); - if (ctx == NULL) { - return ngx_http_next_body_filter(r, in); - } - - dd("Processing in a body filter"); - - conf = ngx_http_get_module_loc_conf(r->parent, ngx_http_tnt_eval_module); - - b = &ctx->buffer; - - if (b->start == NULL) { - b->start = ngx_palloc(r->pool, conf->buffer_size); - if (b->start == NULL) { - return NGX_ERROR; - } - - b->end = b->start + conf->buffer_size; - b->pos = b->last = b->start; - } - - for (cl = in; cl; cl = cl->next) { - - rest = b->end - b->last; - if (rest == 0) { - break; - } - - if (!ngx_buf_in_memory(cl->buf)) { - dd("buf not in memory!"); - continue; - } - - len = cl->buf->last - cl->buf->pos; - - if (len == 0) { - continue; - } - - if (len > (size_t) rest) { - /* Truncating the exceeding part of the response body */ - ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "tnt_eval: need more bytes. aborted. " - "consider increasing your 'tnt_eval_buffer_size' " - "setting"); - len = rest; - } - - b->last = ngx_copy(b->last, cl->buf->pos, len); - } - - return ngx_http_tnt_eval_discard_bufs(r->pool, in); -} - - -static ngx_int_t -ngx_http_tnt_eval_discard_bufs(ngx_pool_t *pool, ngx_chain_t *in) -{ - ngx_chain_t *cl; - - for (cl = in; cl; cl = cl->next) { -#if 0 - if (cl->buf->temporary && cl->buf->memory - && ngx_buf_size(cl->buf) > 0) { - ngx_pfree(pool, cl->buf->start); - } -#endif - - cl->buf->pos = cl->buf->last; - } - - return NGX_OK; -} diff --git a/src/ngx_http_tnt_handlers.h b/src/ngx_http_tnt_handlers.h index 5d67851..2d5e47f 100644 --- a/src/ngx_http_tnt_handlers.h +++ b/src/ngx_http_tnt_handlers.h @@ -108,6 +108,11 @@ typedef struct { */ ngx_uint_t http_methods; + /** If it is set, then the client will recv. a pure result, e.g. {} + * otherwise {"result":[], "id": NUM}id + */ + ngx_uint_t pure_result; + ngx_array_t *headers_source; ngx_http_tnt_headers_t headers; @@ -220,7 +225,8 @@ ngx_http_tnt_overhead(void) "'code':-XXXXX," "'message':''" "}," - "[[]]" + "{ 'result': [[]]," + "'id': 1867996680 }" "}"); } diff --git a/src/ngx_http_tnt_module.c b/src/ngx_http_tnt_module.c index b8d4526..a9296dd 100644 --- a/src/ngx_http_tnt_module.c +++ b/src/ngx_http_tnt_module.c @@ -30,9 +30,10 @@ * please see AUTHORS file. */ +#include + #include -#include #include #include @@ -214,6 +215,13 @@ static ngx_command_t ngx_http_tnt_commands[] = { offsetof(ngx_http_tnt_loc_conf_t, http_rest_methods), &ngx_http_tnt_methods }, + { ngx_string("tnt_pure_result"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_bitmask_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_tnt_loc_conf_t, pure_result), + &ngx_http_tnt_pass_http_request_masks }, + { ngx_string("tnt_http_methods"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_1MORE, ngx_conf_set_bitmask_slot, @@ -454,6 +462,9 @@ ngx_http_tnt_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) (NGX_HTTP_POST |NGX_HTTP_DELETE)); + ngx_conf_merge_bitmask_value(conf->pure_result, prev->pure_result, + NGX_TNT_CONF_OFF); + if (conf->headers_source == NULL) { conf->headers = prev->headers; conf->headers_source = prev->headers_source; @@ -567,6 +578,8 @@ ngx_http_tnt_send_reply(ngx_http_request_t *r, return NGX_ERROR; } + tp_reply_to_json_set_options(&tc, tlcf->pure_result == NGX_TNT_CONF_ON); + rc = tp_transcode(&tc, (char *)ctx->tp_cache->start, ctx->tp_cache->end - ctx->tp_cache->start); if (rc != TP_TRANSCODE_ERROR) { diff --git a/src/ngx_http_tnt_version.h b/src/ngx_http_tnt_version.h index 759c11f..2ff39d2 100644 --- a/src/ngx_http_tnt_version.h +++ b/src/ngx_http_tnt_version.h @@ -33,6 +33,6 @@ #ifndef NGX_HTTP_TNT_VERSION_H #define NGX_HTTP_TNT_VERSION_H 1 -#define NGX_HTTP_TNT_MODULE_VERSION_STRING "v2.4.5-beta" +#define NGX_HTTP_TNT_MODULE_VERSION_STRING "v2.4.6-rc1" #endif diff --git a/src/tp_transcode.c b/src/tp_transcode.c index 57ad9b8..f3f32be 100644 --- a/src/tp_transcode.c +++ b/src/tp_transcode.c @@ -847,6 +847,7 @@ query2tp_complete(void *ctx, size_t *cmpl_msg_size) */ typedef struct tp2json { + struct tpresponse r; char *output; @@ -858,6 +859,12 @@ typedef struct tp2json { tp_transcode_t *tc; int state; + + /* Encode tarantool message w/o protocol.message + * i.e. with protocol: {id:, result:TNT_RESULT, ...}, w/o TNT_RESULT + */ + bool pure_result; + } tp2json_t; static inline int @@ -887,6 +894,10 @@ tp2json_create(tp_transcode_t *tc, char *output, size_t output_size) memset(ctx, 0, sizeof(tp2json_t)); + /* By memset + ctx->pure_result = false; + */ + ctx->pos = ctx->output = output; ctx->end = output + output_size; ctx->tc = tc; @@ -1096,6 +1107,12 @@ tp_reply2json_transcode(void *ctx_, const char *in, size_t in_size) } else { + if (!ctx->pure_result) { + ctx->pos += snprintf(ctx->output, ctx->end - ctx->output, + "{\"id\":%zu,\"result\":", (size_t) tp_getreqid(&ctx->r)); + } + + const char *it = ctx->r.data; rc = tp2json_transcode_internal(ctx, &it, ctx->r.data_end); if (unlikely(rc == TP_TRANSCODE_ERROR)) @@ -1103,7 +1120,11 @@ tp_reply2json_transcode(void *ctx_, const char *in, size_t in_size) } - if (ctx->r.error) { + if (!ctx->pure_result || + /* NOTE https://github.com/tarantool/nginx_upstream_module/issues/44 + */ + ctx->r.error) + { *ctx->pos = '}'; ++ctx->pos; } @@ -1299,6 +1320,15 @@ tp_transcode_bind_data(tp_transcode_t *t, t->data.len = data_end - data_beg; } +void +tp_reply_to_json_set_options(tp_transcode_t *t, bool pure_result) +{ + assert(t); + assert(t->codec.ctx); + tp2json_t *ctx = t->codec.ctx; + ctx->pure_result = pure_result; +} + bool tp_dump(char *output, size_t output_size, const char *input, size_t input_size) diff --git a/src/tp_transcode.h b/src/tp_transcode.h index d8413be..504f337 100644 --- a/src/tp_transcode.h +++ b/src/tp_transcode.h @@ -181,12 +181,10 @@ void tp_transcode_bind_data(tp_transcode_t *t, /** */ void -tp_reply_to_json_set_options(tp_transcode_t *t, - bool pure_result, - int multireturn_skip_count); +tp_reply_to_json_set_options(tp_transcode_t *t, bool pure_result); /** - * WARNING! tp_dump() - is for debug + * WARNING! tp_dump() is for debug! * * Dump Tarantool message in JSON * Returns true, false diff --git a/test/basic_features.py b/test/basic_features.py index c554a20..9bef6df 100755 --- a/test/basic_features.py +++ b/test/basic_features.py @@ -57,7 +57,8 @@ def get_id_i(o, i): return o[i]['id'] def get_result(o): - return o[0] + assert('result' in o), "expected 'result'" + return o['result'][0] # def batch_cases(): @@ -79,8 +80,8 @@ def batch_cases(): assert(rc == 200), 'expected 200' assert(len(res) == 2), 'expected 2 elements, got %i' % len(res) - assert(res[0][0][0][1] == '101234567891234567') - assert(res[1][0][0][1] == '101234567891234567') + assert(res[0]['result'][0][0][1] == '101234567891234567') + assert(res[1]['result'][0][0][1] == '101234567891234567') ## batch = [] @@ -97,12 +98,12 @@ def batch_cases(): assert(len(res) == len(batch)),\ 'expected %i elements, got %i' % (len(batch), len(res)) for i in range(0, len(res)): - rr = res[i][0] - #id = get_id_i(res, i) + rr = res[i]['result'][0] + id = get_id_i(res, i) assert(rr[0] == batch[i]['params'][0]),\ "expected %s, got %s" % (batch[i]['params'][0], rr[0]) - #assert(id == batch[i]['id']),\ - # "expected id %s, got %s" % (batch[i]['id'], id) + assert(id == batch[i]['id']),\ + "expected id %s, got %s" % (batch[i]['id'], id) ## (rc, res) = request([ @@ -125,21 +126,18 @@ def batch_cases(): ]) assert(rc == 200), 'expected 200' - ## This test is broken, since we don't have ID, should it be fixed? [[[ - if False: - for rr in res: - if rr['id'] == 3: - assert('error' in rr or 'message' in rr), \ - 'expected %s returns error/message, got %s' % (rr['id'], rr) - elif rr['id'] == 2: - rr_ = get_result(rr) - assert(rr_[1] == '101234567891234567') - elif rr['id'] == 1: - rr_ = get_result(rr) - assert(rr_ == [{"first":1}, {"second":2}]) - else: - assert False, "unexpected id %s" % rr['id'] - # ]]] + for rr in res: + if rr['id'] == 3: + assert('error' in rr or 'message' in rr), \ + 'expected %s returns error/message, got %s' % (rr['id'], rr) + elif rr['id'] == 2: + rr_ = get_result(rr)[0] + assert(rr_[1] == '101234567891234567') + elif rr['id'] == 1: + rr_ = get_result(rr) + assert(rr_ == [{"first":1}, {"second":2}]) + else: + assert False, "unexpected id %s" % rr['id'] (rc, res) = request_raw('[{"method":"call", "params":["name", __wrong__], ' + '"id":1}, {"method":"call", "params":["name"], "id":2}]'); @@ -173,7 +171,7 @@ def batch_cases(): 'params': bigarray, 'id': 1 }) - assert(res[0][0] == 1), 'expected 1' + assert(res['result'][0][0] == 1), 'expected 1' print ('[+] Big array OK') # diff --git a/test/http_utils.py b/test/http_utils.py index 068dd09..53e63e7 100644 --- a/test/http_utils.py +++ b/test/http_utils.py @@ -182,7 +182,7 @@ def get(url, data, headers): return (False, e) def get_result(o): - return o[0] + return o['result'][0] def assert_if_not_error(s, code = None): assert('error' in s), 'expected error' diff --git a/test/lua.py b/test/lua.py index ab2374b..0d9f06b 100755 --- a/test/lua.py +++ b/test/lua.py @@ -78,53 +78,11 @@ def do_get(url, code, headers): prev_result = None for method in methods: - for code in http_codes: - - curl = BASE_URL + '/eval_basic?status_code=' + str(code) - + curl = BASE_URL + '/lua?status_code=' + str(code) (rcode, result) = method[0](curl, code, {'X-From': 'eval_basic'}) - + # Python does not work if server returns some codes! if rcode == True: - continue - - if VERBOSE: - print ('===============') - print ('req = ', curl, method[1]) - print ('rcore = ', rcode) - print ('expected code', code) - print ('curr = ', result) - if prev_result: - print ('prev = ', prev_result[1]) - - if method[1] != 'GET': - assert(rcode == code), 'expected ' + str(code) - - if prev_result: - # Here is we don't test those fields [[[ - prev_result[1]['args']['status_code'] = str(code) - prev_result[1]['uri'] = result[1]['uri'] - prev_result[1]['method'] = result[1]['method'] - prev_result[1]['headers'] = None - result[1]['headers'] = None - # ]]] - - if prev_result[1] != result[1]: - print ('==== NOT EQUAL ====') - print (prev_result[1]) - print (result[1]) - - assert(prev_result[1] == result[1]) - assert(prev_result[2] == result[2]) - - prev_result = result - -print('[+] OK') - - -# ============= -# -print('[+] Test headers') -loc = BASE_URL + '/eval_headers' -result = post_success(loc, {'params': []}, {'in_h': 1}) + continue; + assert(code == rcode) print('[+] OK') diff --git a/test/ngx_confs/tnt_server_test.conf b/test/ngx_confs/tnt_server_test.conf index 87d0205..bde1925 100644 --- a/test/ngx_confs/tnt_server_test.conf +++ b/test/ngx_confs/tnt_server_test.conf @@ -201,112 +201,61 @@ http { tnt_pass tnt; } - location /eval_basic { - - tnt_eval_buffer_size 1m; - - tnt_eval $tnt_status $tnt_res { - tnt_buffer_size 1m; - tnt_out_multiplier 10; - tnt_pass_http_request on parse_args; - tnt_http_rest_methods all; - tnt_method test_eval; - tnt_pass tnt; - } - - add_header 'X-t' $tnt_status; - - if ($tnt_status = 201){ - return 201 $tnt_res; - } - if ($tnt_status = 202){ - return 202 $tnt_res; - } - if ($tnt_status = 204){ - return 204 $tnt_res; - } - if ($tnt_status = 206){ - return 206 $tnt_res; - } - if ($tnt_status = 300){ - return 300 $tnt_res; - } - if ($tnt_status = 301){ - return 301 $tnt_res; - } - if ($tnt_status = 302){ - return 302 $tnt_res; - } - if ($tnt_status = 303){ - return 303 $tnt_res; - } - if ($tnt_status = 304){ - return 304 $tnt_res; - } - if ($tnt_status = 307){ - return 307 $tnt_res; - } - if ($tnt_status = 400){ - return 400 $tnt_res; - } - if ($tnt_status = 401){ - return 401 $tnt_res; - } - if ($tnt_status = 403){ - return 403 $tnt_res; - } - if ($tnt_status = 404){ - return 404 $tnt_res; - } - if ($tnt_status = 405){ - return 405 $tnt_res; - } - if ($tnt_status = 408){ - return 408 $tnt_res; - } - if ($tnt_status = 409){ - return 409 $tnt_res; - } - if ($tnt_status = 411){ - return 411 $tnt_res; - } - if ($tnt_status = 412){ - return 412 $tnt_res; - } - if ($tnt_status = 413){ - return 413 $tnt_res; - } - if ($tnt_status = 414){ - return 414 $tnt_res; - } - if ($tnt_status = 415){ - return 415 $tnt_res; - } - if ($tnt_status = 416){ - return 416 $tnt_res; - } - if ($tnt_status = 421){ - return 421 $tnt_res; - } - if ($tnt_status = 500){ - return 500 $tnt_res; - } - if ($tnt_status = 501){ - return 501 $tnt_res; - } - if ($tnt_status = 502){ - return 502 $tnt_res; - } - if ($tnt_status = 503){ - return 503 $tnt_res; - } - if ($tnt_status = 504){ - return 504 $tnt_res; - } - if ($tnt_status = 507){ - return 507 $tnt_res; - } - return 200 $tnt_res; + location /tnt_proxy { + tnt_method tnt_proxy; + tnt_buffer_size 1m; + tnt_out_multiplier 10; + tnt_pass_http_request on parse_args; + tnt_pass tnt; + } + + location /lua { + + lua_need_request_body on; + + rewrite_by_lua ' + + local cjson = require("cjson") + + local map = { + GET = ngx.HTTP_GET, + POST = ngx.HTTP_POST, + PUT = ngx.HTTP_PUT, + -- ... + } + + local res = ngx.location.capture("/tnt_proxy", { + args = ngx.var.args, + method = map[ngx.var.request_method], + body = ngx.body + }) + + if res.status == ngx.HTTP_OK then + local answ = cjson.decode(res.body) + + -- Read reply + local result = answ["result"] + + if result ~= nil then + ngx.status = result[1]["ngx"][1] + for k, v in pairs(result[1]["ngx"][2]) do + ngx.header[k] = v + end + + table.remove(result, 1) + ngx.say(cjson.encode(result)) + else + ngx.status = 502 + ngx.say(res.body) + end + + -- Finalize execution + ngx.exit(ngx.OK) + else + ngx.status = 502 + ngx.say("Tarantool does not work") + end + '; } location /echo_big { @@ -317,19 +266,5 @@ http { tnt_http_methods all; tnt_pass tnt; } - - location /eval_headers { - tnt_eval_buffer_size 2m; - tnt_eval $tnt_status $tnt_res { - tnt_buffer_size 2m; - tnt_pass_http_request on parse_args; - tnt_http_rest_methods all; - tnt_method test_eval_headers; - tnt_pass tnt; - } - add_header 'X-t' $tnt_status; - return 200 $tnt_res; - } - } } diff --git a/test/parallel_clients.sh b/test/parallel_clients.sh index a79459b..3924b6c 100755 --- a/test/parallel_clients.sh +++ b/test/parallel_clients.sh @@ -5,7 +5,7 @@ for i in {1..10}; do ./test/v20_features.py & ./test/v23_features.py & ./test/v24_features.py & - ./test/eval_basic.py & + ./test/lua.py & done for i in `jobs -p`; do diff --git a/test/run_all.sh b/test/run_all.sh index 5c10c80..028a1fd 100755 --- a/test/run_all.sh +++ b/test/run_all.sh @@ -22,8 +22,8 @@ for i in {1..10}; do $WORK_DIR/v24_features.py 1> /dev/null || ( echo "[-] $WORK_DIR/v24_features.py failed" && exit 1 ) - $WORK_DIR/eval_basic.py 1> /dev/null || ( - echo "[-] $WORK_DIR/eval_basic.py failed" && exit 1 + $WORK_DIR/lua.py 1> /dev/null || ( + echo "[-] $WORK_DIR/lua.py failed" && exit 1 ) done @@ -45,8 +45,8 @@ for i in {1..3}; do echo "[-] $WORK_DIR/v24_features.py failed" && exit 1 )` & clients_pids="$clients_pids $!" - `$WORK_DIR/eval_basic.py 1> /dev/null || ( - echo "[-] $WORK_DIR/eval_basic.py failed" && exit 1 + `$WORK_DIR/lua.py 1> /dev/null || ( + echo "[-] $WORK_DIR/lua.py failed" && exit 1 )` & clients_pids="$clients_pids $!" done diff --git a/test/test.lua b/test/test.lua index d2bf4e9..891b2b7 100755 --- a/test/test.lua +++ b/test/test.lua @@ -21,7 +21,8 @@ function rest_api_get(a, b) end function echo_big(...) - return ... + local a = {...} + return a end function ret_4096() @@ -102,8 +103,8 @@ function insert(request, a1, a2) return { request, a1, a2 } end -function update(request) - return request +function update(request, ...) + return request, ... end -- ]] @@ -141,39 +142,22 @@ function test_headers_out(req) return true end -function test_eval(req, ...) +function tnt_proxy(req, ...) + local out = {...} for i = 0, 18012 do out[i] = i; end + return { - __ngx = { + ngx = { tonumber(req.args.status_code) or 200, { ["X-Tarantool"] = "FROM_TNT" } } }, - req, - out -end - -function test_eval_headers(req, ...) - - local headers = {} - - for i = 1, 10 do - headers['H' .. tostring(i)] = tostring(i) - end - - return - { - __ngx = { - 200, - headers - } - }, - req, - ... + req +-- out end -- CFG diff --git a/test/v20_features.py b/test/v20_features.py index 484a714..aeea1fd 100755 --- a/test/v20_features.py +++ b/test/v20_features.py @@ -85,22 +85,20 @@ print ('[+] Test "large request"') -BASE_URL = "http://0.0.0.0:8081/issue_59" err_msg = { 'error': { 'message': "Request too large, consider increasing your " + "server's setting 'client_body_buffer_size'", 'code': -32001 } } -preset_method_location = BASE_URL + '/rest_api_parse_query_args' +preset_method_location = BASE_URL + '/issue_59/rest_api_parse_query_args' obj = {} -for i in range(1, 30000): +for i in range(1, 40000): obj[str(i) + 'some_key_name'] = [ i, { 'n': i, 'some_key_name': [[1,2,3],[4]]}] for i in range(1, 10): code, result = post(preset_method_location, { 'params': [obj] }, {}) - assert(code == 400), 'expected 400' - assert(result == err_msg), 'expected error msg (too large)' + assert(code == 500), 'expected 500' expected = obj[str(i) + 'some_key_name'] result = post_success(preset_method_location, { 'params': expected }, {}) diff --git a/test/v23_features.py b/test/v23_features.py index 1afb12c..7f4edab 100755 --- a/test/v23_features.py +++ b/test/v23_features.py @@ -84,7 +84,7 @@ rc, resp = request_raw(preset_method_location, data, {}) params = json.loads(data)['params'] assert(rc == 200), 'expected 200, got ' + str(rc) -assert(params == resp[0]), 'not equal' +assert(params == resp['result'][0]), 'not equal' # =========== # @@ -144,9 +144,9 @@ put_success(preset_method_location, {'id':1}, None) delete_success(preset_method_location, {'params':[]}, None) delete_success(preset_method_location, None, None) - (rc, result) = request(preset_method_location, [{'id': 1}, {'id': 2}]) -assert(result[0] == result[1]) +assert(result[0]['id'] == 1) +assert(result[1]['id'] == 2) data = {"id":0,"params":[ { diff --git a/test/v24_features.py b/test/v24_features.py index 25070c4..4f11e82 100755 --- a/test/v24_features.py +++ b/test/v24_features.py @@ -84,4 +84,3 @@ data['params'][0]['array'].append(i) (code, ret) = post(BASE_URL + '/echo_big', data, None) assert(code == 200), 'expected 200' -assert(ret[1] == data['params'][0])