diff --git a/apisix/discovery/nacos/init.lua b/apisix/discovery/nacos/init.lua index 86e9d4125a83..ebb2484b2bb3 100644 --- a/apisix/discovery/nacos/init.lua +++ b/apisix/discovery/nacos/init.lua @@ -33,14 +33,12 @@ local str_byte = string.byte local str_find = core.string.find local log = core.log -local default_weight -local applications +local applications = {} local auth_path = 'auth/login' local instance_list_path = 'ns/instance/list?healthyOnly=true&serviceName=' local default_namespace_id = "public" local default_group_name = "DEFAULT_GROUP" -local access_key -local secret_key +local default_nacos_name = "default" local events local events_list @@ -49,7 +47,7 @@ local events_list local _M = {} local function discovery_nacos_callback(data, event, source, pid) - applications = data + applications[data.name] = data.data log.notice("update local variable application, event is: ", event, "source: ", source, "server pid:", pid, ", application: ", core.json.encode(applications, true)) @@ -151,14 +149,14 @@ local function get_group_name_param(group_name) end -local function get_signed_param(group_name, service_name) +local function get_signed_param(group_name, service_name, nacos) local param = '' - if access_key ~= '' and secret_key ~= '' then + if nacos.access_key ~= '' and nacos.secret_key ~= '' then local str_to_sign = ngx.now() * 1000 .. '@@' .. group_name .. '@@' .. service_name local args = { - ak = access_key, + ak = nacos.access_key, data = str_to_sign, - signature = ngx.encode_base64(ngx.hmac_sha1(secret_key, str_to_sign)) + signature = ngx.encode_base64(ngx.hmac_sha1(nacos.secret_key, str_to_sign)) } param = '&' .. ngx.encode_args(args) end @@ -166,8 +164,8 @@ local function get_signed_param(group_name, service_name) end -local function get_base_uri() - local host = local_conf.discovery.nacos.host +local function get_base_uri(nacos) + local host = nacos.host -- TODO Add health check to get healthy nodes. local url = host[math_random(#host)] local auth_idx = core.string.rfind_char(url, '@') @@ -185,8 +183,9 @@ local function get_base_uri() url = protocol .. other end - if local_conf.discovery.nacos.prefix then - url = url .. local_conf.discovery.nacos.prefix + local prefix = nacos.prefix + if prefix then + url = url .. prefix end if str_byte(url, #url) ~= str_byte('/') then @@ -208,7 +207,7 @@ local function de_duplication(services, namespace_id, group_name, service_name, end -local function iter_and_add_service(services, values) +local function iter_and_add_service(services, values, nacos) if not values then return end @@ -225,6 +224,12 @@ local function iter_and_add_service(services, values) else up = conf end + local nacos_name_from_args = (up.discovery_args and up.discovery_args.name) + or default_nacos_name + local nacos_name = nacos.name or default_nacos_name + if nacos_name ~= nacos_name_form_args then + goto CONTINUE + end local namespace_id = (up.discovery_args and up.discovery_args.namespace_id) or default_namespace_id @@ -251,7 +256,7 @@ local function iter_and_add_service(services, values) end -local function get_nacos_services() +local function get_nacos_services(nacos) local services = {} -- here we use lazy load to work around circle dependency @@ -260,13 +265,13 @@ local function get_nacos_services() local get_stream_routes = require('apisix.router').stream_routes local get_services = require('apisix.http.service').services local values = get_upstreams() - iter_and_add_service(services, values) + iter_and_add_service(services, values, nacos) values = get_routes() - iter_and_add_service(services, values) + iter_and_add_service(services, values, nacos) values = get_services() - iter_and_add_service(services, values) + iter_and_add_service(services, values, nacos) values = get_stream_routes() - iter_and_add_service(services, values) + iter_and_add_service(services, values, nacos) return services end @@ -279,25 +284,26 @@ local function is_grpc(scheme) end -local function fetch_full_registry(premature) - if premature then - return +local function fetch_from_naocs(nacos) + local nacos_name = default_nacos_name + if nacos.name then + nacos_name = nacos.name end - local up_apps = {} - local base_uri, username, password = get_base_uri() + local base_uri, username, password = get_base_uri(nacos) local token_param, err = get_token_param(base_uri, username, password) if err then log.error('get_token_param error:', err) - if not applications then - applications = up_apps + if not applications[nacos_name] then + applications[nacos_name] = up_apps end return end - local infos = get_nacos_services() + local infos = get_nacos_services(nacos) + if #infos == 0 then - applications = up_apps + applications[nacos_name] = up_apps return end @@ -308,10 +314,11 @@ local function fetch_full_registry(premature) local scheme = service_info.scheme or '' local namespace_param = get_namespace_param(service_info.namespace_id) local group_name_param = get_group_name_param(service_info.group_name) - local signature_param = get_signed_param(service_info.group_name, service_info.service_name) + local signature_param = get_signed_param(service_info.group_name, + service_info.service_name, nacos) local query_path = instance_list_path .. service_info.service_name - .. token_param .. namespace_param .. group_name_param - .. signature_param + .. token_param .. namespace_param .. group_name_param + .. signature_param data, err = get_url(base_uri, query_path) if err then log.error('get_url:', query_path, ' err:', err) @@ -328,17 +335,17 @@ local function fetch_full_registry(premature) for _, host in ipairs(data.hosts) do local nodes = up_apps[namespace_id] - [group_name][service_info.service_name] + [group_name][service_info.service_name] if not nodes then nodes = {} up_apps[namespace_id] - [group_name][service_info.service_name] = nodes + [group_name][service_info.service_name] = nodes end local node = { host = host.ip, port = host.port, - weight = host.weight or default_weight, + weight = host.weight or nacos.weight, } -- docs: https://github.com/yidongnan/grpc-spring-boot-starter/pull/496 @@ -352,16 +359,35 @@ local function fetch_full_registry(premature) ::CONTINUE:: end local new_apps_md5sum = ngx.md5(core.json.encode(up_apps)) - local old_apps_md5sum = ngx.md5(core.json.encode(applications)) + local old_apps_md5sum = ngx.md5(core.json.encode(applications[nacos_name])) if new_apps_md5sum == old_apps_md5sum then return end - applications = up_apps + applications[nacos_name] = up_apps + local arg = { + name = nacos_name, + data = up_apps + } local ok, err = events:post(events_list._source, events_list.updating, - applications) + arg) if not ok then log.error("post_event failure with ", events_list._source, - ", update application error: ", err) + ", update application error: ", err) + end + + +end + +local function fetch_full_registry(premature) + if premature then + return + end + fetch_from_naocs(local_conf.discovery.nacos) + local others_nacos = local_conf.discovery.nacos.hosts + if others_nacos and #others_nacos > 0 then + for _, nacos in ipairs(others_nacos) do + fetch_from_naocs(nacos) + end end end @@ -371,12 +397,14 @@ function _M.nodes(service_name, discovery_args) discovery_args.namespace_id or default_namespace_id local group_name = discovery_args and discovery_args.group_name or default_group_name + local nacos_name = discovery_args + and discovery_args.name or default_nacos_name local logged = false -- maximum waiting time: 5 seconds local waiting_time = 5 local step = 0.1 - while not applications and waiting_time > 0 do + while not applications[nacos_name] and waiting_time > 0 do if not logged then log.warn('wait init') logged = true @@ -385,12 +413,13 @@ function _M.nodes(service_name, discovery_args) waiting_time = waiting_time - step end - if not applications or not applications[namespace_id] - or not applications[namespace_id][group_name] + if not applications or not applications[nacos_name] + or not applications[nacos_name][namespace_id] + or not applications[nacos_name][namespace_id][group_name] then return nil end - return applications[namespace_id][group_name][service_name] + return applications[nacos_name][namespace_id][group_name][service_name] end @@ -405,12 +434,17 @@ function _M.init_worker() return end - default_weight = local_conf.discovery.nacos.weight - log.info('default_weight:', default_weight) + --default_weight = local_conf.discovery.nacos.weight + --log.info('default_weight:', default_weight) + if local_conf.discovery.nacos.host + then + log.warn("config \"naocs.host\" will be deprecated soon.") + end + local fetch_interval = local_conf.discovery.nacos.fetch_interval log.info('fetch_interval:', fetch_interval) - access_key = local_conf.discovery.nacos.access_key - secret_key = local_conf.discovery.nacos.secret_key + --access_key = local_conf.discovery.nacos.access_key + --secret_key = local_conf.discovery.nacos.secret_key ngx_timer_at(0, fetch_full_registry) ngx_timer_every(fetch_interval, fetch_full_registry) end diff --git a/apisix/discovery/nacos/schema.lua b/apisix/discovery/nacos/schema.lua index 294048736e58..ed8cb8c27218 100644 --- a/apisix/discovery/nacos/schema.lua +++ b/apisix/discovery/nacos/schema.lua @@ -54,6 +54,49 @@ return { }, access_key = {type = 'string', default = ''}, secret_key = {type = 'string', default = ''}, + hosts = { + type = 'array', + items = { + type = 'object', + properties = { + name = {type ="string"}, + host = { + type = 'array', + minItems = 1, + items = { + type = 'string', + pattern = host_pattern, + minLength = 2, + maxLength = 100, + }, + }, + + prefix = { + type = 'string', + pattern = prefix_pattern, + maxLength = 100, + default = '/nacos/v1/' + }, + weight = {type = 'integer', minimum = 1, default = 100}, + timeout = { + type = 'object', + properties = { + connect = {type = 'integer', minimum = 1, default = 2000}, + send = {type = 'integer', minimum = 1, default = 2000}, + read = {type = 'integer', minimum = 1, default = 5000}, + }, + default = { + connect = 2000, + send = 2000, + read = 5000, + } + }, + access_key = {type = 'string', default = ''}, + secret_key = {type = 'string', default = ''}, + } + + } + } }, required = {'host'} } diff --git a/ci/pod/docker-compose.first.yml b/ci/pod/docker-compose.first.yml index d203a967ddfc..fda008c6d8fb 100644 --- a/ci/pod/docker-compose.first.yml +++ b/ci/pod/docker-compose.first.yml @@ -109,6 +109,7 @@ services: - consul.cluster ## Nacos cluster + nacos_auth: hostname: nacos1 image: nacos/nacos-server:1.4.1 @@ -133,6 +134,18 @@ services: networks: nacos_net: + + nacos_auth_standalone: + hostname: nacos3 + image: nacos/nacos-server:1.4.1 + env_file: + - ci/pod/nacos/env/standalone.env + restart: unless-stopped + ports: + - "8868:8848" + networks: + nacos_net: + nacos_server_health_check: build: context: ci/pod/nacos/healthcheck @@ -152,6 +165,25 @@ services: networks: nacos_net: + nacos_standalone_server_health_check: + build: + context: ci/pod/nacos/healthcheck + dockerfile: Dockerfile + environment: + CHECK_URI: "http://nacos3:8848/nacos/v1/ns/service/list?pageNo=1&pageSize=2" + tty: true + # debug healthcheck script + # volumes: + # - ./ci/pod/nacos/healthcheck/nacos-server-healthcheck.sh:/nacos-server-healthcheck.sh + healthcheck: + test: [ "CMD", "bash", "/nacos-server-healthcheck.sh" ] + interval: 5s + timeout: 5s + retries: 60 + start_period: 10s + networks: + nacos_net: + nacos_service_health_check: build: context: ci/pod/nacos/healthcheck @@ -296,6 +328,24 @@ services: condition: service_healthy networks: nacos_net: + nacos-service8: + build: + context: ci/pod/nacos/service + dockerfile: Dockerfile + env_file: + - ci/pod/nacos/env/service_standalone.env + environment: + SUFFIX_NUM: 4 + GROUP: test_group + NAMESPACE: test_ns + restart: unless-stopped + ports: + - "18008:18001" + depends_on: + nacos_standalone_server_health_check: + condition: service_healthy + networks: + nacos_net: networks: diff --git a/ci/pod/nacos/env/service_standalone.env b/ci/pod/nacos/env/service_standalone.env new file mode 100644 index 000000000000..ac6f7a865846 --- /dev/null +++ b/ci/pod/nacos/env/service_standalone.env @@ -0,0 +1,2 @@ +SERVICE_NAME=APISIX-NACOS +NACOS_ADDR=nacos3:8848 diff --git a/ci/pod/nacos/env/standalone.env b/ci/pod/nacos/env/standalone.env new file mode 100644 index 000000000000..34738d58ae2f --- /dev/null +++ b/ci/pod/nacos/env/standalone.env @@ -0,0 +1,5 @@ +EMBEDDED_STORAGE=embedded +PREFER_HOST_MODE=hostname +MODE=standalone +JVM_XMS=512m +JVM_XMX=512m diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 953fb1f098a3..61e01eb26e3d 100755 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -296,20 +296,38 @@ nginx_config: # Config for render the template to generate n # connect: 2000 # Default 2000ms # send: 2000 # Default 2000ms # read: 5000 # Default 5000ms -# nacos: # Nacos -# host: # Nacos address(es) + +# nacos: +# name: "default" # Deprecated,see nacos.hosts.name +# host: # Deprecated,see nacos.hosts.host # - "http://${username}:${password}@${host1}:${port1}" -# prefix: "/nacos/v1/" -# fetch_interval: 30 # Default 30s -# `weight` is the `default_weight` that will be attached to each discovered node that -# doesn't have a weight explicitly provided in nacos results -# weight: 100 # Default 100. -# timeout: -# connect: 2000 # Default 2000ms -# send: 2000 # Default 2000ms -# read: 5000 # Default 5000ms -# access_key: "" # Nacos AccessKey ID in Alibaba Cloud, notice that it's for Nacos instances on Microservices Engine (MSE) -# secret_key: "" # Nacos AccessKey Secret in Alibaba Cloud, notice that it's for Nacos instances on Microservices Engine (MSE) +# prefix: "/nacos/v1/" # Deprecated,see nacos.hosts.prefix. +# fetch_interval: 30 # default 30 sec.all nacos in config will use this config +# +# weight: 100 # Deprecated see nacos.hosts.weight +# timeout: # Deprecated see nacos.hosts.timeout +# connect: 2000 # Deprecated see nacos.hosts.timeout +# send: 2000 # Deprecated see nacos.hosts.timeout +# read: 5000 # Deprecated see nacos.hosts.timeout +# access_key: "" # Deprecated see nacos.hosts.access_key +# secret_key: "" # Deprecated see nacos.hosts.secret_key +# hosts: +# - name: "your_nacos_cluster_name" #your nacos cluster name +# host: +# - "http://${username}:${password}@${host1}:${port1}" +# prefix: "/nacos/v1/" +# # `weight` is the `default_weight` that will be attached to each discovered node that +# # doesn't have a weight explicitly provided in nacos results +# weight: 100 # default 100 +# timeout: +# connect: 2000 # default 2000 ms +# send: 2000 # default 2000 ms +# read: 5000 # default 5000 ms +# access_key: "" # Nacos AccessKey ID in Alibaba Cloud, notice that it's for Nacos instances on +# # Microservices Engine (MSE) +# secret_key: "" # Nacos AccessKey Secret in Alibaba Cloud, notice that it's for Nacos instances on +# # Microservices Engine (MSE) + # consul_kv: # Consul KV # servers: # Consul KV address(es) # - "http://127.0.0.1:8500" diff --git a/docs/en/latest/discovery/nacos.md b/docs/en/latest/discovery/nacos.md index 5ebbcee46b49..1cc8d54cef27 100644 --- a/docs/en/latest/discovery/nacos.md +++ b/docs/en/latest/discovery/nacos.md @@ -34,17 +34,35 @@ Add following configuration in `conf/config.yaml` : ```yaml discovery: nacos: - host: + name: "default" # Deprecated,see nacos.hosts.name + host: # Deprecated,see nacos.hosts.host - "http://${username}:${password}@${host1}:${port1}" - prefix: "/nacos/v1/" - fetch_interval: 30 # default 30 sec - # `weight` is the `default_weight` that will be attached to each discovered node that - # doesn't have a weight explicitly provided in nacos results - weight: 100 # default 100 - timeout: - connect: 2000 # default 2000 ms - send: 2000 # default 2000 ms - read: 5000 # default 5000 ms + prefix: "/nacos/v1/" # Deprecated,see nacos.hosts.prefix. + fetch_interval: 30 # default 30 sec.all nacos in config will use this config + + weight: 100 # Deprecated see nacos.hosts.weight + timeout: # Deprecated see nacos.hosts.timeout + connect: 2000 # Deprecated see nacos.hosts.timeout + send: 2000 # Deprecated see nacos.hosts.timeout + read: 5000 # Deprecated see nacos.hosts.timeout + access_key: "" # Deprecated see nacos.hosts.access_key + secret_key: "" # Deprecated see nacos.hosts.secret_key + hosts: + - name: "your_nacos_cluster_name" #your nacos cluster name + host: + - "http://${username}:${password}@${host1}:${port1}" + prefix: "/nacos/v1/" + # `weight` is the `default_weight` that will be attached to each discovered node that + # doesn't have a weight explicitly provided in nacos results + weight: 100 # default 100 + timeout: + connect: 2000 # default 2000 ms + send: 2000 # default 2000 ms + read: 5000 # default 5000 ms + access_key: "" # Nacos AccessKey ID in Alibaba Cloud, notice that it's for Nacos instances on + # Microservices Engine (MSE) + secret_key: "" # Nacos AccessKey Secret in Alibaba Cloud, notice that it's for Nacos instances on + # Microservices Engine (MSE) ``` And you can config it in short by default value: @@ -52,8 +70,9 @@ And you can config it in short by default value: ```yaml discovery: nacos: - host: - - "http://192.168.33.1:8848" + hosts: + - host: + - "http://192.168.33.1:8848" ``` ### Upstream setting @@ -79,10 +98,14 @@ $ curl http://127.0.0.1:9180/apisix/admin/routes/1 -H "X-API-KEY: $admin_key" -X "service_name": "APISIX-NACOS", "type": "roundrobin", "discovery_type": "nacos" + "discovery_args":{ + "name": "your_naocos_cluster_name" + } } }' ``` + The formatted response as below: ```json @@ -128,10 +151,11 @@ $ curl http://127.0.0.1:9180/apisix/admin/stream_routes/1 -H "X-API-KEY: $admin_ ### discovery_args -| Name | Type | Requirement | Default | Valid | Description | -| ------------ | ------ | ----------- | ------- | ----- | ------------------------------------------------------------ | -| namespace_id | string | optional | public | | This parameter is used to specify the namespace of the corresponding service | -| group_name | string | optional | DEFAULT_GROUP | | This parameter is used to specify the group of the corresponding service | +| Name | Type | Requirement | Default | Valid | Description | +|--------------| ------ | ----------- |---------------| ----- |----------------------------------------------------------------------------| +| name | string | optional | default | | This parameter is used to specify nacos cluster in nacos.hosts config | +| namespace_id | string | optional | public | | This parameter is used to specify the namespace of the corresponding service | +| group_name | string | optional | DEFAULT_GROUP | | This parameter is used to specify the group of the corresponding service | #### Specify the namespace diff --git a/t/discovery/nacos.t b/t/discovery/nacos.t index 9af1ee14a814..31a3b503b802 100644 --- a/t/discovery/nacos.t +++ b/t/discovery/nacos.t @@ -32,15 +32,26 @@ deployment: config_provider: yaml discovery: nacos: - host: - - "http://127.0.0.1:8858" - prefix: "/nacos/v1/" - fetch_interval: 1 - weight: 1 - timeout: - connect: 2000 - send: 2000 - read: 5000 + host: + - "http://127.0.0.1:8858" + prefix: "/nacos/v1/" + fetch_interval: 1 + weight: 1 + timeout: + connect: 2000 + send: 2000 + read: 5000 + hosts: + - name: nacos3 + host: + - "http://127.0.0.1:8868" + prefix: "/nacos/v1/" + fetch_interval: 1 + weight: 1 + timeout: + connect: 2000 + send: 2000 + read: 5000 _EOC_ @@ -53,16 +64,15 @@ deployment: config_provider: yaml discovery: nacos: - host: - - "http://nacos:nacos\@127.0.0.1:8848" - prefix: "/nacos/v1/" - fetch_interval: 1 - weight: 1 - timeout: - connect: 2000 - send: 2000 - read: 5000 - + host: + - "http://nacos:nacos\@127.0.0.1:8848" + prefix: "/nacos/v1/" + fetch_interval: 1 + weight: 1 + timeout: + connect: 2000 + send: 2000 + read: 5000 _EOC_ run_tests(); @@ -927,3 +937,28 @@ GET /t --- response_body server 1 server 4 + + + +=== TEST 27: get APISIX-NACOS info from NACOS by nacos name +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: APISIX-NACOS + discovery_type: nacos + discovery_args: + name: nacos3 + namespace_id: test_ns + group_name: test_group + + type: roundrobin +#END +--- request +GET /hello +--- response_body chomp +server 4 +--- no_error_log +[error] diff --git a/t/discovery/nacos2.t b/t/discovery/nacos2.t index e7dc8f97318a..63bac6d21d81 100644 --- a/t/discovery/nacos2.t +++ b/t/discovery/nacos2.t @@ -308,7 +308,7 @@ discovery: local body = json_decode(res.body) local services = body.services - local service = services["public"]["DEFAULT_GROUP"]["APISIX-NACOS"] + local service = services["default"]["public"]["DEFAULT_GROUP"]["APISIX-NACOS"] local number = table.getn(service) ngx.say(number) }