Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: support for Service Discovery Across Multiple Nacos Clusters #10950

Closed
wants to merge 30 commits into from
Closed

feat: support for Service Discovery Across Multiple Nacos Clusters #10950

Show file tree
Hide file tree
Changes from 29 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
8c630ea
Support for Service Discovery Across Multiple Nacos Clusters
ShenFeng312 Feb 21, 2024
a375384
fix test
ShenFeng312 Feb 22, 2024
de48972
fix test
ShenFeng312 Feb 22, 2024
b67062c
fix codestyle
ShenFeng312 Feb 23, 2024
b54d165
add test
ShenFeng312 Apr 16, 2024
1e9ce85
Merge branch 'apache:master' into master
ShenFeng312 Apr 16, 2024
69e5752
fix test
ShenFeng312 Apr 16, 2024
b8062ea
Merge branch 'master' of github.com:ShenFeng312/apisix
ShenFeng312 Apr 16, 2024
ff5f2c5
fix test
ShenFeng312 Apr 16, 2024
f824ed1
add polish test
ShenFeng312 Apr 16, 2024
9ca5361
polish test
ShenFeng312 Apr 16, 2024
f07e9c4
fix test
ShenFeng312 Apr 16, 2024
5acd79a
fix test
ShenFeng312 Apr 17, 2024
3a1ebd1
fix test
ShenFeng312 Apr 17, 2024
a05ff3b
Merge branch 'apache:master' into master
ShenFeng312 Apr 17, 2024
ca1e89a
fix test
ShenFeng312 Apr 19, 2024
ee0b383
Merge branch 'master' of github.com:ShenFeng312/apisix
ShenFeng312 Apr 19, 2024
d7e4991
fix test
ShenFeng312 Apr 19, 2024
a1f5807
fix test
ShenFeng312 Apr 19, 2024
ed80b4c
fix ci
ShenFeng312 Apr 22, 2024
c278e53
Merge branch 'apache:master' into master
ShenFeng312 Apr 22, 2024
daa8239
Update apisix/discovery/nacos/init.lua
ShenFeng312 Apr 22, 2024
7b80d7c
Update apisix/discovery/nacos/init.lua
ShenFeng312 Apr 22, 2024
d9595f4
Update apisix/discovery/nacos/init.lua
ShenFeng312 Apr 22, 2024
a8f6c50
Update apisix/discovery/nacos/init.lua
ShenFeng312 Apr 22, 2024
886e0b9
Update init.lua
ShenFeng312 Apr 22, 2024
b317869
fix code style
ShenFeng312 Apr 29, 2024
accfd55
rename others to hosts
ShenFeng312 Apr 29, 2024
6d2dcc2
update docs and add deprecated warn log
ShenFeng312 Apr 30, 2024
ea3b35c
Update apisix/discovery/nacos/init.lua
ShenFeng312 May 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 80 additions & 46 deletions apisix/discovery/nacos/init.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,12 @@ local str_byte = string.byte
local str_find = core.string.find
local log = core.log

local default_weight
local applications
local applications = {}
local auth_path = 'auth/login'
local instance_list_path = 'ns/instance/list?healthyOnly=true&serviceName='
local default_namespace_id = "public"
local default_group_name = "DEFAULT_GROUP"
local access_key
local secret_key
local default_nacos_name = "default"

local events
local events_list
Expand All @@ -49,7 +47,7 @@ local events_list
local _M = {}

local function discovery_nacos_callback(data, event, source, pid)
applications = data
applications[data.name] = data.data
log.notice("update local variable application, event is: ", event,
"source: ", source, "server pid:", pid,
", application: ", core.json.encode(applications, true))
Expand Down Expand Up @@ -151,23 +149,23 @@ local function get_group_name_param(group_name)
end


local function get_signed_param(group_name, service_name)
local function get_signed_param(group_name, service_name, nacos)
local param = ''
if access_key ~= '' and secret_key ~= '' then
if nacos.access_key ~= '' and nacos.secret_key ~= '' then
local str_to_sign = ngx.now() * 1000 .. '@@' .. group_name .. '@@' .. service_name
local args = {
ak = access_key,
ak = nacos.access_key,
data = str_to_sign,
signature = ngx.encode_base64(ngx.hmac_sha1(secret_key, str_to_sign))
signature = ngx.encode_base64(ngx.hmac_sha1(nacos.secret_key, str_to_sign))
}
param = '&' .. ngx.encode_args(args)
end
return param
end


local function get_base_uri()
local host = local_conf.discovery.nacos.host
local function get_base_uri(nacos)
local host = nacos.host
-- TODO Add health check to get healthy nodes.
local url = host[math_random(#host)]
local auth_idx = core.string.rfind_char(url, '@')
Expand All @@ -185,8 +183,9 @@ local function get_base_uri()
url = protocol .. other
end

if local_conf.discovery.nacos.prefix then
url = url .. local_conf.discovery.nacos.prefix
local prefix = nacos.prefix
if prefix then
url = url .. prefix
end

if str_byte(url, #url) ~= str_byte('/') then
Expand All @@ -208,7 +207,7 @@ local function de_duplication(services, namespace_id, group_name, service_name,
end


local function iter_and_add_service(services, values)
local function iter_and_add_service(services, values, nacos)
if not values then
return
end
Expand All @@ -225,6 +224,12 @@ local function iter_and_add_service(services, values)
else
up = conf
end
local nacos_name_form_args = (up.discovery_args and up.discovery_args.name)
ShenFeng312 marked this conversation as resolved.
Show resolved Hide resolved
or default_nacos_name
local nacos_name = nacos.name or default_nacos_name
if nacos_name ~= nacos_name_form_args then
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if nacos_name ~= nacos_name_form_args then
if nacos_name ~= nacos_name_from_args then

A typo here.

goto CONTINUE
end

local namespace_id = (up.discovery_args and up.discovery_args.namespace_id)
or default_namespace_id
Expand All @@ -251,7 +256,7 @@ local function iter_and_add_service(services, values)
end


local function get_nacos_services()
local function get_nacos_services(nacos)
local services = {}

-- here we use lazy load to work around circle dependency
Expand All @@ -260,13 +265,13 @@ local function get_nacos_services()
local get_stream_routes = require('apisix.router').stream_routes
local get_services = require('apisix.http.service').services
local values = get_upstreams()
iter_and_add_service(services, values)
iter_and_add_service(services, values, nacos)
values = get_routes()
iter_and_add_service(services, values)
iter_and_add_service(services, values, nacos)
values = get_services()
iter_and_add_service(services, values)
iter_and_add_service(services, values, nacos)
values = get_stream_routes()
iter_and_add_service(services, values)
iter_and_add_service(services, values, nacos)
return services
end

Expand All @@ -279,25 +284,26 @@ local function is_grpc(scheme)
end


local function fetch_full_registry(premature)
if premature then
return
local function fetch_from_naocs(nacos)
local nacos_name = default_nacos_name
if nacos.name then
nacos_name = nacos.name
end

local up_apps = {}
local base_uri, username, password = get_base_uri()
local base_uri, username, password = get_base_uri(nacos)
local token_param, err = get_token_param(base_uri, username, password)
if err then
log.error('get_token_param error:', err)
if not applications then
applications = up_apps
if not applications[nacos_name] then
applications[nacos_name] = up_apps
end
return
end

local infos = get_nacos_services()
local infos = get_nacos_services(nacos)

if #infos == 0 then
applications = up_apps
applications[nacos_name] = up_apps
return
end

Expand All @@ -308,10 +314,11 @@ local function fetch_full_registry(premature)
local scheme = service_info.scheme or ''
local namespace_param = get_namespace_param(service_info.namespace_id)
local group_name_param = get_group_name_param(service_info.group_name)
local signature_param = get_signed_param(service_info.group_name, service_info.service_name)
local signature_param = get_signed_param(service_info.group_name,
service_info.service_name, nacos)
Comment on lines +317 to +318
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
local signature_param = get_signed_param(service_info.group_name,
service_info.service_name, nacos)
local signature_param = get_signed_param(service_info.group_name,
service_info.service_name, nacos)

local query_path = instance_list_path .. service_info.service_name
.. token_param .. namespace_param .. group_name_param
.. signature_param
.. token_param .. namespace_param .. group_name_param
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there are many other whitespace changes like this, please revert them

.. signature_param
data, err = get_url(base_uri, query_path)
if err then
log.error('get_url:', query_path, ' err:', err)
Expand All @@ -328,17 +335,17 @@ local function fetch_full_registry(premature)

for _, host in ipairs(data.hosts) do
local nodes = up_apps[namespace_id]
[group_name][service_info.service_name]
[group_name][service_info.service_name]
if not nodes then
nodes = {}
up_apps[namespace_id]
[group_name][service_info.service_name] = nodes
[group_name][service_info.service_name] = nodes
end

local node = {
host = host.ip,
port = host.port,
weight = host.weight or default_weight,
weight = host.weight or nacos.weight,
}

-- docs: https://github.com/yidongnan/grpc-spring-boot-starter/pull/496
Expand All @@ -352,16 +359,35 @@ local function fetch_full_registry(premature)
::CONTINUE::
end
local new_apps_md5sum = ngx.md5(core.json.encode(up_apps))
local old_apps_md5sum = ngx.md5(core.json.encode(applications))
local old_apps_md5sum = ngx.md5(core.json.encode(applications[nacos_name]))
if new_apps_md5sum == old_apps_md5sum then
return
end
applications = up_apps
applications[nacos_name] = up_apps
local arg = {
name = nacos_name,
data = up_apps
}
local ok, err = events:post(events_list._source, events_list.updating,
applications)
arg)
if not ok then
log.error("post_event failure with ", events_list._source,
", update application error: ", err)
", update application error: ", err)
end


end

local function fetch_full_registry(premature)
if premature then
return
end
fetch_from_naocs(local_conf.discovery.nacos)
local others_nacos = local_conf.discovery.nacos.hosts
if others_nacos and #others_nacos > 0 then
for _, nacos in ipairs(others_nacos) do
fetch_from_naocs(nacos)
end
end
Comment on lines +385 to 391
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The old mode will be deprecated later right?
Should we treat them as mutually exclusive now (i.e. disable single registry mode when multiple registries mode are enabled), I think it's clearer for the user rather than mixing them up and suddenly not working anymore when we remove it.

end

Expand All @@ -371,12 +397,14 @@ function _M.nodes(service_name, discovery_args)
discovery_args.namespace_id or default_namespace_id
local group_name = discovery_args
and discovery_args.group_name or default_group_name
local nacos_name = discovery_args
and discovery_args.name or default_nacos_name

local logged = false
-- maximum waiting time: 5 seconds
local waiting_time = 5
local step = 0.1
while not applications and waiting_time > 0 do
while not applications[nacos_name] and waiting_time > 0 do
if not logged then
log.warn('wait init')
logged = true
Expand All @@ -385,12 +413,13 @@ function _M.nodes(service_name, discovery_args)
waiting_time = waiting_time - step
end

if not applications or not applications[namespace_id]
or not applications[namespace_id][group_name]
if not applications or not applications[nacos_name]
or not applications[nacos_name][namespace_id]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
or not applications[nacos_name][namespace_id]
or not applications[nacos_name][namespace_id]

or not applications[nacos_name][namespace_id][group_name]
then
return nil
end
return applications[namespace_id][group_name][service_name]
return applications[nacos_name][namespace_id][group_name][service_name]
end


Expand All @@ -405,12 +434,17 @@ function _M.init_worker()
return
end

default_weight = local_conf.discovery.nacos.weight
log.info('default_weight:', default_weight)
--default_weight = local_conf.discovery.nacos.weight
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why change these?

Copy link
Contributor Author

@ShenFeng312 ShenFeng312 May 10, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because default_weight in different nacos may different @shreemaan-abhishek

--log.info('default_weight:', default_weight)
Comment on lines +437 to +438
Copy link
Contributor

@bzp2010 bzp2010 Jul 13, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should it be deleted? Delete them if they are no longer used.

if local_conf.discovery.nacos.host
then
log.warn("config \"naocs.host\" will be deprecated soon.")
end

local fetch_interval = local_conf.discovery.nacos.fetch_interval
log.info('fetch_interval:', fetch_interval)
access_key = local_conf.discovery.nacos.access_key
secret_key = local_conf.discovery.nacos.secret_key
--access_key = local_conf.discovery.nacos.access_key
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto

Copy link
Contributor Author

@ShenFeng312 ShenFeng312 May 10, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because access_key and secret_key in different nacos may different @shreemaan-abhishek

--secret_key = local_conf.discovery.nacos.secret_key
Comment on lines +446 to +447
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Delete them if they are no longer used too.

ngx_timer_at(0, fetch_full_registry)
ngx_timer_every(fetch_interval, fetch_full_registry)
end
Expand Down
43 changes: 43 additions & 0 deletions apisix/discovery/nacos/schema.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,49 @@ return {
},
access_key = {type = 'string', default = ''},
secret_key = {type = 'string', default = ''},
hosts = {
Copy link
Contributor

@bzp2010 bzp2010 Jul 13, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe like https://github.com/apache/apisix/blob/master/apisix/discovery/kubernetes/schema.lua#L114-L216 would be a better way? Users have the option to continue using the single registry mode, or switch to the multi-registry mode altogether.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we change the configuration file to the above format, the redundant nesting called hosts is unnecessary, i.e. the configuration should be:

discovery:
  nacos:
    host: xxx

Or,

discovery:
  nacos:
    - id: default
      host: xxx
    - id: test
      host: xxx

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

great!

type = 'array',
items = {
type = 'object',
properties = {
name = {type ="string"},
host = {
type = 'array',
minItems = 1,
items = {
type = 'string',
pattern = host_pattern,
minLength = 2,
maxLength = 100,
},
},

prefix = {
type = 'string',
pattern = prefix_pattern,
maxLength = 100,
default = '/nacos/v1/'
},
weight = {type = 'integer', minimum = 1, default = 100},
timeout = {
type = 'object',
properties = {
connect = {type = 'integer', minimum = 1, default = 2000},
send = {type = 'integer', minimum = 1, default = 2000},
read = {type = 'integer', minimum = 1, default = 5000},
},
default = {
connect = 2000,
send = 2000,
read = 5000,
}
},
access_key = {type = 'string', default = ''},
secret_key = {type = 'string', default = ''},
}

}
}
},
required = {'host'}
}
Loading
Loading