diff --git a/lib/uplink/caddy/config/hosts.ex b/lib/uplink/caddy/config/hosts.ex new file mode 100644 index 0000000..ee20193 --- /dev/null +++ b/lib/uplink/caddy/config/hosts.ex @@ -0,0 +1,19 @@ +defmodule Uplink.Caddy.Config.Hosts do + alias Uplink.Packages.Metadata + alias Uplink.Packages.Metadata.Port + + def routable?(%{metadata: %Metadata{main_port: nil}}), do: false + + def routable?(%{ + metadata: %Metadata{ + main_port: %{routing: %Port.Routing{hosts: hosts}} + } + }) + when length(hosts) > 0, + do: true + + def routable?(%{metadata: %Metadata{hosts: hosts}}) when length(hosts) > 0, + do: true + + def routable?(%{metadata: %Metadata{hosts: []}}), do: false +end diff --git a/lib/uplink/caddy/config/port.ex b/lib/uplink/caddy/config/port.ex new file mode 100644 index 0000000..41661c7 --- /dev/null +++ b/lib/uplink/caddy/config/port.ex @@ -0,0 +1,79 @@ +defmodule Uplink.Caddy.Config.Port do + alias Uplink.Packages.Metadata + + alias Uplink.Caddy.Config.Upstreams + + def build(%Metadata{ports: ports} = metadata, install_id) do + ports + |> Enum.map(&build(&1, metadata, install_id)) + |> Enum.reject(&is_nil/1) + end + + def build(%Metadata.Port{} = port, metadata, install_id) do + hosts = Enum.map(metadata.hosts, &merge_slug_and_host(&1, port)) + + routing = Map.get(port, :routing) + + routing_hosts = + if routing do + Enum.map(routing.hosts, &merge_slug_and_host(&1, port)) + else + [] + end + + hosts = + hosts + |> Enum.concat(routing_hosts) + |> Enum.uniq() + |> Enum.sort() + + paths = + if routing && routing.paths != [] do + routing.paths + else + ["*"] + end + + group = + if routing, + do: "router_#{routing.router_id}", + else: "installation_#{metadata.id}" + + if hosts == [] do + nil + else + %{ + group: group, + match: [ + %{ + host: hosts, + path: paths + } + ], + handle: [ + %{ + handler: "reverse_proxy", + load_balancing: %{ + selection_policy: %{ + policy: "least_conn" + } + }, + health_checks: %{ + passive: %{ + fail_duration: "10s", + max_fails: 3, + unhealthy_request_count: 80, + unhealthy_status: [500, 501, 502, 503, 504], + unhealthy_latency: "30s" + } + }, + upstreams: Upstreams.build(metadata, port, install_id) + } + ] + } + end + end + + defp merge_slug_and_host(host, %Metadata.Port{slug: slug}), + do: slug <> "." <> host +end diff --git a/lib/uplink/caddy/config/upstreams.ex b/lib/uplink/caddy/config/upstreams.ex new file mode 100644 index 0000000..29b912a --- /dev/null +++ b/lib/uplink/caddy/config/upstreams.ex @@ -0,0 +1,29 @@ +defmodule Uplink.Caddy.Config.Upstreams do + alias Uplink.Cache + alias Uplink.Packages.Metadata + alias Uplink.Packages.Metadata.Port + + def build(%Metadata{instances: instances}, %Port{} = port, install_id) do + instances + |> filter_valid(install_id) + |> Enum.map(fn instance -> + %{ + dial: "#{instance.slug}:#{port.target}", + max_requests: 100 + } + end) + end + + def filter_valid(instances, install_id) do + completed_instances = Cache.get({:install, install_id, "completed"}) + + if is_list(completed_instances) and Enum.count(completed_instances) > 0 do + instances + |> Enum.filter(fn instance -> + instance.slug in completed_instances + end) + else + instances + end + end +end diff --git a/lib/uplink/clients/caddy/config/builder.ex b/lib/uplink/clients/caddy/config/builder.ex index e1cf75e..714ac0d 100644 --- a/lib/uplink/clients/caddy/config/builder.ex +++ b/lib/uplink/clients/caddy/config/builder.ex @@ -1,10 +1,13 @@ defmodule Uplink.Clients.Caddy.Config.Builder do alias Uplink.Repo - alias Uplink.Cache alias Uplink.Packages alias Uplink.Routings + alias Uplink.Caddy.Config.Hosts + alias Uplink.Caddy.Config.Port + alias Uplink.Caddy.Config.Upstreams + alias Uplink.Clients.Caddy alias Uplink.Clients.Caddy.Admin @@ -17,9 +20,7 @@ defmodule Uplink.Clients.Caddy.Config.Builder do |> Repo.all() |> Repo.preload(deployment: [:app]) |> Enum.map(&Packages.build_install_state/1) - |> Enum.reject(fn %{metadata: metadata} -> - metadata.hosts == [] || is_nil(metadata.main_port) - end) + |> Enum.filter(&Hosts.routable?/1) %{"organization" => %{"storage" => storage_params}} = uplink = Uplink.Clients.Instellar.get_self() @@ -129,8 +130,15 @@ defmodule Uplink.Clients.Caddy.Config.Builder do ) do main_routing = Map.get(metadata.main_port, :routing) - main_paths = + main_routing_hosts = if main_routing do + main_routing.hosts + else + [] + end + + main_paths = + if main_routing && main_routing.paths != [] do main_routing.paths else ["*"] @@ -150,8 +158,6 @@ defmodule Uplink.Clients.Caddy.Config.Builder do [] end - valid_instances = find_valid_instances(metadata.instances, install_id) - proxy_routes = proxies |> Enum.map(fn proxy -> @@ -187,11 +193,17 @@ defmodule Uplink.Clients.Caddy.Config.Builder do } end) + main_hosts = + metadata.hosts + |> Enum.concat(main_routing_hosts) + |> Enum.uniq() + |> Enum.sort() + main_route = %{ group: main_group, match: [ %{ - host: metadata.hosts, + host: main_hosts, path: main_paths } ], @@ -212,75 +224,12 @@ defmodule Uplink.Clients.Caddy.Config.Builder do unhealthy_latency: "30s" } }, - upstreams: - Enum.map(valid_instances, fn instance -> - %{ - dial: "#{instance.slug}:#{metadata.main_port.target}", - max_requests: 100 - } - end) + upstreams: Upstreams.build(metadata, metadata.main_port, install_id) } ] } - sub_routes = - metadata.ports - |> Enum.map(fn port -> - hosts = - Enum.map(metadata.hosts, fn host -> - port.slug <> "." <> host - end) - - routing = Map.get(port, :routing) - - paths = - if routing do - routing.paths - else - ["*"] - end - - group = - if routing, - do: "router_#{routing.router_id}", - else: "installation_#{metadata.id}" - - %{ - group: group, - match: [ - %{ - host: hosts, - path: paths - } - ], - handle: [ - %{ - handler: "reverse_proxy", - load_balancing: %{ - selection_policy: %{ - policy: "least_conn" - } - }, - health_checks: %{ - passive: %{ - fail_duration: "10s", - max_fails: 3, - unhealthy_request_count: 80, - unhealthy_status: [500, 501, 502, 503, 504], - unhealthy_latency: "30s" - } - }, - upstreams: - Enum.map(valid_instances, fn instance -> - %{ - dial: "#{instance.slug}:#{port.target}", - max_requests: 100 - } - end) - } - ] - } - end) + sub_routes = Port.build(metadata, install_id) sub_routes_and_proxies = Enum.concat(sub_routes, proxy_routes) @@ -300,19 +249,6 @@ defmodule Uplink.Clients.Caddy.Config.Builder do ] end - defp find_valid_instances(instances, install_id) do - completed_instances = Cache.get({:install, install_id, "completed"}) - - if is_list(completed_instances) and Enum.count(completed_instances) > 0 do - instances - |> Enum.filter(fn instance -> - instance.slug in completed_instances - end) - else - instances - end - end - defp maybe_merge_tls(params, %{tls: true}) do Map.put(params, :transport, %{ protocol: "http", diff --git a/lib/uplink/packages/metadata/port.ex b/lib/uplink/packages/metadata/port.ex index 02743ae..b0ba68b 100644 --- a/lib/uplink/packages/metadata/port.ex +++ b/lib/uplink/packages/metadata/port.ex @@ -10,6 +10,7 @@ defmodule Uplink.Packages.Metadata.Port do embeds_one :routing, Routing, primary_key: false do field :router_id, :integer + field :hosts, {:array, :string}, default: [] field :paths, {:array, :string}, default: ["*"] end end @@ -23,7 +24,7 @@ defmodule Uplink.Packages.Metadata.Port do defp routing_changeset(routing, params) do routing - |> cast(params, [:router_id, :paths]) + |> cast(params, [:router_id, :hosts, :paths]) |> validate_required([:router_id, :paths]) end end diff --git a/mix.exs b/mix.exs index f769a63..3b7116b 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Uplink.MixProject do def project do [ app: :uplink, - version: "0.17.0", + version: "0.18.0", elixir: "~> 1.13", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, diff --git a/test/scenarios/deployment.ex b/test/scenarios/deployment.ex index 0cded55..ff17097 100644 --- a/test/scenarios/deployment.ex +++ b/test/scenarios/deployment.ex @@ -24,6 +24,7 @@ defmodule Uplink.Scenarios.Deployment do "target" => 4000, "routing" => %{ "router_id" => 1, + "hosts" => ["another.com", "something.com"], "paths" => ["/configure*"] } }, @@ -31,7 +32,12 @@ defmodule Uplink.Scenarios.Deployment do %{ "slug" => "grpc", "source" => 49153, - "target" => 6000 + "target" => 6000, + "routing" => %{ + "router_id" => 1, + "hosts" => ["another.com", "something.com"], + "paths" => ["/*"] + } } ], "hosts" => ["something.com"], diff --git a/test/uplink/clients/caddy/config/builder_test.exs b/test/uplink/clients/caddy/config/builder_test.exs index 25b3546..9707fe2 100644 --- a/test/uplink/clients/caddy/config/builder_test.exs +++ b/test/uplink/clients/caddy/config/builder_test.exs @@ -1,6 +1,13 @@ defmodule Uplink.Clients.Caddy.Config.BuilderTest do use ExUnit.Case + alias Uplink.Members + alias Uplink.Packages + alias Uplink.Secret + alias Uplink.Cache + + alias Uplink.Packages.Metadata + import Uplink.Scenarios.Deployment setup [:setup_endpoints, :setup_base] @@ -66,36 +73,337 @@ defmodule Uplink.Clients.Caddy.Config.BuilderTest do "provider" => %{"api_token" => "something", "name" => "cloudflare"} } = dns - assert %{routes: [first_route, second_route, third_route]} = server + assert %{routes: routes} = server + + routes = Enum.sort(routes) + + [first_route, second_route, third_route] = routes assert %{handle: [handle], match: [match]} = first_route assert %{handle: [second_handle], match: [second_match]} = second_route assert %{handle: [third_handle], match: [third_match]} = third_route + assert match.host == ["another.com", "something.com"] assert match.path == ["/configure*"] - assert third_match.path == ["*"] + assert second_match.path == ["/*"] - assert second_match.path == ["/how-to*"] + assert third_match.path == ["/how-to*"] - assert "grpc.something.com" in third_match.host + assert "grpc.something.com" in second_match.host + assert "grpc.another.com" in second_match.host - [third_upstream] = third_handle.upstreams + [second_upstream] = second_handle.upstreams - assert third_upstream.dial =~ "6000" + assert second_upstream.dial =~ "6000" assert %{handler: "reverse_proxy"} = handle assert %{host: _hosts} = match - [second_upstream] = second_handle.upstreams + [third_upstream] = third_handle.upstreams - assert %{protocol: "http", tls: %{}} = second_handle.transport + assert %{protocol: "http", tls: %{}} = third_handle.transport - assert second_upstream.dial == "proxy.webflow.com:80" + assert third_upstream.dial == "proxy.webflow.com:80" assert %{identity: identity} = admin assert %{identifiers: ["127.0.0.1"]} = identity assert %{module: "s3"} = storage end + + describe "when routing is nil" do + setup do + deployment_params = %{ + "hash" => "a-different-hash", + "archive_url" => "http://localhost/archives/packages.zip", + "stack" => "alpine/3.14", + "channel" => "develop", + "metadata" => %{ + "id" => 1, + "slug" => "uplink-web", + "main_port" => %{ + "slug" => "web", + "source" => 49152, + "target" => 4000 + }, + "ports" => [ + %{ + "slug" => "grpc", + "source" => 49153, + "target" => 6000 + } + ], + "hosts" => ["something.com"], + "variables" => [ + %{"key" => "SOMETHING", "value" => "blah"} + ], + "channel" => %{ + "slug" => "develop", + "package" => %{ + "slug" => "something-1640927800", + "credential" => %{ + "public_key" => "public_key" + }, + "organization" => %{ + "slug" => "upmaru" + } + } + }, + "instances" => [ + %{ + "id" => 1, + "slug" => "something-1", + "node" => %{ + "slug" => "some-node" + } + } + ] + } + } + + {:ok, actor} = + Members.get_or_create_actor(%{ + "identifier" => "zacksiri", + "provider" => "instellar", + "id" => "1" + }) + + metadata = Map.get(deployment_params, "metadata") + + {:ok, metadata} = Packages.parse_metadata(metadata) + + app = + metadata + |> Metadata.app_slug() + |> Packages.get_or_create_app() + + {:ok, deployment} = + Packages.get_or_create_deployment(app, deployment_params) + + {:ok, %{resource: preparing_deployment}} = + Packages.transition_deployment_with(deployment, actor, "prepare") + + {:ok, %{resource: deployment}} = + Packages.transition_deployment_with( + preparing_deployment, + actor, + "complete" + ) + + {:ok, install} = + Packages.create_install(deployment, %{ + "installation_id" => 1, + "deployment" => deployment_params + }) + + signature = Secret.Signature.compute_signature(deployment.hash) + + Cache.put( + {:deployment, signature, install.instellar_installation_id}, + metadata + ) + + {:ok, %{resource: validating_install}} = + Packages.transition_install_with(install, actor, "validate") + + {:ok, %{resource: _executing_install}} = + Packages.transition_install_with(validating_install, actor, "execute") + + :ok + end + + test "render port when routing is nil correctly" do + assert %{apps: apps} = Uplink.Clients.Caddy.build_new_config() + + assert %{http: %{servers: %{"uplink" => server}}} = apps + + assert %{routes: routes} = server + + routes = Enum.sort(routes) + + [first_route, second_route] = routes + + assert %{handle: [handle], match: [match]} = first_route + assert %{handle: [second_handle], match: [second_match]} = second_route + + assert match.host == ["something.com"] + assert match.path == ["*"] + + assert second_match.path == ["*"] + + assert "grpc.something.com" in second_match.host + + [second_upstream] = second_handle.upstreams + + assert second_upstream.dial =~ "6000" + + assert %{handler: "reverse_proxy"} = handle + assert %{host: _hosts} = match + end + end + + describe "when metadata.hosts is empty" do + setup do + deployment_params = %{ + "hash" => "a-different-hash-234", + "archive_url" => "http://localhost/archives/packages.zip", + "stack" => "alpine/3.14", + "channel" => "develop", + "metadata" => %{ + "id" => 1, + "slug" => "uplink-web", + "main_port" => %{ + "slug" => "web", + "source" => 49152, + "target" => 4000, + "routing" => %{ + "router_id" => 1, + "hosts" => ["another.com"], + "paths" => ["*"] + } + }, + "ports" => [ + %{ + "slug" => "grpc", + "source" => 49153, + "target" => 6000 + } + ], + "hosts" => [], + "variables" => [ + %{"key" => "SOMETHING", "value" => "blah"} + ], + "channel" => %{ + "slug" => "develop", + "package" => %{ + "slug" => "something-1640927800", + "credential" => %{ + "public_key" => "public_key" + }, + "organization" => %{ + "slug" => "upmaru" + } + } + }, + "instances" => [ + %{ + "id" => 1, + "slug" => "something-1", + "node" => %{ + "slug" => "some-node" + } + } + ] + } + } + + {:ok, actor} = + Members.get_or_create_actor(%{ + "identifier" => "zacksiri", + "provider" => "instellar", + "id" => "1" + }) + + metadata = Map.get(deployment_params, "metadata") + + {:ok, metadata} = Packages.parse_metadata(metadata) + + app = + metadata + |> Metadata.app_slug() + |> Packages.get_or_create_app() + + {:ok, deployment} = + Packages.get_or_create_deployment(app, deployment_params) + + {:ok, %{resource: preparing_deployment}} = + Packages.transition_deployment_with(deployment, actor, "prepare") + + {:ok, %{resource: deployment}} = + Packages.transition_deployment_with( + preparing_deployment, + actor, + "complete" + ) + + {:ok, install} = + Packages.create_install(deployment, %{ + "installation_id" => 1, + "deployment" => deployment_params + }) + + signature = Secret.Signature.compute_signature(deployment.hash) + + Cache.put( + {:deployment, signature, install.instellar_installation_id}, + metadata + ) + + {:ok, %{resource: validating_install}} = + Packages.transition_install_with(install, actor, "validate") + + {:ok, %{resource: _executing_install}} = + Packages.transition_install_with(validating_install, actor, "execute") + + :ok + end + + test "render port when routing correctly", %{bypass: bypass} do + Uplink.Cache.delete({:proxies, 1}) + + Bypass.expect_once( + bypass, + "GET", + "/uplink/self/routers/1/proxies", + fn conn -> + conn + |> Plug.Conn.put_resp_header("content-type", "application/json") + |> Plug.Conn.send_resp( + 200, + Jason.encode!(%{ + "data" => [ + %{ + "attributes" => %{ + "id" => 1, + "router_id" => 1, + "hosts" => ["opsmaru.com", "www.opsmaru.com"], + "paths" => ["/how-to*"], + "tls" => true, + "target" => "proxy.webflow.com", + "port" => 80 + } + } + ] + }) + ) + end + ) + + assert %{apps: apps} = Uplink.Clients.Caddy.build_new_config() + + assert %{http: %{servers: %{"uplink" => server}}} = apps + + assert %{routes: routes} = server + + routes = Enum.sort(routes) + + [first_route, second_route] = routes + + assert %{handle: [handle], match: [match]} = first_route + assert %{handle: [second_handle], match: [second_match]} = second_route + + assert match.host == ["another.com"] + assert match.path == ["*"] + + assert second_match.path == ["/how-to*"] + + [second_upstream] = second_handle.upstreams + + assert second_upstream.dial =~ "80" + + assert %{handler: "reverse_proxy"} = handle + assert %{host: _hosts} = match + end + end end diff --git a/test/uplink/metrics/pipeline_test.exs b/test/uplink/metrics/pipeline_test.exs index 1976443..b36be1d 100644 --- a/test/uplink/metrics/pipeline_test.exs +++ b/test/uplink/metrics/pipeline_test.exs @@ -26,7 +26,7 @@ defmodule Uplink.Metrics.PipelineTest do } do ref = Broadway.test_message(Uplink.Metrics.Pipeline, message) - assert_receive {:ack, ^ref, [%{data: data}], []} + assert_receive {:ack, ^ref, [%{data: data}], []}, 10_000 assert %{ memory: memory, @@ -65,7 +65,7 @@ defmodule Uplink.Metrics.PipelineTest do } do ref = Broadway.test_message(Uplink.Metrics.Pipeline, message) - assert_receive {:ack, ^ref, [%{data: data}], []} + assert_receive {:ack, ^ref, [%{data: data}], []}, 10_000 assert %{network: network} = data