Merge branch 'release/0.17.0'

.github/workflows/ci.yml

@@ -1,17 +1,17 @@
-name: 'CI'
+name: "CI"
 
 on:
   push:
     branches:
       - master
       - develop
-      - 'feature/**'
-      - 'hotfix/**'
+      - "feature/**"
+      - "hotfix/**"
 
 jobs:
   deps:
     name: Dependencies
-    runs-on: warp-ubuntu-latest-x64-2x
+    runs-on: warp-ubuntu-2204-x64-2x
     container:
       image: alpine:3.18
 
@@ -55,7 +55,7 @@ jobs:
   static_code_analysis:
     name: Static Code Analysis
     needs: deps
-    runs-on: warp-ubuntu-latest-x64-2x
+    runs-on: warp-ubuntu-2204-x64-2x
     container:
       image: alpine:3.18
 
@@ -96,7 +96,7 @@ jobs:
   test:
     name: Unit Tests
     needs: deps
-    runs-on: warp-ubuntu-latest-x64-2x
+    runs-on: warp-ubuntu-2204-x64-2x
     container:
       image: alpine:3.18
 

.github/workflows/deployment.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   build:
     name: Build
-    runs-on: warp-ubuntu-latest-x64-2x
+    runs-on: warp-ubuntu-2204-x64-2x
     if: ${{ github.event.workflow_run.conclusion == 'success' }}
     steps:
       - name: "Checkout"
@@ -49,7 +49,7 @@ jobs:
   deploy:
     name: Deploy
     needs: build
-    runs-on: warp-ubuntu-latest-x64-2x
+    runs-on: warp-ubuntu-2204-x64-2x
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.gitignore

@@ -34,4 +34,6 @@ uplink-*.tar
 
 mnesia
 
-.mnesia
+.mnesia
+
+.envrc

config/config.exs

@@ -33,10 +33,6 @@ config :uplink, Oban,
     components: 1
   ]
 
-config :reverse_proxy_plug,
-       :http_client,
-       ReverseProxyPlug.HTTPClient.Adapters.HTTPoison
-
 config :formation, Formation.Lxd, timeout: 180
 
 config :logger,

config/dev.exs

@@ -1,10 +1,13 @@
 import Config
 
-config :uplink, Uplink.Secret, "secretsomethingsixteen"
+config :uplink,
+       Uplink.Secret,
+       System.get_env("UPLINK_SECRET", "secretsomethingsixteen")
 
 config :uplink, Uplink.Data, mode: "lite"
 
-config :uplink, Uplink.Clients.Instellar, endpoint: "http://localhost/uplink"
+config :uplink, Uplink.Clients.Instellar,
+  endpoint: "http://localhost:4000/uplink"
 
 config :uplink, :environment, :dev
 

config/test.exs

@@ -2,6 +2,14 @@ import Config
 
 config :uplink, Uplink.Data, mode: "pro"
 
+config :uplink, Uplink.Monitors, enabled: false
+
+config :uplink, Uplink.Metrics.Pipeline,
+  producer_module: Broadway.DummyProducer,
+  producer_options: []
+
+config :uplink, Uplink.PipelineSupervisor, sync_interval: 100
+
 config :uplink, Uplink.Repo,
   username:
     System.get_env("UPLINK_DB_USERNAME") || System.get_env("POSTGRES_USERNAME"),
@@ -38,4 +46,5 @@ config :uplink, :drivers, aws_s3: Uplink.Drivers.Bucket.AwsMock
 
 # config :plug, :validate_header_keys_during_test, false
 # Print only warnings and errors during test
+# Disable logging in tests
 config :logger, level: :warn

lib/uplink/application.ex

@@ -5,8 +5,17 @@ defmodule Uplink.Application do
 
   alias Uplink.Web
 
+  @pipeline_supervisor Uplink.PipelineSupervisor
+
   def start(_type, _args) do
     %{key: key, cert: cert} = Web.Certificate.generate()
+
+    pipeline_supervisor_config =
+      Application.get_env(:uplink, @pipeline_supervisor, [])
+
+    sync_interval =
+      Keyword.get(pipeline_supervisor_config, :sync_interval, 5_000)
+
     router_config = Application.get_env(:uplink, Uplink.Router, port: 4040)
 
     internal_router_config =
@@ -22,6 +31,9 @@ defmodule Uplink.Application do
       {Cluster.Supervisor, [topologies, [name: Uplink.ClusterSupervisor]]},
       {Task.Supervisor, name: Uplink.TaskSupervisor},
       {Plug.Cowboy, plug: Uplink.Internal, scheme: :http, port: internal_port},
+      {Pogo.DynamicSupervisor,
+       name: @pipeline_supervisor, scope: :uplink, sync_interval: sync_interval},
+      {Uplink.Monitors, []},
       {
         Plug.Cowboy,
         plug: Uplink.Router,

lib/uplink/clients/instellar.ex

@@ -9,6 +9,7 @@ defmodule Uplink.Clients.Instellar do
     Instance,
     Register,
     Component,
+    Monitor,
     Variable,
     Proxy,
     Self
@@ -35,6 +36,10 @@ defmodule Uplink.Clients.Instellar do
     to: Proxy,
     as: :list
 
+  defdelegate list_monitors,
+    to: Monitor,
+    as: :list
+
   defdelegate deployment_metadata(install),
     to: Installation,
     as: :metadata

lib/uplink/clients/instellar/monitor.ex

@@ -0,0 +1,21 @@
+defmodule Uplink.Clients.Instellar.Monitor do
+  alias Uplink.Clients.Instellar
+
+  def list do
+    headers = Instellar.Self.headers()
+
+    [Instellar.endpoint(), "self", "monitors"]
+    |> Path.join()
+    |> Req.get(headers: headers, max_retries: 1)
+    |> case do
+      {:ok, %{status: 200, body: %{"data" => monitors}}} ->
+        {:ok, monitors}
+
+      {:ok, %{status: _, body: body}} ->
+        {:error, body}
+
+      {:error, error} ->
+        {:error, error}
+    end
+  end
+end

lib/uplink/clients/lxd.ex

@@ -4,6 +4,10 @@ defmodule Uplink.Clients.LXD do
 
   alias Uplink.Clients.LXD
 
+  defdelegate get_node(name),
+    to: __MODULE__.Node.Manager,
+    as: :show
+
   defdelegate list_cluster_members(),
     to: __MODULE__.Cluster.Manager,
     as: :list_members
@@ -16,11 +20,11 @@ defmodule Uplink.Clients.LXD do
     to: __MODULE__.Profile.Manager,
     as: :get
 
-  defdelegate list_instances(project),
-    to: __MODULE__.Instance.Manager,
+  defdelegate list_metrics(options \\ []),
+    to: __MODULE__.Metric.Manager,
     as: :list
 
-  defdelegate list_instances(),
+  defdelegate list_instances(options \\ []),
     to: __MODULE__.Instance.Manager,
     as: :list
 
@@ -33,37 +37,36 @@ defmodule Uplink.Clients.LXD do
     as: :leases
 
   def uplink_leases do
-    Cache.get({:leases, "uplink"}) ||
-      (
-        config = Application.get_env(:uplink, Uplink.Data) || []
-        uplink_project = Keyword.get(config, :project, "default")
-        client = LXD.client()
-
-        uplink_project =
-          client
-          |> Lexdee.get_project(uplink_project)
-          |> case do
-            {:ok, %{body: %{"name" => name}}} -> name
-            {:error, %{"error_code" => 404}} -> "default"
-          end
-
-        case LXD.network_leases(uplink_project) do
-          leases when is_list(leases) ->
-            uplink_addresses =
-              Enum.map(leases, fn lease ->
-                lease.address
-              end)
-
-            Cache.put({:leases, "uplink"}, uplink_addresses,
-              ttl: :timer.hours(3)
-            )
-
-            uplink_addresses
-
-          {:error, error} ->
-            {:error, error}
-        end
-      )
+    Cache.get({:leases, "uplink"}) || fetch_leases()
+  end
+
+  defp fetch_leases do
+    config = Application.get_env(:uplink, Uplink.Data) || []
+    uplink_project = Keyword.get(config, :project, "default")
+    client = LXD.client()
+
+    uplink_project =
+      client
+      |> Lexdee.get_project(uplink_project)
+      |> case do
+        {:ok, %{body: %{"name" => name}}} -> name
+        {:error, %{"error_code" => 404}} -> "default"
+      end
+
+    case LXD.network_leases(uplink_project) do
+      leases when is_list(leases) ->
+        uplink_addresses =
+          Enum.map(leases, fn lease ->
+            lease.address
+          end)
+
+        Cache.put({:leases, "uplink"}, uplink_addresses, ttl: :timer.hours(3))
+
+        uplink_addresses
+
+      {:error, error} ->
+        {:error, error}
+    end
   end
 
   def client do

lib/uplink/clients/lxd/instance.ex

@@ -9,10 +9,13 @@ defmodule Uplink.Clients.LXD.Instance do
     status
     architecture
     profiles
+    project
     description
     created_at
     last_used_at
     expanded_config
+    expanded_devices
+    state
   )a
 
   @required_attrs ~w(
@@ -34,13 +37,16 @@ defmodule Uplink.Clients.LXD.Instance do
     field :status, :string
     field :architecture, :string
     field :profiles, {:array, :string}
+    field :project, :string
     field :description, :string
 
     field :created_at, :utc_datetime_usec
     field :last_used_at, :utc_datetime_usec
 
     field :expanded_config, :map
     field :expanded_devices, :map
+
+    field :state, :map
   end
 
   def changeset(schema, params) do

lib/uplink/clients/lxd/instance/manager.ex

@@ -6,27 +6,25 @@ defmodule Uplink.Clients.LXD.Instance.Manager do
   alias Clients.LXD
   alias LXD.Instance
 
-  def list do
-    LXD.client()
-    |> Lexdee.list_instances(query: [{:recursion, 1}, {"all-projects", true}])
-    |> case do
-      {:ok, %{body: instances}} ->
-        instances =
-          instances
-          |> Enum.map(fn instance ->
-            Instance.parse(instance)
-          end)
+  def list(options \\ []) do
+    project = Keyword.get(options, :project, nil)
+    recursion = Keyword.get(options, :recursion, 1)
 
-        instances
-
-      error ->
-        error
+    if recursion < 1 do
+      raise "recursion must be greater than 0 and less than 3 but got #{recursion}"
     end
-  end
 
-  def list(project) do
+    project_query =
+      if project do
+        [project: project]
+      else
+        [{"all-projects", true}]
+      end
+
+    query = [{:recursion, recursion} | project_query]
+
     LXD.client()
-    |> Lexdee.list_instances(query: [recursion: 1, project: project])
+    |> Lexdee.list_instances(query: query)
     |> case do
       {:ok, %{body: instances}} ->
         instances =

lib/uplink/clients/lxd/metric.ex

@@ -0,0 +1,23 @@
+defmodule Uplink.Clients.LXD.Metric do
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  @primary_key false
+  embedded_schema do
+    field :instance, :string
+    field :label, :string
+    field :value, :string
+  end
+
+  def changeset(metric, params) do
+    metric
+    |> cast(params, [:instance, :type, :value])
+    |> validate_required([:instance, :type, :value])
+  end
+
+  def parse(params) do
+    %__MODULE__{}
+    |> changeset(params)
+    |> apply_action!(:insert)
+  end
+end