ES8

.tool-versions

@@ -1,2 +1,2 @@
-erlang 24.1.6
-elixir 1.13.4-otp-24
+erlang 25.3.2.8
+elixir 1.16.1-otp-25

README.md

@@ -25,7 +25,7 @@ defmodule MySchema do
   use ExlasticSearch.Model
 
   indexes :my_index do
-    settings Application.get_env(:some, :settings)
+    settings Application.compile_env(:some, :settings)
 
     mapping :field
     mapping :other_field, type: :keyword # ecto derived defaults can be overridden
@@ -61,3 +61,13 @@ config :exlasticsearch, :monitoring, ExlasticSearch.Monitoring.Mock
 config :exlasticsearch, ExlasticSearch.Repo,
   url: "http://localhost:9200"
 ```
+
+## Testing
+
+Run integration tests with local ElasticSearch clusters.
+Ensure Docker resources include at least 8 GB of memory.
+
+```sh
+docker-compose up -d
+mix test
+```

config/config.exs

@@ -1,8 +1,5 @@
-use Mix.Config
+import Config
 
 config :exlasticsearch, :type_inference, ExlasticSearch.TypeInference
 
-config :exlasticsearch, :monitoring, ExlasticSearch.Monitoring.Mock
-
-config :exlasticsearch, ExlasticSearch.Repo,
-  url: "http://localhost:9200"
+import_config "#{config_env()}.exs"

config/dev.exs

@@ -1 +0,0 @@
-use Mix.Config

config/prod.exs

@@ -1 +0,0 @@
-use Mix.Config

config/test.exs

@@ -1,5 +1,15 @@
-use Mix.Config
+import Config
 
 config :exlasticsearch, :type_mappings, [
   {DB.CustomType, :integer}
-]
+]
+
+config :exlasticsearch, :monitoring, ExlasticSearch.Monitoring.Mock
+
+config :exlasticsearch, ExlasticSearch.Repo,
+  url: "http://localhost:9200",
+  es8: "http://localhost:9201"
+
+
+config :exlasticsearch, ExlasticSearch.TypelessTestModel,
+  index_type: :es8

docker-compose.yaml

@@ -0,0 +1,24 @@
+version: "3.7"
+volumes:
+  elasticsearch-7-data:
+  elasticsearch-8-data:
+services:
+  elasticsearch-7:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.16
+    restart: always
+    environment:
+      discovery.type: single-node
+    volumes:
+      - elasticsearch-7-data:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+  elasticsearch-8:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.4
+    restart: always
+    environment:
+      discovery.type: single-node
+    volumes:
+      - elasticsearch-8-data:/usr/share/elasticsearch/data
+      - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
+    ports:
+      - 9201:9200

elasticsearch.yml

@@ -0,0 +1,4 @@
+cluster.name: "docker-cluster"
+network.host: 0.0.0.0
+xpack.security.enabled: false
+xpack.security.enrollment.enabled: false

lib/exlasticsearch.ex

@@ -9,7 +9,7 @@ defmodule ExlasticSearch do
     use ExlasticSearch.Model
 
     indexes :my_index do
-      settings Application.get_env(:some, :settings)
+      settings Application.compile_env(:some, :settings)
 
       mapping :field
       mapping :other_field, type: :keyword # ecto derived defaults can be overridden

lib/exlasticsearch/bulk.ex

@@ -28,14 +28,18 @@ defmodule ExlasticSearch.BulkOperation do
     do: bulk_operation_default({op_type, struct, :index})
 
   defp bulk_operation_default({op_type, %{__struct__: model} = struct, index}) do
+    op = %{
+      _id: Indexable.id(struct),
+      _index: model.__es_index__(index)
+    }
+
+    op =
+      if doc_type = model.__doc_type__(),
+        do: Map.put(op, :_type, doc_type),
+        else: op
+
     [
-      %{
-        op_type => %{
-          _id: Indexable.id(struct),
-          _index: model.__es_index__(index),
-          _type: model.__doc_type__()
-        }
-      },
+      %{op_type => op},
       build_document(struct, index)
     ]
   end
@@ -48,13 +52,19 @@ defmodule ExlasticSearch.BulkOperation do
   end
 
   defp bulk_operation_delete({:delete, %{__struct__: model} = struct, index}) do
+    op = %{
+      _id: Indexable.id(struct),
+      _index: model.__es_index__(index)
+    }
+
+    op =
+      if doc_type = model.__doc_type__(),
+        do: Map.put(op, :_type, doc_type),
+        else: op
+
     [
       %{
-        delete: %{
-          _id: Indexable.id(struct),
-          _index: model.__es_index__(index),
-          _type: model.__doc_type__()
-        }
+        delete: op
       }
     ]
   end

lib/exlasticsearch/model.ex

@@ -13,7 +13,7 @@ defmodule ExlasticSearch.Model do
 
   ```
   indexes :my_type do
-    settings Application.get_env(:some, :settings)
+    settings Application.compile_env(:some, :settings)
 
     mapping :column
     mapping :other_column, type: :keyword
@@ -67,13 +67,15 @@ defmodule ExlasticSearch.Model do
   * `type` - the indexes type (and index name will be `type <> "s"`)
   * `block` - the definition of the index
   """
-  defmacro indexes(type, block) do
+  defmacro indexes(type, opts \\ [], block) do
+    doc_type = Keyword.get(opts, :doc_type, type)
+
     quote do
       Module.register_attribute(__MODULE__, :es_mappings, accumulate: true)
       @read_version :ignore
       @index_version :ignore
 
-      def __doc_type__(), do: unquote(type)
+      def __doc_type__(), do: unquote(doc_type)
 
       unquote(block)
 

lib/exlasticsearch/repo.ex

@@ -18,7 +18,7 @@ defmodule ExlasticSearch.Repo do
 
   @chunk_size 2000
   @type response :: {:ok, %HTTPoison.Response{}} | {:error, any}
-  @log_level Application.get_env(:exlasticsearch, __MODULE__, [])
+  @log_level Application.compile_env(:exlasticsearch, __MODULE__, [])
              |> Keyword.get(:log_level, :debug)
 
   @doc """
@@ -60,7 +60,7 @@ defmodule ExlasticSearch.Repo do
   @spec create_mapping(atom) :: response
   def create_mapping(model, index \\ :index, opts \\ []) do
     es_url(index)
-    |> Mapping.put(model.__es_index__(index), model.__doc_type__(), model.__es_mappings__(), opts)
+    |> Mapping.put(model.__es_index__(index), model.__doc_type__() || "", model.__es_mappings__(), opts)
   end
 
   @doc """
@@ -156,7 +156,7 @@ defmodule ExlasticSearch.Repo do
     document = build_document(struct, index)
 
     es_url(index)
-    |> Document.index(model.__es_index__(index), model.__doc_type__(), id, document)
+    |> Document.index(model.__es_index__(index), model.__doc_type__() || "_doc", id, document)
     |> log_response()
     |> mark_failure()
   end
@@ -166,10 +166,21 @@ defmodule ExlasticSearch.Repo do
   """
   @decorate retry()
   def update(model, id, data, index \\ :index) do
-    es_url(index)
-    |> Document.update(model.__es_index__(index), model.__doc_type__(), id, data)
-    |> log_response()
-    |> mark_failure()
+    cond do
+      doc_type = model.__doc_type__() ->
+        es_url(index)
+        |> Document.update(model.__es_index__(index), doc_type, id, data)
+        |> log_response()
+        |> mark_failure()
+
+      %{doc: doc} = data ->
+        model
+        |> struct(Map.put(doc, :id, id))
+        |> index(index)
+
+      true ->
+        {:error, "ExlasticSearch only supports doc updates for ES 8+"}
+    end
   end
 
   @doc """
@@ -215,7 +226,7 @@ defmodule ExlasticSearch.Repo do
   @spec get(struct) ::{:ok, %Response.Record{}} | {:error, any}
   def get(%{__struct__: model} = struct, index_type \\ :read) do
     es_url(index_type)
-    |> Document.get(model.__es_index__(index_type), model.__doc_type__(), Indexable.id(struct))
+    |> Document.get(model.__es_index__(index_type), model.__doc_type__() || "_doc", Indexable.id(struct))
     |> log_response()
     |> decode(Response.Record, model)
   end
@@ -250,9 +261,11 @@ defmodule ExlasticSearch.Repo do
 
   defp model_to_doc_types(models) when is_list(models) do
     models
-    |> Enum.map(& &1.__doc_type__())
+    |> Enum.flat_map(&model_to_doc_types/1)
+  end
+  defp model_to_doc_types(model) do
+    if doc_type = model.__doc_type__(), do: [doc_type], else: []
   end
-  defp model_to_doc_types(model), do: [model.__doc_type__()]
 
   @doc """
   Performs an aggregation against a query, and returns only the aggregation results.
@@ -264,8 +277,11 @@ defmodule ExlasticSearch.Repo do
 
     index_type = query.index_type || :read
 
+    doc_types = if doc_type = model.__doc_type__(), do: [doc_type], else: []
+    es_index = model.__es_index__(index_type)
+
     es_url(index_type)
-    |> Search.search(model.__es_index__(index_type), [model.__doc_type__()], search, size: 0)
+    |> Search.search(es_index, doc_types, search, size: 0)
     # TODO: figure out how to decode these, it's not trivial to type them
     |> log_response()
   end
@@ -277,7 +293,7 @@ defmodule ExlasticSearch.Repo do
   @decorate retry()
   def delete(%{__struct__: model} = struct, index \\ :index) do
     es_url(index)
-    |> Document.delete(model.__es_index__(index), model.__doc_type__(), Indexable.id(struct))
+    |> Document.delete(model.__es_index__(index), model.__doc_type__() || "_doc", Indexable.id(struct))
     |> log_response()
     |> mark_failure()
   end
@@ -306,7 +322,6 @@ defmodule ExlasticSearch.Repo do
     do: struct |> Indexable.preload(index) |> Indexable.document(index)
 
   defp es_url(index) do
-
     case Application.get_env(:exlasticsearch, __MODULE__)[index] do
       nil ->
         Application.get_env(:exlasticsearch, __MODULE__)[:url]

lib/exlasticsearch/retry/decorator.ex

@@ -7,12 +7,12 @@ defmodule ExlasticSearch.Retry.Decorator do
   ```
   """
   use Decorator.Define, retry: 0
-  @config Application.get_env(:exlasticsearch, :retry, [])
+  @config Application.compile_env(:exlasticsearch, :retry, [])
 
   def retry(body, _ctx) do
     {strategy, config} = Keyword.pop(@config, :strategy, ExlasticSearch.Retry.ExponentialBackoff)
     quote do
       unquote(strategy).retry(fn -> unquote(body) end, unquote(config))
     end
   end
-end
+end