[WIP] Support for indexing the codebase

Makefile

@@ -26,7 +26,8 @@ ppxlib \
 ctypes \
 "utop>=2.6.0" \
 "melange>=1.0.0" \
-"rescript-syntax"
+"rescript-syntax" \
+ocaml-uideps
 # Dependencies recommended for developing dune locally,
 # but not wanted in CI
 DEV_DEPS := \
@@ -71,6 +72,10 @@ install-ocamlformat:
 dev-depext:
 	opam depext -y $(TEST_DEPS)
 
+.PHONY: ocaml-uideps
+ocaml-uideps:
+	opam pin add ocaml-uideps https://github.com/voodoos/ocaml-uideps.git#fbc6881e103cf391286893589bce5afaedf2941e
+
 .PHONY: dev-deps
 dev-deps:
 	opam install -y $(TEST_DEPS)

bin/common.ml

@@ -444,6 +444,7 @@ let shared_with_config_file =
   { Dune_config.Partial.display
   ; concurrency
   ; sandboxing_preference = Option.map sandboxing_preference ~f:(fun x -> [ x ])
+  ; workspace_indexation = None
   ; terminal_persistence
   ; cache_enabled
   ; cache_reproducibility_check =

src/dune_config_file/dune_config_file.ml

@@ -38,6 +38,20 @@ module Dune_config = struct
     let decode = enum all
   end
 
+  module Workspace_indexation = struct
+    type t =
+      | Enabled
+      | Disabled
+
+    let all = [ ("enabled", Enabled); ("disabled", Disabled) ]
+
+    let to_dyn = function
+      | Enabled -> Dyn.Variant ("Enabled", [])
+      | Disabled -> Dyn.Variant ("Disabled", [])
+
+    let decode = enum all
+  end
+
   module Concurrency = struct
     type t =
       | Fixed of int
@@ -135,6 +149,7 @@ module Dune_config = struct
       { display : Display.t field
       ; concurrency : Concurrency.t field
       ; terminal_persistence : Terminal_persistence.t field
+      ; workspace_indexation : Workspace_indexation.t field
       ; sandboxing_preference : Sandboxing_preference.t field
       ; cache_enabled : Cache.Enabled.t field
       ; cache_reproducibility_check :
@@ -160,6 +175,8 @@ module Dune_config = struct
       ; concurrency = field a.concurrency b.concurrency
       ; terminal_persistence =
           field a.terminal_persistence b.terminal_persistence
+      ; workspace_indexation =
+          field a.workspace_indexation b.workspace_indexation
       ; sandboxing_preference =
           field a.sandboxing_preference b.sandboxing_preference
       ; cache_enabled = field a.cache_enabled b.cache_enabled
@@ -185,6 +202,7 @@ module Dune_config = struct
         { M.display
         ; concurrency
         ; terminal_persistence
+        ; workspace_indexation
         ; sandboxing_preference
         ; cache_enabled
         ; cache_reproducibility_check
@@ -198,6 +216,8 @@ module Dune_config = struct
         ; ("concurrency", field Concurrency.to_dyn concurrency)
         ; ( "terminal_persistence"
           , field Terminal_persistence.to_dyn terminal_persistence )
+        ; ( "workspace_indexation"
+          , field Workspace_indexation.to_dyn workspace_indexation )
         ; ( "sandboxing_preference"
           , field (Dyn.list Sandbox_mode.to_dyn) sandboxing_preference )
         ; ("cache_enabled", field Cache.Enabled.to_dyn cache_enabled)
@@ -229,6 +249,7 @@ module Dune_config = struct
       { display = None
       ; concurrency = None
       ; terminal_persistence = None
+      ; workspace_indexation = None
       ; sandboxing_preference = None
       ; cache_enabled = None
       ; cache_reproducibility_check = None
@@ -295,6 +316,7 @@ module Dune_config = struct
           { verbosity = Quiet; status_line = not Execution_env.inside_dune }
     ; concurrency = (if Execution_env.inside_dune then Fixed 1 else Auto)
     ; terminal_persistence = Clear_on_rebuild
+    ; workspace_indexation = Disabled
     ; sandboxing_preference = []
     ; cache_enabled = Disabled
     ; cache_reproducibility_check = Skip
@@ -314,6 +336,8 @@ module Dune_config = struct
     and+ concurrency = field_o "jobs" (1, 0) Concurrency.decode
     and+ terminal_persistence =
       field_o "terminal-persistence" (1, 0) Terminal_persistence.decode
+    and+ workspace_indexation =
+      field_o "workspace_indexation" (3, 5) Workspace_indexation.decode
     and+ sandboxing_preference =
       field_o "sandboxing_preference" (1, 0) Sandboxing_preference.decode
     and+ cache_enabled = field_o "cache" (2, 0) Cache.Enabled.decode
@@ -363,6 +387,7 @@ module Dune_config = struct
     { Partial.display
     ; concurrency
     ; terminal_persistence
+    ; workspace_indexation
     ; sandboxing_preference
     ; cache_enabled
     ; cache_reproducibility_check

src/dune_config_file/dune_config_file.mli

@@ -52,6 +52,14 @@ module Dune_config : sig
     val all : (string * t) list
   end
 
+  module Workspace_indexation : sig
+    type t =
+      | Enabled
+      | Disabled
+
+    val all : (string * t) list
+  end
+
   module Action_output_on_success : sig
     include module type of struct
       include Dune_engine.Execution_parameters.Action_output_on_success
@@ -65,6 +73,7 @@ module Dune_config : sig
       { display : Display.t field
       ; concurrency : Concurrency.t field
       ; terminal_persistence : Terminal_persistence.t field
+      ; workspace_indexation : Workspace_indexation.t field
       ; sandboxing_preference : Sandboxing_preference.t field
       ; cache_enabled : Cache.Enabled.t field
       ; cache_reproducibility_check :

src/dune_engine/alias.ml

@@ -18,6 +18,8 @@ module Name = struct
 
   let fmt = of_string "fmt"
 
+  let uideps = of_string "uideps"
+
   let all = of_string "all"
 
   let parse_local_path (loc, p) =
@@ -105,6 +107,8 @@ let runtest = make_standard Name.runtest
 
 let install = make_standard Name.install
 
+let uideps = make_standard Name.uideps
+
 let doc = make_standard (Name.of_string "doc")
 
 let private_doc = make_standard (Name.of_string "doc-private")

src/dune_engine/alias.mli

@@ -12,6 +12,8 @@ module Name : sig
 
   val fmt : t
 
+  val uideps : t
+
   val all : t
 
   val parse_local_path : Loc.t * Path.Local.t -> Path.Local.t * t
@@ -50,6 +52,8 @@ val runtest : dir:Path.Build.t -> t
 
 val install : dir:Path.Build.t -> t
 
+val uideps : dir:Path.Build.t -> t
+
 val doc : dir:Path.Build.t -> t
 
 val private_doc : dir:Path.Build.t -> t

src/dune_rules/exe.ml

@@ -291,6 +291,10 @@ let build_and_link_many ?link_args ?o_files ?embed_in_plugin_libraries ?sandbox
     ~programs ~linkages ~promote cctx =
   let open Memo.O in
   let* () = Module_compilation.build_all cctx in
+  let* () =
+    if Compilation_context.bin_annot cctx then Uideps.cctx_rules cctx
+    else Memo.return ()
+  in
   link_many ?link_args ?o_files ?embed_in_plugin_libraries ?sandbox ~programs
     ~linkages ~promote cctx
 

src/dune_rules/exe_rules.ml

@@ -130,6 +130,7 @@ let executables_rules ~sctx ~dir ~expander ~dir_contents ~scope ~compile_info
   in
   let stdlib_dir = ctx.lib_config.stdlib_dir in
   let* requires_compile = Compilation_context.requires_compile cctx in
+  let* requires_link = Compilation_context.requires_link cctx in
   let* dep_graphs =
     (* Building an archive for foreign stubs, we link the corresponding object
        files directly to improve perf. *)
@@ -204,7 +205,7 @@ let executables_rules ~sctx ~dir ~expander ~dir_contents ~scope ~compile_info
       ~f:(Check_rules.add_cycle_check sctx ~dir)
   in
   ( cctx
-  , Merlin.make ~requires:requires_compile ~stdlib_dir ~flags ~modules
+  , Merlin.make ~requires:requires_link ~stdlib_dir ~flags ~modules
       ~source_dirs:Path.Source.Set.empty ~libname:None ~obj_dir
       ~preprocess:
         (Preprocess.Per_module.without_instrumentation exes.buildable.preprocess)

src/dune_rules/gen_rules.ml

@@ -332,6 +332,7 @@ let gen_project_rules sctx project : unit Memo.t =
   let+ () = Opam_create.add_rules sctx project
   and+ () = Install_rules.gen_project_rules sctx project
   and+ () = Odoc.gen_project_rules sctx project
+  and+ () = Uideps.project_rule sctx project
   and+ () =
     let version = (2, 8) in
     match Dune_project.allow_approximate_merlin project with

src/dune_rules/lib_rules.ml

@@ -500,6 +500,7 @@ let library_rules (lib : Library.t) ~local_lib ~cctx ~source_modules
   let dir = Compilation_context.dir cctx in
   let scope = Compilation_context.scope cctx in
   let* requires_compile = Compilation_context.requires_compile cctx in
+  let* requires_link = Compilation_context.requires_link cctx in
   let stdlib_dir = (Compilation_context.context cctx).lib_config.stdlib_dir in
   let top_sorted_modules =
     let impl_only = Modules.impl_only modules in
@@ -522,6 +523,9 @@ let library_rules (lib : Library.t) ~local_lib ~cctx ~source_modules
     info
   in
   let+ () =
+    if Compilation_context.bin_annot cctx then Uideps.cctx_rules cctx
+    else Memo.return ()
+  and+ () =
     Memo.when_
       (not (Library.is_virtual lib))
       (fun () ->
@@ -545,7 +549,7 @@ let library_rules (lib : Library.t) ~local_lib ~cctx ~source_modules
       }
   in
   ( cctx
-  , Merlin.make ~requires:requires_compile ~stdlib_dir ~flags ~modules
+  , Merlin.make ~requires:requires_link ~stdlib_dir ~flags ~modules
       ~source_dirs:Path.Source.Set.empty
       ~preprocess:
         (Preprocess.Per_module.without_instrumentation lib.buildable.preprocess)

src/dune_rules/merlin/merlin.ml

@@ -43,7 +43,8 @@ module Processed = struct
 
   (* Most of the configuration is shared across a same lib/exe... *)
   type config =
-    { stdlib_dir : Path.t option
+    { build_dir : Path.Build.t
+    ; stdlib_dir : Path.t option
     ; obj_dirs : Path.Set.t
     ; src_dirs : Path.Set.t
     ; flags : string list
@@ -52,10 +53,18 @@ module Processed = struct
     }
 
   let dyn_of_config
-      { stdlib_dir; obj_dirs; src_dirs; flags; extensions; melc_flags } =
+      { build_dir
+      ; stdlib_dir
+      ; obj_dirs
+      ; src_dirs
+      ; flags
+      ; extensions
+      ; melc_flags
+      } =
     let open Dyn in
     record
-      [ ("stdlib_dir", option Path.to_dyn stdlib_dir)
+      [ ("build_dir", Path.Build.to_dyn build_dir)
+      ; ("stdlib_dir", option Path.to_dyn stdlib_dir)
       ; ("obj_dirs", Path.Set.to_dyn obj_dirs)
       ; ("src_dirs", Path.Set.to_dyn src_dirs)
       ; ("flags", list string flags)
@@ -126,11 +135,22 @@ module Processed = struct
     | None, None -> None
 
   let to_sexp ~opens ~pp
-      { stdlib_dir; obj_dirs; src_dirs; flags; extensions; melc_flags } =
+      { build_dir
+      ; stdlib_dir
+      ; obj_dirs
+      ; src_dirs
+      ; flags
+      ; extensions
+      ; melc_flags
+      } =
     let make_directive tag value = Sexp.List [ Atom tag; value ] in
     let make_directive_of_path tag path =
       make_directive tag (Sexp.Atom (serialize_path path))
     in
+    let index_file_path = Uideps.project_index ~build_dir in
+    let index_file =
+      [ make_directive_of_path "INDEX_FILE" (Path.build index_file_path) ]
+    in
     let stdlib_dir =
       match stdlib_dir with
       | None -> []
@@ -185,7 +205,14 @@ module Processed = struct
     in
     Sexp.List
       (List.concat
-         [ stdlib_dir; exclude_query_dir; obj_dirs; src_dirs; flags; suffixes ])
+         [ index_file
+         ; stdlib_dir
+         ; exclude_query_dir
+         ; obj_dirs
+         ; src_dirs
+         ; flags
+         ; suffixes
+         ])
 
   let quote_for_dot_merlin s =
     let s =
@@ -282,7 +309,8 @@ module Processed = struct
                { per_module_config = _
                ; pp_config
                ; config =
-                   { stdlib_dir = _
+                   { build_dir = _
+                   ; stdlib_dir = _
                    ; obj_dirs
                    ; src_dirs
                    ; flags
@@ -527,6 +555,10 @@ module Unprocessed = struct
                     in
                     Path.Set.add obj_dirs public_cmi_dir )))
       in
+
+      let build_dir =
+        Super_context.context sctx |> Context.name |> Context_name.build_dir
+      in
       let src_dirs =
         Path.Set.union src_dirs
           (Path.Set.of_list_map ~f:Path.source more_src_dirs)
@@ -545,7 +577,8 @@ module Unprocessed = struct
             ; Processed.serialize_path path ^ " -as-ppx"
             ])
       in
-      { Processed.stdlib_dir
+      { Processed.build_dir
+      ; stdlib_dir
       ; src_dirs
       ; obj_dirs
       ; flags

src/dune_rules/obj_dir.ml

@@ -521,6 +521,11 @@ module Module = struct
     let cmi_kind = Lib_mode.Cm_kind.cmi cm_kind in
     Option.map file ~f:(fun _ -> obj_file t m ~kind:cmi_kind ~ext)
 
+  let uideps_file t m =
+    let ext = ".uideps" in
+    let kind = Lib_mode.Cm_kind.(Ocaml Cmi) in
+    obj_file t m ~kind ~ext
+
   let cmti_file t m ~cm_kind =
     let ext =
       Ml_kind.cmt_ext

src/dune_rules/obj_dir.mli

@@ -116,6 +116,8 @@ module Module : sig
     -> cm_kind:Lib_mode.Cm_kind.t
     -> 'path option
 
+  val uideps_file : 'path t -> Module.t -> 'path
+
   val obj_file :
     'path t -> Module.t -> kind:Lib_mode.Cm_kind.t -> ext:string -> 'path