[merge] Merge WIP changes from ffi before they get too stale

* ffi:
  [test] Give the import_from_two_modules test its own header file
  [test] Rename a test
  [build] c_compile stage now understands headers are a dependency
  [build] Only write an implicit deps section in ninja files if it is used
  [build] make_temp_filename throws if the filename is empty
  [build] Have the gen_init target generate headers
  [compiler] Write headers for foreign generated code
  [compiler] Add output-header parameter for generating foreign hooks
  [compiler/util] Seperate the write_temp and move operations
  [compiler] Make writing of foreign hooks use a temporary file
  [build] Specify c_compile task name without a path
  [test] Pass --rebuild to all "build" tests
  [test] Print test output in verbose mode
  [build] Support foreign code in more than one module
  [build] Add new plzgeninit program
  [build] Rename the dependency stage to a general "scan" stage
  [ffi] Add some missing newlines in generated foreign code

Makefile

@@ -114,6 +114,7 @@ progs : \
 	src/plzbuild \
 	src/plzc \
 	src/plzdisasm \
+	src/plzgeninit \
 	src/plzlnk
 
 .PHONY: install
@@ -131,6 +132,7 @@ install_progs : install_dirs progs
 	$(INSTALL_STRIP) src/plzbuild $(DEST_DIR)$(BINDIR)
 	$(INSTALL_STRIP) src/plzc $(DEST_DIR)$(BINDIR)
 	$(INSTALL_STRIP) src/plzdisasm $(DEST_DIR)$(BINDIR)
+	$(INSTALL_STRIP) src/plzgeninit $(DEST_DIR)$(BINDIR)
 	$(INSTALL_STRIP) src/plzlnk $(DEST_DIR)$(BINDIR)
 
 .PHONY: install_docs
@@ -169,6 +171,8 @@ src/plzc : .mer_progs
 	touch src/plzc
 src/plzdisasm : .mer_progs
 	touch src/plzdisasm
+src/plzgeninit : .mer_progs
+	touch src/plzgeninit
 src/plzlnk : .mer_progs 
 	touch src/plzlnk
 .mer_progs : $(MERCURY_SOURCES) runtime/pz_config.h $(C_HEADERS)
@@ -177,6 +181,7 @@ src/plzlnk : .mer_progs
 	(cd src; $(MMC_MAKE) --cflags="$(C_CXX_FLAGS_BASE)" $(MCFLAGS) plzbuild)
 	(cd src; $(MMC_MAKE) --cflags="$(C_CXX_FLAGS_BASE)" $(MCFLAGS) plzc)
 	(cd src; $(MMC_MAKE) --cflags="$(C_CXX_FLAGS_BASE)" $(MCFLAGS) plzdisasm)
+	(cd src; $(MMC_MAKE) --cflags="$(C_CXX_FLAGS_BASE)" $(MCFLAGS) plzgeninit)
 	(cd src; $(MMC_MAKE) --cflags="$(C_CXX_FLAGS_BASE)" $(MCFLAGS) plzlnk)
 	touch .mer_progs
 
@@ -258,7 +263,8 @@ realclean : localclean
 	find tests -name *.so \
 		| xargs -r rm
 	rm -f src/tags 
-	rm -f src/plzasm src/plzbuild src/plzc src/plzdisasm src/plzlnk
+	rm -f src/plzasm src/plzbuild src/plzc src/plzdisasm src/plzgeninit \
+		src/plzlnk
 	rm -rf src/Mercury
 	rm -f .mer_progs
 	rm -rf runtime/tags runtime/plzrun

src/.gitignore

@@ -5,5 +5,6 @@ plzasm
 plzbuild
 plzc
 plzdisasm
+plzgeninit
 plzlnk
 tags

src/.vim_mmc_make

@@ -4,7 +4,7 @@ set -e
 
 MCFLAGS=--use-grade-subdirs
 
-for prog in plzasm plzbuild plzc plzdisasm plzlnk; do
+for prog in plzasm plzbuild plzc plzdisasm plzgeninit plzlnk; do
     mmc --output-compile-error-lines 1000 \
         --max-error-line-width 1000 \
         $MCFLAGS -j$JOBS --make $prog

src/build.m

@@ -347,18 +347,25 @@ not is_empty(DupModules)
                 dttr_output         :: string,
                 dttr_input          :: string
             )
-    ;       dt_dep(
-                dtd_name            :: q_name,
-                dtd_dep_file        :: string,
-                dtd_source          :: string,
-                dtd_interface       :: string,
-                dtd_bytecode        :: string
+    ;       dt_scan(
+                dts_name            :: q_name,
+                dts_dep_file        :: string,
+                dts_source          :: string,
+                dts_interface       :: string,
+                dts_bytecode        :: string
             )
     ;       dt_foreign_hooks(
                 dtcg_name           :: q_name,
-                dtcg_output         :: string,
+                dtcg_output_code    :: string,
+                dtcg_output_header  :: string,
                 dtcg_input          :: string
             )
+            % Generate an init file for the FFI from the info files.
+    ;       dt_gen_init(
+                dtgi_name           :: nq_name,
+                dtgi_output         :: string,
+                dtgi_modules        :: list(q_name)
+            )
     ;       dt_c_link(
                 dtcl_name           :: nq_name,
                 dtcl_output         :: string,
@@ -367,6 +374,7 @@ not is_empty(DupModules)
     ;       dt_c_compile(
                 dtcc_output         :: string,
                 dtcc_input          :: string,
+                dtcc_headers        :: list(string),
                 dtcc_generated      :: generated
             ).
 
@@ -389,12 +397,13 @@ not is_empty(DupModules)
       MaybeForeignSources = ok(ForeignSources),
         ModuleTargets = map(make_module_targets, ModuleFiles),
         ProgramTargets = map(make_program_target, Targets),
-        ForeignLinkTargets = map(make_foreign_link_target, Targets),
+        ForeignLinkTargets = condense(map(make_foreign_link_targets,
+            Targets)),
         ForeignCompileTargets = map(make_foreign_target, ForeignSources),
 
         MaybeDeps = ok(condense(ModuleTargets) ++
             ForeignCompileTargets ++
-            list_maybe_to_list(ForeignLinkTargets) ++ ProgramTargets)
+            ForeignLinkTargets ++ ProgramTargets)
 
     ; MaybeModuleFiles = ok(_),
       MaybeForeignSources = errors(Errors),
@@ -473,34 +482,46 @@ not is_empty(DupModules)
     ObjectName = BaseName ++ output_extension,
     DepFile = BaseName ++ depends_extension,
     Targets = [
-        dt_dep(ModuleName, DepFile, SourceName, InterfaceName, ObjectName),
+        dt_scan(ModuleName, DepFile, SourceName, InterfaceName, ObjectName),
         dt_interface(ModuleName, InterfaceName, SourceName, DepFile),
         dt_object(ModuleName, ObjectName, SourceName, DepFile),
         dt_typeres(ModuleName, TyperesName, SourceName),
         dt_foreign_hooks(ModuleName,
-            module_to_foreign_hooks(ModuleName), SourceName),
+            module_to_foreign_hooks_code(ModuleName),
+            module_to_foreign_hooks_header(ModuleName), SourceName),
         dt_c_compile(
             module_to_foreign_object(ModuleName),
-            module_to_foreign_hooks(ModuleName),
+            module_to_foreign_hooks_code(ModuleName),
+            [module_to_foreign_hooks_header(ModuleName)],
             was_generated)
     ].
 
-:- func make_foreign_link_target(target) = maybe(dep_target).
+:- func make_foreign_link_targets(target) = list(dep_target).
 
-make_foreign_link_target(Target) = Dep :-
+make_foreign_link_targets(Target) = Deps :-
     ForeignSources = Target ^ t_c_sources,
     Modules = Target ^ t_modules,
     ( ForeignSources = [],
-        Dep = no
+        Deps = []
     ; ForeignSources = [_ | _],
         Output = make_c_library_name(Target),
         ( if
             map(file_change_extension(cpp_extension, native_object_extension),
                 ForeignSources, ForeignObjects)
         then
             ModuleForeignObjects = map(module_to_foreign_object, Modules),
-            Dep = yes(dt_c_link(Target ^ t_name, Output, ForeignObjects ++
-                ModuleForeignObjects))
+            InitBaseName = nq_name_to_string(Target ^ t_name) ++ "_init",
+            InitSourceName = InitBaseName ++ cpp_extension,
+            InitObjectName = InitBaseName ++ native_object_extension,
+            InitTargetSource = dt_gen_init(Target ^ t_name, InitSourceName,
+                Modules),
+            InitTargetObject = dt_c_compile(InitObjectName, InitSourceName,
+                map(module_to_foreign_hooks_header, Modules), was_generated),
+            LinkTarget = dt_c_link(Target ^ t_name, Output,
+                ForeignObjects ++
+                ModuleForeignObjects ++
+                [InitObjectName]),
+            Deps = [InitTargetSource, InitTargetObject, LinkTarget]
         else
             compile_error($file, $pred, "Unrecognised source file extension")
         )
@@ -518,15 +539,25 @@ not is_empty(DupModules)
         file_change_extension(cpp_extension, native_object_extension,
             CFileName, ObjectName)
     then
-        Target = dt_c_compile(ObjectName, CFileName, hand_written)
+        Target = dt_c_compile(ObjectName, CFileName, [], hand_written)
     else
         compile_error($file, $pred, "Unrecognised source file extension")
     ).
 
-:- func module_to_foreign_hooks(q_name) = string.
+:- func module_to_foreign_hooks_code(q_name) = string.
+
+module_to_foreign_hooks_code(Module) =
+    module_to_foreign_hooks_base(Module) ++ cpp_extension.
+
+:- func module_to_foreign_hooks_header(q_name) = string.
+
+module_to_foreign_hooks_header(Module) =
+    module_to_foreign_hooks_base(Module) ++ c_header_extension.
 
-module_to_foreign_hooks(Module) =
-    canonical_base_name(Module) ++ "_f" ++ cpp_extension.
+:- func module_to_foreign_hooks_base(q_name) = string.
+
+module_to_foreign_hooks_base(Module) =
+    canonical_base_name(Module) ++ "_f".
 
 :- func module_to_foreign_object(q_name) = string.
 
@@ -563,26 +594,36 @@ not is_empty(DupModules)
     foldl(write_target(BuildFile), DepInfo, !IO).
 
 :- pred write_statement(output_stream::in,
-    string::in, string::in, string::in, list(string)::in, maybe(string)::in,
-    maybe(string)::in, list(pair(string, string))::in, io::di, io::uo) is det.
-
-write_statement(File, Command, Name, Output, Inputs, MaybeBinary,
-        MaybeDynDep, Vars, !IO) :-
+    string::in, string::in, string::in, list(string)::in, list(string)::in, maybe(string)::in,
+    list(string)::in, maybe(string)::in, list(pair(string, string))::in, io::di, io::uo) is det.
+
+write_statement(File, Command, Name, Output, ImplicitOutputs, Inputs, MaybeBinary,
+        ImplicitDeps, MaybeDynDep, Vars, !IO) :-
+    ( ImplicitOutputs = [],
+        ImplicitOutput = ""
+    ; ImplicitOutputs = [_ | _],
+        ImplicitOutput = " | " ++ string_join(" ", ImplicitOutputs)
+    ),
     InputsStr = string_join(" ", Inputs),
     ( MaybeBinary = yes(Binary),
         BinaryInput = ["$path/" ++ Binary]
     ; MaybeBinary = no,
         BinaryInput = []
     ),
-    ExtraDepsStr = string_join(" ", BinaryInput),
+    ExtraDeps = BinaryInput ++ ImplicitDeps,
+    ( ExtraDeps = [_ | _],
+        ExtraDepsStr = " | " ++ string_join(" ", ExtraDeps)
+    ; ExtraDeps = [],
+        ExtraDepsStr = ""
+    ),
     ( MaybeDynDep = yes(DynDep),
         DynDepStr = " || " ++ DynDep
     ; MaybeDynDep = no,
         DynDepStr = ""
     ),
     write_string(File,
-        "build " ++ Output ++ " : " ++ Command ++ " " ++ InputsStr ++ " | "
-            ++ ExtraDepsStr ++ DynDepStr ++ "\n",
+        "build " ++ Output ++ ImplicitOutput ++ " : " ++ Command ++ " " ++
+            InputsStr ++ ExtraDepsStr ++ DynDepStr ++ "\n",
         !IO),
     write_var(File, "name" - Name, !IO),
     ( MaybeDynDep = yes(DynDep_),
@@ -604,8 +645,17 @@ not is_empty(DupModules)
 
 write_build_statement(File, Command, Name, Output, Path, Input, MaybeBinary,
         !IO) :-
-    write_statement(File, Command, Name, Output, [Path ++ Input],
-        MaybeBinary, no, [], !IO).
+    write_statement(File, Command, Name, Output, [], [Path ++ Input],
+        MaybeBinary, [], no, [], !IO).
+
+:- pred write_c_compile_statement(output_stream::in, string::in,
+    string::in, string::in, string::in, list(string)::in,
+    io::di, io::uo) is det.
+
+write_c_compile_statement(File, Name, Output, Path, Input, Headers,
+        !IO) :-
+    write_statement(File, "c_compile", Name, Output, [], [Path ++ Input],
+        no, Headers, no, [], !IO).
 
 :- pred write_plzc_statement(output_stream::in, string::in, q_name::in,
     string::in, string::in, string::in, list(pair(string, string))::in,
@@ -614,7 +664,7 @@ not is_empty(DupModules)
 write_plzc_statement(File, Command, Name, Output, Input,
         DepFile, Vars, !IO) :-
     write_statement(File, Command, q_name_to_string(Name),
-        Output, ["../" ++ Input], yes("plzc"), yes(DepFile), Vars, !IO).
+        Output, [], ["../" ++ Input], yes("plzc"), [], yes(DepFile), Vars, !IO).
 
 :- pred write_link_statement(output_stream::in, string::in, nq_name::in,
     string::in, list(string)::in, maybe(string)::in,
@@ -623,7 +673,7 @@ not is_empty(DupModules)
 write_link_statement(File, Command, Name, Output, Objects, MaybeBinary,
         !IO) :-
     write_statement(File, Command, nq_name_to_string(Name),
-        "../" ++ Output, Objects, MaybeBinary, no, [], !IO).
+        "../" ++ Output, [], Objects, MaybeBinary, [], no, [], !IO).
 
 :- pred write_target(output_stream::in, dep_target::in, io::di, io::uo) is det.
 
@@ -647,28 +697,35 @@ not is_empty(DupModules)
     write_build_statement(File, "plztyperes", q_name_to_string(ModuleName),
         TyperesFile, "../", SourceFile, yes("plzc"), !IO).
 write_target(File,
-        dt_dep(ModuleName, DepFile, SourceFile, InterfaceFile, BytecodeFile),
+        dt_scan(ModuleName, DepFile, SourceFile, InterfaceFile, BytecodeFile),
         !IO) :-
     Inputs = ["../" ++ SourceFile],
-    write_statement(File, "plzdep", q_name_to_string(ModuleName),
-        DepFile, Inputs, yes("plzc"), no,
+    write_statement(File, "plzscan", q_name_to_string(ModuleName),
+        DepFile, [], Inputs, yes("plzc"), [], no,
         ["target"       - BytecodeFile,
          "interface"    - InterfaceFile],
         !IO).
-write_target(File, dt_foreign_hooks(ModuleName, Output, Source), !IO) :-
-    write_build_statement(File, "plzgf", q_name_to_string(ModuleName),
-        Output, "../", Source, no, !IO).
+write_target(File, dt_foreign_hooks(ModuleName, OutCode, OutHeader, Source),
+        !IO) :-
+    write_statement(File, "plzgf", q_name_to_string(ModuleName),
+        OutCode, [OutHeader], ["../" ++ Source], no, [], no,
+        ["header"       - OutHeader], !IO).
+write_target(File, dt_gen_init(ModuleName, Output, Inputs), !IO) :-
+    InputsString = string_join(" ", map(q_name_to_string, Inputs)),
+    write_statement(File, "gen_init", nq_name_to_string(ModuleName),
+        Output, [], [], yes("plzgeninit"), [], no,
+        ["modules" - InputsString], !IO).
 write_target(File, dt_c_link(ModuleName, Output, Inputs), !IO) :-
     write_link_statement(File, "c_link", ModuleName, Output, Inputs,
         no, !IO).
-write_target(File, dt_c_compile(Object, Source, SrcWasGenerated), !IO) :-
+write_target(File, dt_c_compile(Object, Source, Headers, SrcWasGenerated),
+        !IO) :-
     ( SrcWasGenerated = was_generated,
         Path = ""
     ; SrcWasGenerated = hand_written,
         Path = "../"
     ),
-    write_build_statement(File, "c_compile", Path ++ Source,
-        Object, Path, Source, no, !IO).
+    write_c_compile_statement(File, Source, Object, Path, Source, Headers, !IO).
 
 %-----------------------------------------------------------------------%
 
@@ -796,13 +853,13 @@ not is_empty(DupModules)
 		$in -o $out
     description = Making interface for $name
 
-rule plzdep
-    command = $path/plzc $pcflags --mode make-depends $
+rule plzscan
+    command = $path/plzc $pcflags --mode scan $
 		--target-bytecode $target --target-interface $interface $
 		--module-name-check $name $
 		--source-path $source_path $
 		$in -o $out
-    description = Calculating dependencies for $name
+    description = Scanning $name for dependencies
 
 rule plzc
     command = $path/plzc $pcflags --mode compile $
@@ -816,9 +873,15 @@ not is_empty(DupModules)
     command = $path/plzc $pcflags --mode generate-foreign $
 		--module-name-check $name $
 		--source-path $source_path $
+		--output-header $header $
 		$in -o $out
     description = Generating foreign hooks for $name
 
+rule gen_init
+    command = $path/plzgeninit $
+        $modules -o $out
+    description = Generating foreign initialisation code for $name
+
 rule plzlink
     command = $path/plzlnk $plflags -n $name -o $out $in
     description = Linking $name

src/constant.m

@@ -25,6 +25,7 @@
 :- func native_object_extension = string.
 :- func native_dylib_extension = string.
 :- func cpp_extension = string.
+:- func c_header_extension = string.
 
 :- func build_file = string.
 :- func build_directory = string.
@@ -61,6 +62,7 @@
 native_object_extension = ".o".
 native_dylib_extension = ".so".
 cpp_extension = ".cpp".
+c_header_extension = ".h".
 
 build_file = "BUILD.plz".
 build_directory = "_build".