diff --git a/CHANGES.md b/CHANGES.md index cf28c0831c..46a6856ecb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,7 @@ that follows the semantic of the backend (js or wasm) * Compiler: warn on joo_global_object * Compiler: revisit static env handling (#1708) +* Compiler: Emit index map when linking multiple js files together (#1714) * Runtime: change Sys.os_type on windows (Cygwin -> Win32) * Runtime: backtraces are really expensive, they need to be be explicitly requested at compile time (--enable with-js-error) or at startup (OCAMLRUNPARAM=b=1) diff --git a/compiler/bin-js_of_ocaml/build_fs.ml b/compiler/bin-js_of_ocaml/build_fs.ml index 3465edd4f9..5911760091 100644 --- a/compiler/bin-js_of_ocaml/build_fs.ml +++ b/compiler/bin-js_of_ocaml/build_fs.ml @@ -74,7 +74,7 @@ function jsoo_create_file_extern(name,content){ let code = Code.prepend Code.empty instr in Filename.gen_file output_file (fun chan -> let pfs_fmt = Pretty_print.to_out_channel chan in - let (_ : Source_map.t option) = + let (_ : Source_map.Standard.t option) = Driver.f ~standalone:true ~wrap_with_fun:`Iife diff --git a/compiler/bin-js_of_ocaml/cmd_arg.ml b/compiler/bin-js_of_ocaml/cmd_arg.ml index fbaf4411ce..98d1cfee8a 100644 --- a/compiler/bin-js_of_ocaml/cmd_arg.ml +++ b/compiler/bin-js_of_ocaml/cmd_arg.ml @@ -43,7 +43,7 @@ type t = { common : Jsoo_cmdline.Arg.t ; (* compile option *) profile : Driver.profile option - ; source_map : (string option * Source_map.t) option + ; source_map : (string option * Source_map.Standard.t) option ; runtime_files : string list ; no_runtime : bool ; include_runtime : bool @@ -302,13 +302,13 @@ let options = then let file, sm_output_file = match output_file with - | `Name file, _ when sourcemap_inline_in_js -> file, None - | `Name file, _ -> file, Some (chop_extension file ^ ".map") - | `Stdout, _ -> "STDIN", None + | `Name file, _ when sourcemap_inline_in_js -> Some file, None + | `Name file, _ -> Some file, Some (chop_extension file ^ ".map") + | `Stdout, _ -> None, None in Some ( sm_output_file - , { Source_map.version = 3 + , { Source_map.Standard.version = 3 ; file ; sourceroot = sourcemap_root ; sources = [] @@ -531,13 +531,13 @@ let options_runtime_only = then let file, sm_output_file = match output_file with - | `Name file, _ when sourcemap_inline_in_js -> file, None - | `Name file, _ -> file, Some (chop_extension file ^ ".map") - | `Stdout, _ -> "STDIN", None + | `Name file, _ when sourcemap_inline_in_js -> Some file, None + | `Name file, _ -> Some file, Some (chop_extension file ^ ".map") + | `Stdout, _ -> None, None in Some ( sm_output_file - , { Source_map.version = 3 + , { Source_map.Standard.version = 3 ; file ; sourceroot = sourcemap_root ; sources = [] diff --git a/compiler/bin-js_of_ocaml/cmd_arg.mli b/compiler/bin-js_of_ocaml/cmd_arg.mli index c63aae2933..ec756685b5 100644 --- a/compiler/bin-js_of_ocaml/cmd_arg.mli +++ b/compiler/bin-js_of_ocaml/cmd_arg.mli @@ -23,7 +23,7 @@ type t = { common : Jsoo_cmdline.Arg.t ; (* compile option *) profile : Driver.profile option - ; source_map : (string option * Source_map.t) option + ; source_map : (string option * Source_map.Standard.t) option ; runtime_files : string list ; no_runtime : bool ; include_runtime : bool diff --git a/compiler/bin-js_of_ocaml/compile.ml b/compiler/bin-js_of_ocaml/compile.ml index cc7b2c34c6..2a71d0f24e 100644 --- a/compiler/bin-js_of_ocaml/compile.ml +++ b/compiler/bin-js_of_ocaml/compile.ml @@ -47,6 +47,7 @@ let output_gen ~standalone ~custom_header ~build_info ~source_map output_file f match source_map, sm with | None, _ | _, None -> () | Some (output_file, _), Some sm -> + let sm = `Standard sm in let urlData = match output_file with | None -> diff --git a/compiler/bin-js_of_ocaml/link.ml b/compiler/bin-js_of_ocaml/link.ml index 0cd273b600..6b0f8add7f 100644 --- a/compiler/bin-js_of_ocaml/link.ml +++ b/compiler/bin-js_of_ocaml/link.ml @@ -23,7 +23,7 @@ open Cmdliner type t = { common : Jsoo_cmdline.Arg.t - ; source_map : (string option * Source_map.t) option + ; source_map : (string option * Source_map.Standard.t) option ; js_files : string list ; output_file : string option ; resolve_sourcemap_url : bool @@ -96,13 +96,13 @@ let options = then let file, sm_output_file = match output_file with - | Some file when sourcemap_inline_in_js -> file, None - | Some file -> file, Some (chop_extension file ^ ".map") - | None -> "STDIN", None + | Some file when sourcemap_inline_in_js -> Some file, None + | Some file -> Some file, Some (chop_extension file ^ ".map") + | None -> None, None in Some ( sm_output_file - , { Source_map.version = 3 + , { Source_map.Standard.version = 3 ; file ; sourceroot = sourcemap_root ; sources = [] diff --git a/compiler/bin-jsoo_minify/jsoo_minify.ml b/compiler/bin-jsoo_minify/jsoo_minify.ml index f607fa369b..b05145386f 100644 --- a/compiler/bin-jsoo_minify/jsoo_minify.ml +++ b/compiler/bin-jsoo_minify/jsoo_minify.ml @@ -92,7 +92,7 @@ let f { Cmd_arg.common; output_file; use_stdin; files } = if t () then (m ())#program p else p) in let p = Js_assign.program p in - let (_ : Source_map.t option) = Js_output.program pp p in + let (_ : Source_map.Standard.t option) = Js_output.program pp p in () in with_output (fun out_channel -> diff --git a/compiler/lib/driver.ml b/compiler/lib/driver.ml index 4d2aa5ba90..6dbeb4f772 100644 --- a/compiler/lib/driver.ml +++ b/compiler/lib/driver.ml @@ -713,7 +713,7 @@ let full ~standalone ~wrap_with_fun ~profile ~link ~source_map ~formatter d p = emit formatter optimized_code let full_no_source_map ~formatter ~standalone ~wrap_with_fun ~profile ~link d p = - let (_ : Source_map.t option) = + let (_ : Source_map.Standard.t option) = full ~standalone ~wrap_with_fun ~profile ~link ~source_map:None ~formatter d p in () diff --git a/compiler/lib/driver.mli b/compiler/lib/driver.mli index 91f846b989..5472edf5ae 100644 --- a/compiler/lib/driver.mli +++ b/compiler/lib/driver.mli @@ -35,11 +35,11 @@ val f : -> ?wrap_with_fun:[ `Iife | `Anonymous | `Named of string ] -> ?profile:profile -> link:[ `All | `All_from of string list | `Needed | `No ] - -> ?source_map:Source_map.t + -> ?source_map:Source_map.Standard.t -> formatter:Pretty_print.t -> Parse_bytecode.Debug.t -> Code.program - -> Source_map.t option + -> Source_map.Standard.t option val f' : ?standalone:bool diff --git a/compiler/lib/js_assign.ml b/compiler/lib/js_assign.ml index c12257d207..ca42b2ee9d 100644 --- a/compiler/lib/js_assign.ml +++ b/compiler/lib/js_assign.ml @@ -425,7 +425,7 @@ let program' (module Strategy : Strategy) p = "Some variables escaped (#%d). Use [--debug js_assign] for more info.@." (IdentSet.cardinal free) else - let (_ : Source_map.t option) = + let (_ : Source_map.Standard.t option) = Js_output.program ~accept_unnamed_var:true (Pretty_print.to_out_channel stderr) diff --git a/compiler/lib/js_output.ml b/compiler/lib/js_output.ml index 64dcdcab64..1c2fa741dc 100644 --- a/compiler/lib/js_output.ml +++ b/compiler/lib/js_output.ml @@ -1905,8 +1905,8 @@ let program ?(accept_unnamed_var = false) f ?source_map p = let names = Hashtbl.create 17 in let contents : Source_map.Source_content.t option list ref option = match source_map with - | None | Some { Source_map.sources_content = None; _ } -> None - | Some { Source_map.sources_content = Some _; _ } -> Some (ref []) + | None | Some { Source_map.Standard.sources_content = None; _ } -> None + | Some { Source_map.Standard.sources_content = Some _; _ } -> Some (ref []) in let push_mapping, get_file_index, get_name_index, source_map_enabled = let source_map_enabled = @@ -1926,7 +1926,7 @@ let program ?(accept_unnamed_var = false) f ?source_map p = loop xs ys in loop sm.sources (Option.value ~default:[] sm.sources_content); - List.iter sm.Source_map.names ~f:(fun f -> + List.iter sm.Source_map.Standard.names ~f:(fun f -> Hashtbl.add names f (Hashtbl.length names)); true in @@ -2014,7 +2014,7 @@ let program ?(accept_unnamed_var = false) f ?source_map p = { gen_line; gen_col; ori_source; ori_line; ori_col; ori_name }) in let mappings = Source_map.Mappings.encode mappings in - Some { sm with Source_map.sources; names; sources_content; mappings } + Some { sm with Source_map.Standard.sources; names; sources_content; mappings } in PP.check f; (if stats () diff --git a/compiler/lib/js_output.mli b/compiler/lib/js_output.mli index 5f8b95adb0..3b69d3de24 100644 --- a/compiler/lib/js_output.mli +++ b/compiler/lib/js_output.mli @@ -21,6 +21,6 @@ val program : ?accept_unnamed_var:bool -> Pretty_print.t - -> ?source_map:Source_map.t + -> ?source_map:Source_map.Standard.t -> Javascript.program - -> Source_map.t option + -> Source_map.Standard.t option diff --git a/compiler/lib/link_js.ml b/compiler/lib/link_js.ml index e4d3d2989e..3c17bfedae 100644 --- a/compiler/lib/link_js.ml +++ b/compiler/lib/link_js.ml @@ -154,7 +154,7 @@ type action = | Drop | Unit | Build_info of Build_info.t - | Source_map of Source_map.t + | Source_map of Source_map.Standard.t let prefix_kind line = match String.is_prefix ~prefix:sourceMappingURL line with @@ -170,6 +170,10 @@ let prefix_kind line = | true -> `Json_base64 (String.length sourceMappingURL_base64) | false -> `Url (String.length sourceMappingURL)) +let rule_out_index_map = function + | `Standard sm -> sm + | `Index _ -> failwith "unexpected index map at this stage" + let action ~resolve_sourcemap_url ~drop_source_map file line = match prefix_kind line, drop_source_map with | `Other, (true | false) -> Keep @@ -177,7 +181,8 @@ let action ~resolve_sourcemap_url ~drop_source_map file line = | `Build_info bi, _ -> Build_info bi | (`Json_base64 _ | `Url _), true -> Drop | `Json_base64 offset, false -> - Source_map (Source_map.of_string (Base64.decode_exn ~off:offset line)) + Source_map + (rule_out_index_map (Source_map.of_string (Base64.decode_exn ~off:offset line))) | `Url _, false when not resolve_sourcemap_url -> Drop | `Url offset, false -> let url = String.sub line ~pos:offset ~len:(String.length line - offset) in @@ -186,7 +191,7 @@ let action ~resolve_sourcemap_url ~drop_source_map file line = let l = in_channel_length ic in let content = really_input_string ic l in close_in ic; - Source_map (Source_map.of_string content) + Source_map (rule_out_index_map (Source_map.of_string content)) module Units : sig val read : Line_reader.t -> Unit_info.t -> Unit_info.t @@ -319,11 +324,12 @@ let link ~output ~linkall ~mklib ~toplevel ~files ~resolve_sourcemap_url ~source let sm_for_file = ref None in let ic = Line_reader.open_ file in let skip ic = Line_reader.drop ic in + let line_offset = Line_writer.lnum oc in let reloc = ref [] in let copy ic oc = let line = Line_reader.next ic in Line_writer.write ~source:ic oc line; - reloc := (Line_reader.lnum ic, Line_writer.lnum oc) :: !reloc + reloc := (Line_reader.lnum ic, Line_writer.lnum oc - line_offset) :: !reloc in let rec read () = match Line_reader.peek ic with @@ -427,7 +433,7 @@ let link ~output ~linkall ~mklib ~toplevel ~files ~resolve_sourcemap_url ~source Line_writer.write_lines oc content); (match !sm_for_file with | None -> () - | Some x -> sm := (x, !reloc) :: !sm); + | Some x -> sm := (x, !reloc, line_offset) :: !sm); match !build_info, build_info_for_file with | None, None -> () | Some _, None -> () @@ -440,32 +446,32 @@ let link ~output ~linkall ~mklib ~toplevel ~files ~resolve_sourcemap_url ~source match source_map with | None -> () | Some (file, init_sm) -> - let sm = - List.rev_map !sm ~f:(fun (sm, reloc) -> + let sections = + List.rev_map !sm ~f:(fun (sm, reloc, offset) -> let tbl = Hashtbl.create 17 in List.iter reloc ~f:(fun (a, b) -> Hashtbl.add tbl a b); - Source_map.filter_map sm ~f:(Hashtbl.find_opt tbl)) + ( { Source_map.Index.gen_line = offset; gen_column = 0 } + , `Map (Source_map.Standard.filter_map sm ~f:(Hashtbl.find_opt tbl)) )) in - (match Source_map.merge (init_sm :: sm) with - | None -> () - | Some sm -> ( - (* preserve some info from [init_sm] *) - let sm = - { sm with - version = init_sm.version - ; file = init_sm.file - ; sourceroot = init_sm.sourceroot - } - in - match file with - | None -> - let data = Source_map.to_string sm in - let s = sourceMappingURL_base64 ^ Base64.encode_exn data in - Line_writer.write oc s - | Some file -> - Source_map.to_file sm file; - let s = sourceMappingURL ^ Filename.basename file in - Line_writer.write oc s)); + let sm = + { Source_map.Index.version = init_sm.Source_map.Standard.version + ; file = init_sm.file + ; sections = + (* preserve some info from [init_sm] *) + List.map sections ~f:(fun (ofs, `Map sm) -> + ofs, `Map { sm with sourceroot = init_sm.sourceroot }) + } + in + let sm = `Index sm in + (match file with + | None -> + let data = Source_map.to_string sm in + let s = sourceMappingURL_base64 ^ Base64.encode_exn data in + Line_writer.write oc s + | Some file -> + Source_map.to_file sm file; + let s = sourceMappingURL ^ Filename.basename file in + Line_writer.write oc s); if times () then Format.eprintf " sourcemap: %a@." Timer.print t let link ~output ~linkall ~mklib ~toplevel ~files ~resolve_sourcemap_url ~source_map = diff --git a/compiler/lib/link_js.mli b/compiler/lib/link_js.mli index fbb70ef824..7e21a9fbbf 100644 --- a/compiler/lib/link_js.mli +++ b/compiler/lib/link_js.mli @@ -24,5 +24,5 @@ val link : -> toplevel:bool -> files:string list -> resolve_sourcemap_url:bool - -> source_map:(string option * Source_map.t) option + -> source_map:(string option * Source_map.Standard.t) option -> unit diff --git a/compiler/lib/source_map.ml b/compiler/lib/source_map.ml index a2d30d9f85..1ae1b2f099 100644 --- a/compiler/lib/source_map.ml +++ b/compiler/lib/source_map.ml @@ -210,151 +210,13 @@ module Mappings = struct readline 1 0 [] end -type t = - { version : int - ; file : string - ; sourceroot : string option - ; sources : string list - ; sources_content : Source_content.t option list option - ; names : string list - ; mappings : Mappings.t - } - -let empty ~filename = - { version = 3 - ; file = filename - ; sourceroot = None - ; sources = [] - ; sources_content = None - ; names = [] - ; mappings = Mappings.empty - } - -let maps ~sources_offset ~names_offset x = - match x with - | Gen _ -> x - | Gen_Ori { gen_line; gen_col; ori_source; ori_line; ori_col } -> - let ori_source = ori_source + sources_offset in - Gen_Ori { gen_line; gen_col; ori_source; ori_line; ori_col } - | Gen_Ori_Name { gen_line; gen_col; ori_source; ori_line; ori_col; ori_name } -> - let ori_source = ori_source + sources_offset in - let ori_name = ori_name + names_offset in - Gen_Ori_Name { gen_line; gen_col; ori_source; ori_line; ori_col; ori_name } - -let filter_map sm ~f = - let a = Array.of_list (Mappings.decode sm.mappings) in - Array.stable_sort - ~cmp:(fun t1 t2 -> - match compare (gen_line t1) (gen_line t2) with - | 0 -> compare (gen_col t1) (gen_col t2) - | n -> n) - a; - let l = Array.to_list a |> List.group ~f:(fun a b -> gen_line a = gen_line b) in - - let rec loop acc mapping = - match mapping with - | [] -> List.rev acc - | x :: xs -> - let gen_line = gen_line (List.hd x) in - let acc = - match f gen_line with - | None -> acc - | Some gen_line -> - List.rev_append_map - x - ~f:(function - | Gen { gen_line = _; gen_col } -> Gen { gen_line; gen_col } - | Gen_Ori { gen_line = _; gen_col; ori_source; ori_line; ori_col } -> - Gen_Ori { gen_line; gen_col; ori_source; ori_line; ori_col } - | Gen_Ori_Name - { gen_line = _; gen_col; ori_source; ori_line; ori_col; ori_name } - -> - Gen_Ori_Name - { gen_line; gen_col; ori_source; ori_line; ori_col; ori_name }) - acc - in - loop acc xs - in - let mappings = loop [] l in - { sm with mappings = Mappings.encode mappings } - -let merge = function - | [] -> None - | _ :: _ as l -> - let rec loop acc_rev mappings_rev ~sources_offset ~names_offset l = - match l with - | [] -> acc_rev, mappings_rev - | sm :: rest -> - let acc_rev, mappings_rev = - ( { acc_rev with - sources = List.rev_append sm.sources acc_rev.sources - ; names = List.rev_append sm.names acc_rev.names - ; sources_content = - (match sm.sources_content, acc_rev.sources_content with - | Some x, Some acc_rev -> Some (List.rev_append x acc_rev) - | None, _ | _, None -> None) - ; mappings = Mappings.empty - } - , List.rev_append_map - ~f:(maps ~sources_offset ~names_offset) - (Mappings.decode sm.mappings) - mappings_rev ) - in - loop - acc_rev - mappings_rev - ~sources_offset:(sources_offset + List.length sm.sources) - ~names_offset:(names_offset + List.length sm.names) - rest - in - let acc_rev, mappings_rev = - loop - { (empty ~filename:"") with sources_content = Some [] } - [] - ~sources_offset:0 - ~names_offset:0 - l - in - Some - { acc_rev with - mappings = Mappings.encode (List.rev mappings_rev) - ; sources = List.rev acc_rev.sources - ; names = List.rev acc_rev.names - ; sources_content = Option.map ~f:List.rev acc_rev.sources_content - } - -(* IO *) - -let json t = - let rewrite_path path = - if Filename.is_relative path - then path - else - match Build_path_prefix_map.get_build_path_prefix_map () with - | Some map -> Build_path_prefix_map.rewrite map path - | None -> path - in - let stringlit s = `Stringlit (Yojson.Safe.to_string (`String s)) in - `Assoc - [ "version", `Intlit (string_of_int t.version) - ; "file", stringlit (rewrite_path t.file) - ; ( "sourceRoot" - , stringlit - (match t.sourceroot with - | None -> "" - | Some s -> rewrite_path s) ) - ; "names", `List (List.map t.names ~f:(fun s -> stringlit s)) - ; "sources", `List (List.map t.sources ~f:(fun s -> stringlit (rewrite_path s))) - ; "mappings", stringlit (Mappings.to_string t.mappings) - ; ( "sourcesContent" - , `List - (match t.sources_content with - | None -> [] - | Some l -> - List.map l ~f:(function - | None -> `Null - | Some x -> Source_content.to_json x)) ) - ] +let rewrite_path path = + if Filename.is_relative path + then path + else + match Build_path_prefix_map.get_build_path_prefix_map () with + | Some map -> Build_path_prefix_map.rewrite map path + | None -> path let invalid () = invalid_arg "Source_map.of_json" @@ -394,52 +256,324 @@ let list_stringlit_opt name rest = | _ -> invalid () with Not_found -> None -let of_json (json : Yojson.Raw.t) = - match json with - | `Assoc (("version", `Intlit version) :: rest) when int_of_string version = 3 -> - let string name json = Option.map ~f:string_of_stringlit (stringlit name json) in - let file = - match string "file" rest with - | None -> "" - | Some s -> s - in - let sourceroot = string "sourceRoot" rest in - let names = - match list_stringlit "names" rest with - | None -> [] - | Some l -> List.map ~f:string_of_stringlit l - in - let sources = - match list_stringlit "sources" rest with - | None -> [] - | Some l -> List.map ~f:string_of_stringlit l - in - let sources_content = - match list_stringlit_opt "sourcesContent" rest with - | None -> None - | Some l -> - Some - (List.map l ~f:(function - | None -> None - | Some s -> Some (Source_content.of_stringlit s))) - in - let mappings = - match string "mappings" rest with - | None -> Mappings.empty - | Some s -> Mappings.of_string s - in - { version = int_of_float (float_of_string version) - ; file - ; sourceroot - ; names - ; sources_content - ; sources - ; mappings - } - | _ -> invalid () +module Standard = struct + type t = + { version : int + ; file : string option + ; sourceroot : string option + ; sources : string list + ; sources_content : Source_content.t option list option + ; names : string list + ; mappings : Mappings.t + } + + let empty = + { version = 3 + ; file = None + ; sourceroot = None + ; sources = [] + ; sources_content = None + ; names = [] + ; mappings = Mappings.empty + } + + let maps ~sources_offset ~names_offset x = + match x with + | Gen _ -> x + | Gen_Ori { gen_line; gen_col; ori_source; ori_line; ori_col } -> + let ori_source = ori_source + sources_offset in + Gen_Ori { gen_line; gen_col; ori_source; ori_line; ori_col } + | Gen_Ori_Name { gen_line; gen_col; ori_source; ori_line; ori_col; ori_name } -> + let ori_source = ori_source + sources_offset in + let ori_name = ori_name + names_offset in + Gen_Ori_Name { gen_line; gen_col; ori_source; ori_line; ori_col; ori_name } + + let filter_map sm ~f = + let a = Array.of_list (Mappings.decode sm.mappings) in + Array.stable_sort + ~cmp:(fun t1 t2 -> + match compare (gen_line t1) (gen_line t2) with + | 0 -> compare (gen_col t1) (gen_col t2) + | n -> n) + a; + let l = Array.to_list a |> List.group ~f:(fun a b -> gen_line a = gen_line b) in + + let rec loop acc mapping = + match mapping with + | [] -> List.rev acc + | x :: xs -> + let gen_line = gen_line (List.hd x) in + let acc = + match f gen_line with + | None -> acc + | Some gen_line -> + List.rev_append_map + x + ~f:(function + | Gen { gen_line = _; gen_col } -> Gen { gen_line; gen_col } + | Gen_Ori { gen_line = _; gen_col; ori_source; ori_line; ori_col } -> + Gen_Ori { gen_line; gen_col; ori_source; ori_line; ori_col } + | Gen_Ori_Name + { gen_line = _; gen_col; ori_source; ori_line; ori_col; ori_name } + -> + Gen_Ori_Name + { gen_line; gen_col; ori_source; ori_line; ori_col; ori_name }) + acc + in + loop acc xs + in + let mappings = loop [] l in + { sm with mappings = Mappings.encode mappings } + + let merge = function + | [] -> None + | _ :: _ as l -> + let rec loop acc_rev mappings_rev ~sources_offset ~names_offset l = + match l with + | [] -> acc_rev, mappings_rev + | sm :: rest -> + let acc_rev, mappings_rev = + ( { acc_rev with + sources = List.rev_append sm.sources acc_rev.sources + ; names = List.rev_append sm.names acc_rev.names + ; sources_content = + (match sm.sources_content, acc_rev.sources_content with + | Some x, Some acc_rev -> Some (List.rev_append x acc_rev) + | None, _ | _, None -> None) + ; mappings = Mappings.empty + } + , List.rev_append_map + ~f:(maps ~sources_offset ~names_offset) + (Mappings.decode sm.mappings) + mappings_rev ) + in + loop + acc_rev + mappings_rev + ~sources_offset:(sources_offset + List.length sm.sources) + ~names_offset:(names_offset + List.length sm.names) + rest + in + let acc_rev, mappings_rev = + loop + { empty with sources_content = Some [] } + [] + ~sources_offset:0 + ~names_offset:0 + l + in + Some + { acc_rev with + mappings = Mappings.encode (List.rev mappings_rev) + ; sources = List.rev acc_rev.sources + ; names = List.rev acc_rev.names + ; sources_content = Option.map ~f:List.rev acc_rev.sources_content + } + + let json t = + let stringlit s = `Stringlit (Yojson.Safe.to_string (`String s)) in + `Assoc + (List.filter_map + ~f:(fun (name, v) -> + match v with + | None -> None + | Some v -> Some (name, v)) + [ "version", Some (`Intlit (string_of_int t.version)) + ; ( "file" + , match t.file with + | None -> None + | Some file -> Some (stringlit (rewrite_path file)) ) + ; ( "sourceRoot" + , match t.sourceroot with + | None -> None + | Some s -> Some (stringlit (rewrite_path s)) ) + ; "names", Some (`List (List.map t.names ~f:(fun s -> stringlit s))) + ; ( "sources" + , Some (`List (List.map t.sources ~f:(fun s -> stringlit (rewrite_path s)))) ) + ; "mappings", Some (stringlit (Mappings.to_string t.mappings)) + ; ( "sourcesContent" + , match t.sources_content with + | None -> None + | Some l -> + Some + (`List + (List.map l ~f:(function + | None -> `Null + | Some x -> Source_content.to_json x))) ) + ]) + + let of_json (json : Yojson.Raw.t) = + match json with + | `Assoc (("version", `Intlit version) :: rest) when int_of_string version = 3 -> + let string name json = Option.map ~f:string_of_stringlit (stringlit name json) in + let file = string "file" rest in + let sourceroot = string "sourceRoot" rest in + let names = + match list_stringlit "names" rest with + | None -> [] + | Some l -> List.map ~f:string_of_stringlit l + in + let sources = + match list_stringlit "sources" rest with + | None -> [] + | Some l -> List.map ~f:string_of_stringlit l + in + let sources_content = + match list_stringlit_opt "sourcesContent" rest with + | None -> None + | Some l -> + Some + (List.map l ~f:(function + | None -> None + | Some s -> Some (Source_content.of_stringlit s))) + in + let mappings = + match string "mappings" rest with + | None -> Mappings.empty + | Some s -> Mappings.of_string s + in + { version = int_of_float (float_of_string version) + ; file + ; sourceroot + ; names + ; sources_content + ; sources + ; mappings + } + | _ -> invalid () + + let to_string m = Yojson.Raw.to_string (json m) + + let to_file m file = Yojson.Raw.to_file file (json m) +end +(* IO *) + +module Index = struct + type offset = + { gen_line : int + ; gen_column : int + } + + type t = + { version : int + ; file : string option + ; sections : (offset * [ `Map of Standard.t ]) list + } + + let json t = + let stringlit s = `Stringlit (Yojson.Safe.to_string (`String s)) in + `Assoc + (List.filter_map + ~f:(fun (name, v) -> + match v with + | None -> None + | Some v -> Some (name, v)) + [ "version", Some (`Intlit (string_of_int t.version)) + ; ( "file" + , match t.file with + | None -> None + | Some file -> Some (stringlit (rewrite_path file)) ) + ; ( "sections" + , Some + (`List + (List.map + ~f:(fun ({ gen_line; gen_column }, `Map sm) -> + `Assoc + [ ( "offset" + , `Assoc + [ "line", `Intlit (string_of_int gen_line) + ; "column", `Intlit (string_of_int gen_column) + ] ) + ; "map", Standard.json sm + ]) + t.sections)) ) + ]) + + let intlit ~errmsg name json = + match List.assoc name json with + | `Intlit i -> int_of_string i + | _ -> invalid_arg errmsg + | exception Not_found -> invalid_arg errmsg + + let section_of_json : Yojson.Raw.t -> offset * [ `Map of Standard.t ] = function + | `Assoc json -> + let offset = + match List.assoc "offset" json with + | `Assoc fields -> + let gen_line = + intlit + "line" + fields + ~errmsg: + "Source_map_io.Index.of_json: field 'line' absent or invalid from \ + section" + in + let gen_column = + intlit + "column" + fields + ~errmsg: + "Source_map_io.Index.of_json: field 'column' absent or invalid from \ + section" + in + { gen_line; gen_column } + | _ -> + invalid_arg "Source_map_io.Index.of_json: 'offset' field of unexpected type" + in + (match List.assoc "url" json with + | _ -> + invalid_arg + "Source_map_io.Index.of_json: URLs in index maps are not currently \ + supported" + | exception Not_found -> ()); + let map = + try Standard.of_json (List.assoc "map" json) with + | Not_found -> invalid_arg "Source_map_io.Index.of_json: field 'map' absent" + | Invalid_argument _ -> + invalid_arg "Source_map_io.Index.of_json: invalid sub-map object" + in + offset, `Map map + | _ -> invalid_arg "Source_map_io.Index.of_json: section of unexpected type" + + let of_json = function + | `Assoc fields -> ( + let string name json = Option.map ~f:string_of_stringlit (stringlit name json) in + let file = string "file" fields in + match List.assoc "sections" fields with + | `List sections -> + let sections = List.map ~f:section_of_json sections in + { version = 3; file; sections } + | _ -> invalid_arg "Source_map_io.Index.of_json: `sections` is not an array" + | exception Not_found -> + invalid_arg "Source_map_io.Index.of_json: no `sections` field") + | _ -> invalid_arg "Source_map_io.of_json: map is not an object" + + let to_string m = Yojson.Raw.to_string (json m) + + let to_file m file = Yojson.Raw.to_file file (json m) +end + +type t = + [ `Standard of Standard.t + | `Index of Index.t + ] + +let of_json = function + | `Assoc fields as json -> ( + match List.assoc "sections" fields with + | _ -> `Index (Index.of_json json) + | exception Not_found -> `Standard (Standard.of_json json)) + | _ -> invalid_arg "Source_map_io.of_json: map is not an object" let of_string s = of_json (Yojson.Raw.from_string s) -let to_string m = Yojson.Raw.to_string (json m) +let of_file f = of_json (Yojson.Raw.from_file f) -let to_file m file = Yojson.Raw.to_file file (json m) +let to_string = function + | `Standard m -> Standard.to_string m + | `Index i -> Index.to_string i + +let to_file x f = + match x with + | `Standard m -> Standard.to_file m f + | `Index i -> Index.to_file i f diff --git a/compiler/lib/source_map.mli b/compiler/lib/source_map.mli index a5d6278329..c00e8872b9 100644 --- a/compiler/lib/source_map.mli +++ b/compiler/lib/source_map.mli @@ -67,27 +67,54 @@ module Mappings : sig function is mostly a no-op and is very cheap. *) end -type t = - { version : int - ; file : string - ; sourceroot : string option - ; sources : string list - ; sources_content : Source_content.t option list option - ; names : string list - ; mappings : Mappings.t - (** Left uninterpreted, since most useful operations can be performed efficiently +module Standard : sig + type t = + { version : int + ; file : string option + ; sourceroot : string option + ; sources : string list + ; sources_content : Source_content.t option list option + ; names : string list + ; mappings : Mappings.t + (** Left uninterpreted, since most useful operations can be performed efficiently directly on the encoded form, and a full decoding can be costly for big sourcemaps. *) - } + } + + val filter_map : t -> f:(int -> int option) -> t + (** If [f l] returns [Some l'], map line [l] to [l'] (in the generated file) in + the returned debug mappings. If [f l] returns [None], remove debug mappings + which concern line [l] of the generated file. *) -val filter_map : t -> f:(int -> int option) -> t + val merge : t list -> t option + (** Merge two lists of debug mappings. The time cost of the merge is more than + linear in function of the size of the input mappings. *) + + val empty : t +end -val merge : t list -> t option +module Index : sig + type offset = + { gen_line : int + ; gen_column : int + } + + type nonrec t = + { version : int + ; file : string option + ; sections : (offset * [ `Map of Standard.t ]) list + } +end -val empty : filename:string -> t +type t = + [ `Standard of Standard.t + | `Index of Index.t + ] val to_string : t -> string val to_file : t -> string -> unit val of_string : string -> t + +val of_file : string -> t diff --git a/compiler/tests-compiler/build_path_prefix_map.ml b/compiler/tests-compiler/build_path_prefix_map.ml index 6e2ce10fdc..29f9f7c8c3 100644 --- a/compiler/tests-compiler/build_path_prefix_map.ml +++ b/compiler/tests-compiler/build_path_prefix_map.ml @@ -20,6 +20,12 @@ open Js_of_ocaml_compiler.Stdlib open Util let%expect_test _ = + let print_section (sm : Js_of_ocaml_compiler.Source_map.Standard.t) = + Printf.printf "file: %s\n" (Option.value ~default:"" sm.file); + Printf.printf "sourceRoot: %s\n" (Option.value ~default:"" sm.sourceroot); + Printf.printf "sources:\n"; + List.iter sm.sources ~f:(fun source -> Printf.printf "- %s\n" (normalize_path source)) + in with_temp_dir ~f:(fun () -> let name = "test.ml" in Filetype.write_file name "let id x = x"; @@ -29,17 +35,14 @@ let%expect_test _ = |> compile_cmo_to_javascript ~sourcemap:true ~pretty:false |> extract_sourcemap |> function - | Some (sm : Js_of_ocaml_compiler.Source_map.t) -> - Printf.printf "file: %s\n" sm.file; - Printf.printf "sourceRoot: %s\n" (Option.value ~default:"" sm.sourceroot); - Printf.printf "sources:\n"; - List.iter sm.sources ~f:(fun source -> - Printf.printf "- %s\n" (normalize_path source)) + | Some (`Standard (sm : Js_of_ocaml_compiler.Source_map.Standard.t)) -> + print_section sm + | Some (`Index i) -> List.iter i.sections ~f:(fun (_, `Map sm) -> print_section sm) | None -> failwith "no sourcemap generated!"); [%expect {| - file: test.js - sourceRoot: - sources: - - /dune-root/test.ml + file: test.js + sourceRoot: + sources: + - /dune-root/test.ml |}] diff --git a/compiler/tests-compiler/macro.ml b/compiler/tests-compiler/macro.ml index fe000c8e77..843583c4fa 100644 --- a/compiler/tests-compiler/macro.ml +++ b/compiler/tests-compiler/macro.ml @@ -31,7 +31,9 @@ let print_macro_transformed source = in let parsed = Util.parse_js source in let transformed, _ = Jsoo.Macro.f ~flags:false parsed in - let (_ : Jsoo.Source_map.t option) = Jsoo.Js_output.program pp transformed in + let (_ : Jsoo.Source_map.Standard.t option) = + Jsoo.Js_output.program pp transformed + in print_endline (Buffer.contents buffer)) let print_macro_transformed source = diff --git a/compiler/tests-compiler/sourcemap.ml b/compiler/tests-compiler/sourcemap.ml index 6625996539..2aa2cdb9e1 100644 --- a/compiler/tests-compiler/sourcemap.ml +++ b/compiler/tests-compiler/sourcemap.ml @@ -20,7 +20,7 @@ open Js_of_ocaml_compiler open Stdlib open Util -let print_mapping (sm : Source_map.t) = +let print_mapping ~line_offset ~col_offset (sm : Source_map.Standard.t) = let sources = Array.of_list sm.sources in let _names = Array.of_list sm.names in let mappings = Source_map.Mappings.decode sm.mappings in @@ -34,9 +34,10 @@ let print_mapping (sm : Source_map.t) = (file ori_source) ori_line ori_col - gen_line - gen_col - | Gen { gen_line; gen_col } -> Printf.printf "null -> %d:%d\n" gen_line gen_col) + (gen_line + line_offset) + (gen_col + col_offset) + | Gen { gen_line; gen_col } -> + Printf.printf "null -> %d:%d\n" (gen_line + line_offset) (gen_col + col_offset)) let%expect_test _ = with_temp_dir ~f:(fun () -> @@ -55,7 +56,14 @@ let%expect_test _ = print_file (Filetype.path_of_js_file js_file); match extract_sourcemap js_file with | None -> Printf.printf "No sourcemap found\n" - | Some sm -> print_mapping sm); + | Some (`Standard sm) -> print_mapping ~line_offset:0 ~col_offset:0 sm + | Some (`Index i) -> + List.iter + i.sections + ~f:(fun + ( ({ gen_line; gen_column } : Js_of_ocaml_compiler.Source_map.Index.offset) + , `Map sm ) + -> print_mapping ~line_offset:gen_line ~col_offset:gen_column sm)); [%expect {| $ cat "test.ml" @@ -125,31 +133,35 @@ let%expect_test _ = Source_map.Gen_Ori { gen_line; gen_col; ori_source = source; ori_line = line; ori_col = col } in - let s1 : Source_map.t = - { (Source_map.empty ~filename:"1.map") with + let s1 : Source_map.Standard.t = + { Source_map.Standard.empty with names = [ "na"; "nb"; "nc" ] ; sources = [ "sa"; "sb" ] ; mappings = Source_map.Mappings.encode [ gen (1, 1) (10, 10) 0; gen (3, 3) (20, 20) 1 ] } in - let s2 : Source_map.t = - { (Source_map.empty ~filename:"2.map") with + let s2 : Source_map.Standard.t = + { Source_map.Standard.empty with names = [ "na2"; "nb2" ] ; sources = [ "sa2" ] ; mappings = Source_map.Mappings.encode [ gen (3, 3) (5, 5) 0 ] } in - let m = Source_map.merge [ s1; Source_map.filter_map s2 ~f:(fun x -> Some (x + 20)) ] in + let m = + Source_map.Standard.merge + [ s1; Source_map.Standard.filter_map s2 ~f:(fun x -> Some (x + 20)) ] + in (match m with | None -> () | Some sm -> - let encoded_mappings = sm.Source_map.mappings in + let encoded_mappings = sm.Source_map.Standard.mappings in print_endline (Source_map.Mappings.to_string encoded_mappings); - print_mapping sm); + print_mapping ~line_offset:0 ~col_offset:0 sm); [%expect {| CASU;;GCUU;;;;;;;;;;;;;;;;;;;;GCff sa:10:10 -> 1:1 sb:20:20 -> 3:3 - sa2:5:5 -> 23:3 |}] + sa2:5:5 -> 23:3 + |}] diff --git a/compiler/tests-compiler/util/util.ml b/compiler/tests-compiler/util/util.ml index 946b1472ec..965e3e0519 100644 --- a/compiler/tests-compiler/util/util.ml +++ b/compiler/tests-compiler/util/util.ml @@ -421,7 +421,7 @@ let program_to_string ?(compact = false) p = let buffer = Buffer.create 17 in let pp = Jsoo.Pretty_print.to_buffer buffer in Jsoo.Pretty_print.set_compact pp compact; - let (_ : Jsoo.Source_map.t option) = Jsoo.Js_output.program pp p in + let (_ : Jsoo.Source_map.Standard.t option) = Jsoo.Js_output.program pp p in (* This final comment should help to keep merge-confict inside {| .. |}, allowing to resolve confict with [dune promote]. *) Buffer.add_string buffer "//end\n"; diff --git a/compiler/tests-sourcemap/dump.reference b/compiler/tests-sourcemap/dump.reference index d9a8cf150c..be88f6df65 100644 --- a/compiler/tests-sourcemap/dump.reference +++ b/compiler/tests-sourcemap/dump.reference @@ -1,7 +1,7 @@ sourcemap for test.bc.js -b.ml:1:4 -> 12: function <>f(x){return x - 1 | 0;} -b.ml:1:6 -> 14: function f(<>x){return x - 1 | 0;} -b.ml:1:10 -> 17: function f(x){<>return x - 1 | 0;} -b.ml:1:6 -> 24: function f(x){return <>x - 1 | 0;} -b.ml:1:15 -> 34: function f(x){return x - 1 | 0;<>} -b.ml:1:4 -> 23: var Testlib_B = [0, <>f]; +/my/sourceRoot#b.ml:1:4 -> 12: function <>f(x){return x - 1 | 0;} +/my/sourceRoot#b.ml:1:6 -> 14: function f(<>x){return x - 1 | 0;} +/my/sourceRoot#b.ml:1:10 -> 17: function f(x){<>return x - 1 | 0;} +/my/sourceRoot#b.ml:1:6 -> 24: function f(x){return <>x - 1 | 0;} +/my/sourceRoot#b.ml:1:15 -> 34: function f(x){return x - 1 | 0;<>} +/my/sourceRoot#b.ml:1:4 -> 23: var Testlib_B = [0, <>f]; diff --git a/compiler/tests-sourcemap/dump_sourcemap.ml b/compiler/tests-sourcemap/dump_sourcemap.ml index 27dfa136f3..59bb828bd9 100644 --- a/compiler/tests-sourcemap/dump_sourcemap.ml +++ b/compiler/tests-sourcemap/dump_sourcemap.ml @@ -36,7 +36,7 @@ let extract_sourcemap lines = Some (Source_map.of_string content) | _ -> None -let print_mapping lines (sm : Source_map.t) = +let print_mapping lines ?(line_offset = 0) (sm : Source_map.Standard.t) = let lines = Array.of_list lines in let sources = Array.of_list sm.sources in let _names = Array.of_list sm.names in @@ -62,15 +62,30 @@ let print_mapping lines (sm : Source_map.t) = -> ( match file ori_source with | "a.ml" | "b.ml" | "c.ml" | "d.ml" -> + let root = + match sm.sourceroot with + | None -> "" + | Some root -> root ^ "#" + in Printf.printf - "%s:%d:%d -> %d:%s\n" + "%s%s:%d:%d -> %d:%s\n" + root (file ori_source) ori_line ori_col gen_col - (mark gen_col lines.(gen_line - 1)) + (mark gen_col lines.(gen_line - 1 + line_offset)) | _ -> ())) +let print_sourcemap lines = function + | `Standard sm -> print_mapping lines sm + | `Index l -> + List.iter + l.Source_map.Index.sections + ~f:(fun (Source_map.Index.{ gen_line; gen_column }, `Map sm) -> + assert (gen_column = 0); + print_mapping lines ~line_offset:gen_line sm) + let files = Sys.argv |> Array.to_list |> List.tl let () = @@ -80,4 +95,4 @@ let () = | None -> Printf.printf "not sourcemap for %s\n" f | Some sm -> Printf.printf "sourcemap for %s\n" f; - print_mapping lines sm) + print_sourcemap lines sm) diff --git a/compiler/tests-sourcemap/dune b/compiler/tests-sourcemap/dune index e690f5a3ee..6c96b3aef1 100644 --- a/compiler/tests-sourcemap/dune +++ b/compiler/tests-sourcemap/dune @@ -7,6 +7,9 @@ (name test) (modules test) (modes js) + (js_of_ocaml + (link_flags + (:standard --source-map-root /my/sourceRoot))) (libraries testlib)) (library