diff --git a/CHANGES.md b/CHANGES.md index 56919dd197..4362e7609a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,8 @@ +# dev (2024-??) - ?? + +## Features/Changes +* Compiler: optimize sourcemap processing, improving linking performance #1617 + # 5.8.2 (2024-05-26) - Luc ## Bug fixes diff --git a/compiler/bin-js_of_ocaml/cmd_arg.ml b/compiler/bin-js_of_ocaml/cmd_arg.ml index 5b9eff75b1..56dcd74491 100644 --- a/compiler/bin-js_of_ocaml/cmd_arg.ml +++ b/compiler/bin-js_of_ocaml/cmd_arg.ml @@ -312,9 +312,10 @@ let options = ; file ; sourceroot = sourcemap_root ; sources = [] - ; sources_content = (if sourcemap_don't_inline_content then None else Some []) + ; sources_contents = + (if sourcemap_don't_inline_content then None else Some []) ; names = [] - ; mappings = [] + ; mappings = Source_map.Mappings.empty } ) else None in @@ -551,9 +552,10 @@ let options_runtime_only = ; file ; sourceroot = sourcemap_root ; sources = [] - ; sources_content = (if sourcemap_don't_inline_content then None else Some []) + ; sources_contents = + (if sourcemap_don't_inline_content then None else Some []) ; names = [] - ; mappings = [] + ; mappings = Source_map.Mappings.empty } ) else None in diff --git a/compiler/bin-js_of_ocaml/link.ml b/compiler/bin-js_of_ocaml/link.ml index 090913d20b..2719e7fb10 100644 --- a/compiler/bin-js_of_ocaml/link.ml +++ b/compiler/bin-js_of_ocaml/link.ml @@ -106,9 +106,9 @@ let options = ; file ; sourceroot = sourcemap_root ; sources = [] - ; sources_content = Some [] + ; sources_contents = Some [] ; names = [] - ; mappings = [] + ; mappings = Source_map.Mappings.empty } ) else None in diff --git a/compiler/lib/js_output.ml b/compiler/lib/js_output.ml index 33d49bb7da..af093fb05e 100644 --- a/compiler/lib/js_output.ml +++ b/compiler/lib/js_output.ml @@ -1903,10 +1903,10 @@ let program ?(accept_unnamed_var = false) f ?source_map p = let temp_mappings = ref [] in let files = Hashtbl.create 17 in let names = Hashtbl.create 17 in - let contents : string option list ref option = + let contents : Source_map.Source_text.t list ref option = match source_map with - | None | Some { Source_map.sources_content = None; _ } -> None - | Some { Source_map.sources_content = Some _; _ } -> Some (ref []) + | None | Some { Source_map.sources_contents = None; _ } -> None + | Some { Source_map.sources_contents = Some _; _ } -> Some (ref []) in let push_mapping, get_file_index, get_name_index, source_map_enabled = let source_map_enabled = @@ -1918,27 +1918,28 @@ let program ?(accept_unnamed_var = false) f ?source_map p = | [], _ -> () | x :: xs, [] -> Hashtbl.add files x (Hashtbl.length files); - Option.iter contents ~f:(fun r -> r := None :: !r); + Option.iter contents ~f:(fun r -> r := Source_map.Source_text.empty :: !r); loop xs [] | x :: xs, y :: ys -> Hashtbl.add files x (Hashtbl.length files); Option.iter contents ~f:(fun r -> r := y :: !r); loop xs ys in - loop sm.sources (Option.value ~default:[] sm.sources_content); + loop sm.sources (Option.value ~default:[] sm.sources_contents); List.iter sm.Source_map.names ~f:(fun f -> Hashtbl.add names f (Hashtbl.length names)); true in let find_source file = - match Builtins.find file with - | Some f -> Some (Builtins.File.content f) - | None -> - if Sys.file_exists file && not (Sys.is_directory file) - then - let content = Fs.read_file file in - Some content - else None + Source_map.Source_text.encode + (match Builtins.find file with + | Some f -> Some (Builtins.File.content f) + | None -> + if Sys.file_exists file && not (Sys.is_directory file) + then + let content = Fs.read_file file in + Some content + else None) in ( (fun pos m -> temp_mappings := (pos, m) :: !temp_mappings) , (fun file -> @@ -1979,10 +1980,10 @@ let program ?(accept_unnamed_var = false) f ?source_map p = | Some sm -> let sources = hashtbl_to_list files in let names = hashtbl_to_list names in - let sources_content = - match contents with - | None -> None - | Some r -> Some (List.rev !r) + let sources_contents = + let open Option.Syntax in + let* r = contents in + Option.return (List.rev !r) in let sources = List.map sources ~f:(fun filename -> @@ -1990,8 +1991,9 @@ let program ?(accept_unnamed_var = false) f ?source_map p = | None -> filename | Some _ -> Filename.concat "/builtin" filename) in + let sm_mappings = Source_map.Mappings.decode sm.mappings in let mappings = - List.rev_append_map !temp_mappings sm.mappings ~f:(fun (pos, m) -> + List.rev_append_map !temp_mappings sm_mappings ~f:(fun (pos, m) -> let gen_line = pos.PP.p_line + 1 in let gen_col = pos.PP.p_col in match m with @@ -2006,7 +2008,8 @@ let program ?(accept_unnamed_var = false) f ?source_map p = Source_map.Gen_Ori_Name { gen_line; gen_col; ori_source; ori_line; ori_col; ori_name }) in - Some { sm with Source_map.sources; names; sources_content; mappings } + let mappings = Source_map.Mappings.encode mappings in + Some { sm with Source_map.sources; names; sources_contents; mappings } in PP.check f; (if stats () diff --git a/compiler/lib/link_js.ml b/compiler/lib/link_js.ml index 3fb7f14035..a2bdd19d59 100644 --- a/compiler/lib/link_js.ml +++ b/compiler/lib/link_js.ml @@ -35,7 +35,9 @@ module Line_reader : sig val peek : t -> string option - val drop : t -> unit + val drop : t -> drop_action:(unit -> unit) -> unit + (** [drop_action] is the function to call if a line was effectively dropped + (if EOF is reached, this function may return without dropping a line). *) val close : t -> unit @@ -78,15 +80,17 @@ end = struct Some s with End_of_file -> None) - let drop t = + let drop t ~drop_action = match t.next with | Some _ -> t.next <- None; - t.lnum <- t.lnum + 1 + t.lnum <- t.lnum + 1; + drop_action () | None -> ( try let (_ : string) = input_line t.ic in - t.lnum <- t.lnum + 1 + t.lnum <- t.lnum + 1; + drop_action () with End_of_file -> ()) let lnum t = t.lnum @@ -99,9 +103,15 @@ module Line_writer : sig val of_channel : out_channel -> t - val write : ?source:Line_reader.t -> t -> string -> unit + val write : ?source:Line_reader.t -> t -> add:(int -> unit) -> string -> unit + (** [write ~source t s ~add] writes [s], followed by a newline, and calls + [edit], giving it in argument the number of "line number" pragma lines + emitted before writing [s]. *) - val write_lines : ?source:Line_reader.t -> t -> string -> unit + val write_lines : ?source:Line_reader.t -> t -> total:(int -> unit) -> string -> unit + (** [write_lines ~source t s ~total] writes all lines in [s], ensures that the last + line ends with a newline, and calls [total], giving it in argument the total number + of lines written, including "line number" pragma lines. *) val lnum : t -> int end = struct @@ -113,7 +123,7 @@ end = struct let of_channel oc = { oc; source = None; lnum = 0 } - let write ?source t s = + let write ?source t ~add s = let source = match source with | None -> None @@ -130,21 +140,24 @@ end = struct | Some (fname1, lnum1), Some (fname2, lnum2) -> if String.equal fname1 fname2 && lnum1 + 1 = lnum2 then 0 else emit fname2 lnum2 in + add lnum_off; output_string t.oc s; output_string t.oc "\n"; let lnum_off = lnum_off + 1 in t.source <- source; t.lnum <- t.lnum + lnum_off - let write_lines ?source t lines = + let write_lines ?source t ~total lines = let l = String.split_on_char ~sep:'\n' lines in + let lcount = ref 0 in let rec w = function | [ "" ] | [] -> () | s :: xs -> - write ?source t s; + let _ = write ?source t s ~add:(fun n -> lcount := !lcount + n + 1) in w xs in - w l + w l; + total !lcount let lnum t = t.lnum end @@ -170,6 +183,10 @@ let prefix_kind line = | true -> `Json_base64 (String.length sourceMappingURL_base64) | false -> `Url (String.length sourceMappingURL)) +let rule_out_index_map = function + | `Standard sm -> sm + | `Index _ -> failwith "unexpected index map at this stage" + let action ~resolve_sourcemap_url ~drop_source_map file line = match prefix_kind line, drop_source_map with | `Other, (true | false) -> Keep @@ -177,7 +194,9 @@ let action ~resolve_sourcemap_url ~drop_source_map file line = | `Build_info bi, _ -> Build_info bi | (`Json_base64 _ | `Url _), true -> Drop | `Json_base64 offset, false -> - Source_map (Source_map_io.of_string (Base64.decode_exn ~off:offset line)) + Source_map + (rule_out_index_map + (Source_map_io.of_string (Base64.decode_exn ~off:offset line))) | `Url _, false when not resolve_sourcemap_url -> Drop | `Url offset, false -> let url = String.sub line ~pos:offset ~len:(String.length line - offset) in @@ -186,44 +205,44 @@ let action ~resolve_sourcemap_url ~drop_source_map file line = let l = in_channel_length ic in let content = really_input_string ic l in close_in ic; - Source_map (Source_map_io.of_string content) + Source_map (rule_out_index_map (Source_map_io.of_string content)) module Units : sig - val read : Line_reader.t -> Unit_info.t -> Unit_info.t + val read : Line_reader.t -> drop_action:(unit -> unit) -> Unit_info.t -> Unit_info.t val scan_file : string -> Build_info.t option * Unit_info.t list end = struct - let rec read ic uinfo = + let rec read ic ~drop_action uinfo = match Line_reader.peek ic with | None -> uinfo | Some line -> ( match Unit_info.parse uinfo line with | None -> uinfo | Some uinfo -> - Line_reader.drop ic; - read ic uinfo) + Line_reader.drop ~drop_action ic; + read ic ~drop_action uinfo) - let find_unit_info ic = + let find_unit_info ~drop_action ic = let rec find_next ic = match Line_reader.peek ic with | None -> None | Some line -> ( match prefix_kind line with | `Json_base64 _ | `Url _ | `Other | `Build_info _ -> - Line_reader.drop ic; + Line_reader.drop ~drop_action ic; find_next ic - | `Unit -> Some (read ic Unit_info.empty)) + | `Unit -> Some (read ic ~drop_action Unit_info.empty)) in find_next ic - let find_build_info ic = + let find_build_info ~drop_action ic = let rec find_next ic = match Line_reader.peek ic with | None -> None | Some line -> ( match prefix_kind line with | `Json_base64 _ | `Url _ | `Other -> - Line_reader.drop ic; + Line_reader.drop ~drop_action ic; find_next ic | `Build_info bi -> Some bi | `Unit -> None) @@ -232,12 +251,13 @@ end = struct let scan_file file = let ic = Line_reader.open_ file in + let drop_action () = () in let rec scan_all ic acc = - match find_unit_info ic with + match find_unit_info ~drop_action ic with | None -> List.rev acc | Some x -> scan_all ic (x :: acc) in - let build_info = find_build_info ic in + let build_info = find_build_info ~drop_action ic in let units = scan_all ic [] in Line_reader.close ic; build_info, units @@ -318,12 +338,28 @@ let link ~output ~linkall ~mklib ~toplevel ~files ~resolve_sourcemap_url ~source in let sm_for_file = ref None in let ic = Line_reader.open_ file in - let skip ic = Line_reader.drop ic in - let reloc = ref [] in + let old_line_count = Line_writer.lnum oc in + let edits = ref [] in + let emit_drop_action edits () = edits := Source_map.Line_edits.Drop :: !edits in + let skip ic = Line_reader.drop ~drop_action:(emit_drop_action edits) ic in let copy ic oc = let line = Line_reader.next ic in - Line_writer.write ~source:ic oc line; - reloc := (Line_reader.lnum ic, Line_writer.lnum oc) :: !reloc + Line_writer.write + ~source:ic + ~add:(fun count -> edits := Add { count } :: !edits) + oc + line; + (* Note: line actions are in reverse order compared to the actual generated + lines *) + edits := Source_map.Line_edits.Keep :: !edits + in + let write_line oc str = + Line_writer.write oc str ~add:(fun count -> + edits := Source_map.Line_edits.(Add { count = count + 1 }) :: !edits) + in + let write_lines oc str = + Line_writer.write_lines oc str ~total:(fun count -> + edits := Source_map.Line_edits.(Add { count }) :: !edits) in let rec read () = match Line_reader.peek ic with @@ -342,11 +378,13 @@ let link ~output ~linkall ~mklib ~toplevel ~files ~resolve_sourcemap_url ~source if not !build_info_emitted then ( let bi = Build_info.with_kind bi (if mklib then `Cma else `Unknown) in - Line_writer.write_lines oc (Build_info.to_string bi); + write_lines oc (Build_info.to_string bi); build_info_emitted := true) | Drop -> skip ic | Unit -> - let u = Units.read ic Unit_info.empty in + let u = + Units.read ic ~drop_action:(emit_drop_action edits) Unit_info.empty + in if StringSet.cardinal (StringSet.inter u.Unit_info.provides to_link) > 0 then ( if u.effects_without_cps && not !warn_effects @@ -358,7 +396,7 @@ let link ~output ~linkall ~mklib ~toplevel ~files ~resolve_sourcemap_url ~source (if mklib then let u = if linkall then { u with force_link = true } else u in - Line_writer.write_lines oc (Unit_info.to_string u)); + write_lines oc (Unit_info.to_string u)); let size = ref 0 in while match Line_reader.peek ic with @@ -402,7 +440,7 @@ let link ~output ~linkall ~mklib ~toplevel ~files ~resolve_sourcemap_url ~source read () in read (); - Line_writer.write oc ""; + write_line oc ""; Line_reader.close ic; (match is_runtime with | None -> () @@ -424,10 +462,11 @@ let link ~output ~linkall ~mklib ~toplevel ~files ~resolve_sourcemap_url ~source (Parse_bytecode.Debug.create ~include_cmis:false false) code; let content = Buffer.contents b in - Line_writer.write_lines oc content); + write_lines oc content); (match !sm_for_file with | None -> () - | Some x -> sm := (x, !reloc) :: !sm); + | Some x -> + sm := (file, x, List.rev !edits, Line_writer.lnum oc - old_line_count) :: !sm); match !build_info, build_info_for_file with | None, None -> () | Some _, None -> () @@ -440,32 +479,51 @@ let link ~output ~linkall ~mklib ~toplevel ~files ~resolve_sourcemap_url ~source match source_map with | None -> () | Some (file, init_sm) -> - let sm = - List.rev_map !sm ~f:(fun (sm, reloc) -> - let tbl = Hashtbl.create 17 in - List.iter reloc ~f:(fun (a, b) -> Hashtbl.add tbl a b); - Source_map.filter_map sm ~f:(Hashtbl.find_opt tbl)) + let sourcemaps_and_line_counts = + List.rev_map !sm ~f:(fun (file, sm, edits, lcount) -> + if debug () + then ( + Format.eprintf "@[line actions for '%s' (lcount %d)@," file lcount; + Format.eprintf "%a@," Source_map.Line_edits.pp edits; + Format.eprintf "@]"); + let mappings = sm.Source_map.mappings in + let mappings = Source_map.Mappings.edit ~strict:false mappings edits in + { sm with mappings }, lcount) in - (match Source_map.merge (init_sm :: sm) with - | None -> () - | Some sm -> ( - (* preserve some info from [init_sm] *) - let sm = - { sm with - version = init_sm.version - ; file = init_sm.file - ; sourceroot = init_sm.sourceroot - } - in - match file with - | None -> - let data = Source_map_io.to_string sm in - let s = sourceMappingURL_base64 ^ Base64.encode_exn data in - Line_writer.write oc s - | Some file -> - Source_map_io.to_file sm file; - let s = sourceMappingURL ^ Filename.basename file in - Line_writer.write oc s)); + let merged_sourcemap = + let open Source_map in + assert (String.equal (Mappings.to_string init_sm.mappings) ""); + { version = init_sm.version + ; file = init_sm.file + ; Index.sections = + (let _, sections = + List.fold_left + sourcemaps_and_line_counts + ~f:(fun (cur_ofs, sections) (sm, generated_line_count) -> + let offset = Index.{ gen_line = cur_ofs; gen_column = 0 } in + cur_ofs + generated_line_count, (offset, `Map sm) :: sections) + ~init:(0, []) + in + List.rev sections) + } + in + (* preserve some info from [init_sm] *) + let merged_sourcemap = + { merged_sourcemap with + sections = + List.map merged_sourcemap.sections ~f:(fun (ofs, `Map sm) -> + ofs, `Map { sm with sourceroot = init_sm.sourceroot }) + } + in + (match file with + | None -> + let data = Source_map_io.Index.to_string merged_sourcemap in + let s = sourceMappingURL_base64 ^ Base64.encode_exn data in + Line_writer.write oc s ~add:(fun _ -> ()) |> ignore + | Some file -> + Source_map_io.Index.to_file merged_sourcemap file; + let s = sourceMappingURL ^ Filename.basename file in + Line_writer.write oc s ~add:(fun _ -> ()) |> ignore); if times () then Format.eprintf " sourcemap: %a@." Timer.print t let link ~output ~linkall ~mklib ~toplevel ~files ~resolve_sourcemap_url ~source_map = diff --git a/compiler/lib/source_map.ml b/compiler/lib/source_map.ml index 3d88b0d1bd..8da81c5fe6 100644 --- a/compiler/lib/source_map.ml +++ b/compiler/lib/source_map.ml @@ -40,16 +40,565 @@ type map = ; ori_name : int } -type mapping = map list +module Line_edits = struct + type action = + | Keep + | Drop + | Add of { count : int } + + let pp_action fmt = + let open Format in + function + | Keep -> pp_print_string fmt "Keep" + | Drop -> pp_print_string fmt "Drop" + | Add { count } -> fprintf fmt "@[Add@ {@ count =@ %d@ }@]" count + + type t = action list + + let pp fmt = Format.(pp_print_list pp_action fmt) +end + +module Mappings = struct + type t = Uninterpreted of string [@@unboxed] + + let empty = Uninterpreted "" + + external of_string : string -> t = "%identity" + + external to_string : t -> string = "%identity" + + let update_carries_from_segment + ~carry_source + ~carry_line + ~carry_col + ~carry_name + ~pos + ~len + str = + (* Note: we don't care about the first field since we do linewise editing, + and it is reset for every line. *) + match Vlq64.decode_l str ~pos ~len with + | [ _gen_col ] -> carry_source, carry_line, carry_col, carry_name + | [ _gen_col; source; line; col ] -> + carry_source + source, carry_line + line, carry_col + col, carry_name + | [ _gen_col; source; line; col; name ] -> + carry_source + source, carry_line + line, carry_col + col, carry_name + name + | _ -> invalid_arg "Mapping.update_carries_from_segment" + + let update_carries_and_write_segment + ~carry_source + ~carry_line + ~carry_col + ~carry_name + ~pos + ~len + str + ~buf = + match Vlq64.decode_l str ~pos ~len with + | [ gen_col ] -> + Vlq64.encode_l buf [ gen_col ]; + carry_source, carry_line, carry_col, carry_name + | [ gen_col; source; line; col ] -> + Vlq64.encode_l + buf + [ gen_col; source + carry_source; line + carry_line; col + carry_col ]; + 0, 0, 0, carry_name + | [ gen_col; source; line; col; name ] -> + Vlq64.encode_l + buf + [ gen_col + ; source + carry_source + ; line + carry_line + ; col + carry_col + ; name + carry_name + ]; + 0, 0, 0, 0 + | _ -> + invalid_arg + (Format.sprintf + "Mapping.update_carries_from_segment %s" + (String.sub ~pos ~len str)) + + (* Fold [f] over the segments in string [str.(pos..len - 1)]. *) + let fold_on_segments ~str ~pos ~len f ~init = + let rec loop acc pos end_ = + if pos >= end_ + then acc + else + let next_delim = + try min (String.index_from str pos ',') end_ with Not_found -> end_ + in + let len = next_delim - pos in + if len <= 0 then acc else loop (f acc str ~pos ~len) (next_delim + 1) end_ + in + loop init pos (pos + len) + + type carries = + { carry_source : int + ; carry_line : int + ; carry_col : int + ; carry_name : int + } + + let update_carries_from_line + ~carry_source + ~carry_line + ~carry_col + ~carry_name + ~pos + ~len + str = + fold_on_segments + ~str + ~pos + ~len + ~init:{ carry_source; carry_line; carry_col; carry_name } + (fun acc str ~pos ~len -> + let { carry_source; carry_line; carry_col; carry_name } = acc in + let carry_source, carry_line, carry_col, carry_name = + update_carries_from_segment + ~carry_source + ~carry_line + ~carry_col + ~carry_name + ~pos + ~len + str + in + { carry_source; carry_line; carry_col; carry_name }) + + let update_carries_and_write_line + ~carry_source + ~carry_line + ~carry_col + ~carry_name + ~pos + ~len + str + ~buf = + let _, carries = + fold_on_segments + ~str + ~pos + ~len + ~init:(true, { carry_source; carry_line; carry_col; carry_name }) + (fun (is_first, acc) str ~pos ~len -> + let { carry_source; carry_line; carry_col; carry_name } = acc in + if not is_first then Buffer.add_char buf ','; + let carry_source, carry_line, carry_col, carry_name = + update_carries_and_write_segment + ~carry_source + ~carry_line + ~carry_col + ~carry_name + ~pos + ~len + str + ~buf + in + false, { carry_source; carry_line; carry_col; carry_name }) + in + Buffer.add_char buf ';'; + carries + + (* If [strict], then the number of [Keep] and [Drop] elementes in [edits] + should be the same as the number of generated lines covered by the + mappings [orig]. Otherwise, there may be more edit actions, in which case + [Keep] just adds a line without mappings and [Drop] does nothing. *) + let rec edit_loop + ~orig + ~carry_source + ~carry_line + ~carry_col + ~carry_name + ~strict + buf + offset_in_orig + edits = + let open Line_edits in + if offset_in_orig >= String.length orig + then ( + (* No more lines in the mappings string *) + match edits with + | [] -> { carry_source; carry_line; carry_col; carry_name } + | _ :: _ -> + List.iter edits ~f:(function + | Add { count } -> Buffer.add_string buf (String.make count ';') + | Keep -> + if strict + then + invalid_arg + "Mapping.edit: more Keep or Drop edits than lines in mappings"; + Buffer.add_char buf ';' + | Drop -> + if strict + then + invalid_arg + "Mapping.edit: more Keep or Drop edits than lines in mappings"); + { carry_source; carry_line; carry_col; carry_name }) + else + match edits with + | [] -> { carry_source; carry_line; carry_col; carry_name } + | Keep :: rem -> + let next_group_delim = + try String.index_from orig offset_in_orig ';' + with Not_found -> String.length orig + in + let { carry_source; carry_line; carry_col; carry_name } = + update_carries_and_write_line + ~carry_source + ~carry_line + ~carry_col + ~carry_name + ~pos:offset_in_orig + ~len:(next_group_delim - offset_in_orig) + orig + ~buf + in + edit_loop + ~orig + ~carry_source + ~carry_line + ~carry_col + ~carry_name + ~strict + buf + (* Skip the ';' *) + (next_group_delim + 1) + rem + | Drop :: rem -> + let next_group_delim = + try String.index_from orig offset_in_orig ';' + with Not_found -> String.length orig + in + let { carry_source; carry_line; carry_col; carry_name } = + update_carries_from_line + ~carry_source + ~carry_line + ~carry_col + ~carry_name + ~pos:offset_in_orig + ~len:(next_group_delim - offset_in_orig) + orig + in + edit_loop + ~orig + ~carry_source + ~carry_line + ~carry_col + ~carry_name + ~strict + buf + (next_group_delim + 1) + rem + | Add { count } :: rem -> + Buffer.add_string buf (String.make count ';'); + edit_loop + ~orig + ~carry_source + ~carry_line + ~carry_col + ~carry_name + ~strict + buf + offset_in_orig + rem + + let edit ~strict (Uninterpreted orig) edits = + let buf = Buffer.create 8_192 in + let _ = + edit_loop + ~strict + ~orig + ~carry_source:0 + ~carry_line:0 + ~carry_col:0 + ~carry_name:0 + buf + 0 + edits + in + Uninterpreted (Buffer.contents buf) + + let num_gen_lines m = + let rec loop count pos = + if pos >= String.length m + then count + else + let next_delim = + try String.index_from m pos ';' with Not_found -> String.length m + in + if next_delim >= String.length m - 1 + then (* This was the last line *) + count + 1 + else loop (count + 1) (next_delim + 1) + in + loop 0 0 + + let sum_offsets_segment ~carry_source ~carry_line ~carry_col ~carry_name ~pos ~len str = + match Vlq64.decode_l str ~pos ~len with + | [ _gen_col ] -> { carry_source; carry_line; carry_col; carry_name } + | [ _gen_col; source; line; col ] -> + { carry_source = carry_source + source + ; carry_line = carry_line + line + ; carry_col = carry_col + col + ; carry_name + } + | [ _gen_col; source; line; col; name ] -> + { carry_source = carry_source + source + ; carry_line = carry_line + line + ; carry_col = carry_col + col + ; carry_name = carry_name + name + } + | _ -> invalid_arg "Mapping.sum_offsets_segment: invalid segment" + + let sum_offsets_line ~carry_source ~carry_line ~carry_col ~carry_name ~pos ~len str = + fold_on_segments + ~str + ~pos + ~len + ~init:{ carry_source; carry_line; carry_col; carry_name } + (fun { carry_source; carry_line; carry_col; carry_name } str ~pos ~len -> + sum_offsets_segment ~carry_source ~carry_line ~carry_col ~carry_name ~pos ~len str) + + (* Fold [f] over the ';'-separated groups in string [str.(pos..len - 1)]. *) + let fold_on_lines ~str f ~init = + let rec loop acc pos = + if pos >= String.length str + then acc + else + let next_delim = + try min (String.index_from str pos ';') (String.length str) + with Not_found -> String.length str + in + let len = next_delim - pos in + loop (f acc str ~pos ~len) (next_delim + 1) + in + loop init 0 + + let sum_offsets mapping = + fold_on_lines + ~str:mapping + ~init:{ carry_source = 0; carry_line = 0; carry_col = 0; carry_name = 0 } + (fun { carry_source; carry_line; carry_col; carry_name } str ~pos ~len -> + sum_offsets_line ~carry_source ~carry_line ~carry_col ~carry_name ~pos ~len str) + + let concat ~source_count1 ~name_count1 (Uninterpreted m1) (Uninterpreted m2) = + match m1, m2 with + | "", m2 -> Uninterpreted m2 + | m1, "" -> Uninterpreted m1 + | _, _ -> + let buf = Buffer.create 8_192 in + (* First do a pass on [m1] to accumulate its carries. *) + let { carry_source; carry_line; carry_col; carry_name } = sum_offsets m1 in + Buffer.add_string buf m1; + if not (Char.equal m1.[String.length m1 - 1] ';') then Buffer.add_char buf ';'; + let _ = + edit_loop + ~orig:m2 + (* Make the initial absolute offsets in [m2] relative. Also account + for the fact that fields [sources] and [names] of [m2] will be + concatenated to those of [m1]. *) + ~carry_source:(source_count1 - carry_source) + ~carry_line:~-carry_line + ~carry_col:~-carry_col + ~carry_name:(name_count1 - carry_name) + ~strict:true + buf + 0 + (List.init ~len:(num_gen_lines m2) ~f:(Fun.const Line_edits.Keep)) + in + Uninterpreted (Buffer.contents buf) + + let gen_line = function + | Gen { gen_line; _ } | Gen_Ori { gen_line; _ } | Gen_Ori_Name { gen_line; _ } -> + gen_line + + let gen_col = function + | Gen { gen_col; _ } | Gen_Ori { gen_col; _ } | Gen_Ori_Name { gen_col; _ } -> gen_col + + let encode mapping = + let a = Array.of_list mapping in + let len = Array.length a in + Array.stable_sort + ~cmp:(fun t1 t2 -> + match compare (gen_line t1) (gen_line t2) with + | 0 -> compare (gen_col t1) (gen_col t2) + | n -> n) + a; + let buf = Buffer.create 1024 in + (* The binary format encodes lines starting at zero, but + [ori_line] and [gen_line] are 1 based. *) + let gen_line_r = ref 1 in + let gen_col_r = ref 0 in + let ori_source_r = ref 0 in + let ori_line_r = ref 1 in + let ori_col_r = ref 0 in + let ori_name_r = ref 0 in + let rec loop prev i = + if i < len + then + let c = a.(i) in + if i + 1 < len && gen_line c = gen_line a.(i + 1) && gen_col c = gen_col a.(i + 1) + then (* Only keep one source location per generated location *) + loop prev (i + 1) + else ( + if !gen_line_r <> gen_line c + then ( + assert (!gen_line_r < gen_line c); + for _i = !gen_line_r to gen_line c - 1 do + Buffer.add_char buf ';' + done; + gen_col_r := 0; + gen_line_r := gen_line c) + else if i > 0 + then Buffer.add_char buf ','; + let l = + match c with + | Gen { gen_line = _; gen_col } -> + let res = [ gen_col - !gen_col_r ] in + gen_col_r := gen_col; + res + | Gen_Ori { gen_line = _; gen_col; ori_source; ori_line; ori_col } -> + let res = + [ gen_col - !gen_col_r + ; ori_source - !ori_source_r + ; ori_line - !ori_line_r + ; ori_col - !ori_col_r + ] + in + gen_col_r := gen_col; + ori_col_r := ori_col; + ori_line_r := ori_line; + ori_source_r := ori_source; + res + | Gen_Ori_Name + { gen_line = _; gen_col; ori_source; ori_line; ori_col; ori_name } -> + let res = + [ gen_col - !gen_col_r + ; ori_source - !ori_source_r + ; ori_line - !ori_line_r + ; ori_col - !ori_col_r + ; ori_name - !ori_name_r + ] + in + gen_col_r := gen_col; + ori_col_r := ori_col; + ori_line_r := ori_line; + ori_source_r := ori_source; + ori_name_r := ori_name; + res + in + Vlq64.encode_l buf l; + loop i (i + 1)) + in + loop (-1) 0; + Uninterpreted (Buffer.contents buf) + + let decode (Uninterpreted str) = + let total_len = String.length str in + let gen_col = ref 0 in + let ori_source = ref 0 in + let ori_line = ref 1 in + let ori_col = ref 0 in + let ori_name = ref 0 in + let rec readline line pos acc = + if pos >= total_len + then List.rev acc + else + let last = try String.index_from str pos ';' with Not_found -> total_len in + gen_col := 0; + let pos, acc = + if pos = last then pos + 1, acc else read_tokens line pos last acc + in + readline (succ line) pos acc + and read_tokens line start stop acc = + let last = + try min (String.index_from str start ',') stop with Not_found -> stop + in + let v = Vlq64.decode_l str ~pos:start ~len:(last - start) in + match v with + | [] -> last + 1, acc + | v -> + let v = + match v with + | [ g ] -> + gen_col := !gen_col + g; + Gen { gen_line = line; gen_col = !gen_col } + | [ g; os; ol; oc ] -> + gen_col := !gen_col + g; + ori_source := !ori_source + os; + ori_line := !ori_line + ol; + ori_col := !ori_col + oc; + Gen_Ori + { gen_line = line + ; gen_col = !gen_col + ; ori_source = !ori_source + ; ori_line = !ori_line + ; ori_col = !ori_col + } + | [ g; os; ol; oc; on ] -> + gen_col := !gen_col + g; + ori_source := !ori_source + os; + ori_line := !ori_line + ol; + ori_col := !ori_col + oc; + ori_name := !ori_name + on; + Gen_Ori_Name + { gen_line = line + ; gen_col = !gen_col + ; ori_source = !ori_source + ; ori_line = !ori_line + ; ori_col = !ori_col + ; ori_name = !ori_name + } + | _ -> invalid_arg "Source_map.mapping_of_string" + in + let acc = v :: acc in + if last = stop then last + 1, acc else read_tokens line (last + 1) stop acc + in + (* The binary format encodes lines starting at zero, but + [ori_line] and [gen_line] are 1 based. *) + readline 1 0 [] +end + +module Source_text = struct + type t = Uninterpreted of string [@@unboxed] + + external of_json_string : string -> t = "%identity" + + external to_json_string : t -> string = "%identity" + + let empty = Uninterpreted "" + + let to_json = function + | None -> `Null + | Some text -> `String text + + let encode t = + let json = Yojson.Basic.to_string (to_json t) in + Uninterpreted json + + let of_json = function + | `String s -> Some s + | `Null -> None + | _ -> invalid_arg "Source_map.Sources_contents.of_json: expected string or null" + + let decode (Uninterpreted s) : string option = + (* The two stages of the encoding, in reverse. *) + try of_json (Yojson.Basic.from_string s) + with Yojson.Json_error s -> + invalid_arg + ("Source_map.Sources_contents.decode: This is not a valid JSON object: " ^ s) +end type t = { version : int ; file : string ; sourceroot : string option ; sources : string list - ; sources_content : string option list option + ; sources_contents : Source_text.t list option ; names : string list - ; mappings : mapping + ; mappings : Mappings.t } let empty ~filename = @@ -57,244 +606,40 @@ let empty ~filename = ; file = filename ; sourceroot = None ; sources = [] - ; sources_content = None + ; sources_contents = None ; names = [] - ; mappings = [] + ; mappings = Mappings.empty } -let gen_line = function - | Gen { gen_line; _ } | Gen_Ori { gen_line; _ } | Gen_Ori_Name { gen_line; _ } -> - gen_line - -let gen_col = function - | Gen { gen_col; _ } | Gen_Ori { gen_col; _ } | Gen_Ori_Name { gen_col; _ } -> gen_col - -let string_of_mapping mapping = - let a = Array.of_list mapping in - let len = Array.length a in - Array.stable_sort - ~cmp:(fun t1 t2 -> - match compare (gen_line t1) (gen_line t2) with - | 0 -> compare (gen_col t1) (gen_col t2) - | n -> n) - a; - let buf = Buffer.create 1024 in - (* The binary format encodes lines starting at zero, but - [ori_line] and [gen_line] are 1 based. *) - let gen_line_r = ref 1 in - let gen_col_r = ref 0 in - let ori_source_r = ref 0 in - let ori_line_r = ref 1 in - let ori_col_r = ref 0 in - let ori_name_r = ref 0 in - let rec loop prev i = - if i < len - then - let c = a.(i) in - if i + 1 < len && gen_line c = gen_line a.(i + 1) && gen_col c = gen_col a.(i + 1) - then (* Only keep one source location per generated location *) - loop prev (i + 1) - else ( - if !gen_line_r <> gen_line c - then ( - assert (!gen_line_r < gen_line c); - for _i = !gen_line_r to gen_line c - 1 do - Buffer.add_char buf ';' - done; - gen_col_r := 0; - gen_line_r := gen_line c) - else if i > 0 - then Buffer.add_char buf ','; - let l = - match c with - | Gen { gen_line = _; gen_col } -> - let res = [ gen_col - !gen_col_r ] in - gen_col_r := gen_col; - res - | Gen_Ori { gen_line = _; gen_col; ori_source; ori_line; ori_col } -> - let res = - [ gen_col - !gen_col_r - ; ori_source - !ori_source_r - ; ori_line - !ori_line_r - ; ori_col - !ori_col_r - ] - in - gen_col_r := gen_col; - ori_col_r := ori_col; - ori_line_r := ori_line; - ori_source_r := ori_source; - res - | Gen_Ori_Name - { gen_line = _; gen_col; ori_source; ori_line; ori_col; ori_name } -> - let res = - [ gen_col - !gen_col_r - ; ori_source - !ori_source_r - ; ori_line - !ori_line_r - ; ori_col - !ori_col_r - ; ori_name - !ori_name_r - ] - in - gen_col_r := gen_col; - ori_col_r := ori_col; - ori_line_r := ori_line; - ori_source_r := ori_source; - ori_name_r := ori_name; - res - in - Vlq64.encode_l buf l; - loop i (i + 1)) - in - loop (-1) 0; - Buffer.contents buf - -let mapping_of_string str = - let total_len = String.length str in - let gen_col = ref 0 in - let ori_source = ref 0 in - let ori_line = ref 1 in - let ori_col = ref 0 in - let ori_name = ref 0 in - let rec readline line pos acc = - if pos >= total_len - then List.rev acc - else - let last = try String.index_from str pos ';' with Not_found -> total_len in - gen_col := 0; - let pos, acc = if pos = last then pos + 1, acc else read_tokens line pos last acc in - readline (succ line) pos acc - and read_tokens line start stop acc = - let last = try min (String.index_from str start ',') stop with Not_found -> stop in - let v = Vlq64.decode_l str ~pos:start ~len:(last - start) in - match v with - | [] -> last + 1, acc - | v -> - let v = - match v with - | [ g ] -> - gen_col := !gen_col + g; - Gen { gen_line = line; gen_col = !gen_col } - | [ g; os; ol; oc ] -> - gen_col := !gen_col + g; - ori_source := !ori_source + os; - ori_line := !ori_line + ol; - ori_col := !ori_col + oc; - Gen_Ori - { gen_line = line - ; gen_col = !gen_col - ; ori_source = !ori_source - ; ori_line = !ori_line - ; ori_col = !ori_col - } - | [ g; os; ol; oc; on ] -> - gen_col := !gen_col + g; - ori_source := !ori_source + os; - ori_line := !ori_line + ol; - ori_col := !ori_col + oc; - ori_name := !ori_name + on; - Gen_Ori_Name - { gen_line = line - ; gen_col = !gen_col - ; ori_source = !ori_source - ; ori_line = !ori_line - ; ori_col = !ori_col - ; ori_name = !ori_name - } - | _ -> invalid_arg "Source_map.mapping_of_string" - in - let acc = v :: acc in - if last = stop then last + 1, acc else read_tokens line (last + 1) stop acc - in - (* The binary format encodes lines starting at zero, but - [ori_line] and [gen_line] are 1 based. *) - readline 1 0 [] - -let maps ~sources_offset ~names_offset x = - match x with - | Gen _ -> x - | Gen_Ori { gen_line; gen_col; ori_source; ori_line; ori_col } -> - let ori_source = ori_source + sources_offset in - Gen_Ori { gen_line; gen_col; ori_source; ori_line; ori_col } - | Gen_Ori_Name { gen_line; gen_col; ori_source; ori_line; ori_col; ori_name } -> - let ori_source = ori_source + sources_offset in - let ori_name = ori_name + names_offset in - Gen_Ori_Name { gen_line; gen_col; ori_source; ori_line; ori_col; ori_name } - -let filter_map sm ~f = - let a = Array.of_list sm.mappings in - Array.stable_sort - ~cmp:(fun t1 t2 -> - match compare (gen_line t1) (gen_line t2) with - | 0 -> compare (gen_col t1) (gen_col t2) - | n -> n) - a; - let l = Array.to_list a |> List.group ~f:(fun a b -> gen_line a = gen_line b) in - - let rec loop acc mapping = - match mapping with - | [] -> List.rev acc - | x :: xs -> - let gen_line = gen_line (List.hd x) in - let acc = - match f gen_line with - | None -> acc - | Some gen_line -> - List.rev_append_map - x - ~f:(function - | Gen { gen_line = _; gen_col } -> Gen { gen_line; gen_col } - | Gen_Ori { gen_line = _; gen_col; ori_source; ori_line; ori_col } -> - Gen_Ori { gen_line; gen_col; ori_source; ori_line; ori_col } - | Gen_Ori_Name - { gen_line = _; gen_col; ori_source; ori_line; ori_col; ori_name } - -> - Gen_Ori_Name - { gen_line; gen_col; ori_source; ori_line; ori_col; ori_name }) - acc - in - loop acc xs - in - let mappings = loop [] l in - { sm with mappings } - -let merge = function - | [] -> None - | _ :: _ as l -> - let rec loop acc_rev ~sources_offset ~names_offset l = - match l with - | [] -> acc_rev - | sm :: rest -> - let acc_rev = - { acc_rev with - sources = List.rev_append sm.sources acc_rev.sources - ; names = List.rev_append sm.names acc_rev.names - ; sources_content = - (match sm.sources_content, acc_rev.sources_content with - | Some x, Some acc_rev -> Some (List.rev_append x acc_rev) - | None, _ | _, None -> None) - ; mappings = - List.rev_append_map - ~f:(maps ~sources_offset ~names_offset) - sm.mappings - acc_rev.mappings - } - in - loop - acc_rev - ~sources_offset:(sources_offset + List.length sm.sources) - ~names_offset:(names_offset + List.length sm.names) - rest - in - let acc_rev = - loop - { (empty ~filename:"") with sources_content = Some [] } - ~sources_offset:0 - ~names_offset:0 - l - in - Some - { acc_rev with - mappings = List.rev acc_rev.mappings - ; sources = List.rev acc_rev.sources - ; names = List.rev acc_rev.names - ; sources_content = Option.map ~f:List.rev acc_rev.sources_content - } +let concat ~file ~sourceroot s1 s2 = + if not (Int.equal s1.version s2.version) + then invalid_arg "Source_map.concat: different versions"; + { version = s1.version + ; file + ; sourceroot + ; sources = s1.sources @ s2.sources + ; sources_contents = + (match s1.sources_contents, s2.sources_contents with + | None, contents | contents, None -> contents + | Some c1, Some c2 -> Some (c1 @ c2)) + ; names = s1.names @ s2.names + ; mappings = + Mappings.concat + ~source_count1:(List.length s1.sources) + ~name_count1:(List.length s1.names) + s1.mappings + s2.mappings + } + +module Index = struct + type offset = + { gen_line : int + ; gen_column : int + } + + type nonrec t = + { version : int + ; file : string + ; sections : (offset * [ `Map of t ]) list + } +end diff --git a/compiler/lib/source_map.mli b/compiler/lib/source_map.mli index b394fe8970..c40a0e8dc5 100644 --- a/compiler/lib/source_map.mli +++ b/compiler/lib/source_map.mli @@ -38,24 +38,99 @@ type map = ; ori_name : int } -type mapping = map list +module Line_edits : sig + type action = + | Keep + | Drop + | Add of { count : int } + + val pp_action : Format.formatter -> action -> unit + + type t = action list + + val pp : Format.formatter -> t -> unit +end + +module Mappings : sig + type t + + val empty : t + (** Represents the empty mapping. *) + + val of_string : string -> t + (** By default, mappings are left uninterpreted, since many operations can be + performed efficiently directly on the encoded form. It is guaranteed that + {!val:of_string} and {!val:to_string} are inverse functions. *) + + val decode : t -> map list + + val encode : map list -> t + + val to_string : t -> string + (** Returns the mappings as a string in the Source map v3 format. *) + + val edit : strict:bool -> t -> Line_edits.t -> t + (** Apply line edits in order. If the number of {!const:Line_edits.Keep} and + {!const:Line_edits.Drop} actions does not match the number of lines in + the domain of the input mapping, only the lines affected by an edit are + included in the result. *) +end + +module Source_text : sig + type t + + val empty : t + + val of_json_string : string -> t + (** By default, sources contents are left uninterpreted as decoding this field can be + costly if the amount of code is large, and is seldom required. It is guaranteed that + {!val:of_json_string} and {!val:to_json_string} are inverse functions. *) + + val decode : t -> string option + + val encode : string option -> t + + val to_json_string : t -> string + (** Returns a valid JSON object (in this instance, a string literal, double quotes + included) representing the source text. *) +end type t = { version : int ; file : string ; sourceroot : string option ; sources : string list - ; sources_content : string option list option + ; sources_contents : Source_text.t list option + (** Left uninterpreted by default, since decoding it requires to handle special + characters, which can be costly for huge codebases. *) ; names : string list - ; mappings : mapping + ; mappings : Mappings.t + (** Left uninterpreted, since most useful operations can be performed efficiently + directly on the encoded form, and a full decoding can be costly for big + sourcemaps. *) } -val filter_map : t -> f:(int -> int option) -> t - -val merge : t list -> t option +val empty : filename:string -> t -val mapping_of_string : string -> mapping +val concat : file:string -> sourceroot:string option -> t -> t -> t +(** If [s1] encodes a mapping for a generated file [f1], and [s2] for a + generated file [f2], then [concat ~file ~sourceroot s1 s2] encodes the + union of these mappings for the concatenation of [f1] and [f2], with name + [file] and source root [sourceroot). *) -val string_of_mapping : mapping -> string +module Index : sig + type offset = + { gen_line : int + ; gen_column : int + } -val empty : filename:string -> t + type nonrec t = + { version : int + ; file : string + ; sections : (offset * [ `Map of t ]) list + (** List of [(offset, map)] pairs. The sourcemap spec allows for [map] to be + either a sourcemap object or a URL, but we don't need to generate + composite sourcemaps with URLs for now, and it is therefore not + implemented. *) + } +end diff --git a/compiler/lib/source_map_io.mli b/compiler/lib/source_map_io.mli index 65c6b905b2..41c41cba1b 100644 --- a/compiler/lib/source_map_io.mli +++ b/compiler/lib/source_map_io.mli @@ -25,4 +25,10 @@ val to_string : t -> string val to_file : t -> string -> unit -val of_string : string -> t +module Index : sig + val to_string : Index.t -> string + + val to_file : Index.t -> string -> unit +end + +val of_string : string -> [ `Standard of Source_map.t | `Index of Source_map.Index.t ] diff --git a/compiler/lib/source_map_io.unsupported.ml b/compiler/lib/source_map_io.unsupported.ml index dbdb35816c..9199c0fa45 100644 --- a/compiler/lib/source_map_io.unsupported.ml +++ b/compiler/lib/source_map_io.unsupported.ml @@ -25,4 +25,10 @@ let of_string _ = fail () let to_file _ _ = fail () +module Composite = struct + let to_string _ _ = fail () + + let to_file _ _ = fail () +end + let enabled = false diff --git a/compiler/lib/source_map_io.yojson.ml b/compiler/lib/source_map_io.yojson.ml index 05f0975e63..ae0267de0a 100644 --- a/compiler/lib/source_map_io.yojson.ml +++ b/compiler/lib/source_map_io.yojson.ml @@ -19,47 +19,58 @@ open Source_map +let rewrite_path path = + if Filename.is_relative path + then path + else + match Build_path_prefix_map.get_build_path_prefix_map () with + | Some map -> Build_path_prefix_map.rewrite map path + | None -> path + +(* Escapes special characters and wrap in double quotes *) +let stringlit_of_string s = `Stringlit (Yojson.Basic.to_string (`String s)) + let json t = - let rewrite_path path = - if Filename.is_relative path - then path - else - match Build_path_prefix_map.get_build_path_prefix_map () with - | Some map -> Build_path_prefix_map.rewrite map path - | None -> path + let mappings = + `Stringlit ("\"" ^ Mappings.to_string t.mappings ^ "\"") + (* Nothing to escape *) in - `Assoc - [ "version", `Float (float_of_int t.version) - ; "file", `String (rewrite_path t.file) + let fields = + [ "version", `Intlit (Int.to_string t.version) + ; "file", stringlit_of_string (rewrite_path t.file) ; ( "sourceRoot" - , `String + , stringlit_of_string (match t.sourceroot with | None -> "" | Some s -> rewrite_path s) ) - ; "names", `List (List.map (fun s -> `String s) t.names) - ; "sources", `List (List.map (fun s -> `String (rewrite_path s)) t.sources) - ; "mappings", `String (string_of_mapping t.mappings) - ; ( "sourcesContent" - , `List - (match t.sources_content with - | None -> [] - | Some l -> - List.map - (function - | None -> `Null - | Some s -> `String s) - l) ) + ; "names", `List (List.map (fun s -> stringlit_of_string s) t.names) + ; ( "sources" + , `List (List.map (fun s -> stringlit_of_string (rewrite_path s)) t.sources) ) + ; "mappings", mappings ] + in + match t.sources_contents with + | None -> `Assoc fields + | Some cs -> + `Assoc + (fields + @ [ ( "sourcesContent" + , `List (List.map (fun t -> `Stringlit (Source_text.to_json_string t)) cs) ) + ]) let invalid () = invalid_arg "Source_map.of_json" +let string_of_stringlit (`Stringlit s) = + match Yojson.Basic.from_string s with + | `String s -> s + | _ -> invalid_arg "Source_map_io.string_of_stringlit: not a JSON string literal" + let string name rest = - try - match List.assoc name rest with - | `String s -> Some s - | `Null -> None - | _ -> invalid () - with Not_found -> None + match List.assoc name rest with + | `Stringlit _ as s -> Some (string_of_stringlit s) + | `Null -> None + | _ -> invalid () + | exception Not_found -> None let list_string name rest = try @@ -68,54 +79,161 @@ let list_string name rest = Some (List.map (function - | `String s -> s + | `Stringlit _ as lit -> string_of_stringlit lit | _ -> invalid ()) l) | _ -> invalid () with Not_found -> None -let list_string_opt name rest = - try - match List.assoc name rest with - | `List l -> - Some - (List.map - (function - | `String s -> Some s - | `Null -> None - | _ -> invalid ()) - l) - | _ -> invalid () - with Not_found -> None +let stringlit_opt name assoc = + match List.assoc name assoc with + | `Stringlit s -> Some s + | _ | (exception Not_found) -> None + +let stringlit_list_opt name assoc = + match List.assoc name assoc with + | `List l -> + Some + (List.map + (function + | `Stringlit lit -> lit + | _ -> invalid ()) + l) + | _ -> invalid () + | exception Not_found -> None -let of_json json = +let standard_map_of_json json = match json with - | `Assoc (("version", `Float version) :: rest) when int_of_float version = 3 -> - let def v d = - match v with - | None -> d - | Some v -> v - in + | `Assoc (("version", version) :: rest) -> + (match version with + | `Floatlit version when Float.equal (Float.of_string version) 3.0 -> () + | `Intlit version when Int.equal (int_of_string version) 3 -> () + | `Floatlit _ | `Intlit _ -> invalid_arg "Source_map_io.of_json: version != 3" + | _ -> invalid_arg "Source_map_io.of_json: version field is not a number"); + (match List.assoc "sections" rest with + | _ -> invalid_arg "Source_map_io.standard_map_of_json: not a standard map" + | exception Not_found -> ()); let file = string "file" rest in let sourceroot = string "sourceRoot" rest in let names = list_string "names" rest in let sources = list_string "sources" rest in - let sources_content = list_string_opt "sourcesContent" rest in - let mappings = string "mappings" rest in - { version = int_of_float version - ; file = def file "" + let sources_contents = stringlit_list_opt "sourcesContent" rest in + let mappings = stringlit_opt "mappings" rest in + let mappings = + Option.map + (fun mappings -> + assert ( + String.length mappings >= 2 + && Char.equal mappings.[0] '"' + && Char.equal mappings.[String.length mappings - 1] '"'); + Mappings.of_string (String.sub mappings 1 (String.length mappings - 2))) + mappings + in + { version = 3 + ; file = Option.value file ~default:"" ; sourceroot - ; names = def names [] - ; sources_content - ; sources = def sources [] - ; mappings = mapping_of_string (def mappings "") + ; names = Option.value names ~default:[] + ; sources_contents = + Option.map (List.map Source_text.of_json_string) sources_contents + ; sources = Option.value sources ~default:[] + ; mappings = Option.value mappings ~default:Mappings.empty } | _ -> invalid () -let of_string s = of_json (Yojson.Basic.from_string s) - -let to_string m = Yojson.Basic.to_string (json m) +let to_string m = Yojson.Raw.to_string (json m) -let to_file m file = Yojson.Basic.to_file file (json m) +let to_file m file = Yojson.Raw.to_file file (json m) let enabled = true + +module Index = struct + let json t = + `Assoc + [ "version", `Intlit (Int.to_string t.Index.version) + ; "file", stringlit_of_string (rewrite_path t.file) + ; ( "sections" + , `List + (List.map + (fun ({ Index.gen_line; gen_column }, `Map sm) -> + `Assoc + [ ( "offset" + , `Assoc + [ "line", `Intlit (Int.to_string gen_line) + ; "column", `Intlit (Int.to_string gen_column) + ] ) + ; "map", json sm + ]) + t.sections) ) + ] + + let intlit ~errmsg name json = + match List.assoc name json with + | `Intlit i -> int_of_string i + | _ -> invalid_arg errmsg + | exception Not_found -> invalid_arg errmsg + + let section_of_json : Yojson.Raw.t -> Index.offset * [ `Map of t ] = function + | `Assoc json -> + let offset = + match List.assoc "offset" json with + | `Assoc fields -> + let gen_line = + intlit + "line" + fields + ~errmsg: + "Source_map_io.Index.of_json: field 'line' absent or invalid from \ + section" + in + let gen_column = + intlit + "column" + fields + ~errmsg: + "Source_map_io.Index.of_json: field 'column' absent or invalid from \ + section" + in + Index.{ gen_line; gen_column } + | _ -> + invalid_arg "Source_map_io.Index.of_json: 'offset' field of unexpected type" + in + (match List.assoc "url" json with + | _ -> + invalid_arg + "Source_map_io.Index.of_json: URLs in index maps are not currently \ + supported" + | exception Not_found -> ()); + let map = + try standard_map_of_json (List.assoc "map" json) with + | Not_found -> invalid_arg "Source_map_io.Index.of_json: field 'map' absent" + | Invalid_argument _ -> + invalid_arg "Source_map_io.Index.of_json: invalid sub-map object" + in + offset, `Map map + | _ -> invalid_arg "Source_map_io.Index.of_json: section of unexpected type" + + let of_json = function + | `Assoc fields -> ( + let file = string "file" fields in + match List.assoc "sections" fields with + | `List sections -> + let sections = List.map section_of_json sections in + { Index.version = 3; file = Option.value file ~default:""; sections } + | _ -> invalid_arg "Source_map_io.Index.of_json: `sections` is not an array" + | exception Not_found -> + invalid_arg "Source_map_io.Index.of_json: no `sections` field") + | _ -> invalid_arg "Source_map_io.of_json: map is not an object" + + let to_string m = Yojson.Raw.to_string (json m) + + let to_file m file = Yojson.Raw.to_file file (json m) +end + +let of_json = function + | `Assoc fields as json -> ( + match List.assoc "sections" fields with + | _ -> `Index (Index.of_json json) + | exception Not_found -> `Standard (standard_map_of_json json)) + | _ -> invalid_arg "Source_map_io.of_json: map is not an object" + +let of_string s = of_json (Yojson.Raw.from_string s) diff --git a/compiler/lib/stdlib.ml b/compiler/lib/stdlib.ml index e1a108e6a6..58ae474597 100644 --- a/compiler/lib/stdlib.ml +++ b/compiler/lib/stdlib.ml @@ -390,6 +390,12 @@ module Option = struct let value ~default = function | None -> default | Some s -> s + + let return v = Some v + + module Syntax = struct + let ( let* ) x f = bind ~f x + end end module Int64 = struct diff --git a/compiler/tests-compiler/build_path_prefix_map.ml b/compiler/tests-compiler/build_path_prefix_map.ml index 6e2ce10fdc..00117ee13e 100644 --- a/compiler/tests-compiler/build_path_prefix_map.ml +++ b/compiler/tests-compiler/build_path_prefix_map.ml @@ -29,12 +29,13 @@ let%expect_test _ = |> compile_cmo_to_javascript ~sourcemap:true ~pretty:false |> extract_sourcemap |> function - | Some (sm : Js_of_ocaml_compiler.Source_map.t) -> + | Some (`Standard (sm : Js_of_ocaml_compiler.Source_map.t)) -> Printf.printf "file: %s\n" sm.file; Printf.printf "sourceRoot: %s\n" (Option.value ~default:"" sm.sourceroot); Printf.printf "sources:\n"; List.iter sm.sources ~f:(fun source -> Printf.printf "- %s\n" (normalize_path source)) + | Some (`Index _) -> failwith "unexpected index map" | None -> failwith "no sourcemap generated!"); [%expect {| diff --git a/compiler/tests-compiler/sourcemap.ml b/compiler/tests-compiler/sourcemap.ml index 791cb94d27..65469a029e 100644 --- a/compiler/tests-compiler/sourcemap.ml +++ b/compiler/tests-compiler/sourcemap.ml @@ -23,7 +23,8 @@ open Util let print_mapping (sm : Source_map.t) = let sources = Array.of_list sm.sources in let _names = Array.of_list sm.names in - List.iter sm.mappings ~f:(fun (m : Source_map.map) -> + let mappings = Source_map.Mappings.decode sm.mappings in + List.iter mappings ~f:(fun (m : Source_map.map) -> match m with | Gen_Ori { gen_line; gen_col; ori_line; ori_col; ori_source } | Gen_Ori_Name { gen_line; gen_col; ori_line; ori_col; ori_source; ori_name = _ } -> @@ -54,7 +55,8 @@ let%expect_test _ = print_file (Filetype.path_of_js_file js_file); match extract_sourcemap js_file with | None -> Printf.printf "No sourcemap found\n" - | Some sm -> print_mapping sm); + | Some (`Standard sm) -> print_mapping sm + | Some (`Index _) -> failwith "unexpected index map"); [%expect {| $ cat "test.ml" @@ -110,8 +112,8 @@ function x (a, b) { let%expect_test _ = let map_str = ";;;;EAEE,EAAE,EAAC,CAAE;ECQY,UACC" in - let map = Source_map.mapping_of_string map_str in - let map_str' = Source_map.string_of_mapping map in + let map = Source_map.Mappings.(decode (of_string map_str)) in + let map_str' = Source_map.Mappings.(to_string (encode map)) in print_endline map_str; print_endline map_str'; [%expect @@ -128,25 +130,30 @@ let%expect_test _ = { (Source_map.empty ~filename:"1.map") with names = [ "na"; "nb"; "nc" ] ; sources = [ "sa"; "sb" ] - ; mappings = [ gen (1, 1) (10, 10) 0; gen (3, 3) (20, 20) 1 ] + ; mappings = + Source_map.Mappings.encode [ gen (1, 1) (10, 10) 0; gen (3, 3) (20, 20) 1 ] } in let s2 : Source_map.t = { (Source_map.empty ~filename:"2.map") with names = [ "na2"; "nb2" ] ; sources = [ "sa2" ] - ; mappings = [ gen (3, 3) (5, 5) 0 ] + ; mappings = Source_map.Mappings.encode [ gen (3, 3) (5, 5) 0 ] } in - let m = Source_map.merge [ s1; Source_map.filter_map s2 ~f:(fun x -> Some (x + 20)) ] in - (match m with - | None -> () - | Some sm -> - print_endline (Source_map.string_of_mapping sm.mappings); - print_mapping sm); + let edits = + Source_map.Line_edits.([ Add { count = 17 } ] @ List.init ~len:3 ~f:(Fun.const Keep)) + in + let s2 = + { s2 with mappings = Source_map.Mappings.edit ~strict:true s2.mappings edits } + in + let m = Source_map.concat ~file:"" ~sourceroot:None s1 s2 in + let encoded_mappings = m.Source_map.mappings in + print_endline (Source_map.Mappings.to_string encoded_mappings); + print_mapping m; [%expect {| - CASU;;GCUU;;;;;;;;;;;;;;;;;;;;GCff + CASU;;GCUU;;;;;;;;;;;;;;;;;;;;GCff; sa:10:10 -> 1:1 sb:20:20 -> 3:3 sa2:5:5 -> 23:3 |}] diff --git a/compiler/tests-compiler/util/util.mli b/compiler/tests-compiler/util/util.mli index 5788400928..99b0581637 100644 --- a/compiler/tests-compiler/util/util.mli +++ b/compiler/tests-compiler/util/util.mli @@ -53,7 +53,12 @@ val compile_bc_to_javascript : val jsoo_minify : ?flags:string list -> pretty:bool -> Filetype.js_file -> Filetype.js_file -val extract_sourcemap : Filetype.js_file -> Js_of_ocaml_compiler.Source_map.t option +val extract_sourcemap : + Filetype.js_file + -> [ `Standard of Js_of_ocaml_compiler.Source_map.t + | `Index of Js_of_ocaml_compiler.Source_map.Index.t + ] + option val run_javascript : Filetype.js_file -> string diff --git a/compiler/tests-sourcemap/dump_sourcemap.ml b/compiler/tests-sourcemap/dump_sourcemap.ml index eec55e5a8e..78729c7cf1 100644 --- a/compiler/tests-sourcemap/dump_sourcemap.ml +++ b/compiler/tests-sourcemap/dump_sourcemap.ml @@ -36,11 +36,12 @@ let extract_sourcemap lines = Some (Source_map_io.of_string content) | _ -> None -let print_mapping lines (sm : Source_map.t) = +let print_mapping lines ?(line_offset = 0) (sm : Source_map.t) = let lines = Array.of_list lines in let sources = Array.of_list sm.sources in let _names = Array.of_list sm.names in - List.iter sm.mappings ~f:(fun (m : Source_map.map) -> + let mappings = Source_map.Mappings.decode sm.mappings in + List.iter mappings ~f:(fun (m : Source_map.map) -> let file = function | -1 -> "null" | n -> normalize_path sources.(n) @@ -67,9 +68,22 @@ let print_mapping lines (sm : Source_map.t) = ori_line ori_col gen_col - (mark gen_col lines.(gen_line - 1)) + (mark gen_col lines.(gen_line - 1 + line_offset)) | _ -> ())) +let print_sourcemap lines = function + | `Standard sm -> print_mapping lines sm + | `Index l -> + List.iter + l.Source_map.Index.sections + ~f:(fun (Source_map.Index.{ gen_line; gen_column }, `Map sm) -> + assert (gen_column = 0); + (* + Format.eprintf "[@SOURCEMAP (line = %d)@," gen_line; + Yojson.Raw.pretty_print Format.err_formatter sm; + *) + print_mapping lines ~line_offset:gen_line sm) + let files = Sys.argv |> Array.to_list |> List.tl let () = @@ -79,4 +93,4 @@ let () = | None -> Printf.printf "not sourcemap for %s\n" f | Some sm -> Printf.printf "sourcemap for %s\n" f; - print_mapping lines sm) + print_sourcemap lines sm) diff --git a/compiler/tests-sourcemap/dune b/compiler/tests-sourcemap/dune index e690f5a3ee..efaf86e885 100644 --- a/compiler/tests-sourcemap/dune +++ b/compiler/tests-sourcemap/dune @@ -1,6 +1,10 @@ (env (_ (js_of_ocaml + (flags + (:standard --source-map)) + (link_flags + (:standard --source-map --resolve-sourcemap-url true)) (compilation_mode separate)))) (executable diff --git a/tools/sourcemap/jsoo_sourcemap.ml b/tools/sourcemap/jsoo_sourcemap.ml index 309358705d..9c359c6587 100644 --- a/tools/sourcemap/jsoo_sourcemap.ml +++ b/tools/sourcemap/jsoo_sourcemap.ml @@ -44,5 +44,6 @@ let () = | Some base64 -> Js_of_ocaml_compiler.Base64.decode_exn base64) | _ -> failwith "unable to find sourcemap" in - let sm = Js_of_ocaml_compiler.Source_map_io.of_string content in - print_endline (Js_of_ocaml_compiler.Source_map_io.to_string sm) + match Js_of_ocaml_compiler.Source_map_io.of_string content with + | `Standard sm -> print_endline (Js_of_ocaml_compiler.Source_map_io.to_string sm) + | `Index im -> print_endline (Js_of_ocaml_compiler.Source_map_io.Index.to_string im)