Merge pull request #8 from OCamlPro/z-2020-08-01-use-patches-only

patches_url supports archives

.gitignore

@@ -2,3 +2,4 @@
 /opam-bin
 _build
 .merlin
+/test/root

src/opambinCommandClean.ml

@@ -16,7 +16,7 @@ let clean_log () =
   Sys.remove OpambinGlobals.opambin_log ;
   ()
 
-let clean_all () =
+let clean_store () =
   List.iter (fun dir ->
       Printf.eprintf "Cleaning %s\n%!" dir;
       OpambinMisc.call [| "rm"; "-rf" ; dir |];
@@ -26,9 +26,13 @@ let clean_all () =
       OpambinGlobals.opambin_store_repo_packages_dir ;
       OpambinGlobals.opambin_store_archives_dir ;
     ];
+  OpambinMisc.call [| "opam"; "update" |];
+  ()
+
+let clean_all () =
   clean_log ();
+  clean_store ();
   (* flush the copy of the repo that opam keeps *)
-  OpambinMisc.call [| "opam"; "update" |];
   ()
 
 let action args =
@@ -38,6 +42,7 @@ let action args =
     List.iter (function
         | "all" -> clean_all ()
         | "log" -> clean_log ()
+        | "store" -> clean_store ()
         | s ->
           Printf.eprintf "Unexpected argument %S.\n%!" s;
           exit 2) args
@@ -49,7 +54,7 @@ let cmd =
   cmd_action = (fun () -> action !anon_args) ;
   cmd_args = [
     [], Arg.Anons (fun list -> anon_args := list),
-    Ezcmd.info "What to clean (`all` or `log`)";
+    Ezcmd.info "What to clean (`all`, `log` or `store`)";
   ];
   cmd_man = [];
   cmd_doc = "clear all packages and archives from the cache and store";

src/opambinCommandConfig.ml

@@ -117,7 +117,7 @@ let cmd = {
       ),
     Ezcmd.info @@
     Printf.sprintf
-      "location of relocation patches (git@ or file://)" ;
+      "location of relocation patches (git@, file:// or https://)" ;
 
     [ "title" ], Arg.String (fun s ->
         OpambinConfig.title =:= s;

src/opambinCommandInstall.ml

@@ -71,7 +71,7 @@ let install_hooks () =
 
 let install_repos () =
 
-  add_repo ~repo:"local-bin"
+  add_repo ~repo:OpambinGlobals.opam_opambin_repo
     ~url:( Printf.sprintf "file://%s"
              OpambinGlobals.opambin_store_repo_dir )
 
@@ -81,23 +81,49 @@ let install_patches () =
     (* nothing to do *)
     ()
   else
-  if EzString.starts_with patches_url ~prefix:"git@"
-  || EzString.starts_with patches_url ~prefix:"https://"
-  || EzString.starts_with patches_url ~prefix:"http://"
-  then
     let opambin_patches_dir = OpambinGlobals.opambin_patches_dir in
-    OpambinMisc.call [| "rm"; "-rf"; opambin_patches_dir ^ ".tmp" |];
-    OpambinMisc.call
-      [| "git"; "clone" ; patches_url ; opambin_patches_dir ^ ".tmp" |];
-    OpambinMisc.call [| "rm"; "-rf"; opambin_patches_dir |];
-    OpambinMisc.call
-      [| "mv"; opambin_patches_dir ^ ".tmp"; opambin_patches_dir |]
-  else
-    begin
-      Printf.eprintf
-        "Error: patches_url '%s' should either be local (file://) or git (git@, http[s]://)\n%!" patches_url;
-      exit 2
-    end
+    let tmp_dir = opambin_patches_dir ^ ".tmp" in
+
+    if EzString.starts_with patches_url ~prefix:"git@" then begin
+      OpambinMisc.call [| "rm"; "-rf"; tmp_dir |];
+      OpambinMisc.call [| "git"; "clone" ; patches_url ; tmp_dir |];
+      OpambinMisc.call [| "rm"; "-rf"; opambin_patches_dir |];
+      OpambinMisc.call [| "mv"; tmp_dir; opambin_patches_dir |]
+    end else
+
+    if EzString.starts_with patches_url ~prefix:"https://"
+    || EzString.starts_with patches_url ~prefix:"http://" then begin
+        let output = OpambinGlobals.opambin_dir // "relocation-patches.tar.gz" in
+        match OpambinMisc.wget ~url:patches_url ~output with
+        | None ->
+          Printf.kprintf failwith "Could not retrieve archive at %s" patches_url
+        | Some output ->
+
+          OpambinMisc.call [| "rm"; "-rf"; tmp_dir |];
+          EzFile.make_dir ~p:true tmp_dir ;
+
+          Unix.chdir tmp_dir ;
+          OpambinMisc.call [| "tar" ; "zxf" ; output |] ;
+          Unix.chdir OpambinGlobals.curdir;
+
+          let patches_subdir = tmp_dir // "patches" in
+          if not ( Sys.file_exists patches_subdir ) then
+            Printf.kprintf failwith
+              "archive %s does not contain 'patches/' subdir" patches_url;
+
+          OpambinMisc.call [| "rm"; "-rf"; opambin_patches_dir |];
+          EzFile.make_dir ~p:true opambin_patches_dir;
+          Sys.rename patches_subdir (opambin_patches_dir // "patches");
+          OpambinMisc.call [| "rm"; "-rf"; tmp_dir |];
+          Sys.remove output
+
+      end
+    else
+      begin
+        Printf.eprintf
+          "Error: patches_url '%s' should either be local (file://) or git (git@, http[s]://)\n%!" patches_url;
+        exit 2
+      end
 
 
 let action args =

src/opambinCommandPostInstall.ml

@@ -108,7 +108,7 @@ let digest_sources () =
   in
   iter ".";
   let s = Buffer.contents b in
-  OpambinMisc.global_log "buffer: %s" s ;
+  (*  OpambinMisc.global_log "buffer: %s" s ; *)
   EzFile.write_file
     ( Printf.sprintf "/tmp/buffer.%d" (Unix.getpid ()) ) s;
   digest s
@@ -143,31 +143,31 @@ let compute_hash ?source_md5 ~name ~version ~depends () =
   let switch = OpambinMisc.current_switch () in
   let temp_dir = OpambinGlobals.opambin_switch_temp_dir () in
   EzFile.make_dir ~p:true temp_dir ;
-  let opam_file = OpambinGlobals.marker_opam ~name in
-  if not ( Sys.file_exists opam_file ) then begin
-    let oc = Unix.openfile opam_file
-        [ Unix.O_CREAT; Unix.O_WRONLY ; Unix.O_TRUNC ] 0o644 in
-    let nv = Printf.sprintf "%s.%s" name version in
-    OpambinMisc.call ~stdout:oc
-      [| "opam" ; "show" ; nv ; "--raw" ; "--safe" ; "--switch" ; switch |];
-    Unix.close oc;
-  end ;
-  let opam_content = EzFile.read_file opam_file in
-  let source_md5 = match source_md5 with
-    | Some source_md5 -> source_md5
+  let ( source_md5, opam_file ) = match source_md5 with
+    | Some source_md5 ->
+      source_md5, OpambinGlobals.backup_opam ~name
     | None ->
+      let digest_sources = digest_sources () in
+
+      let opam_file = OpambinGlobals.marker_opam in
+      let oc = Unix.openfile opam_file
+          [ Unix.O_CREAT; Unix.O_WRONLY ; Unix.O_TRUNC ] 0o644 in
+      let nv = Printf.sprintf "%s.%s" name version in
+      OpambinMisc.call ~stdout:oc
+        [| "opam" ; "show" ; nv ; "--raw" ; "--safe" ; "--switch" ; switch |];
+      Unix.close oc;
+      let opam_content = EzFile.read_file opam_file in
       OpambinMisc.global_log "File (%s):\n%s" name opam_content;
       let digest_opam = digest opam_content in
       OpambinMisc.global_log "Opam (%s): %s" name digest_opam;
 
-      let package_uid = digest (
-          digest_opam ^ digest_sources ()
-        ) in
+      let package_uid = digest ( digest_opam ^ digest_sources ) in
+
       OpambinMisc.global_log "package_uid(%s): %s" name package_uid;
       let s = Printf.sprintf "%s.%s|%s|%s"
           name version package_uid (String.concat "," depends_nv) in
       OpambinMisc.global_log "source(%s) : %s" name s ;
-      short ( digest s )
+      short ( digest s ), opam_file
   in
   OpambinMisc.global_log "source_md5 (%s): %s" name source_md5;
   ( source_md5, depends, !dependset, !missing_versions, opam_file )
@@ -180,7 +180,7 @@ let error_on_missing =
 let commit ~name ~version ~depends files =
   if
     not !!OpambinConfig.create_enabled
-    || Sys.file_exists ( OpambinGlobals.marker_skip ~name )
+    || Sys.file_exists ( OpambinGlobals.backup_skip ~name )
   then
     OpambinMisc.global_log "package %s: create disabled" name
   else
@@ -198,7 +198,7 @@ let commit ~name ~version ~depends files =
       OpambinMisc.global_log "creating binary archive...";
       EzFile.make_dir ~p:true temp_dir ;
       let source_md5 =
-        EzFile.read_file ( OpambinGlobals.marker_source ~name ) in
+        EzFile.read_file ( OpambinGlobals.backup_source ~name ) in
       let ( source_md5, depends, dependset, missing_versions, opam_file ) =
         compute_hash ~source_md5 ~name ~version ~depends () in
       if missing_versions <> [] then begin

src/opambinCommandPreBuild.ml

@@ -61,21 +61,6 @@ archive-mirrors: "../../cache"
 *)
       None
 
-let wget ~url ~md5 =
-  let temp_dir = OpambinGlobals.opambin_switch_temp_dir () in
-  EzFile.make_dir ~p:true temp_dir;
-  let output = temp_dir // md5 in
-  OpambinMisc.call [| "curl" ;
-          "--write-out" ; "%{http_code}\\n" ;
-          "--retry" ; "3" ;
-          "--retry-delay" ; "2" ;
-          "--user-agent" ; "opam-bin/2.0.5" ;
-          "-L" ;
-          "-o" ; output ;
-          url
-       |];
-  Some output
-
 let check_cached_binary_archive ~version ~repo ~package =
   OpambinMisc.global_log "found binary package in repo %s" repo;
   let package_dir = repo // "packages" // package // version in
@@ -110,7 +95,10 @@ let check_cached_binary_archive ~version ~repo ~package =
           Printf.eprintf "error: url.src not found\n%!";
           exit 2
         | Some url ->
-          match wget ~url ~md5 with
+
+          let temp_dir = OpambinGlobals.opambin_switch_temp_dir () in
+          let output = temp_dir // md5 in
+          match OpambinMisc.wget ~url ~output with
           | None ->
             Printf.eprintf "Error: could not download archive at %S\n%!" url;
             exit 2
@@ -230,10 +218,10 @@ Error: patches dir '%s' does not exist.\n
           OpambinMisc.global_log "Using patch %s for %s.%s"
             patch name keep_version ;
           OpambinMisc.call [| "cp" ; "-f";
-                              patch ; OpambinGlobals.marker_patch ~name |];
+                              patch ; OpambinGlobals.marker_patch |];
           OpambinMisc.call [| "patch" ; "-p1"; "-i"; patch |] ;
           if Sys.file_exists "reloc-patch.sh" then
-            OpambinMisc.call [| "sh"; "-c"; "./reloc_patch.sh" |];
+            OpambinMisc.call [| "sh"; "./reloc_patch.sh" |];
           true
     else
       true
@@ -280,27 +268,36 @@ let action args =
     ( String.concat "\n    " ( cmd_name :: args) ) ;
   match args with
   | name :: version :: depends :: [] ->
-    let marker_skip = OpambinGlobals.marker_skip ~name in
+    let marker_skip = OpambinGlobals.marker_skip in
     if not !!OpambinConfig.enabled
     || OpambinMisc.not_this_switch () then begin
       OpambinMisc.global_log "opam-bin is disabled";
-      OpambinGlobals.write_marker marker_skip
+      EzFile.write_file marker_skip
         "opam-bin is disabled";
     end else
-      let marker_source = OpambinGlobals.marker_source ~name in
-      let marker_opam = OpambinGlobals.marker_opam ~name in
-      let marker_patch = OpambinGlobals.marker_patch ~name in
-      if Sys.file_exists marker_source then Sys.remove marker_source ;
-      if Sys.file_exists marker_opam then Sys.remove marker_opam ;
-      if Sys.file_exists marker_patch then Sys.remove marker_patch ;
+      let marker_source = OpambinGlobals.marker_source in
+      let marker_opam = OpambinGlobals.marker_opam in
+      let marker_patch = OpambinGlobals.marker_patch in
+      if Sys.file_exists marker_source then begin
+        OpambinMisc.global_log "removing marker_source";
+        Sys.remove marker_source ;
+      end;
+      if Sys.file_exists marker_opam then begin
+        OpambinMisc.global_log "removing marker_opam";
+        Sys.remove marker_opam ;
+      end;
+      if Sys.file_exists marker_patch then begin
+        OpambinMisc.global_log "removing marker_patch";
+        Sys.remove marker_patch ;
+      end;
       if Sys.file_exists OpambinGlobals.marker_cached then begin
         OpambinMisc.global_log "%s should not already exist!"
           OpambinGlobals.marker_cached;
         exit 2
       end else
       if Sys.file_exists OpambinGlobals.package_version then begin
         OpambinMisc.global_log "already a binary package";
-        OpambinGlobals.write_marker marker_source "already-a-binary-package";
+        EzFile.write_file marker_source "already-a-binary-package";
       end else begin
         OpambinMisc.global_log "checking for cached archive";
         match cached_binary_archive ~name ~version ~depends with
@@ -314,9 +311,9 @@ let action args =
               "Error: opam-bin is configured to prevent compilation.\n%!";
             exit 2
           end;
-          OpambinGlobals.write_marker marker_source source_md5
+          EzFile.write_file marker_source source_md5
         | `MissingVersions missing_versions ->
-          OpambinGlobals.write_marker marker_skip
+          EzFile.write_file marker_skip
             ( Printf.sprintf "Missing binary deps: %s"
                 ( String.concat " " missing_versions ) )
         | `NotRelocatable ->
@@ -325,7 +322,7 @@ let action args =
               "Error: opam-bin is configured to force relocation.\n%!";
             exit 2
           end;
-          OpambinGlobals.write_marker marker_skip
+          EzFile.write_file marker_skip
             "Missing relocation patch for unrelocatable package"
       end
   | _ ->

src/opambinCommandPreInstall.ml

@@ -31,14 +31,30 @@ let cmd_name = "pre-install"
    NOTE: we could move the installation part in a pre-install command.
  *)
 
+(* During the pre-install, we should copy local markers to switch
+   markers, to avoid them being copied by install steps. We could not
+   have done it before because they would be removed by the removal of
+   a previous version of the package happening between build and
+   install. *)
+
 let action args =
   OpambinMisc.global_log "CMD: %s\n%!"
     ( String.concat "\n    " ( cmd_name :: args) ) ;
   OpambinMisc.make_cache_dir ();
   match args with
   | name :: _version :: _depends :: [] ->
-    if Sys.file_exists ( OpambinGlobals.marker_source ~name )
-    || Sys.file_exists ( OpambinGlobals.marker_skip ~name )
+    List.iter (fun (marker, backup) ->
+        if Sys.file_exists marker then
+          Sys.rename marker ( backup ~name )
+      ) [
+      OpambinGlobals.marker_skip , OpambinGlobals.backup_skip;
+      OpambinGlobals.marker_source , OpambinGlobals.backup_source;
+      OpambinGlobals.marker_opam , OpambinGlobals.backup_opam;
+      OpambinGlobals.marker_patch ,OpambinGlobals.backup_patch;
+    ];
+
+    if Sys.file_exists ( OpambinGlobals.backup_source ~name )
+    || Sys.file_exists ( OpambinGlobals.backup_skip ~name )
     then
       ()
     else

src/opambinCommandPreRemove.ml

@@ -24,10 +24,10 @@ let action args =
         with _ -> ()
       ) [
       OpambinGlobals.opambin_switch_packages_dir () // name ;
-      OpambinGlobals.marker_source ~name ;
-      OpambinGlobals.marker_opam ~name ;
-      OpambinGlobals.marker_skip ~name ;
-      OpambinGlobals.marker_patch ~name ;
+      OpambinGlobals.backup_marker ~name ".source" ;
+      OpambinGlobals.backup_marker ~name ".skip" ;
+      OpambinGlobals.backup_marker ~name ".opam" ;
+      OpambinGlobals.backup_marker ~name ".patch" ;
     ]
   | _ ->
     Printf.eprintf

src/opambinCommandWrapBuild.ml

@@ -31,9 +31,9 @@ let action args =
     ( String.concat "\n    " ( cmd_name :: args) ) ;
   OpambinMisc.make_cache_dir ();
   match args with
-  | name :: _version :: _depends :: cmd ->
-    if Sys.file_exists ( OpambinGlobals.marker_source ~name )
-    || Sys.file_exists ( OpambinGlobals.marker_skip ~name )
+  | _name :: _version :: _depends :: cmd ->
+    if Sys.file_exists OpambinGlobals.marker_source
+    || Sys.file_exists OpambinGlobals.marker_skip
     then
       OpambinMisc.call (Array.of_list cmd)
   | _ ->

src/opambinCommandWrapInstall.ml

@@ -36,8 +36,8 @@ let action args =
   OpambinMisc.make_cache_dir ();
   match args with
   | name :: _version :: _depends :: cmd ->
-    if Sys.file_exists ( OpambinGlobals.marker_source ~name )
-    || Sys.file_exists ( OpambinGlobals.marker_skip ~name )
+    if Sys.file_exists ( OpambinGlobals.backup_source ~name )
+    || Sys.file_exists ( OpambinGlobals.backup_skip ~name )
     then
       OpambinMisc.call (Array.of_list cmd)
   | _ ->