diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b005b9c..9819cdd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,13 +12,18 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Java - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: '8' distribution: 'zulu' @@ -29,6 +34,15 @@ jobs: with: python-version: '3.10' + - name: Cache Appose environments + id: cache-appose + uses: actions/cache@v4 + with: + path: ~/.local/share/appose + key: ${{ runner.os }}-build-appose-${{ hashFiles('*') }} + restore-keys: | + ${{ runner.os }}-build-appose- + - name: Set up CI environment run: .github/setup.sh diff --git a/notes.md b/notes.md new file mode 100644 index 0000000..07c0ebf --- /dev/null +++ b/notes.md @@ -0,0 +1,291 @@ +## Builder API + +* Want an API to create an environment from an envFile: i.e. `pixi.toml` or `environment.yml`. +* Want an API to build up an environment piecemeal: adding dependencies one by one. +* Do we need an API to mix and match these two things? I.e. start from envFile but then add on? + - An argument for this: what if you want to mix in Java JARs? pixi.toml can't do that yet. +* Want an API to create an environment from a *string* representation of an envFile. + +Most flexible to put all these things into the Builder, not only directly in Appose (like `system()`). + +What sorts of dependencies do we want to support adding? + +1. conda-forge packages +2. PyPI packages +3. Maven coords +4. Java itself + +Pixi gets us (1) and (2). + +* Maven coords can be gotten by Groovy Grape in Java, by jgo in Python. (jgo needs work) +* Java itself can be gotten by cjdk in Python; what about from Java? Port cjdk? Or hack it with openjdk conda for now? + +Should we make the API more general than the above? Yes! We can use `ServiceLoader`, same as we do with `ShmFactory`. + +The interface is `BuildHandler`: +* `boolean include(String content, String scheme)` +* `boolean channel(String name, String location)` + +And the implementations + supported schemes are: +* `PixiHandler` -- `environment.yml`, `pixi.toml`, `pypi`, `conda`, and null. +* `MavenHandler` -- `maven` +* `JDKHandler` -- `openjdk` + +Although the term "scheme" might be confused with URI scheme, other terms are problematic too: +* "platform" will be confused with OS/arch. +* "method" will be confused with functions of a class. +* "system" might be confused with the computer itself, and/or system environment, path, etc. +* "paradigm" sounds too pretentious. +* "repoType" is rather clunky. + +The `Builder` then has its own `include` and `channel` methods that delegate to +all discovered `BuildHandler` plugins. The `Builder` can also have more +convenience methods: + +* `Builder file(String filePath) { return file(new File(filePath)); }` +* `Builder file(String filePath, String scheme) { return file(new File(filePath), scheme); }` +* `Builder file(File file) { return file(file, file.getName()); }` +* `Builder file(File file, String scheme) { return include(readContentsAsString(file), scheme); }` + +For the `file`-to-`include` trick to work with files like `requirements.txt`, +the handling of `conda`/null scheme should split the content string into lines, +and process them in a loop. + +Here are some example API calls made possible by the above design: +```java +Appose.env() + .file("/path/to/environment.yml") + // OR: .file("/path/to/pixi.toml") + // OR: .file("/path/to/requirements.txt", "pypi") + .include("cowsay", "pypi") + .include("openjdk>=17") // Install OpenJDK from conda-forge! + .include("maven") // Install Maven from conda-forge... confusing, yeah? + .include("conda-forge::maven") // Specify channel explicitly with environment.yml syntax. + .include("org.scijava:parsington", "maven") + // OR: .include("org.scijava:parsington") i.e. infer `maven` from the colon? + // OR: .include("org.scijava:parsington:2.0.0", "maven") + .channel("scijava", "maven:https://maven.scijava.org/content/groups/public") + .include("sc.fiji:fiji", "maven") + .include("zulu:17", "openjdk") // Install an OpenJDK from the Coursier index. + + .channel("bioconda") // Add a conda channel + .channel(name: str, location: str = None) + .build() // Whew! +``` + +### 2024-08-20 update + +One tricky thing is the base directory when combining paradigms: +* Get rid of "base directory" naming (in conda, the "base" environment is something else, so it's a confusing word here) in favor of `${appose-cache-dir}/${env-name}` convention. By default, `appose-cache-dir` equals `~/.local/share/appose`, but we could provide a way to override it... +* Similarly, get rid of the `base(...)` builder method in favor of adding a `build(String envName) -> Environment` signature. +* But what to name `Environment#base` property now? I really like `base`... Maybe `basedir`? Or `prefix``? +* Each `BuildHandler` catalogs the `include` and `channel` calls it feels are relevant, but does not do anything until `build` is finally called. +* If `build()` is called with no args, then the build handlers are queried sequentially (`default String envName() { return null; }`?). The first non-null name that comes back is taken as truth and then `build(thatName)` is passed to all handlers. Otherwise, an exception is raised "No environment name given". +* Environments all live in `~/.local/share/appose/`, where `` is the name of the environment. If `name:` is given in a `pixi.toml` or `environment.yml`, great, the `PixiBuildHandler` can parse out that string when its `envName()` method is called. + +What about starting child processes via `pixi run`? That's not agnostic of the builder... +* What if the build handlers are also involved in child process launches? The `service(exes, args)` method could delegate to them... +* `BuildHandler` → `EnvHandler`? +* In `Environment`, how about replacing `use_system_path` with just a `path` list of dirs to check for executables being launched by `service`? Then we could dispense with the boilerplate `python`, `bin/python` repetition. +* Each env handler gets a chance to influence the worker launch args... and/or bin dirs... + - The pixi handler could prepend `.../pixi run` when a `pixi.toml` is present. + But this is tricky, because it goes *before* the selected exe... pre-args vs post-args uhhh + +So, environment has: +* path (list of directories -- only used if `all_args[0]` is not already an absolute path to an executable program already?) +* launcher (list of string args to prepend) +* classpath (list of elements to include when running java) + +* `Map>` is what's returned by the `build(...)` step of each `BuildHandler`. + - Relevant keys include: "path", "classpath", "launcher" + - The `new Environment() { ... }` invocation will aggregate the values given here into its accessors. + +* When running a service, it first uses the path to search for the requested exe, before prepending the launcher args. + - What about pixi + cjdk? We'll need the full path to java... + - How can we tell the difference between that and pixi alone with openjdk from conda-forge? + - In the former case, we need the full path, in the latter case, no. + - Pixi should just put `${envDir}/bin` onto the path, no? + - There is an edge case where `pixi run foo` works, even though `foo` is not an executable on the path... in which case, the environment service can just plow ahead with it when it can't find a `foo` executable on the path. But it should *first* make an attempt to reify the `foo` from the environment `path` before punting in that way. + +#### pixi + +During its build step, it prepends `[/path/to/pixi, run]` to the environment's launcher list, and `${env-dir}/bin` to the environment's path list. + +#### cjdk + +``` +cjdk -j adoptium:21 java-home +/home/curtis/.cache/cjdk/v0/jdks/d217ee819493b9c56beed2e4d481e4c370de993d/jdk-21.0.4+7 +/home/curtis/.cache/cjdk/v0/jdks/d217ee819493b9c56beed2e4d481e4c370de993d/jdk-21.0.4+7/bin/java -version +openjdk version "21.0.4" 2024-07-16 LTS +OpenJDK Runtime Environment Temurin-21.0.4+7 (build 21.0.4+7-LTS) +OpenJDK 64-Bit Server VM Temurin-21.0.4+7 (build 21.0.4+7-LTS, mixed mode, sharing) +``` + +So `JDKHandler`, during its build step, prepends the java-home directory to the environment's path: `$(cjdk -j adoptium:21 java-home)/bin` + +#### Maven + +No need to add any directories to the environment path! + +However, if Maven artifacts are added via `includes`, they should not only be downloaded, but also be part of the class path when launching java-based programs. We could do that by putting classpath into the `Environment` class directly, along side `path`... it's only a little hacky ;_; + +## Pixi + +Is even better than micromamba. It's a great fit for Appose's requirements. + +### Setup for Appose + +```shell +# Install a copy of Pixi into Appose's workspace. +mkdir -p ~/.local/share/appose/tmp +cd ~/.local/share/appose/tmp +curl -fsLO https://github.com/prefix-dev/pixi/releases/download/v0.27.1/pixi-x86_64-unknown-linux-musl.tar.gz +mkdir -p ../.pixi/bin +cd ../.pixi/bin +tar xf ../tmp/pixi-x86_64-unknown-linux-musl.tar.gz +alias pixi=~/.local/share/appose/.pixi/bin/pixi +``` + +And/or consider setting `PIXI_HOME` to `$HOME/.local/share/appose` +when using the `$HOME/.local/share/appose/.pixi/bin/pixi` binary. +This would let us, in the future, tweak Pixi's Appose-wide configuration +by adding a `$HOME/.local/share/appose/.pixi/config.toml` file. + +#### Create an Appose environment + +```shell +mkdir -p ~/.local/share/appose/sc-fiji-spiff +pixi init ~/.local/share/appose/sc-fiji-spiff +``` + +#### Add channels to the project/environment + +```shell +cd ~/.local/share/appose/sc-fiji-spiff +pixi project channel add bioconda pytorch +``` + +Doing them all in one command will have less overhead. + +#### Add dependencies to the project/environment + +```shell +cd ~/.local/share/appose/sc-fiji-spiff +pixi add python pip +pixi add --pypi cowsay +``` + +Doing them all in two commands (one for conda, one for pypi) will have less overhead. + +#### Use it + +```shell +pixi run python -c 'import cowsay; cowsay.cow("moo")' +``` + +One awesome thing is that Appose will be able to launch the +child process using `pixi run ...`, which takes care of running +activation scripts before launch—so the child program should +work as though run from an activated environment (or `pixi shell`). + +### Bugs + +#### Invalid environment names do not fail fast + +```shell +pixi project environment add sc.fiji.spiff +pixi tree +``` +Fails with: `Failed to parse environment name 'sc.fiji.spiff', please use only lowercase letters, numbers and dashes` + +### Multiple environments in one pixi project? + +I explored making a single Appose project and using pixi's multi-environment +support to manage Appose environments, all within that one project, as follows: + +```shell +# Initialize the shared Appose project. +pixi init +pixi project description set "Appose: multi-language interprocess cooperation with shared memory." + +# Create a new environment within the Appose project. +pixi project environment add sc-fiji-spiff +# Install dependencies into a feature with matching name. +pixi add --feature sc-fiji-spiff python pip +pixi add --feature sc-fiji-spiff --pypi cowsay +# No known command to link sc-fiji-spiff feature with sc-fiji-spiff project... +mv pixi.toml pixi.toml.old +sed 's/sc-fiji-spiff = \[\]/sc-fiji-spiff = ["sc-fiji-spiff"]/' pixi.toml.old > pixi.toml +# Finally, we can use the environment! +pixi run --environment sc-fiji-spiff python -c 'import cowsay; cowsay.cow("moo")' +``` + +This works, but a single `pixi.toml` file for all of Appose is probably too +fragile, whereas a separate project folder for each Appose environment should +be more robust, reducing the chance that one Appose-based project (e.g. JDLL) +might stomp on another Appose-based project (e.g. TrackMate) due to their usage +of the same `pixi.toml`. + +So we'll just settle for pixi's standard behavior here: a single environment +named `default` per pixi project, with one pixi project per Appose environment. +Unfortunately, that means our environment directory structure will be: +``` +~/.local/share/appose/sc-fiji-spiff/.pixi/envs/default +``` +for an Appose environment named `sc-fiji-spiff`. +(Note that environment names cannot contain dots, only alphameric and dash.) + +To attain a better structure, I tried creating a `~/.local/share/appose/sc-fiji-spiff/.pixi/config.toml` with contents: +```toml +detached-environments: "/home/curtis/.local/share/appose" +``` + +It works, but then the environment folder from above ends up being: +``` +~/.local/share/appose/sc-fiji-spiff-/envs/sc-fiji-spiff +``` +Looks like pixi creates one folder under `envs` for each project, with a +numerical hash to reduce collisions between multiple projects with the same +name... and then still makes an `envs` folder for that project beneath it. +So there is no escape from pixi's directory convention of: +``` +/envs/ +``` +Which of these is the least annoying? +``` +~/.local/share/appose/sc-fiji-spiff/.pixi/envs/default +~/.local/share/appose/sc-fiji-spiff-782634298734/envs/default +~/.local/share/appose/sc-fiji-spiff/envs/sc-fiji-spiff +~/.local/share/appose/sc-fiji-spiff-782634298734/envs/sc-fiji-spiff +``` +The detached-environments approach is actually longer, and entails +additional configuration and more potential for confusion; the +shortest path ends up being the first one, which is pixi's standard +behavior anyway. + +The only shorter one would be: +``` +~/.local/share/appose/.pixi/envs/sc-fiji-spiff +``` +if we opted to keep `~/.local/share/appose` as a single Pixi project +root with multiple environments... but the inconvenience and risks +around a single shared `pixi.toml`, and hassle of multi-environment +configuration, outweigh the benefit of slightly shorter paths. + +With separate Pixi projects we can also let Appose users specify their own +`pixi.toml` (maybe a partial one?), directly. Or an `environment.yml` that gets +used via `pixi init --import`. Maybe someday even a `requirements.txt`, if the +request (https://github.com/prefix-dev/pixi/issues/1410) gets implemented. + +## Next steps + +1. Add tests for the current Mamba builder. +2. Make the tests pass. +3. Introduce `BuildHandler` design and migrate Mamba logic to a build handler. +4. Implement a build handler built on pixi, to replace the micromamba one. +5. Implement build handlers for maven and openjdk. +6. Implement pixi, maven, and openjdk build handlers in appose-python, too. +7. Once it all works: release 0.3.0. + +And: update https://github.com/imglib/imglib2-appose to work with appose 0.2.0+. diff --git a/pom.xml b/pom.xml index 9202965..0376c0d 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.apposed appose - 0.2.1-SNAPSHOT + 0.3.0-SNAPSHOT Appose Appose: multi-language interprocess cooperation with shared memory. @@ -93,7 +93,7 @@ @@ -108,7 +108,6 @@ org.apache.ivy ivy - ${ivy.version} @@ -121,6 +120,12 @@ jna-platform + + + org.apache.commons + commons-compress + + org.junit.jupiter @@ -132,5 +137,11 @@ junit-jupiter-engine test + + commons-io + commons-io + test + + diff --git a/src/main/java/org/apposed/appose/Appose.java b/src/main/java/org/apposed/appose/Appose.java index 8bf7dd4..53f75e4 100644 --- a/src/main/java/org/apposed/appose/Appose.java +++ b/src/main/java/org/apposed/appose/Appose.java @@ -30,6 +30,7 @@ package org.apposed.appose; import java.io.File; +import java.io.IOException; /** * Appose is a library for interprocess cooperation with shared memory. The @@ -49,10 +50,6 @@ * {@link Service.Task#listen via callbacks}. * *

Examples

- *
    - *
  • TODO - move the below code somewhere linkable, for succinctness - * here.
    • - *
*

* Here is a very simple example written in Java: *

@@ -143,40 +140,166 @@ *
  • The worker must issue responses in Appose's response format on its * standard output (stdout) stream.
    • * + *

    Requests to worker from service

    *

    - * TODO - write up the request and response formats in detail here! - * JSON, one line per request/response. + * A request is a single line of JSON sent to the worker process via + * its standard input stream. It has a {@code task} key taking the form of a + * UUID, + * and a {@code requestType} key with one of the following values: *

    + *

    EXECUTE

    + *

    + * Asynchronously execute a script within the worker process. E.g.: + *

    + * 
    
+ * {
+ *    "task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
+ *    "requestType" : "EXECUTE",
+ *    "script" : "task.outputs[\"result\"] = computeResult(gamma)\n",
+ *    "inputs" : {"gamma": 2.2}
+ * }
+ *
    + *

    CANCEL

    + *

    + * Cancel a running script. E.g.: + *

    + * 
    
+ * {
+ *    "task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
+ *    "requestType" : "CANCEL"
+ * }
+ *
    * + *

    Responses from worker to service

    + *

    + * A response is a single line of JSON with a {@code task} key + * taking the form of a + * UUID, + * and a {@code responseType} key with one of the following values: + *

    + *

    LAUNCH

    + *

    + * A LAUNCH response is issued to confirm the success of an EXECUTE request. + *

    + * 
    
+ * {
+ *    "task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
+ *    "responseType" : "LAUNCH"
+ * }
+ *
    + *

    UPDATE

    + *

    + * An UPDATE response is issued to convey that a task has somehow made + * progress. The UPDATE response typically comes bundled with a {@code message} + * string indicating what has changed, {@code current} and/or {@code maximum} + * progress indicators conveying the step the task has reached, or both. + *

    + * 
    
+ * {
+ *    "task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
+ *    "responseType" : "UPDATE",
+ *    "message" : "Processing step 0 of 91",
+ *    "current" : 0,
+ *    "maximum" : 91
+ * }
+ *
    + *

    COMPLETION

    + *

    + * A COMPLETION response is issued to convey that a task has successfully + * completed execution, as well as report the values of any task outputs. + *

    + * 
    
+ * {
+ *    "task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
+ *    "responseType" : "COMPLETION",
+ *    "outputs" : {"result" : 91}
+ * }
+ *
    + *

    CANCELATION

    + *

    + * A CANCELATION response is issued to confirm the success of a CANCEL request. + *

    + * 
    
+ * {
+ *    "task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
+ *    "responseType" : "CANCELATION"
+ * }
+ *
    + *

    FAILURE

    + *

    + * A FAILURE response is issued to convey that a task did not completely + * and successfully execute, such as an exception being raised. + *

    + * 
    
+ * {
+ *    "task" : "87427f91-d193-4b25-8d35-e1292a34b5c4",
+ *    "responseType" : "FAILURE",
+ *    "error", "Invalid gamma value"
+ * }
+ *
    + * * @author Curtis Rueden */ public class Appose { - public static Builder base(File directory) { - return new Builder().base(directory); + public static Builder scheme(String scheme) { + return new Builder().scheme(scheme); + } + + public static Builder file(String filePath) throws IOException { + return new Builder().file(filePath); + } + + public static Builder file(String filePath, String scheme) throws IOException { + return new Builder().file(filePath, scheme); + } + + public static Builder file(File file) throws IOException { + return new Builder().file(file); + } + + public static Builder file(File file, String scheme) throws IOException { + return new Builder().file(file, scheme); + } + + public static Builder channel(String name) { + return new Builder().channel(name); + } + + public static Builder channel(String name, String location) { + return new Builder().channel(name, location); + } + + public static Builder include(String content) { + return new Builder().include(content); + } + + public static Builder include(String content, String scheme) { + return new Builder().include(content, scheme); } - public static Builder base(String directory) { - return base(new File(directory)); + @Deprecated + public static Builder conda(File environmentYaml) throws IOException { + return file(environmentYaml, "environment.yml"); } - public static Builder java(String vendor, String version) { - return new Builder().java(vendor, version); + public static Environment build(File directory) throws IOException { + return new Builder().build(directory); } - public static Builder conda(File environmentYaml) { - return new Builder().conda(environmentYaml); + public static Environment build(String directory) throws IOException { + return build(new File(directory)); } - public static Environment system() { + public static Environment system() throws IOException { return system(new File(".")); } - public static Environment system(File directory) { - return new Builder().base(directory).useSystemPath().build(); + public static Environment system(File directory) throws IOException { + return new Builder().useSystemPath().build(directory); } - public static Environment system(String directory) { + public static Environment system(String directory) throws IOException { return system(new File(directory)); } } diff --git a/src/main/java/org/apposed/appose/BuildHandler.java b/src/main/java/org/apposed/appose/BuildHandler.java new file mode 100644 index 0000000..440bbe5 --- /dev/null +++ b/src/main/java/org/apposed/appose/BuildHandler.java @@ -0,0 +1,89 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 - 2024 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.apposed.appose; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface BuildHandler { + + /** + * Registers a channel from which elements of the environment can be obtained. + * + * @param name The name of the channel to register. + * @param location The location of the channel (e.g. a URI), or {@code null} if the + * name alone is sufficient to unambiguously identify the channel. + * @return true iff the channel is understood by this build handler implementation. + * @see Builder#channel + */ + boolean channel(String name, String location); + + /** + * Registers content to be included within the environment. + * + * @param content The content to include in the environment, fetching if needed. + * @param scheme The type of content, which serves as a hint for + * how to interpret the content in some scenarios. + * @see Builder#include + */ + boolean include(String content, String scheme); + + /** Suggests a name for the environment currently being built. */ + String envName(); + + /** + * Executes the environment build, according to the configured channels and includes. + * + * @param envDir The directory into which the environment will be built. + * @param builder The {@link Builder} instance managing the build process. + * Contains event subscribers and output configuration table. + * @throws IOException If something goes wrong building the environment. + * @see Builder#build(String) + */ + void build(File envDir, Builder builder) throws IOException; + + default void progress(Builder builder, String title, long current) { + progress(builder, title, current, -1); + } + + default void progress(Builder builder, String title, long current, long maximum) { + builder.progressSubscribers.forEach(subscriber -> subscriber.accept(title, current, maximum)); + } + + default void output(Builder builder, String message) { + builder.outputSubscribers.forEach(subscriber -> subscriber.accept(message)); + } + + default void error(Builder builder, String message) { + builder.errorSubscribers.forEach(subscriber -> subscriber.accept(message)); + } +} diff --git a/src/main/java/org/apposed/appose/Builder.java b/src/main/java/org/apposed/appose/Builder.java index 10cc756..8b13496 100644 --- a/src/main/java/org/apposed/appose/Builder.java +++ b/src/main/java/org/apposed/appose/Builder.java @@ -30,56 +30,336 @@ package org.apposed.appose; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.ServiceLoader; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; +/** + * TODO + * + * @author Curtis Rueden + */ public class Builder { - public Environment build() { - // TODO Build the thing!~ - // Hash the state to make a base directory name. - // - Construct conda environment from condaEnvironmentYaml. - // - Download and unpack JVM of the given vendor+version. - // - Populate ${baseDirectory}/jars with Maven artifacts? - String base = baseDir.getPath(); - boolean useSystemPath = systemPath; - return new Environment() { - @Override public String base() { return base; } - @Override public boolean useSystemPath() { return useSystemPath; } - }; - } + public final Map> config = new HashMap<>(); + public final List progressSubscribers = new ArrayList<>(); + public final List> outputSubscribers = new ArrayList<>(); + public final List> errorSubscribers = new ArrayList<>(); - // -- Configuration -- + private final List handlers; - private boolean systemPath; + private boolean includeSystemPath; + private String scheme = "conda"; - public Builder useSystemPath() { - systemPath = true; + Builder() { + handlers = new ArrayList<>(); + ServiceLoader.load(BuildHandler.class).forEach(handlers::add); + } + + /** + * Registers a callback method to be invoked when progress happens during environment building. + * + * @param subscriber Party to inform when build progress happens. + * @return This {@code Builder} instance, for fluent-style programming. + * @see ProgressConsumer#accept + */ + public Builder subscribeProgress(ProgressConsumer subscriber) { + progressSubscribers.add(subscriber); return this; } - private File baseDir; + public Builder subscribeOutput(Consumer subscriber) { + outputSubscribers.add(subscriber); + return this; + } - public Builder base(File directory) { - baseDir = directory; + public Builder subscribeError(Consumer subscriber) { + errorSubscribers.add(subscriber); return this; } - // -- Conda -- + /** + * Shorthand for {@link #subscribeProgress}, {@link #subscribeOutput}, + * and {@link #subscribeError} calls registering subscribers that + * emit their arguments to stdout. Useful for debugging environment + * construction, e.g. complex environments with many conda packages. + * + * @return This {@code Builder} instance, for fluent-style programming. + */ + public Builder logDebug() { + String reset = "\u001b[0m"; + String yellow = "\u001b[0;33m"; + String red = "\u001b[0;31m"; + return subscribeProgress((title, cur, max) -> System.out.printf("%s: %d/%d\n", title, cur, max)) + .subscribeOutput(msg -> System.out.printf("%s%s%s", yellow, msg.isEmpty() ? "." : msg, reset)) + .subscribeError(msg -> System.out.printf("%s%s%s", red, msg.isEmpty() ? "." : msg, reset)); + } - private File condaEnvironmentYaml; + /** + * TODO + * + * @return This {@code Builder} instance, for fluent-style programming. + */ + public Builder useSystemPath() { + includeSystemPath = true; + return this; + } - public Builder conda(File environmentYaml) { - this.condaEnvironmentYaml = environmentYaml; + /** + * Sets the scheme to use with subsequent {@link #channel(String)} and + * {@link #include(String)} directives. + * + * @param scheme TODO + * @return This {@code Builder} instance, for fluent-style programming. + */ + public Builder scheme(String scheme) { + this.scheme = scheme; return this; } - // -- Java -- + /** + * TODO + * + * @return This {@code Builder} instance, for fluent-style programming. + */ + public Builder file(String filePath) throws IOException { + return file(new File(filePath)); + } - private String javaVendor; - private String javaVersion; + /** + * TODO + * + * @return This {@code Builder} instance, for fluent-style programming. + */ + public Builder file(String filePath, String scheme) throws IOException { + return file(new File(filePath), scheme); + } - public Builder java(String vendor, String version) { - this.javaVendor = vendor; - this.javaVersion = version; - return this; + /** + * TODO + * + * @return This {@code Builder} instance, for fluent-style programming. + */ + public Builder file(File file) throws IOException { + return file(file, file.getName()); + } + + /** + * TODO + * + * @return This {@code Builder} instance, for fluent-style programming. + */ + public Builder file(File file, String scheme) throws IOException { + byte[] bytes = Files.readAllBytes(file.toPath()); + return include(new String(bytes), scheme); + } + + /** + * Registers a channel that provides components of the environment, + * according to the currently configured scheme ("conda" by default). + *

    + * For example, {@code channel("bioconda")} registers the {@code bioconda} + * channel as a source for conda packages. + *

    + * + * @param name The name of the channel to register. + * @return This {@code Builder} instance, for fluent-style programming. + * @see #channel(String, String) + * @see #scheme(String) + */ + public Builder channel(String name) { + return channel(name, scheme); + } + + /** + * Registers a channel that provides components of the environment. + * How to specify a channel is implementation-dependent. Examples: + * + *
      + *
    • {@code channel("bioconda")} - + * to register the {@code bioconda} channel as a source for conda packages.
      • + *
    • {@code channel("scijava", "maven:https://maven.scijava.org/content/groups/public")} - + * to register the SciJava Maven repository as a source for Maven artifacts.
      • + *
    + * + * @param name The name of the channel to register. + * @param location The location of the channel (e.g. a URI), or {@code null} if the + * name alone is sufficient to unambiguously identify the channel. + * @return This {@code Builder} instance, for fluent-style programming. + * @throws IllegalArgumentException if the channel is not understood by any of the available build handlers. + */ + public Builder channel(String name, String location) { + // Pass the channel directive to all handlers. + if (handle(handler -> handler.channel(name, location))) return this; + // None of the handlers accepted the directive. + throw new IllegalArgumentException("Unsupported channel: " + name + + (location == null ? "" : "=" + location)); + } + + /** + * TODO + * + * @param content TODO + * @return This {@code Builder} instance, for fluent-style programming. + * @see #include(String, String) + * @see #scheme(String) + */ + public Builder include(String content) { + return include(content, scheme); + } + + /** + * Registers content to be included within the environment. + * How to specify the content is implementation-dependent. Examples: + *
      + *
    • {@code include("cowsay", "pypi")} - + * Install {@code cowsay} from the Python package index.
      • + *
    • {@code include("openjdk=17")} - + * Install {@code openjdk} version 17 from conda-forge.
      • + *
    • {@code include("bioconda::sourmash")} - + * Specify a conda channel explicitly using environment.yml syntax.
      • + *
    • {@code include("org.scijava:parsington", "maven")} - + * Install the latest version of Parsington from Maven Central.
      • + *
    • {@code include("org.scijava:parsington:2.0.0", "maven")} - + * Install Parsington 2.0.0 from Maven Central.
      • + *
    • {@code include("sc.fiji:fiji", "maven")} - + * Install the latest version of Fiji from registered Maven repositories.
      • + *
    • {@code include("zulu:17", "jdk")} - + * Install version 17 of Azul Zulu OpenJDK.
      • + *
    • {@code include(yamlString, "environment.yml")} - + * Provide the literal contents of a conda {@code environment.yml} file, + * indicating a set of packages to include. + *
    + *

    + * Note that content is not actually fetched or installed until + * {@link #build} is called at the end of the builder chain. + *

    + * + * @param content The content (e.g. a package name, or perhaps the contents of an environment + * configuration file) to include in the environment, fetching if needed. + * @param scheme The type of content, which serves as a hint for how to interpret + * the content in some scenarios; see above for examples. + * @return This {@code Builder} instance, for fluent-style programming. + * @throws IllegalArgumentException if the include directive is not understood by any of the available build handlers. + */ + public Builder include(String content, String scheme) { + // Pass the include directive to all handlers. + if (handle(handler -> handler.include(content, scheme))) return this; + // None of the handlers accepted the directive. + throw new IllegalArgumentException("Unsupported '" + scheme + "' content: " + content); + } + + /** + * Executes the environment build, according to the configured channels and includes, + * with a name inferred from the configuration registered earlier. For example, if + * {@code environment.yml} content was registered, the name from that configuration will be used. + * + * @return The newly constructed Appose {@link Environment}, + * from which {@link Service}s can be launched. + * @see #build(String) + * @throws IllegalStateException if no name can be inferred from included content. + * @throws IOException If something goes wrong building the environment. + */ + public Environment build() throws IOException { + // Autodetect the environment name from the available build handlers. + return build(handlers.stream() + .map(BuildHandler::envName) + .filter(Objects::nonNull) + .findFirst() + .orElse(null)); + } + + /** + * Executes the environment build, according to the configured channels and includes. + * with a base directory inferred from the given name. + * + * @param envName The name of the environment to build. + * @return The newly constructed Appose {@link Environment}, + * from which {@link Service}s can be launched. + * @throws IOException If something goes wrong building the environment. + */ + public Environment build(String envName) throws IOException { + if (envName == null || envName.isEmpty()) { + throw new IllegalArgumentException("No environment name given."); + } + // TODO: Make Appose's root directory configurable. + Path apposeRoot = Paths.get(System.getProperty("user.home"), ".local", "share", "appose"); + return build(apposeRoot.resolve(envName).toFile()); + } + + /** + * Executes the environment build, according to the configured channels and includes. + * with the given base directory. + * + * @param envDir The directory in which to construct the environment. + * @return The newly constructed Appose {@link Environment}, + * from which {@link Service}s can be launched. + * @throws IOException If something goes wrong building the environment. + */ + public Environment build(File envDir) throws IOException { + if (envDir == null) { + throw new IllegalArgumentException("No environment directory given."); + } + if (!envDir.exists()) { + if (!envDir.mkdirs()) { + throw new RuntimeException("Failed to create environment directory: " + envDir); + } + } + if (!envDir.isDirectory()) { + throw new IllegalArgumentException("Not a directory: " + envDir); + } + + config.clear(); + for (BuildHandler handler : handlers) handler.build(envDir, this); + + String base = envDir.getAbsolutePath(); + + List launchArgs = listFromConfig("launchArgs", config); + List binPaths = listFromConfig("binPaths", config); + List classpath = listFromConfig("classpath", config); + + // Always add the environment directory itself to the binPaths. + // Especially important on Windows, where python.exe is not tucked into a bin subdirectory. + binPaths.add(envDir.getAbsolutePath()); + + if (includeSystemPath) { + List systemPaths = Arrays.asList(System.getenv("PATH").split(File.pathSeparator)); + binPaths.addAll(systemPaths); + } + + return new Environment() { + @Override public String base() { return base; } + @Override public List binPaths() { return binPaths; } + @Override public List classpath() { return classpath; } + @Override public List launchArgs() { return launchArgs; } + }; + } + + private boolean handle(Function handlerFunction) { + boolean handled = false; + for (BuildHandler handler : handlers) + handled |= handlerFunction.apply(handler); + return handled; + } + + private static List listFromConfig(String key, Map> config) { + List value = config.getOrDefault(key, Collections.emptyList()); + return value.stream().map(Object::toString).collect(Collectors.toList()); + } + + public interface ProgressConsumer { + void accept(String title, long current, long maximum); } } diff --git a/src/main/java/org/apposed/appose/Environment.java b/src/main/java/org/apposed/appose/Environment.java index 1fe7204..4313939 100644 --- a/src/main/java/org/apposed/appose/Environment.java +++ b/src/main/java/org/apposed/appose/Environment.java @@ -40,8 +40,10 @@ public interface Environment { - default String base() { return "."; } - default boolean useSystemPath() { return false; } + String base(); + List binPaths(); + List classpath(); + List launchArgs(); /** * Creates a Python script service. @@ -56,10 +58,7 @@ public interface Environment { * @throws IOException If something goes wrong starting the worker process. */ default Service python() throws IOException { - List pythonExes = Arrays.asList( - "python", "python3", "python.exe", - "bin/python", "bin/python.exe" - ); + List pythonExes = Arrays.asList("python", "python3", "python.exe"); return service(pythonExes, "-c", "import appose.python_worker; appose.python_worker.main()"); } @@ -128,14 +127,15 @@ default Service java(String mainClass, List classPath, // Ensure that the classpath includes Appose and its dependencies. // NB: This list must match Appose's dependencies in pom.xml! - List> apposeDeps = Arrays.asList(// - org.apposed.appose.GroovyWorker.class, // ------> org.apposed:appose - org.apache.groovy.util.ScriptRunner.class, // --> org.codehaus.groovy:groovy - groovy.json.JsonOutput.class, // ---------------> org.codehaus.groovy:groovy-json - org.apache.ivy.Ivy.class, // -------------------> org.apache.ivy:ivy - com.sun.jna.Pointer.class, // ------------------> com.sun.jna:jna - com.sun.jna.platform.linux.LibRT.class, // -----> com.sun.jna:jna-platform - com.sun.jna.platform.win32.Kernel32.class // ---> com.sun.jna:jna-platform + List> apposeDeps = Arrays.asList( + org.apposed.appose.GroovyWorker.class, // ------------------------> org.apposed:appose + org.apache.groovy.util.ScriptRunner.class, // --------------------> org.codehaus.groovy:groovy + groovy.json.JsonOutput.class, // ---------------------------------> org.codehaus.groovy:groovy-json + org.apache.ivy.Ivy.class, // -------------------------------------> org.apache.ivy:ivy + com.sun.jna.Pointer.class, // ------------------------------------> com.sun.jna:jna + com.sun.jna.platform.linux.LibRT.class, // -----------------------> com.sun.jna:jna-platform + com.sun.jna.platform.win32.Kernel32.class, // --------------------> com.sun.jna:jna-platform + org.apache.commons.compress.archivers.ArchiveException.class // --> org.apache.commons:commons-compress ); for (Class depClass : apposeDeps) { File location = FilePaths.location(depClass); @@ -172,26 +172,38 @@ default Service java(String mainClass, List classPath, * * @param exes List of executables to try for launching the worker process. * @param args Command line arguments to pass to the worker process - * (e.g. {"-v", "--enable-everything"}. + * (e.g. {"-v", "--enable-everything"}). * @return The newly created service. * @see #groovy To create a service for Groovy script execution. * @see #python() To create a service for Python script execution. * @throws IOException If something goes wrong starting the worker process. */ default Service service(List exes, String... args) throws IOException { - if (args.length == 0) throw new IllegalArgumentException("No executable given"); - - List dirs = useSystemPath() // - ? Arrays.asList(System.getenv("PATH").split(File.pathSeparator)) // - : Arrays.asList(base()); - - File exeFile = FilePaths.findExe(dirs, exes); - if (exeFile == null) throw new IllegalArgumentException("No executables found amongst candidates: " + exes); + if (exes == null || exes.isEmpty()) throw new IllegalArgumentException("No executable given"); + + // Discern path to executable by searching the environment's binPaths. + File exeFile = FilePaths.findExe(binPaths(), exes); + + // Calculate exe string. + List launchArgs = launchArgs(); + final String exe; + if (exeFile == null) { + if (launchArgs.isEmpty()) { + throw new IllegalArgumentException("No executables found amongst candidates: " + exes); + } + // No exeFile was found in the binPaths, but there are prefixed launchArgs. + // So we now try to use the first executable bare, because in this scenario + // we may have a situation like `pixi run python` where the intended executable + // become available on the system path while the environment is activated. + exe = exes.get(0); + } + else exe = exeFile.getCanonicalPath(); - String[] allArgs = new String[args.length + 1]; - System.arraycopy(args, 0, allArgs, 1, args.length); - allArgs[0] = exeFile.getCanonicalPath(); + // Construct final args list: launchArgs + exe + args + List allArgs = new ArrayList<>(launchArgs); + allArgs.add(exe); + allArgs.addAll(Arrays.asList(args)); - return new Service(new File(base()), allArgs); + return new Service(new File(base()), allArgs.toArray(new String[0])); } } diff --git a/src/main/java/org/apposed/appose/FilePaths.java b/src/main/java/org/apposed/appose/FilePaths.java index 1a3d2ee..876db7e 100644 --- a/src/main/java/org/apposed/appose/FilePaths.java +++ b/src/main/java/org/apposed/appose/FilePaths.java @@ -30,12 +30,16 @@ package org.apposed.appose; import java.io.File; +import java.io.IOException; import java.net.URISyntaxException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; /** - * Utility methods for working with file paths. + * Utility methods for working with files. */ public final class FilePaths { @@ -78,4 +82,112 @@ public static File findExe(List dirs, List exes) { } return null; } + + /** + * Merges the files of the given source directory into the specified destination directory. + *

    + * For example, {@code moveDirectory(foo, bar)} would move: + *

    + *
      + *
    • {@code foo/a.txt} → {@code bar/a.txt}
      • + *
    • {@code foo/b.dat} → {@code bar/b.dat}
      • + *
    • {@code foo/c.csv} → {@code bar/c.csv}
      • + *
    • {@code foo/subfoo/d.doc} → {@code bar/subfoo/d.doc}
      • + *
    • etc.
      • + *
    + * + * @param srcDir TODO + * @param destDir TODO + * @param overwrite TODO + */ + public static void moveDirectory(File srcDir, File destDir, boolean overwrite) throws IOException { + if (!srcDir.isDirectory()) throw new IllegalArgumentException("Not a directory: " + srcDir); + if (!destDir.isDirectory()) throw new IllegalArgumentException("Not a directory: " + destDir); + try (DirectoryStream stream = Files.newDirectoryStream(srcDir.toPath())) { + for (Path srcPath : stream) moveFile(srcPath.toFile(), destDir, overwrite); + } + if (!srcDir.delete()) throw new IOException("Could not remove directory " + destDir); + } + + /** + * Moves the given source file to the destination directory, + * creating intermediate destination directories as needed. + *

    + * If the destination {@code file.ext} already exists, one of two things will happen: either + * A) the existing destination file will be renamed as a backup to {@code file.ext.old}—or + * {@code file.ext.0.old}, {@code file.ext.1.old}, etc., if {@code file.ext.old} also already exists—or + * B) the source file will be renamed as a backup in this manner. + * Which behavior occurs depends on the value of the {@code overwrite} flag: + * true to back up the destination file, or false to back up the source file. + *

    + * + * @param srcFile Source file to move. + * @param destDir Destination directory into which the file will be moved. + * @param overwrite If true, "overwrite" the destination file with the source file, + * backing up any existing destination file first; if false, + * leave the original destination file in place, instead moving + * the source file to a backup destination as a "previous" version. + * @throws IOException If something goes wrong with the needed I/O operations. + */ + public static void moveFile(File srcFile, File destDir, boolean overwrite) throws IOException { + File destFile = new File(destDir, srcFile.getName()); + if (srcFile.isDirectory()) { + // Create matching destination directory as needed. + if (!destFile.exists() && !destFile.mkdirs()) + throw new IOException("Failed to create destination directory: " + destDir); + // Recurse over source directory contents. + moveDirectory(srcFile, destFile, overwrite); + return; + } + // Source file is not a directory; move it into the destination directory. + if (destDir.exists() && !destDir.isDirectory()) throw new IllegalArgumentException("Non-directory destination path: " + destDir); + if (!destDir.exists() && !destDir.mkdirs()) throw new IOException("Failed to create destination directory: " + destDir); + if (destFile.exists() && !overwrite) { + // Destination already exists, and we aren't allowed to rename it. So we instead + // rename the source file directly to a backup filename in the destination directory. + renameToBackup(srcFile, destDir); + return; + } + + // Rename the existing destination file (if any) to a + // backup file, then move the source file into place. + renameToBackup(destFile); + if (!srcFile.renameTo(destFile)) throw new IOException("Failed to move file: " + srcFile + " -> " + destFile); + } + + /** + * TODO + * + * @param srcFile TODO + * @throws IOException If something goes wrong with the needed I/O operations. + */ + public static void renameToBackup(File srcFile) throws IOException { + renameToBackup(srcFile, srcFile.getParentFile()); + } + + /** + * TODO + * + * @param srcFile TODO + * @param destDir TODO + * @throws IOException If something goes wrong with the needed I/O operations. + */ + public static void renameToBackup(File srcFile, File destDir) throws IOException { + if (!srcFile.exists()) return; // Nothing to back up! + String prefix = srcFile.getName(); + String suffix = "old"; + File backupFile = new File(destDir, prefix + "." + suffix); + for (int i = 0; i < 1000; i++) { + if (!backupFile.exists()) break; + // The .old backup file already exists! Try .0.old, .1.old, and so on. + backupFile = new File(destDir, prefix + "." + i + "." + suffix); + } + if (backupFile.exists()) { + File failedTarget = new File(destDir, prefix + "." + suffix); + throw new UnsupportedOperationException("Too many backup files already exist for target: " + failedTarget); + } + if (!srcFile.renameTo(backupFile)) { + throw new IOException("Failed to rename file:" + srcFile + " -> " + backupFile); + } + } } diff --git a/src/main/java/org/apposed/appose/GroovyWorker.java b/src/main/java/org/apposed/appose/GroovyWorker.java index 0ac4b7f..585d624 100644 --- a/src/main/java/org/apposed/appose/GroovyWorker.java +++ b/src/main/java/org/apposed/appose/GroovyWorker.java @@ -118,6 +118,7 @@ public Task(String uuid) { this.uuid = uuid; } + @SuppressWarnings("unused") public void update(String message, Long current, Long maximum) { Map args = new HashMap<>(); if (message != null) args.put("message", message); @@ -126,6 +127,7 @@ public void update(String message, Long current, Long maximum) { respond(ResponseType.UPDATE, args); } + @SuppressWarnings("unused") public void cancel() { respond(ResponseType.CANCELATION, null); } diff --git a/src/main/java/org/apposed/appose/NDArray.java b/src/main/java/org/apposed/appose/NDArray.java index 62e19b4..525b915 100644 --- a/src/main/java/org/apposed/appose/NDArray.java +++ b/src/main/java/org/apposed/appose/NDArray.java @@ -143,7 +143,8 @@ private static int safeInt(final long value) { /** * Enumerates possible data type of {@link NDArray} elements. */ - public static enum DType { + @SuppressWarnings("unused") + public enum DType { INT8("int8", Byte.BYTES), // INT16("int16", Short.BYTES), // INT32("int32", Integer.BYTES), // @@ -179,8 +180,8 @@ public int bytesPerElement() /** * Get the label of this {@code DType}. *

    - * The label can used as a {@code dtype} in Python. It is also used for JSON - * serialization. + * The label can be used as a {@code dtype} in Python. + * It is also used for JSON serialization. * * @return the label. */ @@ -203,7 +204,7 @@ public static DType fromLabel(final String label) throws IllegalArgumentExceptio } /** - * The shape of a multi-dimensional array. + * The shape of a multidimensional array. */ public static class Shape { @@ -282,6 +283,7 @@ public long numElements() { * * @return dimensions array */ + @SuppressWarnings("unused") public int[] toIntArray() { return shape; } @@ -296,9 +298,9 @@ public int[] toIntArray(final Order order) { return shape; } else { - final int[] ishape = new int[shape.length]; - Arrays.setAll(ishape, i -> shape[shape.length - i - 1]); - return ishape; + final int[] iShape = new int[shape.length]; + Arrays.setAll(iShape, i -> shape[shape.length - i - 1]); + return iShape; } } @@ -308,6 +310,7 @@ public int[] toIntArray(final Order order) { * * @return dimensions array */ + @SuppressWarnings("unused") public long[] toLongArray() { return toLongArray(order); } @@ -318,14 +321,14 @@ public long[] toLongArray() { * @return dimensions array */ public long[] toLongArray(final Order order) { - final long[] lshape = new long[shape.length]; + final long[] lShape = new long[shape.length]; if (order.equals(this.order)) { - Arrays.setAll(lshape, i -> shape[i]); + Arrays.setAll(lShape, i -> shape[i]); } else { - Arrays.setAll(lshape, i -> shape[shape.length - i - 1]); + Arrays.setAll(lShape, i -> shape[shape.length - i - 1]); } - return lshape; + return lShape; } /** diff --git a/src/main/java/org/apposed/appose/Types.java b/src/main/java/org/apposed/appose/Types.java index e2b842e..372615a 100644 --- a/src/main/java/org/apposed/appose/Types.java +++ b/src/main/java/org/apposed/appose/Types.java @@ -47,15 +47,34 @@ private Types() { // NB: Prevent instantiation of utility class. } + /** + * Converts a Map into a JSON string. + * @param data + * data that wants to be encoded + * @return string containing the info of the data map + */ public static String encode(Map data) { return GENERATOR.toJson(data); } + /** + * Converts a JSON string into a map. + * @param json + * json string + * @return a map of with the information of the json + */ + @SuppressWarnings("unchecked") public static Map decode(String json) { return postProcess(new JsonSlurper().parseText(json)); } - /** Dumps the given exception, including stack trace, to a string. */ + /** + * Dumps the given exception, including stack trace, to a string. + * + * @param t + * the given exception {@link Throwable} + * @return the String containing the whole exception trace + */ public static String stackTrace(Throwable t) { StringWriter sw = new StringWriter(); t.printStackTrace(new PrintWriter(sw)); diff --git a/src/main/java/org/apposed/appose/mamba/FileDownloader.java b/src/main/java/org/apposed/appose/mamba/FileDownloader.java new file mode 100644 index 0000000..315e6b4 --- /dev/null +++ b/src/main/java/org/apposed/appose/mamba/FileDownloader.java @@ -0,0 +1,84 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 - 2024 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.apposed.appose.mamba; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.channels.ReadableByteChannel; + +class FileDownloader { + private static final long CHUNK_SIZE = 1024 * 1024 * 5; + + private final ReadableByteChannel rbc; + private final FileOutputStream fos; + + public FileDownloader(ReadableByteChannel rbc, FileOutputStream fos) { + this.rbc = rbc; + this.fos = fos; + } + + /** + * Download a file without the possibility of interrupting the download + * @throws IOException if there is any error downloading the file from the url + */ + public void call() throws IOException { + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + } + + /** + * Download a file with the possibility of interrupting the download if the parentThread is + * interrupted + * + * @param parentThread + * thread from where the download was launched, it is the reference used to stop the download + * @throws IOException if there is any error downloading the file from the url + * @throws InterruptedException if the download is interrupted because the parentThread is interrupted + */ + public void call(Thread parentThread) throws IOException, InterruptedException { + long position = 0; + while (true) { + long transferred = fos.getChannel().transferFrom(rbc, position, CHUNK_SIZE); + if (transferred == 0) { + break; + } + + position += transferred; + if (!parentThread.isAlive()) { + // Close resources if needed and exit + closeResources(); + throw new InterruptedException("File download was interrupted."); + } + } + } + + private void closeResources() throws IOException { + if (rbc != null) rbc.close(); + if (fos != null) fos.close(); + } +} diff --git a/src/main/java/org/apposed/appose/mamba/Mamba.java b/src/main/java/org/apposed/appose/mamba/Mamba.java new file mode 100644 index 0000000..f671731 --- /dev/null +++ b/src/main/java/org/apposed/appose/mamba/Mamba.java @@ -0,0 +1,624 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 - 2024 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +// Adapted from JavaConda (https://github.com/javaconda/javaconda), +// which has the following license: + +/*-***************************************************************************** + * Copyright (C) 2021, Ko Sugawara + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ****************************************************************************-*/ + +package org.apposed.appose.mamba; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.List; +import java.util.UUID; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +/** + * Conda-based environment manager, implemented by delegating to micromamba. + * + * @author Ko Sugawara + * @author Carlos Garcia + */ +class Mamba { + + /** + * String containing the path that points to the micromamba executable + */ + final String mambaCommand; + + /** + * Root directory of micromamba that also contains the environments folder + * + * 

    +	 * rootdir
+	 * ├── bin
+	 * │   ├── micromamba(.exe)
+	 * │   ...
+	 * ├── envs
+	 * │   ├── your_env
+	 * │   │   ├── python(.exe)
+	 *
    + */ + private final String rootdir; + + /** + * Consumer that tracks the progress in the download of micromamba, the software used + * by this class to manage Conda environments. + */ + private BiConsumer mambaDownloadProgressConsumer; + + /** + * Consumer that tracks the standard output stream produced by the micromamba process when it is executed. + */ + private Consumer outputConsumer; + + /** + * Consumer that tracks the standard error stream produced by the micromamba process when it is executed. + */ + private Consumer errorConsumer; + + /** + * Relative path to the micromamba executable from the micromamba {@link #rootdir} + */ + private final static Path MICROMAMBA_RELATIVE_PATH = isWindowsOS() ? + Paths.get("Library", "bin", "micromamba.exe") : + Paths.get("bin", "micromamba"); + + /** + * Path where Appose installs Micromamba by default + */ + final public static String BASE_PATH = Paths.get(System.getProperty("user.home"), ".local", "share", "appose", "micromamba").toString(); + + /** + * URL from where Micromamba is downloaded to be installed + */ + public final static String MICROMAMBA_URL = + "https://micro.mamba.pm/api/micromamba/" + microMambaPlatform() + "/latest"; + + /** + * ID used to identify the text retrieved from the error stream when a consumer is used + */ + public final static String ERR_STREAM_UUUID = UUID.randomUUID().toString(); + + /** + * + * @return a String that identifies the current OS to download the correct Micromamba version + */ + private static String microMambaPlatform() { + String osName = System.getProperty("os.name"); + if (osName.startsWith("Windows")) osName = "Windows"; + String osArch = System.getProperty("os.arch"); + switch (osName + "|" + osArch) { + case "Linux|amd64": return "linux-64"; + case "Linux|aarch64": return "linux-aarch64"; + case "Linux|ppc64le": return "linux-ppc64le"; + case "Mac OS X|x86_64": return "osx-64"; + case "Mac OS X|aarch64": return "osx-arm64"; + case "Windows|amd64": return "win-64"; + default: return null; + } + } + + private void updateMambaDownloadProgress(long current, long total) { + if (mambaDownloadProgressConsumer != null) + mambaDownloadProgressConsumer.accept(current, total); + } + + private void updateOutputConsumer(String str) { + if (outputConsumer != null) + outputConsumer.accept(str == null ? "" : str); + } + + private void updateErrorConsumer(String str) { + if (errorConsumer != null) + errorConsumer.accept(str == null ? "" : str); + } + + /** + * Returns a {@link ProcessBuilder} with the working directory specified in the + * constructor. + * + * @param isInheritIO + * Sets the source and destination for subprocess standard I/O to be + * the same as those of the current Java process. + * @return The {@link ProcessBuilder} with the working directory specified in + * the constructor. + */ + private ProcessBuilder getBuilder( final boolean isInheritIO ) + { + final ProcessBuilder builder = new ProcessBuilder().directory( new File( rootdir ) ); + if ( isInheritIO ) + builder.inheritIO(); + return builder; + } + + /** + * Create a new {@link Mamba} object. The root dir for the Micromamba installation + * will be the default base path defined at {@link #BASE_PATH} + * If there is no Micromamba found at the base path {@link #BASE_PATH}, an {@link IllegalStateException} will be thrown + *

    + * It is expected that the Micromamba installation has executable commands as shown below: + *

    + * 
    +	 * MAMBA_ROOT
+	 * ├── bin
+	 * │   ├── micromamba(.exe)
+	 * │   ...
+	 * ├── envs
+	 * │   ├── your_env
+	 * │   │   ├── python(.exe)
+	 *
    + */ + public Mamba() { + this(BASE_PATH); + } + + /** + * Create a new Conda object. The root dir for Conda installation can be + * specified as {@code String}. + * If there is no Micromamba found at the specified path, it will be installed automatically + * if the parameter 'installIfNeeded' is true. If not an {@link IllegalStateException} will be thrown. + *

    + * It is expected that the Conda installation has executable commands as shown below: + *

    + * 
    +	 * MAMBA_ROOT
+	 * ├── bin
+	 * │   ├── micromamba(.exe)
+	 * │   ...
+	 * ├── envs
+	 * │   ├── your_env
+	 * │   │   ├── python(.exe)
+	 *
    + * + * @param rootdir + * The root dir for Mamba installation. + */ + public Mamba(final String rootdir) { + if (rootdir == null) + this.rootdir = BASE_PATH; + else + this.rootdir = rootdir; + this.mambaCommand = Paths.get(this.rootdir).resolve(MICROMAMBA_RELATIVE_PATH).toAbsolutePath().toString(); + } + + /** + * Gets whether micromamba is installed or not to be able to use the instance of {@link Mamba} + * @return whether micromamba is installed or not to be able to use the instance of {@link Mamba} + */ + public boolean isMambaInstalled() { + try { + getVersion(); + return true; + } catch (IOException | InterruptedException e) { + return false; + } + } + + /** + * Check whether micromamba is installed or not to be able to use the instance of {@link Mamba} + * @throws IllegalStateException if micromamba is not installed + */ + private void checkMambaInstalled() { + if (!isMambaInstalled()) throw new IllegalStateException("Micromamba is not installed"); + } + + /** + * Registers the consumer for the standard error stream of every micromamba call. + * @param consumer + * callback function invoked for each stderr line of every micromamba call + */ + public void setMambaDownloadProgressConsumer(BiConsumer consumer) { + this.mambaDownloadProgressConsumer = consumer; + } + + /** + * Registers the consumer for the standard output stream of every micromamba call. + * @param consumer + * callback function invoked for each stdout line of every micromamba call + */ + public void setOutputConsumer(Consumer consumer) { + this.outputConsumer = consumer; + } + + /** + * Registers the consumer for the standard error stream of every micromamba call. + * @param consumer + * callback function invoked for each stderr line of every micromamba call + */ + public void setErrorConsumer(Consumer consumer) { + this.errorConsumer = consumer; + } + + private File downloadMicromamba() throws IOException, InterruptedException, URISyntaxException { + final File tempFile = File.createTempFile( "micromamba", ".tar.bz2" ); + tempFile.deleteOnExit(); + URL website = MambaInstallerUtils.redirectedURL(new URL(MICROMAMBA_URL)); + long size = MambaInstallerUtils.getFileSize(website); + Thread currentThread = Thread.currentThread(); + IOException[] ioe = {null}; + InterruptedException[] ie = {null}; + Thread dwnldThread = new Thread(() -> { + try ( + ReadableByteChannel rbc = Channels.newChannel(website.openStream()); + FileOutputStream fos = new FileOutputStream(tempFile) + ) { + new FileDownloader(rbc, fos).call(currentThread); + } + catch (IOException e) { ioe[0] = e; } + catch (InterruptedException e) { ie[0] = e; } + }); + dwnldThread.start(); + while (dwnldThread.isAlive()) { + Thread.sleep(20); // 50 FPS update rate + updateMambaDownloadProgress(tempFile.length(), size); + } + if (ioe[0] != null) throw ioe[0]; + if (ie[0] != null) throw ie[0]; + if (tempFile.length() < size) + throw new IOException("Error downloading micromamba from: " + MICROMAMBA_URL); + return tempFile; + } + + private void decompressMicromamba(final File tempFile) throws IOException, InterruptedException { + final File tempTarFile = File.createTempFile( "micromamba", ".tar" ); + tempTarFile.deleteOnExit(); + MambaInstallerUtils.unBZip2(tempFile, tempTarFile); + File mambaBaseDir = new File(rootdir); + if (!mambaBaseDir.isDirectory() && !mambaBaseDir.mkdirs()) + throw new IOException("Failed to create Micromamba default directory " + + mambaBaseDir.getParentFile().getAbsolutePath() + + ". Please try installing it in another directory."); + MambaInstallerUtils.unTar(tempTarFile, mambaBaseDir); + File mmFile = new File(mambaCommand); + if (!mmFile.exists()) throw new IOException("Expected micromamba binary is missing: " + mambaCommand); + if (!mmFile.canExecute()) { + boolean executableSet = new File(mambaCommand).setExecutable(true); + if (!executableSet) + throw new IOException("Cannot set file as executable due to missing permissions, " + + "please do it manually: " + mambaCommand); + } + } + + /** + * Downloads and installs Micromamba. + * + * @throws IOException + * If an I/O error occurs. + * @throws InterruptedException + * If the current thread is interrupted by another thread while it + * is waiting, then the wait is ended and an InterruptedException is + * thrown. + * @throws URISyntaxException if there is any error with the micromamba url + */ + public void installMicromamba() throws IOException, InterruptedException, URISyntaxException { + if (isMambaInstalled()) return; + decompressMicromamba(downloadMicromamba()); + } + + /** + * Returns {@code \{"cmd.exe", "/c"\}} for Windows and an empty list for + * Mac/Linux. + * + * @return {@code \{"cmd.exe", "/c"\}} for Windows and an empty list for + * Mac/Linux. + */ + private static List< String > getBaseCommand() + { + final List< String > cmd = new ArrayList<>(); + if ( isWindowsOS() ) + cmd.addAll( Arrays.asList( "cmd.exe", "/c" ) ); + return cmd; + } + + /** + * Run {@code conda update} in the specified environment. A list of packages to + * update and extra parameters can be specified as {@code args}. + * + * @param envDir + * The directory within which the environment will be updated. + * @param args + * The list of packages to be updated and extra parameters as + * {@code String...}. + * @throws IOException + * If an I/O error occurs. + * @throws InterruptedException + * If the current thread is interrupted by another thread while it + * is waiting, then the wait is ended and an InterruptedException is + * thrown. + * @throws IllegalStateException if Micromamba has not been installed, thus the instance of {@link Mamba} cannot be used + */ + public void updateIn( final File envDir, final String... args ) throws IOException, InterruptedException + { + checkMambaInstalled(); + final List< String > cmd = new ArrayList<>( Arrays.asList( "update", "--prefix", envDir.getAbsolutePath() ) ); + cmd.addAll( Arrays.asList( args ) ); + if (!cmd.contains("--yes") && !cmd.contains("-y")) cmd.add("--yes"); + runMamba(cmd.toArray(new String[0])); + } + + /** + * Run {@code conda create} to create a Conda environment defined by the input environment yaml file. + * + * @param envDir + * The directory within which the environment will be created. + * @param envYaml + * The environment yaml file containing the information required to build it + * @throws IOException + * If an I/O error occurs. + * @throws InterruptedException + * If the current thread is interrupted by another thread while it + * is waiting, then the wait is ended and an InterruptedException is + * thrown. + * @throws IllegalStateException if Micromamba has not been installed, thus the instance of {@link Mamba} cannot be used + */ + public void createWithYaml( final File envDir, final String envYaml ) throws IOException, InterruptedException + { + checkMambaInstalled(); + runMamba("env", "create", "--prefix", + envDir.getAbsolutePath(), "-f", envYaml, "-y", "-vv" ); + } + + /** + * Returns Conda version as a {@code String}. + * + * @return The Conda version as a {@code String}. + * @throws IOException + * If an I/O error occurs. + * @throws InterruptedException + * If the current thread is interrupted by another thread while it + * is waiting, then the wait is ended and an InterruptedException is + * thrown. + */ + public String getVersion() throws IOException, InterruptedException { + final List< String > cmd = getBaseCommand(); + if (mambaCommand.contains(" ") && isWindowsOS()) + cmd.add( surroundWithQuotes(Arrays.asList( coverArgWithDoubleQuotes(mambaCommand), "--version" )) ); + else + cmd.addAll( Arrays.asList( coverArgWithDoubleQuotes(mambaCommand), "--version" ) ); + final Process process = getBuilder( false ).command( cmd ).start(); + if ( process.waitFor() != 0 ) + throw new RuntimeException("Error getting Micromamba version"); + return new BufferedReader( new InputStreamReader( process.getInputStream() ) ).readLine(); + } + + /** + * Run a Conda command with one or more arguments. + * + * @param isInheritIO + * Sets the source and destination for subprocess standard I/O to be + * the same as those of the current Java process. + * @param args + * One or more arguments for the Mamba command. + * @throws RuntimeException + * If there is any error running the commands + * @throws IOException + * If an I/O error occurs. + * @throws InterruptedException + * If the current thread is interrupted by another thread while it + * is waiting, then the wait is ended and an InterruptedException is + * thrown. + * @throws IllegalStateException if Micromamba has not been installed, thus the instance of {@link Mamba} cannot be used + */ + public void runMamba(boolean isInheritIO, final String... args ) throws RuntimeException, IOException, InterruptedException + { + checkMambaInstalled(); + Thread mainThread = Thread.currentThread(); + SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss"); + + final List< String > cmd = getBaseCommand(); + List argsList = new ArrayList<>(); + argsList.add( coverArgWithDoubleQuotes(mambaCommand) ); + argsList.addAll( Arrays.stream( args ).map(aa -> { + if (aa.contains(" ") && isWindowsOS()) return coverArgWithDoubleQuotes(aa); + else return aa; + }).collect(Collectors.toList()) ); + boolean containsSpaces = argsList.stream().anyMatch(aa -> aa.contains(" ")); + + if (!containsSpaces || !isWindowsOS()) cmd.addAll(argsList); + else cmd.add(surroundWithQuotes(argsList)); + + ProcessBuilder builder = getBuilder(isInheritIO).command(cmd); + Process process = builder.start(); + // Use separate threads to read each stream to avoid a deadlock. + updateOutputConsumer(sdf.format(Calendar.getInstance().getTime()) + " -- STARTING INSTALLATION" + System.lineSeparator()); + long updatePeriod = 300; + Thread outputThread = new Thread(() -> { + try ( + InputStream inputStream = process.getInputStream(); + InputStream errStream = process.getErrorStream() + ){ + byte[] buffer = new byte[1024]; // Buffer size can be adjusted + StringBuilder processBuff = new StringBuilder(); + StringBuilder errBuff = new StringBuilder(); + String processChunk = ""; + String errChunk = ""; + int newLineIndex; + long t0 = System.currentTimeMillis(); + while (process.isAlive() || inputStream.available() > 0) { + if (!mainThread.isAlive()) { + process.destroyForcibly(); + return; + } + if (inputStream.available() > 0) { + processBuff.append(new String(buffer, 0, inputStream.read(buffer))); + while ((newLineIndex = processBuff.indexOf(System.lineSeparator())) != -1) { + processChunk += sdf.format(Calendar.getInstance().getTime()) + " -- " + + processBuff.substring(0, newLineIndex + 1).trim() + System.lineSeparator(); + processBuff.delete(0, newLineIndex + 1); + } + } + if (errStream.available() > 0) { + errBuff.append(new String(buffer, 0, errStream.read(buffer))); + while ((newLineIndex = errBuff.indexOf(System.lineSeparator())) != -1) { + errChunk += ERR_STREAM_UUUID + errBuff.substring(0, newLineIndex + 1).trim() + System.lineSeparator(); + errBuff.delete(0, newLineIndex + 1); + } + } + // Sleep for a bit to avoid busy waiting + Thread.sleep(60); + if (System.currentTimeMillis() - t0 > updatePeriod) { + updateOutputConsumer(processChunk); + processChunk = ""; + errChunk = ""; + t0 = System.currentTimeMillis(); + } + } + if (inputStream.available() > 0) { + processBuff.append(new String(buffer, 0, inputStream.read(buffer))); + processChunk += sdf.format(Calendar.getInstance().getTime()) + " -- " + processBuff.toString().trim(); + } + if (errStream.available() > 0) { + errBuff.append(new String(buffer, 0, errStream.read(buffer))); + errChunk += ERR_STREAM_UUUID + errBuff.toString().trim(); + } + updateErrorConsumer(errChunk); + updateOutputConsumer(processChunk + System.lineSeparator() + + sdf.format(Calendar.getInstance().getTime()) + " -- TERMINATED PROCESS\n"); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } + }); + // Start reading threads + outputThread.start(); + int processResult; + try { + processResult = process.waitFor(); + } catch (InterruptedException ex) { + throw new InterruptedException("Mamba process stopped. The command being executed was: " + cmd); + } + // Wait for all output to be read + outputThread.join(); + if (processResult != 0) + throw new RuntimeException("Exit code " + processResult + " from command execution: " + builder.command()); + } + + /** + * Run a Conda command with one or more arguments. + * + * @param args + * One or more arguments for the Conda command. + * @throws RuntimeException + * If there is any error running the commands + * @throws IOException + * If an I/O error occurs. + * @throws InterruptedException + * If the current thread is interrupted by another thread while it + * is waiting, then the wait is ended and an InterruptedException is + * thrown. + * @throws IllegalStateException if Micromamba has not been installed, thus the instance of {@link Mamba} cannot be used + */ + public void runMamba(final String... args ) throws RuntimeException, IOException, InterruptedException + { + checkMambaInstalled(); + runMamba(false, args); + } + + /** + * In Windows, if a command prompt argument contains and space " " it needs to + * start and end with double quotes + * @param arg + * the cmd argument + * @return a robust argument + */ + private static String coverArgWithDoubleQuotes(String arg) { + String[] specialChars = new String[] {" "}; + for (String schar : specialChars) { + if (arg.startsWith("\"") && arg.endsWith("\"")) + continue; + if (arg.contains(schar) && isWindowsOS()) { + return "\"" + arg + "\""; + } + } + return arg; + } + + /** + * When an argument of a command prompt argument in Windows contains an space, not + * only the argument needs to be surrounded by double quotes, but the whole sentence + * @param args + * arguments to be executed by the windows cmd + * @return a complete Sting containing all the arguments and surrounded by double quotes + */ + private static String surroundWithQuotes(List args) { + String arg = "\""; + for (String aa : args) { + arg += aa + " "; + } + arg = arg.substring(0, arg.length() - 1); + arg += "\""; + return arg; + } + + private static boolean isWindowsOS() { + return System.getProperty("os.name").startsWith("Windows"); + } +} diff --git a/src/main/java/org/apposed/appose/mamba/MambaHandler.java b/src/main/java/org/apposed/appose/mamba/MambaHandler.java new file mode 100644 index 0000000..5cb6979 --- /dev/null +++ b/src/main/java/org/apposed/appose/mamba/MambaHandler.java @@ -0,0 +1,215 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 - 2024 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.apposed.appose.mamba; + +import org.apposed.appose.BuildHandler; +import org.apposed.appose.Builder; +import org.apposed.appose.FilePaths; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.stream.Collectors; + +/** A {@link BuildHandler} plugin powered by micromamba. */ +public class MambaHandler implements BuildHandler { + + private final List channels = new ArrayList<>(); + private final List condaIncludes = new ArrayList<>(); + private final List yamlIncludes = new ArrayList<>(); + private final List pypiIncludes = new ArrayList<>(); + + @Override + public boolean channel(String name, String location) { + if (location == null) { + // Assume it's a conda channel. + channels.add(name); + return true; + } + return false; + } + + @Override + public boolean include(String content, String scheme) { + if (content == null) throw new NullPointerException("content must not be null"); + if (scheme == null) throw new NullPointerException("scheme must not be null"); + switch (scheme) { + case "conda": + // It's a conda package (or newline-separated package list). + condaIncludes.addAll(lines(content)); + return true; + case "pypi": + // It's a PyPI package (or newline-separated package list). + pypiIncludes.addAll(lines(content)); + return true; + case "environment.yml": + yamlIncludes.add(content); + return true; + } + return false; + } + + @Override + public String envName() { + for (String yaml : yamlIncludes) { + String[] lines = yaml.split("(\r\n|\n|\r)"); + Optional name = Arrays.stream(lines) + .filter(line -> line.startsWith("name:")) + .map(line -> line.substring(5).trim().replace("\"", "")) + .findFirst(); + if (name.isPresent()) return name.get(); + } + return null; + } + + @Override + public void build(File envDir, Builder builder) throws IOException { + if (!channels.isEmpty() || !condaIncludes.isEmpty() || !pypiIncludes.isEmpty()) { + throw new UnsupportedOperationException( + "Sorry, I don't know how to mix in additional packages from Conda or PyPI yet." + + " Please put them in your environment.yml for now."); + } + + Mamba conda = new Mamba(Mamba.BASE_PATH); + boolean isCondaDir = new File(envDir, "conda-meta").isDirectory(); + + if (yamlIncludes.isEmpty()) { + // Nothing for this handler to do. + if (isCondaDir) { + // If directory already exists and is a conda environment prefix, + // inject needed micromamba stuff into the configuration. + fillConfig(conda, envDir, builder.config); + } + return; + } + if (yamlIncludes.size() > 1) { + throw new UnsupportedOperationException( + "Sorry, I can't synthesize Conda environments from multiple environment.yml files yet." + + " Please use a single environment.yml for now."); + } + + // Is this envDir an already-existing conda directory? + // If so, we can update it. + if (isCondaDir) { + // This environment has already been populated. + // TODO: Should we update it? For now, we just use it. + fillConfig(conda, envDir, builder.config); + return; + } + + // Micromamba refuses to create an environment into an existing non-conda directory: + // + // "Non-conda folder exists at prefix" + // + // So if a non-conda directory already exists, we need to perform some + // contortions to make micromamba integrate with other build handlers: + // + // 1. If envDir already exists, rename it temporarily. + // 2. Run the micromamba command to create the environment. + // 3. Recursively move any previously existing contents from the + // temporary directory into the newly constructed one. + // 4. If moving an old file would overwrite a new file, put the old + // file back with a .old extension, so nothing is permanently lost. + // 5. As part of the move, remove the temp directories as they empty out. + + // Write out environment.yml from input content. + // We cannot write it into envDir, because mamba needs the directory to + // not exist yet in order to create the environment there. So we write it + // into a temporary work directory with a hopefully unique name. + Path envPath = envDir.getAbsoluteFile().toPath(); + String antiCollision = "" + (new Random().nextInt(90000000) + 10000000); + File workDir = envPath.resolveSibling(envPath.getFileName() + "." + antiCollision + ".tmp").toFile(); + if (envDir.exists()) { + if (envDir.isDirectory()) { + // Move aside the existing non-conda directory. + if (!envDir.renameTo(workDir)) { + throw new IOException("Failed to rename directory: " + envDir + " -> " + workDir); + } + } + else throw new IllegalArgumentException("Non-directory file already exists: " + envDir.getAbsolutePath()); + } + else if (!workDir.mkdirs()) { + throw new IOException("Failed to create work directory: " + workDir); + } + + // At this point, workDir exists and envDir does not. + // We want to write the environment.yml file into the work dir. + // But what if there is an existing environment.yml file in the work dir? + // Let's move it out of the way, rather than stomping on it. + File environmentYaml = new File(workDir, "environment.yml"); + FilePaths.renameToBackup(environmentYaml); + + // It should be safe to write out the environment.yml file now. + try (FileWriter fout = new FileWriter(environmentYaml)) { + fout.write(yamlIncludes.get(0)); + } + + // Finally, we can build the environment from the environment.yml file. + try { + conda.setOutputConsumer(msg -> builder.outputSubscribers.forEach(sub -> sub.accept(msg))); + conda.setErrorConsumer(msg -> builder.errorSubscribers.forEach(sub -> sub.accept(msg))); + conda.setMambaDownloadProgressConsumer((cur, max) -> { + builder.progressSubscribers.forEach(subscriber -> subscriber.accept("Downloading micromamba", cur, max)); + }); + + conda.installMicromamba(); + conda.createWithYaml(envDir, environmentYaml.getAbsolutePath()); + fillConfig(conda, envDir, builder.config); + } + catch (InterruptedException | URISyntaxException e) { + throw new IOException(e); + } + finally { + // Lastly, we merge the contents of workDir into envDir. This will be + // at least the environment.yml file, and maybe other files from other handlers. + FilePaths.moveDirectory(workDir, envDir, false); + } + } + + private List lines(String content) { + return Arrays.stream(content.split("(\r\n|\n|\r)")) + .map(String::trim) + .filter(s -> !s.isEmpty() && !s.startsWith("#")) + .collect(Collectors.toList()); + } + + private void fillConfig(Mamba conda, File envDir, Map> config) { + // Use `mamba run -p $envDir ...` to run within this environment. + config.computeIfAbsent("launchArgs", k -> new ArrayList<>()); + config.get("launchArgs").addAll(Arrays.asList(conda.mambaCommand, "run", "-p", envDir.getAbsolutePath())); + } +} diff --git a/src/main/java/org/apposed/appose/mamba/MambaInstallerUtils.java b/src/main/java/org/apposed/appose/mamba/MambaInstallerUtils.java new file mode 100644 index 0000000..3fe3ce8 --- /dev/null +++ b/src/main/java/org/apposed/appose/mamba/MambaInstallerUtils.java @@ -0,0 +1,252 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 - 2024 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.apposed.appose.mamba; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; + +import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.archivers.ArchiveStreamFactory; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; + +/** + * Utility methods unzip bzip2 files and to enable the download of micromamba + */ +final class MambaInstallerUtils { + + private MambaInstallerUtils() { + // Prevent instantiation of utility class. + } + + /** + * DEcompress a bzip2 file into a new file. + * The method is needed because Micromamba is distributed as a .tr.bz2 file and + * many distributions do not have tools readily available to extract the required files + * @param source + * .bzip2 file + * @param destination + * destination folder where the contents of the file are going to be decompressed + * @throws FileNotFoundException if the .bzip2 file is not found or does not exist + * @throws IOException if the source file already exists or there is any error with the decompression + * @throws InterruptedException if the therad where the decompression is happening is interrupted + */ + public static void unBZip2(File source, File destination) throws FileNotFoundException, IOException, InterruptedException { + try ( + BZip2CompressorInputStream input = new BZip2CompressorInputStream(new BufferedInputStream(new FileInputStream(source))); + FileOutputStream output = new FileOutputStream(destination); + ) { + copy(input, output); + } + } + + /** + * Copies the content of a InputStream into an OutputStream + * + * @param input + * the InputStream to copy + * @param output + * the target, may be null to simulate output to dev/null on Linux and NUL on Windows + * @return the number of bytes copied + * @throws IOException if an error occurs copying the streams + * @throws InterruptedException if the thread where this is happening is interrupted + */ + private static long copy(final InputStream input, final OutputStream output) throws IOException, InterruptedException { + int bufferSize = 4096; + final byte[] buffer = new byte[bufferSize]; + int n = 0; + long count = 0; + while (-1 != (n = input.read(buffer))) { + if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Decompressing stopped."); + if (output != null) { + output.write(buffer, 0, n); + } + count += n; + } + return count; + } + + /** Untar an input file into an output file. + + * The output file is created in the output folder, having the same name + * as the input file, minus the '.tar' extension. + * + * @param inputFile the input .tar file + * @param outputDir the output directory file. + * @throws IOException + * @throws FileNotFoundException + */ + public static void unTar(final File inputFile, final File outputDir) throws FileNotFoundException, IOException, InterruptedException { + + try ( + InputStream is = new FileInputStream(inputFile); + TarArchiveInputStream debInputStream = (TarArchiveInputStream) new ArchiveStreamFactory().createArchiveInputStream("tar", is); + ) { + TarArchiveEntry entry = null; + while ((entry = (TarArchiveEntry)debInputStream.getNextEntry()) != null) { + final File outputFile = new File(outputDir, entry.getName()); + if (entry.isDirectory()) { + if (!outputFile.exists()) { + if (!outputFile.mkdirs()) { + throw new IllegalStateException(String.format("Couldn't create directory %s.", outputFile.getAbsolutePath())); + } + } + } else { + if (!outputFile.getParentFile().exists()) { + if (!outputFile.getParentFile().mkdirs()) + throw new IOException("Failed to create directory " + outputFile.getParentFile().getAbsolutePath()); + } + try (OutputStream outputFileStream = new FileOutputStream(outputFile)) { + copy(debInputStream, outputFileStream); + } + } + } + } catch (ArchiveException e) { + throw new IOException(e); + } + + } + + /** + * Example main method + * @param args + * no args are required + * @throws FileNotFoundException if some file is not found + * @throws IOException if there is any error reading or writting + * @throws URISyntaxException if the url is wrong or there is no internet connection + * @throws InterruptedException if there is interrruption + */ + public static void main(String[] args) throws FileNotFoundException, IOException, URISyntaxException, InterruptedException { + String url = Mamba.MICROMAMBA_URL; + final File tempFile = File.createTempFile( "miniconda", ".tar.bz2" ); + tempFile.deleteOnExit(); + URL website = MambaInstallerUtils.redirectedURL(new URL(url)); + ReadableByteChannel rbc = Channels.newChannel(website.openStream()); + try (FileOutputStream fos = new FileOutputStream(tempFile)) { + long transferred = fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + System.out.print(tempFile.length()); + } + String tarPath = "C:\\Users\\angel\\OneDrive\\Documentos\\pasteur\\git\\micromamba-1.5.1-1.tar"; + String mambaPath = "C:\\Users\\angel\\OneDrive\\Documentos\\pasteur\\git\\mamba"; + unBZip2(new File("C:\\Users\\angel\\OneDrive\\Documentos\\pasteur\\git\\micromamba-1.5.1-1.tar.bz2"), + new File(tarPath)); + unTar(new File(tarPath), new File(mambaPath)); + } + + /** + * This method shuold be used when we get the following response codes from + * a {@link HttpURLConnection}: + * - {@link HttpURLConnection#HTTP_MOVED_TEMP} + * - {@link HttpURLConnection#HTTP_MOVED_PERM} + * - {@link HttpURLConnection#HTTP_SEE_OTHER} + * + * If that is not the response code or the connection does not work, the url + * returned will be the same as the provided. + * If the method is used corretly, it will return the URL to which the original URL + * has been redirected + * @param url + * original url. Connecting to that url must give a 301, 302 or 303 response code + * @return the redirected url + * @throws MalformedURLException if the url does not fulfil the requirements for an url to be correct + * @throws URISyntaxException if the url is incorrect or there is no internet connection + */ + public static URL redirectedURL(URL url) throws MalformedURLException, URISyntaxException { + int statusCode; + HttpURLConnection conn; + try { + conn = (HttpURLConnection) url.openConnection(); + statusCode = conn.getResponseCode(); + } catch (IOException ex) { + return url; + } + if (statusCode < 300 || statusCode > 308) + return url; + String newURL = conn.getHeaderField("Location"); + try { + return redirectedURL(new URL(newURL)); + } catch (MalformedURLException ex) { + } + try { + if (newURL.startsWith("//")) + return redirectedURL(new URL("http:" + newURL)); + else + throw new MalformedURLException(); + } catch (MalformedURLException ex) { + } + URI uri = url.toURI(); + String scheme = uri.getScheme(); + String host = uri.getHost(); + String mainDomain = scheme + "://" + host; + return redirectedURL(new URL(mainDomain + newURL)); + } + + /** + * Get the size of the file stored in the given URL + * @param url + * url where the file is stored + * @return the size of the file + */ + public static long getFileSize(URL url) { + HttpURLConnection conn = null; + try { + conn = (HttpURLConnection) url.openConnection(); + // TODO: Fix user agent. + conn.setRequestProperty("User-Agent", "Appose/0.1.0(" + System.getProperty("os.name") + "; Java " + System.getProperty("java.version")); + if (conn.getResponseCode() >= 300 && conn.getResponseCode() <= 308) + return getFileSize(redirectedURL(url)); + if (conn.getResponseCode() != 200) + throw new Exception("Unable to connect to: " + url.toString()); + long size = conn.getContentLengthLong(); + conn.disconnect(); + return size; + } catch (IOException e) { + throw new RuntimeException(e); + } catch (Exception ex) { + ex.printStackTrace(); + String msg = "Unable to connect to " + url.toString(); + System.out.println(msg); + return 1; + } + } +} diff --git a/src/main/java/org/apposed/appose/shm/ShmLinux.java b/src/main/java/org/apposed/appose/shm/ShmLinux.java index 85f021d..ac5e06f 100644 --- a/src/main/java/org/apposed/appose/shm/ShmLinux.java +++ b/src/main/java/org/apposed/appose/shm/ShmLinux.java @@ -155,7 +155,6 @@ private static long getSHMSize(final int shmFd) { final long size = LibRtOrC.lseek(shmFd, 0, LibRtOrC.SEEK_END); if (size == -1) { - // TODO remove LibRtOrC.close(shmFd); throw new RuntimeException("Failed to get shared memory segment size. Errno: " + Native.getLastError()); } return size; diff --git a/src/main/java/org/apposed/appose/shm/ShmMacOS.java b/src/main/java/org/apposed/appose/shm/ShmMacOS.java index bb4d405..388a2cc 100644 --- a/src/main/java/org/apposed/appose/shm/ShmMacOS.java +++ b/src/main/java/org/apposed/appose/shm/ShmMacOS.java @@ -146,7 +146,6 @@ private static long getSHMSize(final int shmFd) { final long size = MacosHelpers.INSTANCE.get_shared_memory_size(shmFd); if (size == -1) { - // TODO remove macosInstance.unlink_shared_memory(null);; throw new RuntimeException("Failed to get shared memory segment size. Errno: " + Native.getLastError()); } return size; diff --git a/src/main/java/org/apposed/appose/shm/ShmWindows.java b/src/main/java/org/apposed/appose/shm/ShmWindows.java index b52c91e..c558d4a 100644 --- a/src/main/java/org/apposed/appose/shm/ShmWindows.java +++ b/src/main/java/org/apposed/appose/shm/ShmWindows.java @@ -42,9 +42,6 @@ /** * Windows-specific shared memory implementation. - *

    - * TODO separate unlink and close - *

    * * @author Carlos Garcia Lopez de Haro * @author Tobias Pietzsch @@ -85,7 +82,7 @@ private static ShmInfo prepareShm(String name, boolean create, int prevSize = getSHMSize(shm_name); } while (prevSize >= 0); } else { - shm_name = nameMangle_TODO(name); + shm_name = name; prevSize = getSHMSize(shm_name); } ShmUtils.checkSize(shm_name, prevSize, size); @@ -126,7 +123,7 @@ private static ShmInfo prepareShm(String name, boolean create, int ShmInfo info = new ShmInfo<>(); info.size = shm_size; - info.name = nameUnmangle_TODO(shm_name); + info.name = shm_name; info.pointer = pointer; info.writePointer = writePointer; info.handle = hMapFile; @@ -134,19 +131,6 @@ private static ShmInfo prepareShm(String name, boolean create, int return info; } - // TODO equivalent of removing slash - private static String nameUnmangle_TODO (String memoryName){ - return memoryName; - } - - // TODO equivalent of adding slash - // Do we need the "Local\" prefix? - private static String nameMangle_TODO (String memoryName){ - // if (!memoryName.startsWith("Local" + File.separator) && !memoryName.startsWith("Global" + File.separator)) - // memoryName = "Local" + File.separator + memoryName; - return memoryName; - } - // name is WITH prefix etc private static boolean checkSHMExists ( final String name){ final WinNT.HANDLE hMapFile = Kernel32.INSTANCE.OpenFileMapping(WinNT.FILE_MAP_READ, false, name); diff --git a/src/main/resources/META-INF/services/org.apposed.appose.BuildHandler b/src/main/resources/META-INF/services/org.apposed.appose.BuildHandler new file mode 100644 index 0000000..4a96693 --- /dev/null +++ b/src/main/resources/META-INF/services/org.apposed.appose.BuildHandler @@ -0,0 +1 @@ +org.apposed.appose.mamba.MambaHandler diff --git a/src/test/java/org/apposed/appose/ApposeTest.java b/src/test/java/org/apposed/appose/ApposeTest.java index 3b82c55..599d73d 100644 --- a/src/test/java/org/apposed/appose/ApposeTest.java +++ b/src/test/java/org/apposed/appose/ApposeTest.java @@ -34,6 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.fail; +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -72,7 +73,7 @@ public class ApposeTest { public void testGroovy() throws IOException, InterruptedException { Environment env = Appose.system(); try (Service service = env.groovy()) { - //service.debug(System.err::println); + maybeDebug(service); executeAndAssert(service, COLLATZ_GROOVY); } } @@ -81,28 +82,73 @@ public void testGroovy() throws IOException, InterruptedException { public void testPython() throws IOException, InterruptedException { Environment env = Appose.system(); try (Service service = env.python()) { - //service.debug(System.err::println); + maybeDebug(service); executeAndAssert(service, COLLATZ_PYTHON); } } @Test - public void testServiceStartupFailure() throws IOException { - Environment env = Appose.base("no-pythons-to-be-found-here").build(); + public void testConda() throws IOException, InterruptedException { + Environment env = Appose + .conda(new File("src/test/resources/envs/cowsay.yml")) + .logDebug() + .build(); try (Service service = env.python()) { - fail("Python worker process started successfully!?"); + maybeDebug(service); + Task task = service.task( + "import cowsay\n" + + "task.outputs['moo'] = cowsay.get_output_string('cow', 'moo')\n" + ); + task.waitFor(); + assertComplete(task); + String expectedMoo = + " ___\n" + + "| moo |\n" + + " ===\n" + + " \\\n" + + " \\\n" + + " ^__^\n" + + " (oo)\\_______\n" + + " (__)\\ )\\/\\\n" + + " ||----w |\n" + + " || ||"; + String actualMoo = (String) task.outputs.get("moo"); + assertEquals(expectedMoo, actualMoo); + } + } + + @Test + public void testServiceStartupFailure() throws IOException, InterruptedException { + String tempNonExistingDir = "no-pythons-to-be-found-here"; + new File(tempNonExistingDir).deleteOnExit(); + Environment env = Appose.build(tempNonExistingDir); + try (Service service = env.python()) { + String info = ""; + try { + Task task = service.task( + "import sys\n" + + "task.outputs['executable'] = sys.executable\n" + + "task.outputs['version'] = sys.version" + ); + task.waitFor(); + info += "\n- sys.executable = " + task.outputs.get("executable"); + info += "\n- sys.version = " + task.outputs.get("version"); + } + finally { + fail("Python worker process started successfully!?" + info); + } } catch (IllegalArgumentException exc) { assertEquals( "No executables found amongst candidates: " + - "[python, python3, python.exe, bin/python, bin/python.exe]", + "[python, python3, python.exe]", exc.getMessage() ); } } public void executeAndAssert(Service service, String script) - throws InterruptedException, IOException + throws IOException, InterruptedException { Task task = service.task(script); @@ -128,9 +174,10 @@ class TaskState { // Wait for task to finish. task.waitFor(); + assertComplete(task); // Validate the execution result. - assertSame(TaskStatus.COMPLETE, task.status); + assertComplete(task); Number result = (Number) task.outputs.get("result"); assertEquals(91, result.intValue()); @@ -156,10 +203,34 @@ class TaskState { } TaskState completion = events.get(92); assertSame(ResponseType.COMPLETION, completion.responseType); - assertSame(TaskStatus.COMPLETE, completion.status); assertEquals("[90] -> 1", completion.message); assertEquals(90, completion.current); assertEquals(1, completion.maximum); assertNull(completion.error); } + + private void maybeDebug(Service service) { + String debug1 = System.getenv("DEBUG"); + String debug2 = System.getProperty("appose.debug"); + if (falsy(debug1) && falsy(debug2)) return; + service.debug(System.err::println); + } + + private boolean falsy(String value) { + if (value == null) return true; + String tValue = value.trim(); + if (tValue.isEmpty()) return true; + if (tValue.equalsIgnoreCase("false")) return true; + if (tValue.equals("0")) return true; + return false; + } + private void assertComplete(Task task) { + String errorMessage = ""; + if (task.status != TaskStatus.COMPLETE) { + String caller = new RuntimeException().getStackTrace()[1].getMethodName(); + errorMessage = "TASK ERROR in method " + caller + ":\n" + task.error; + System.err.println(); + } + assertEquals(TaskStatus.COMPLETE, task.status, errorMessage); + } } diff --git a/src/test/java/org/apposed/appose/FilePathsTest.java b/src/test/java/org/apposed/appose/FilePathsTest.java new file mode 100644 index 0000000..3b50a23 --- /dev/null +++ b/src/test/java/org/apposed/appose/FilePathsTest.java @@ -0,0 +1,195 @@ +/*- + * #%L + * Appose: multi-language interprocess cooperation with shared memory. + * %% + * Copyright (C) 2023 - 2024 Appose developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.apposed.appose; + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests {@link FilePaths}. + * + * @author Curtis Rueden + */ +public class FilePathsTest { + + /** Tests {@link FilePaths#findExe}. */ + @Test + public void testFindExe() throws IOException { + File tmpDir = Files.createTempDirectory("appose-FilePathsTest-testFindExe-").toFile(); + try { + // Set up some red herrings. + createStubFile(tmpDir, "walk"); + createStubFile(tmpDir, "fly"); + File binDir = createDirectory(tmpDir, "bin"); + File binFly = createStubFile(binDir, "fly"); + // Mark the desired match as executable. + assertTrue(binFly.setExecutable(true)); + assertTrue(binFly.canExecute()); + + // Search for the desired match. + List dirs = Arrays.asList(tmpDir.getAbsolutePath(), binDir.getAbsolutePath()); + List exes = Arrays.asList("walk", "fly", "swim"); + File exe = FilePaths.findExe(dirs, exes); + + // Check that we found the right file. + assertEquals(binFly, exe); + } + finally { + FileUtils.deleteDirectory(tmpDir); + } + } + + /** Tests {@link FilePaths#location}. */ + @Test + public void testLocation() { + // NB: Will fail if this test is run in a weird way (e.g. + // from inside the tests JAR), but I don't care right now. :-P + File expected = Paths.get(System.getProperty("user.dir"), "target", "test-classes").toFile(); + File actual = FilePaths.location(getClass()); + assertEquals(expected, actual); + } + + /** Tests {@link FilePaths#moveDirectory}. */ + @Test + public void testMoveDirectory() throws IOException { + File tmpDir = Files.createTempDirectory("appose-FilePathsTest-testMoveDirectory-").toFile(); + try { + // Set up a decently weighty directory structure. + File srcDir = createDirectory(tmpDir, "src"); + File breakfast = createStubFile(srcDir, "breakfast"); + File lunchDir = createDirectory(srcDir, "lunch"); + File lunchFile1 = createStubFile(lunchDir, "apples", "fuji"); + File lunchFile2 = createStubFile(lunchDir, "bananas"); + File dinnerDir = createDirectory(srcDir, "dinner"); + File dinnerFile1 = createStubFile(dinnerDir, "bread"); + File dinnerFile2 = createStubFile(dinnerDir, "wine"); + File destDir = createDirectory(tmpDir, "dest"); + File destLunchDir = createDirectory(destDir, "lunch"); + createStubFile(destLunchDir, "apples", "gala"); + + // Move the source directory to the destination. + FilePaths.moveDirectory(srcDir, destDir, false); + + // Check whether everything worked. + assertFalse(srcDir.exists()); + assertMoved(breakfast, destDir, ""); + assertMoved(lunchFile1, destLunchDir, "gala"); + File backupLunchFile1 = new File(destLunchDir, "apples.old"); + assertContent(backupLunchFile1, "fuji"); + assertMoved(lunchFile2, destLunchDir, ""); + File destDinnerDir = new File(destDir, dinnerDir.getName()); + assertMoved(dinnerFile1, destDinnerDir, ""); + assertMoved(dinnerFile2, destDinnerDir, ""); + } + finally { + FileUtils.deleteDirectory(tmpDir); + } + } + + /** Tests {@link FilePaths#moveFile}. */ + @Test + public void testMoveFile() throws IOException { + File tmpDir = Files.createTempDirectory("appose-FilePathsTest-testMoveFile-").toFile(); + try { + File srcDir = createDirectory(tmpDir, "from"); + File srcFile = createStubFile(srcDir, "stuff.txt", "shiny"); + File destDir = createDirectory(tmpDir, "to"); + File destFile = createStubFile(destDir, "stuff.txt", "obsolete"); + boolean overwrite = true; + + FilePaths.moveFile(srcFile, destDir, overwrite); + + assertTrue(srcDir.exists()); + assertFalse(srcFile.exists()); + assertContent(destFile, "shiny"); + File backupFile = new File(destDir, "stuff.txt.old"); + assertContent(backupFile, "obsolete"); + } + finally { + FileUtils.deleteDirectory(tmpDir); + } + } + + /** Tests {@link FilePaths#renameToBackup}. */ + @Test + public void testRenameToBackup() throws IOException { + File tmpFile = Files.createTempFile("appose-FilePathsTest-testRenameToBackup-", "").toFile(); + assertTrue(tmpFile.exists()); + tmpFile.deleteOnExit(); + FilePaths.renameToBackup(tmpFile); + File backupFile = new File(tmpFile.getParent(), tmpFile.getName() + ".old"); + backupFile.deleteOnExit(); + assertFalse(tmpFile.exists()); + assertTrue(backupFile.exists()); + } + + private File createDirectory(File parent, String name) { + File dir = new File(parent, name); + assertTrue(dir.mkdir()); + assertTrue(dir.exists()); + return dir; + } + + private File createStubFile(File dir, String name) throws IOException { + return createStubFile(dir, name, "<" + name + ">"); + } + + private File createStubFile(File dir, String name, String content) throws IOException { + File stubFile = new File(dir, name); + try (PrintWriter pw = new PrintWriter(new FileWriter(stubFile))) { + pw.print(content); + } + assertTrue(stubFile.exists()); + return stubFile; + } + + private void assertMoved(File srcFile, File destDir, String expectedContent) throws IOException { + assertFalse(srcFile.exists()); + File destFile = new File(destDir, srcFile.getName()); + assertContent(destFile, expectedContent); + } + + private void assertContent(File file, String expectedContent) throws IOException { + assertTrue(file.exists()); + String actualContent = new String(Files.readAllBytes(file.toPath())); + assertEquals(expectedContent, actualContent); + } +} diff --git a/src/test/java/org/apposed/appose/NDArrayExamplePython.java b/src/test/java/org/apposed/appose/NDArrayExamplePython.java index e37582c..bf8fe92 100644 --- a/src/test/java/org/apposed/appose/NDArrayExamplePython.java +++ b/src/test/java/org/apposed/appose/NDArrayExamplePython.java @@ -52,7 +52,7 @@ public static void main(String[] args) throws Exception { } // pass to python (will be wrapped as numpy ndarray - final Environment env = Appose.base( "/opt/homebrew/Caskroom/miniforge/base/envs/appose/" ).build(); + final Environment env = Appose.build("/opt/homebrew/Caskroom/miniforge/base/envs/appose/"); try ( Service service = env.python() ) { final Map< String, Object > inputs = new HashMap<>(); inputs.put( "img", ndArray); diff --git a/src/test/java/org/apposed/appose/SharedMemoryTest.java b/src/test/java/org/apposed/appose/SharedMemoryTest.java index 59b4e6a..05fd7c2 100644 --- a/src/test/java/org/apposed/appose/SharedMemoryTest.java +++ b/src/test/java/org/apposed/appose/SharedMemoryTest.java @@ -41,6 +41,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests {@link SharedMemory}. @@ -54,7 +55,7 @@ public void testShmCreate() throws IOException { int size = 456; try (SharedMemory shm = SharedMemory.create(null, size)) { assertNotNull(shm.name()); - assertEquals(size, shm.size()); + assertTrue(shm.size() >= size); assertNotNull(shm.pointer()); // Modify the memory contents. @@ -74,8 +75,8 @@ public void testShmCreate() throws IOException { String output = runPython( "from multiprocessing.shared_memory import SharedMemory\n" + "from sys import stdout\n" + - "shm = SharedMemory(name='" + shm.name() + "', size=" + shm.size() + ")\n" + - "matches = sum(1 for i in range(shm.size) if shm.buf[i] == (shm.size - i) % 256)\n" + + "shm = SharedMemory(name='" + shm.name() + "', size=" + size + ")\n" + + "matches = sum(1 for i in range(" + size + ") if shm.buf[i] == (" + size + " - i) % 256)\n" + "stdout.write(f'{matches}\\n')\n" + "stdout.flush()\n" + "shm.unlink()\n" // HACK: to satisfy Python's overly aggressive resource tracker @@ -112,7 +113,7 @@ public void testShmAttach() throws IOException { assertNotNull(shmName); assertFalse(shmName.isEmpty()); int shmSize = Integer.parseInt(shmInfo[1]); - assertEquals(345, shmSize); + assertTrue(shmSize >= 345); // Attach to the shared memory and verify it matches expectations. try (SharedMemory shm = SharedMemory.attach(shmName, shmSize)) { diff --git a/src/test/java/org/apposed/appose/TypesTest.java b/src/test/java/org/apposed/appose/TypesTest.java index 06e7e39..2d218fe 100644 --- a/src/test/java/org/apposed/appose/TypesTest.java +++ b/src/test/java/org/apposed/appose/TypesTest.java @@ -70,7 +70,7 @@ public class TypesTest { "\"shm\":{" + "\"appose_type\":\"shm\"," + "\"name\":\"SHM_NAME\"," + - "\"size\":4000" + + "\"size\":SHM_SIZE" + "}" + "}" + "}"; @@ -110,7 +110,9 @@ public void testEncode() { data.put("ndArray", ndArray); String json = Types.encode(data); assertNotNull(json); - String expected = JSON.replaceAll("SHM_NAME", ndArray.shm().name()); + String expected = JSON + .replaceAll("SHM_NAME", ndArray.shm().name()) + .replaceAll("SHM_SIZE", "" + ndArray.shm().size()); assertEquals(expected, json); } } @@ -119,11 +121,16 @@ public void testEncode() { public void testDecode() { Map data; String shmName; + int shmSize; // Create name shared memory segment and decode JSON block. try (SharedMemory shm = SharedMemory.create(null, 4000)) { shmName = shm.name(); - data = Types.decode(JSON.replaceAll("SHM_NAME", shmName)); + shmSize = shm.size(); + String json = JSON + .replaceAll("SHM_NAME", shmName) + .replaceAll("SHM_SIZE", "" + shmSize); + data = Types.decode(json); } // Validate results. @@ -158,7 +165,7 @@ public void testDecode() { assertEquals(20, ndArray.shape().get(1)); assertEquals(25, ndArray.shape().get(2)); assertEquals(shmName, ndArray.shm().name()); - assertEquals(4000, ndArray.shm().size()); + assertEquals(shmSize, ndArray.shm().size()); } } diff --git a/src/test/resources/envs/cowsay.yml b/src/test/resources/envs/cowsay.yml new file mode 100644 index 0000000..a35704e --- /dev/null +++ b/src/test/resources/envs/cowsay.yml @@ -0,0 +1,9 @@ +name: appose-cowsay +channels: + - conda-forge +dependencies: + - python >= 3.8 + - pip + - appose + - pip: + - cowsay==6.1