Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

allow dataset connection to sample, improve error handling #7

Merged
merged 3 commits into from
Sep 2, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ public class AuthenticationOptions {
@Option(
names = {"-u", "--user"},
description = "openBIS user name")
private String user;
private String openbisUser;
@ArgGroup(multiplicity = "1") // ensures the password is provided once with at least one of the possible options.
PasswordOptions passwordOptions;
PasswordOptions openbisPasswordOptions;

@Option(
names = {"-as", "-as_url"},
Expand All @@ -43,11 +43,23 @@ public class AuthenticationOptions {
scope = CommandLine.ScopeType.INHERIT)
public String configPath;

public String getUser() {
if(user == null & configPath!=null && !configPath.isBlank()) {
user = ReadProperties.getProperties(configPath).get("user");
@Option(
names = {"-seek-server", "-seek_url"},
description = "SEEK API URL",
scope = CommandLine.ScopeType.INHERIT)
private String seek_url;

public String getOpenbisUser() {
if(openbisUser == null & configPath!=null && !configPath.isBlank()) {
openbisUser = ReadProperties.getProperties(configPath).get("user");
}
return user;
return openbisUser;
}

public String getSeekURL() {
log.error("No URL to the SEEK address provided.");
System.exit(2);
return seek_url;
}

public String getDSS() {
Expand All @@ -64,8 +76,8 @@ public String getAS() {
return as_url;
}

public char[] getPassword() {
return passwordOptions.getPassword();
public char[] getOpenbisPassword() {
return openbisPasswordOptions.getPassword();
}

/**
Expand Down Expand Up @@ -110,7 +122,7 @@ char[] getPassword() {
@Override
public String toString() {
return new StringJoiner(", ", AuthenticationOptions.class.getSimpleName() + "[", "]")
.add("user='" + user + "'")
.add("user='" + openbisUser + "'")
.toString();
//ATTENTION: do not expose the password here!
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public class DownloadPetabCommand implements Runnable {

@Override
public void run() {
OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS());
OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS());
OpenbisConnector openbis = new OpenbisConnector(authentication);

List<DataSet> datasets = openbis.findDataSets(Collections.singletonList(datasetCode));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public void run() {
} else {
System.out.println("Querying experiment in all available spaces...");
}
OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS());
OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS());
OpenbisConnector openbis = new OpenbisConnector(authentication);
List<DataSet> datasets = openbis.listDatasetsOfExperiment(spaces, experimentCode).stream()
.sorted(Comparator.comparing(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public void run() {
} else {
summary.add("Querying samples in all available spaces...");
}
OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS());
OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS());
OpenbisConnector openbis = new OpenbisConnector(authentication);
Map<SampleTypeConnection, Integer> hierarchy = openbis.queryFullSampleHierarchy(spaces);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public void run() {
} else {
summary.add("Querying samples in all available spaces...");
}
OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS());
OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS());
OpenbisConnector openbis = new OpenbisConnector(authentication);

if (spaces.isEmpty()) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package life.qbic.io.commandline;

import ch.ethz.sis.openbis.generic.OpenBIS;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet;
import java.io.File;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import life.qbic.App;
import life.qbic.model.DatasetWithProperties;
import life.qbic.model.download.OpenbisConnector;
import picocli.CommandLine.Command;
import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
import picocli.CommandLine.Parameters;

@Command(name = "openbis-to-seek",
description = "Transfers data or metadata from openBIS to SEEK.")
public class TransferDataToSeekCommand implements Runnable {

@Parameters(arity = "1", paramLabel = "dataset id", description = "The code of the dataset (or its metadata) to transfer. Can be found via list-data.")
private String datasetCode;
@Parameters(arity = "1", paramLabel = "seek node", description = "The node in SEEK to which to transfer the dataset.")
private String seekNode;
@Option(names = { "-d", "--data"}, usageHelp = true, description = "Transfers the data itself to SEEK along with the metadata")
private boolean transferData;
@Mixin
AuthenticationOptions auth = new AuthenticationOptions();

@Override
public void run() {
OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS());
OpenbisConnector openbis = new OpenbisConnector(authentication);

List<DataSet> datasets = openbis.findDataSets(Collections.singletonList(datasetCode));

if(datasets.isEmpty()) {
System.out.println(datasetCode+" not found");
return;
}
DatasetWithProperties result = new DatasetWithProperties(datasets.get(0));
Optional<String> patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID",

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could there be other PatientIds classifiers? If so i'd recommend to move this into a global identifier.

result.getExperiment().getIdentifier());
patientID.ifPresent(s -> result.addProperty("patientID", s));

System.out.println("Found dataset, downloading.");
System.out.println();

final String tmpPath = "tmp/";

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm confused of the purpose of this command? It downloads the data into a temp folder and afterwards deletes it again?


File downloadFolder = openbis.downloadDataset(tmpPath, datasetCode);



cleanupTemp(new File(tmpPath));

System.out.println("Done");
}

private void cleanupTemp(File tmpFolder) {
File[] files = tmpFolder.listFiles();
if (files != null) { //some JVMs return null for empty dirs
for (File f : files) {
if (f.isDirectory()) {
cleanupTemp(f);
} else {
f.delete();
}
}
}
}

}
43 changes: 31 additions & 12 deletions src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,6 @@
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId;
import java.io.File;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import life.qbic.App;
Expand All @@ -23,9 +19,9 @@ public class UploadDatasetCommand implements Runnable {

@Parameters(arity = "1", paramLabel = "file/folder", description = "The path to the file or folder to upload")
private String dataPath;
@Parameters(arity = "1", paramLabel = "experiment ID", description = "The full identifier of the experiment the data should be attached to. "
+ "The identifier must be of the format: /space/project/experiment")
private String experimentID;
@Parameters(arity = "1", paramLabel = "object ID", description = "The full identifier of the experiment or sample the data should be attached to. "
+ "The identifier must be of the format: /space/project/experiment for experiments or /space/sample for samples")
private String objectID;
@Option(arity = "1..*", paramLabel = "<parent_datasets>", description = "Optional list of dataset codes to act"
+ " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"})
private List<String> parents = new ArrayList<>();
Expand All @@ -36,15 +32,28 @@ public class UploadDatasetCommand implements Runnable {

@Override
public void run() {
OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS());
OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS());
openbis = new OpenbisConnector(authentication);

if(!pathValid(dataPath)) {
System.out.printf("Path %s could not be found%n", dataPath);
return;
}
if(!experimentExists(experimentID)) {
System.out.printf("Experiment %s could not be found%n", experimentID);
boolean attachToSample = OpenbisConnector.sampleIdPattern.matcher(objectID).find();
boolean attachToExperiment = false;
if(!attachToSample) {
attachToExperiment = OpenbisConnector.experimentIdPattern.matcher(objectID).find();
}
if(!attachToExperiment && !attachToSample) {
System.out.printf("%s is neither a valid experiment nor sample identifier%n", objectID);
return;
}
if(attachToExperiment && !experimentExists(objectID)) {
System.out.printf("Experiment with identifier %s could not be found%n", objectID);
return;
}
if(attachToSample && !sampleExists(objectID)) {
System.out.printf("Sample object with identifier %s could not be found%n", objectID);
return;
}
if(!datasetsExist(parents)) {
Expand All @@ -54,10 +63,19 @@ public void run() {
System.out.println();
System.out.println("Parameters verified, uploading dataset...");
System.out.println();
DataSetPermId result = openbis.registerDataset(Path.of(dataPath), experimentID, parents);
System.out.printf("Dataset %s was successfully created%n", result.getPermId());
if(attachToExperiment) {
DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), objectID, parents);
System.out.printf("Dataset %s was successfully created%n", result.getPermId());
wow-such-code marked this conversation as resolved.
Show resolved Hide resolved
} else {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NIT: I think it would help to have 2 different output messages here, since one result attaches the dataset to an experiment while the other registers it to a sample

DataSetPermId result = openbis.registerDatasetForSample(Path.of(dataPath), objectID, parents);
System.out.printf("Dataset %s was successfully created%n", result.getPermId());
wow-such-code marked this conversation as resolved.
Show resolved Hide resolved
}
}

private boolean sampleExists(String objectID) {
return openbis.sampleExists(objectID);
}

private boolean datasetsExist(List<String> datasetCodes) {
return openbis.findDataSets(datasetCodes).size() == datasetCodes.size();
}
Expand All @@ -70,4 +88,5 @@ private boolean pathValid(String dataPath) {
return new File(dataPath).exists();
}


}
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,17 @@ public class UploadPetabResultCommand implements Runnable {

@Override
public void run() {
OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS());
OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS());
openbis = new OpenbisConnector(authentication);

if(!pathValid(dataPath)) {
System.out.printf("Path %s could not be found%n", dataPath);
return;
}
if(!new File(dataPath).isDirectory()) {
System.out.printf("%s is not a directory. Please specify the PETab directory root%n", dataPath);
return;
}
if(!experimentExists(experimentID)) {
System.out.printf("Experiment %s could not be found%n", experimentID);
return;
Expand All @@ -60,7 +64,7 @@ public void run() {
}
System.out.println("Uploading dataset...");
//TODO copy and remove source references here
DataSetPermId result = openbis.registerDataset(Path.of(dataPath), experimentID, parents);
DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), experimentID, parents);
System.out.printf("Dataset %s was successfully created%n", result.getPermId());
Comment on lines +67 to 68

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you need to differentiate between registration to an experiment and to a sample here as well?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the use case here was specified as always using an experiment

}

Expand Down
61 changes: 0 additions & 61 deletions src/main/java/life/qbic/model/download/Authentication.java

This file was deleted.

Loading
Loading