Skip to content

Commit

Permalink
Overwrite checker changes for each protocol
Browse files Browse the repository at this point in the history
  • Loading branch information
Amisha Sinha committed Apr 28, 2023
1 parent d0777be commit a569729
Show file tree
Hide file tree
Showing 9 changed files with 163 additions and 25 deletions.
17 changes: 15 additions & 2 deletions src/main/java/com/rabbitMq/rabbitmqscheduler/Services/BoxExpander.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public void createClient(EndpointCredential credential) {
}

@Override
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String basePath) {
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String basePath, boolean overwrite) {
List<EntityInfo> transferFiles = new ArrayList<>();
Stack<BoxFolder> travStack = new Stack<>();//this will only hold folders to traverse
if(userSelectedResources.isEmpty()) return new ArrayList<>(); //we need to signal the cancellation of this transferjob request.
Expand Down Expand Up @@ -54,7 +54,20 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
if (child instanceof BoxFile.Info) {
BoxFile.Info fileInfo = (BoxFile.Info) child;
BoxFile boxFile = new BoxFile(this.connection, fileInfo.getID());
transferFiles.add(boxFileToEntityInfo(boxFile));
EntityInfo entityInfo = boxFileToEntityInfo(boxFile);
boolean shouldTransfer = true;
if (!overwrite) {
for (EntityInfo existingFile : transferFiles) {
if (existingFile.getName().equals(entityInfo.getName()) &&
existingFile.getPath().equals(entityInfo.getPath())) {
shouldTransfer = false;
break;
}
}
}
if (shouldTransfer) {
transferFiles.add(boxFileToEntityInfo(boxFile));
}
} else if (child instanceof BoxFolder.Info) {
BoxFolder.Info folderInfo = (BoxFolder.Info) child;
BoxFolder childFolder = new BoxFolder(this.connection, folderInfo.getID());
Expand Down
20 changes: 19 additions & 1 deletion src/main/java/com/rabbitMq/rabbitmqscheduler/Services/DropBoxExpander.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public void createClient(EndpointCredential credential) {
}

@Override
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String parentPath) {
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String parentPath, boolean overwrite) {
Stack<Metadata> traversalQueue = new Stack<>();
List<EntityInfo> expandedFiles = new ArrayList<>();
if (parentPath == null || parentPath.isEmpty()) parentPath = "";
Expand All @@ -39,6 +39,9 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
List<Metadata> resources = listOp(parentPath);
for (Metadata resource : resources) {
if (resource instanceof FileMetadata) {
if (!overwrite && destinationFileExists(resource.getName(),parentPath)) {
continue; // Skip this file
}
expandedFiles.add(metaDataToFileInfo((FileMetadata) resource));
} else if (resource instanceof FolderMetadata) {
traversalQueue.push(resource);
Expand All @@ -49,6 +52,9 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
List<Metadata> dropBoxFiles = listOp(fileInfo.getPath());
dropBoxFiles.forEach(metadata -> {
if (metadata instanceof FileMetadata) {
if (!overwrite && destinationFileExists(metadata.getName(),fileInfo.getPath())) {
return; // Skip this file
}
expandedFiles.add(metaDataToFileInfo((FileMetadata) metadata));
} else if (metadata instanceof FolderMetadata) {
traversalQueue.push(metadata);
Expand All @@ -61,6 +67,9 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
List<Metadata> folderList = listOp(folderMetadata.getPathLower());
for (Metadata res : folderList) {
if (res instanceof FileMetadata) {
if (!overwrite && destinationFileExists(res.getName(),parentPath)) {
continue; // Skip this file
}
expandedFiles.add(metaDataToFileInfo((FileMetadata) res));
} else if (res instanceof FolderMetadata) {
traversalQueue.push(res);
Expand All @@ -69,6 +78,15 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
}
return expandedFiles;
}
private boolean destinationFileExists(String fileName, String destinationPath) {
List<Metadata> destinationFiles = listOp(destinationPath);
for (Metadata file : destinationFiles) {
if (file.getName().equals(fileName) && file instanceof FileMetadata) {
return true;
}
}
return false;
}

public EntityInfo metaDataToFileInfo(FileMetadata file) {
EntityInfo fileInfo = new EntityInfo();
Expand Down
11 changes: 10 additions & 1 deletion src/main/java/com/rabbitMq/rabbitmqscheduler/Services/FTPExpander.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public void createClient(EndpointCredential credential) {

@SneakyThrows
@Override
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String basePath) {
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String basePath, boolean overwrite) {
this.infoList = userSelectedResources;
List<EntityInfo> filesToTransferList = new LinkedList<>();
Stack<FileObject> traversalStack = new Stack<>();
Expand All @@ -68,6 +68,15 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
traversalStack.push(fObject);
}
}
// If overwrite flag is not set, delete all files on the destination
if (!overwrite) {
for (EntityInfo e : this.infoList) {
FileObject fObject = fsm.resolveFile(this.vfsCredential.getUri() + basePath + e.getId(), this.options);
if (fObject.exists() && fObject.getType() == FileType.FILE) {
fObject.delete();
}
}
}
for (int files = Integer.MAX_VALUE; files > 0 && !traversalStack.isEmpty(); --files) {
FileObject curr = traversalStack.pop();
FileName fileName = curr.getName();
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/rabbitMq/rabbitmqscheduler/Services/FileExpander.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ public interface FileExpander {
* @param basePath: This is the path to the file we want to expand. This can also be an id for an OAuth flat file system basically
* @return
*/
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources,String basePath);
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources,String basePath, boolean overwrite);

}
14 changes: 13 additions & 1 deletion src/main/java/com/rabbitMq/rabbitmqscheduler/Services/GDriveExpander.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,21 @@ public void createClient(EndpointCredential credential) {
}

@Override
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String basePath) {
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String basePath, boolean overwrite) {
Stack<File> fileListStack = new Stack<>();
List<EntityInfo> fileInfoList = new ArrayList<>();
// Remove files on the destination if overwrite flag is not set
if (!overwrite) {
for (EntityInfo fileInfo : userSelectedResources) {
String fileQuery = "'" + fileInfo.getId() + "' in parents and trashed=false";
try {
// Delete the file
client.files().delete(fileInfo.getId()).execute();
} catch (IOException e) {
logger.error("Error deleting file with ID: " + fileInfo.getId(), e);
}
}
}
for(EntityInfo fileInfo : userSelectedResources){
String fileQuery = "'" + fileInfo.getId() + "' in parents and trashed=false";
EntityInfo info = getMetadataForfile(fileInfo.getId());
Expand Down
62 changes: 57 additions & 5 deletions src/main/java/com/rabbitMq/rabbitmqscheduler/Services/HttpExpander.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public void createClient(EndpointCredential credential) {

@SneakyThrows
@Override
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String basePath) {
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String basePath, boolean overwrite) {
List<EntityInfo> filesToSend = new ArrayList<>();
Stack<Element> directoriesToTraverse = new Stack<>();
if (basePath.isEmpty()) basePath = "/";
Expand All @@ -54,7 +54,20 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
if (elem.text().endsWith("/")) { //directory to expand
directoriesToTraverse.push(elem);
} else { //we have a file
filesToSend.add(fromElement(elem));
EntityInfo fileInfo = fromElement(elem);
boolean shouldTransfer = true;
if (!overwrite) {
for (EntityInfo existingFile : filesToSend) {
if (existingFile.getName().equals(fileInfo.getName()) &&
existingFile.getPath().equals(fileInfo.getPath())) {
shouldTransfer = false;
break;
}
}
}
if (shouldTransfer) {
filesToSend.add(fileInfo);
}
}
}
} else { //move only files/folders the user selected
Expand All @@ -70,11 +83,37 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
if (elem.text().endsWith("/")) { //directory to expand
directoriesToTraverse.push(elem);
} else { //we have a file
filesToSend.add(fromElement(elem));
EntityInfo entityInfo = fromElement(elem);
boolean shouldTransfer = true;
if (!overwrite) {
for (EntityInfo existingFile : filesToSend) {
if (existingFile.getName().equals(entityInfo.getName()) &&
existingFile.getPath().equals(entityInfo.getPath())) {
shouldTransfer = false;
break;
}
}
}
if (shouldTransfer) {
filesToSend.add(entityInfo);
}
}
}
}else{
filesToSend.add(this.fileToInfo(this.credential.getUri() + basePath + selectedFiles.getPath()));
EntityInfo entityInfo = this.fileToInfo(this.credential.getUri() + basePath + selectedFiles.getPath());
boolean shouldTransfer = true;
if (!overwrite) {
for (EntityInfo existingFile : filesToSend) {
if (existingFile.getName().equals(entityInfo.getName()) &&
existingFile.getPath().equals(entityInfo.getPath())) {
shouldTransfer = false;
break;
}
}
}
if (shouldTransfer) {
filesToSend.add(entityInfo);
}
}
}
}
Expand All @@ -91,7 +130,20 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
if (elem.text().endsWith("/")) { //directory to expand
directoriesToTraverse.push(elem);
} else { //we have a file
filesToSend.add(fromElement(elem));
EntityInfo fileInfo = fromElement(elem);
boolean shouldTransfer = true;
if (!overwrite) {
for (EntityInfo existingFile : filesToSend) {
if (existingFile.getName().equals(fileInfo.getName()) &&
existingFile.getPath().equals(fileInfo.getPath())) {
shouldTransfer = false;
break;
}
}
}
if (shouldTransfer) {
filesToSend.add(fileInfo);
}
}
}
}
Expand Down
18 changes: 9 additions & 9 deletions src/main/java/com/rabbitMq/rabbitmqscheduler/Services/RequestModifier.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,33 +39,33 @@ public class RequestModifier {
Set<String> nonOautUsingType = new HashSet<>(Arrays.asList(new String[]{"ftp", "sftp", "http", "s3"}));
Set<String> oautUsingType = new HashSet<>(Arrays.asList(new String[]{"dropbox", "box", "gdrive", "gftp"}));

public List<EntityInfo> selectAndExpand(TransferJobRequest.Source source, List<EntityInfo> selectedResources) {
public List<EntityInfo> selectAndExpand(TransferJobRequest.Source source, List<EntityInfo> selectedResources, boolean overwrite) {
logger.info("The info list in select and expand is \n" + selectedResources.toString());
switch (source.getType()) {
case ftp:
ftpExpander.createClient(source.getVfsSourceCredential());
return ftpExpander.expandedFileSystem(selectedResources, source.getParentInfo().getPath());
return ftpExpander.expandedFileSystem(selectedResources, source.getParentInfo().getPath(), overwrite);
case s3:
s3Expander.createClient(source.getVfsSourceCredential());
return s3Expander.expandedFileSystem(selectedResources, source.getParentInfo().getPath());
return s3Expander.expandedFileSystem(selectedResources, source.getParentInfo().getPath(), overwrite);
case sftp:
case scp:
sftpExpander.createClient(source.getVfsSourceCredential());
return sftpExpander.expandedFileSystem(selectedResources, source.getParentInfo().getPath());
return sftpExpander.expandedFileSystem(selectedResources, source.getParentInfo().getPath(), overwrite);
case http:
httpExpander.createClient(source.getVfsSourceCredential());
return httpExpander.expandedFileSystem(selectedResources, source.getParentInfo().getPath());
return httpExpander.expandedFileSystem(selectedResources, source.getParentInfo().getPath(), overwrite);
case box:
boxExpander.createClient(source.getOauthSourceCredential());
return boxExpander.expandedFileSystem(selectedResources, source.getParentInfo().getId());
return boxExpander.expandedFileSystem(selectedResources, source.getParentInfo().getId(), overwrite);
case dropbox:
dropBoxExpander.createClient(source.getOauthSourceCredential());
return dropBoxExpander.expandedFileSystem(selectedResources, source.getParentInfo().getId());
return dropBoxExpander.expandedFileSystem(selectedResources, source.getParentInfo().getId(), overwrite);
case vfs:
return selectedResources;
case gdrive:
gDriveExpander.createClient(source.getOauthSourceCredential());
return gDriveExpander.expandedFileSystem(selectedResources, source.getParentInfo().getId());
return gDriveExpander.expandedFileSystem(selectedResources, source.getParentInfo().getId(), overwrite);

}
return null;
Expand Down Expand Up @@ -136,7 +136,7 @@ public TransferJobRequest createRequest(RequestFromODS odsTransferRequest) {
OAuthEndpointCredential destinationCredential = credentialService.fetchOAuthCredential(odsTransferRequest.getDestination().getType(), odsTransferRequest.getOwnerId(), odsTransferRequest.getDestination().getCredId());
d.setOauthDestCredential(destinationCredential);
}
List<EntityInfo> expandedFiles = this.selectAndExpand(s, odsTransferRequest.getSource().getInfoList());
List<EntityInfo> expandedFiles = this.selectAndExpand(s, odsTransferRequest.getSource().getInfoList(), odsTransferRequest.getOptions().isOverwrite());
expandedFiles = this.checkDestinationChunkSize(expandedFiles, d, odsTransferRequest.getOptions().getChunkSize());
s.setInfoList(expandedFiles);
transferJobRequest.setSource(s);
Expand Down
30 changes: 27 additions & 3 deletions src/main/java/com/rabbitMq/rabbitmqscheduler/Services/S3Expander.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public void createClient(EndpointCredential cred) {
}

@Override
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String basePath) {
public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String basePath, boolean overwrite) {
List<EntityInfo> traversedFiles = new LinkedList<>();
//trim leading forward slashes from base path (s3 doesn't recognise it as root)
basePath = StringUtils.stripStart(basePath, "/");
Expand All @@ -53,13 +53,37 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
entityInfo.setId(obj.getKey());
entityInfo.setPath(obj.getKey());
entityInfo.setSize(obj.getSize());
traversedFiles.add(entityInfo);
boolean shouldTransfer = true;
if (!overwrite) {
for (EntityInfo existingFile : traversedFiles) {
if (existingFile.getName().equals(entityInfo.getName()) &&
existingFile.getPath().equals(entityInfo.getPath())) {
shouldTransfer = false;
break;
}
}
}
if (shouldTransfer) {
traversedFiles.add(entityInfo);
}
}
// the case where the user selected a file
} else if(this.s3Client.doesObjectExist(this.regionAndBucket[1], userSelectedResource.getPath())){
ObjectMetadata metadata = this.s3Client.getObjectMetadata(this.regionAndBucket[1],userSelectedResource.getPath());
userSelectedResource.setSize(metadata.getContentLength());
traversedFiles.add(userSelectedResource);
boolean shouldTransfer = true;
if (!overwrite) {
for (EntityInfo existingFile : traversedFiles) {
if (existingFile.getName().equals(userSelectedResource.getName()) &&
existingFile.getPath().equals(userSelectedResource.getPath())) {
shouldTransfer = false;
break;
}
}
}
if (shouldTransfer) {
traversedFiles.add(userSelectedResource);
}
}
}

Expand Down
Loading

0 comments on commit a569729

Please sign in to comment.