diff --git a/.gitignore b/.gitignore index 123f81b..89b6b1f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ release.properties local-cache maven-repository-provisioner.log .checkstyle +**/*.iml \ No newline at end of file diff --git a/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/Configuration.java b/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/Configuration.java index cc7cbf5..aa084ae 100644 --- a/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/Configuration.java +++ b/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/Configuration.java @@ -1,4 +1,4 @@ -/** +/** * Copyright simpligility technologies inc. http://www.simpligility.com * Licensed under Eclipse Public License - v 1.0 http://www.eclipse.org/legal/epl-v10.html */ @@ -15,23 +15,23 @@ public class Configuration @Parameter( names = { "-h", "-help" }, help = true, description = "Display the help information." ) private boolean help; - @Parameter( names = { "-s", "-sourceUrl" }, - description = + @Parameter( names = { "-s", "-sourceUrl" }, + description = "URL for the source repository from which artifacts are resolved, " + "example for a Nexus install is http://localhost:8081/content/groups/public" ) private String sourceUrl = "https://repo.maven.apache.org/maven2"; - @Parameter( names = { "-t", "-targetUrl" }, + @Parameter( names = { "-t", "-targetUrl" }, description = "Folder or URL for the target repository e.g. dist-repo or " - + " http://localhost:8081/content/repositories/release", + + " http://localhost:8081/content/repositories/release", required = true ) private String targetUrl; - @Parameter( names = { "-a", "-artifactCoordinates" }, + @Parameter( names = { "-a", "-artifactCoordinates" }, description = "GroupId/ArtifactId/Version (GAV) coordinates of the desired artifacts using the " + "syntax (values in [] are optional): " + "g:a[:extension][:classifier]:v|g:a[:extension][:classifier]:v " - + " e.g. org.apache.commons:commons-lang3:3.3.2|com.google.inject:guice:jar:no_aop:3.0", + + " e.g. org.apache.commons:commons-lang3:3.3.2|com.google.inject:guice:jar:no_aop:3.0", required = false ) private String artifactCoordinate; @@ -43,7 +43,7 @@ public class Configuration @Parameter( names = { "-su", "-sourceUsername" }, description = "Username for the source repository, if required." ) private String sourceUsername; - + @Parameter( names = { "-sp", "-sourcePassword" }, description = "Password for the source repository, if required." ) private String sourcePassword; @@ -54,7 +54,7 @@ public class Configuration @Parameter( names = { "-ij", "-includeJavadoc" }, description = "Flag to enable/disable download of javadoc artifacts.", arity = 1 ) private Boolean includeJavadoc = true; - + @Parameter( names = { "-ip", "-includeProvidedScope" }, description = "Flag to include/exclude provisioning of dependencies with provided scope.", arity = 1 ) private Boolean includeProvidedScope = false; @@ -67,12 +67,12 @@ public class Configuration description = "Flag to include/exclude provisioning of dependencies with runtime scope.", arity = 1 ) private Boolean includeRuntimeScope = false; - @Parameter( names = { "-cd", "-cacheDirectory" }, + @Parameter( names = { "-cd", "-cacheDirectory" }, description = "Local directory used as a cache between resolving and deploying or as the " + "source repository that should be transferred." ) private String cacheDirectory = "local-cache"; - @Parameter( names = { "-ct", "-checkTarget" }, + @Parameter( names = { "-ct", "-checkTarget" }, description = "Check target repository before deploying, if target GAV pom exists no deployment will be" + " attempted", arity = 1 ) private Boolean checkTarget = true; @@ -82,6 +82,10 @@ public class Configuration + "deployments of a second execution are logged.", arity = 1 ) private Boolean verifyOnly = false; + @Parameter( names = { "-dt", "-deployThreads" }, + description = "Number of threads to use for deploying artifacts, default is 5." ) + private int deployThreads = 5; + public void setSourceUrl( String sourceUrl ) { this.sourceUrl = sourceUrl; @@ -111,12 +115,12 @@ public void setSourceUsername( String sourceUsername ) { this.sourceUsername = sourceUsername; } - + public void seSourcetPassword( String sourcePassword ) { this.sourcePassword = sourcePassword; } - + public void setIncludeSources( Boolean includeSources ) { this.includeSources = includeSources; @@ -204,12 +208,12 @@ public String getSourceUsername() { return sourceUsername; } - + public String getSourcePassword() { return sourcePassword; } - + public Boolean getIncludeSources() { return includeSources; @@ -224,7 +228,7 @@ public Boolean getIncludeProvidedScope() { return includeProvidedScope; } - + public Boolean getIncludeTestScope() { return includeTestScope; @@ -250,6 +254,15 @@ public Boolean getVerifyOnly() return verifyOnly; } + public void setDeployThreads(int deployThreads) + { + this.deployThreads = deployThreads; + } + + public int getDeployThreads() { + return deployThreads; + } + public List getArtifactCoordinates() { List coords = Arrays.asList( artifactCoordinate.split( "\\|" ) ); @@ -261,14 +274,14 @@ public boolean hasArtifactsCoordinates() return artifactCoordinate != null && !artifactCoordinate.isEmpty(); } - public String getConfigSummary() + public String getConfigSummary() { StringBuilder builder = new StringBuilder(); builder.append( "\nProvisioning artifacts: " + this.getArtifactCoordinate() + "\n" ) .append( "Source: " + getSourceUrl() + "\n" ) .append( "Target: " + getTargetUrl() + "\n" ) .append( "Username: " + getUsername() + "\n" ); - if ( this.getPassword() != null ) + if ( this.getPassword() != null ) { builder.append( "Password: " + getPassword().replaceAll( ".", "***" ) + "\n" ); } diff --git a/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/MavenRepositoryDeployer.java b/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/MavenRepositoryDeployer.java index 806c3f5..208374d 100644 --- a/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/MavenRepositoryDeployer.java +++ b/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/MavenRepositoryDeployer.java @@ -1,4 +1,4 @@ -/** +/** * Copyright simpligility technologies inc. http://www.simpligility.com * Licensed under Eclipse Public License - v 1.0 http://www.eclipse.org/legal/epl-v10.html */ @@ -11,10 +11,13 @@ import java.net.HttpURLConnection; import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.TreeSet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.filefilter.AndFileFilter; import org.apache.commons.io.filefilter.DirectoryFileFilter; import org.apache.commons.io.filefilter.IOFileFilter; @@ -30,9 +33,6 @@ import org.apache.maven.model.io.xpp3.MavenXpp3Reader; import org.eclipse.aether.DefaultRepositorySystemSession; import org.eclipse.aether.RepositorySystem; -import org.eclipse.aether.artifact.Artifact; -import org.eclipse.aether.artifact.DefaultArtifact; -import org.eclipse.aether.deployment.DeployRequest; import org.eclipse.aether.repository.Authentication; import org.eclipse.aether.repository.RemoteRepository; import org.eclipse.aether.util.repository.AuthenticationBuilder; @@ -48,12 +48,52 @@ public class MavenRepositoryDeployer { private static Logger logger = LoggerFactory.getLogger( "MavenRepositoryHelper" ); + private static MavenRepositoryDeployer instance; + + public static MavenRepositoryDeployer getInstance(File repositoryPath, Configuration configuration) { + if (instance == null) { + instance = new MavenRepositoryDeployer(repositoryPath, configuration); + } + return instance; + } + private File repositoryPath; private RepositorySystem system; private DefaultRepositorySystemSession session; + private final String targetUrl; + private final String username; + private final String password; + private final Boolean checkTarget; + private final Boolean verifyOnly; + private final int threads; + + public static class DeploymentResult { + private final TreeSet successfulDeploys = new TreeSet(); + private final TreeSet failedDeploys = new TreeSet(); + private final TreeSet potentialDeploys = new TreeSet(); + + public DeploymentResult(TreeSet successfulDeploys, TreeSet failedDeploys, TreeSet potentialDeploys) { + this.successfulDeploys.addAll(successfulDeploys); + this.failedDeploys.addAll(failedDeploys); + this.potentialDeploys.addAll(potentialDeploys); + } + + public TreeSet getSuccessfulDeploys() { + return successfulDeploys; + } + + public TreeSet getFailedDeploys() { + return failedDeploys; + } + + public TreeSet getPotentialDeploys() { + return potentialDeploys; + } + } + private final TreeSet successfulDeploys = new TreeSet(); private final TreeSet failedDeploys = new TreeSet(); @@ -62,9 +102,15 @@ public class MavenRepositoryDeployer private final TreeSet potentialDeploys = new TreeSet(); - public MavenRepositoryDeployer( File repositoryPath ) + public MavenRepositoryDeployer( File repositoryPath, Configuration configuration ) { this.repositoryPath = repositoryPath; + this.targetUrl = configuration.getTargetUrl(); + this.username = configuration.getUsername(); + this.password = configuration.getPassword(); + this.checkTarget = configuration.getCheckTarget(); + this.verifyOnly = configuration.getVerifyOnly(); + this.threads = configuration.getDeployThreads(); initialize(); } @@ -125,127 +171,53 @@ public static Collection getPomFiles( File repoPath ) } - public void deployToRemote( String targetUrl, String username, String password, Boolean checkTarget, - Boolean verifyOnly ) - { - Collection leafDirectories = getLeafDirectories( repositoryPath ); + public void run() { + Collection leafDirectories = getLeafDirectories(repositoryPath); - for ( File leafDirectory : leafDirectories ) - { - String leafAbsolutePath = leafDirectory.getAbsoluteFile().toString(); - int repoAbsolutePathLength = repositoryPath.getAbsoluteFile().toString().length(); - String leafRepoPath = leafAbsolutePath.substring( repoAbsolutePathLength + 1, leafAbsolutePath.length() ); + try (ExecutorService executorService = Executors.newFixedThreadPool(threads)) { + List> futures = new ArrayList<>(); - Gav gav = GavUtil.getGavFromRepositoryPath( leafRepoPath ); + for (File leafDirectory : leafDirectories) { + String leafAbsolutePath = leafDirectory.getAbsoluteFile().toString(); + int repoAbsolutePathLength = repositoryPath.getAbsoluteFile().toString().length(); + String leafRepoPath = leafAbsolutePath.substring(repoAbsolutePathLength + 1, leafAbsolutePath.length()); - boolean pomInTarget = false; - if ( checkTarget ) - { - pomInTarget = checkIfPomInTarget( targetUrl, username, password, gav ); - } - - if ( pomInTarget ) - { - logger.info( "Found POM for " + gav + " already in target. Skipping deployment." ); - skippedDeploys.add( gav.toString() ); - } - else - { - // only interested in files using the artifactId-version* pattern - // don't bother with .sha1 files - IOFileFilter fileFilter = - new AndFileFilter( new WildcardFileFilter( gav.getArtifactId() + "-" + gav.getVersion() + "*" ), - new NotFileFilter( new SuffixFileFilter( "sha1" ) ) ); - Collection artifacts = FileUtils.listFiles( leafDirectory, fileFilter, null ); - - Authentication auth = new AuthenticationBuilder().addUsername( username ).addPassword( password ) - .build(); - - RemoteRepository distRepo = new RemoteRepository.Builder( "repositoryIdentifier", "default", targetUrl ) - .setProxy( ProxyHelper.getProxy( targetUrl ) ) - .setAuthentication( auth ).build(); - - DeployRequest deployRequest = new DeployRequest(); - deployRequest.setRepository( distRepo ); - for ( File file : artifacts ) - { - String extension; - if ( file.getName().endsWith( "tar.gz" ) ) - { - extension = "tar.gz"; - } - else - { - extension = FilenameUtils.getExtension( file.getName() ); - } + Gav gav = GavUtil.getGavFromRepositoryPath(leafRepoPath); - String baseFileName = gav.getFilenameStart() + "." + extension; - String fileName = file.getName(); - String g = gav.getGroupId(); - String a = gav.getArtifactId(); - String v = gav.getVersion(); - - Artifact artifact = null; - if ( gav.getPomFilename().equals( fileName ) ) - { - artifact = new DefaultArtifact( g, a, MavenConstants.POM, v ); - } - else if ( gav.getJarFilename().equals( fileName ) ) - { - artifact = new DefaultArtifact( g, a, MavenConstants.JAR, v ); - } - else if ( gav.getSourceFilename().equals( fileName ) ) - { - artifact = new DefaultArtifact( g, a, MavenConstants.SOURCES, MavenConstants.JAR, v ); - } - else if ( gav.getJavadocFilename().equals( fileName ) ) - { - artifact = new DefaultArtifact( g, a, MavenConstants.JAVADOC, MavenConstants.JAR, v ); - } - else if ( baseFileName.equals( fileName ) ) - { - artifact = new DefaultArtifact( g, a, extension, v ); - } - else - { - String classifier = - file.getName().substring( gav.getFilenameStart().length() + 1, - file.getName().length() - ( "." + extension ).length() ); - artifact = new DefaultArtifact( g, a, classifier, extension, v ); - } - - if ( artifact != null ) - { - artifact = artifact.setFile( file ); - deployRequest.addArtifact( artifact ); - } + boolean pomInTarget = false; + if (checkTarget) { + pomInTarget = checkIfPomInTarget(targetUrl, username, password, gav); } - try - { - if ( verifyOnly ) - { - for ( Artifact artifact : deployRequest.getArtifacts() ) - { - potentialDeploys.add( artifact.toString() ); - } - } - else - { - system.deploy( session, deployRequest ); - for ( Artifact artifact : deployRequest.getArtifacts() ) - { - successfulDeploys.add( artifact.toString() ); - } - } + if (pomInTarget) { + logger.info("Found POM for " + gav + " already in target. Skipping deployment."); + addSkippedDeploy(gav.toString()); + } else { + IOFileFilter fileFilter = new AndFileFilter(new WildcardFileFilter(gav.getArtifactId() + "-" + gav.getVersion() + "*"), + new NotFileFilter(new SuffixFileFilter("sha1"))); + Collection artifacts = FileUtils.listFiles(leafDirectory, fileFilter, null); + + Authentication auth = new AuthenticationBuilder().addUsername(username).addPassword(password).build(); + + RemoteRepository distRepo = new RemoteRepository.Builder("repositoryIdentifier", "default", targetUrl) + .setProxy(ProxyHelper.getProxy(targetUrl)) + .setAuthentication(auth).build(); + + MavenRepositoryDeploymentCallable deploymentCallable = new MavenRepositoryDeploymentCallable(gav, artifacts, distRepo, verifyOnly, system, session); + futures.add(executorService.submit(deploymentCallable)); } - catch ( Exception e ) - { - logger.info( "Deployment failed with " + e.getMessage() + ", artifact might be deployed already." ); - for ( Artifact artifact : deployRequest.getArtifacts() ) - { - failedDeploys.add( artifact.toString() ); + } + + for (Future future : futures) { + try { + MavenRepositoryDeployer.DeploymentResult result = future.get(); + synchronized (this) { + successfulDeploys.addAll(result.getSuccessfulDeploys()); + failedDeploys.addAll(result.getFailedDeploys()); + potentialDeploys.addAll(result.getPotentialDeploys()); } + } catch (Exception e) { + logger.error("Error while getting deployment result", e); } } } @@ -389,4 +361,28 @@ public String getFailureMessage() { return "Failed to deploy some artifacts."; } + + public void addSuccessfulDeploy(String artifact) { + successfulDeploys.add(artifact); + } + + public void addFailedDeploy(String artifact) { + failedDeploys.add(artifact); + } + + public void addSkippedDeploy(String artifact) { + skippedDeploys.add(artifact); + } + + public void addVerifiedDeploy(String artifact) { + potentialDeploys.add(artifact); + } + + public void summarize() { + logger.info("Summary of Deployment Results:"); + logger.info("Successful Deployments {}: {}", successfulDeploys.size(), String.join(",", successfulDeploys)); + logger.info("Failed Deployments {}: {}", failedDeploys.size(), String.join(",", failedDeploys)); + logger.info("Skipped Deployments {}: {}", skippedDeploys.size(), String.join(",", skippedDeploys)); + logger.info("Potential Deployments {}: {}", potentialDeploys.size(), String.join(",", potentialDeploys)); + } } diff --git a/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/MavenRepositoryDeploymentCallable.java b/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/MavenRepositoryDeploymentCallable.java new file mode 100644 index 0000000..040e7cb --- /dev/null +++ b/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/MavenRepositoryDeploymentCallable.java @@ -0,0 +1,138 @@ +package com.simpligility.maven.provisioner; + +import com.simpligility.maven.Gav; +import com.simpligility.maven.MavenConstants; +import org.apache.commons.io.FilenameUtils; +import org.eclipse.aether.DefaultRepositorySystemSession; +import org.eclipse.aether.RepositorySystem; +import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.artifact.DefaultArtifact; +import org.eclipse.aether.deployment.DeployRequest; +import org.eclipse.aether.repository.RemoteRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Collection; +import java.util.TreeSet; +import java.util.concurrent.Callable; + + +public class MavenRepositoryDeploymentCallable implements Callable { + private static Logger logger = LoggerFactory.getLogger( "MavenRepositoryDeploymentRunnable" ); + + private final RemoteRepository distRepo; + private final Collection artifacts; + private final Gav gav; + private final boolean verifyOnly; + + private RepositorySystem system; + private DefaultRepositorySystemSession session; + + private final TreeSet successfulDeploys = new TreeSet(); + private final TreeSet failedDeploys = new TreeSet(); + private final TreeSet potentialDeploys = new TreeSet(); + + public MavenRepositoryDeploymentCallable(Gav gav, Collection artifacts, RemoteRepository distRepo, boolean verifyOnly, RepositorySystem system, DefaultRepositorySystemSession session) { + this.gav = gav; + this.artifacts = artifacts; + this.distRepo = distRepo; + this.verifyOnly = verifyOnly; + this.system = system; + this.session = session; + } + + public MavenRepositoryDeployer.DeploymentResult call() { + DeployRequest deployRequest = new DeployRequest(); + deployRequest.setRepository( distRepo ); + for ( File file : artifacts ) + { + Artifact artifact = getArtifact(gav, file); + if ( artifact != null ) + { + artifact = artifact.setFile( file ); + deployRequest.addArtifact( artifact ); + } + } + + try + { + if ( verifyOnly ) + { + for ( Artifact artifact : deployRequest.getArtifacts() ) + { + potentialDeploys.add( artifact.toString() ); + } + } + else + { + system.deploy( session, deployRequest ); + for ( Artifact artifact : deployRequest.getArtifacts() ) + { + successfulDeploys.add( artifact.toString() ); + } + } + } + catch ( Exception e ) + { + logger.warn("Deployment failed with {}, artifact might be deployed already.", e.getMessage()); + for ( Artifact artifact : deployRequest.getArtifacts() ) + { + failedDeploys.add( artifact.toString() ); + } + } + return new MavenRepositoryDeployer.DeploymentResult( successfulDeploys, failedDeploys, potentialDeploys ); + } + + public String getExtension(File file) { + String extension; + if ( file.getName().endsWith( "tar.gz" ) ) + { + extension = "tar.gz"; + } + else + { + extension = FilenameUtils.getExtension( file.getName() ); + } + return extension; + } + + public Artifact getArtifact(Gav gav, File file) { + String extension = getExtension(file); + String baseFileName = gav.getFilenameStart() + "." + extension; + String fileName = file.getName(); + String g = gav.getGroupId(); + String a = gav.getArtifactId(); + String v = gav.getVersion(); + + Artifact artifact = null; + if ( gav.getPomFilename().equals( fileName ) ) + { + artifact = new DefaultArtifact( g, a, MavenConstants.POM, v ); + } + else if ( gav.getJarFilename().equals( fileName ) ) + { + artifact = new DefaultArtifact( g, a, MavenConstants.JAR, v ); + } + else if ( gav.getSourceFilename().equals( fileName ) ) + { + artifact = new DefaultArtifact( g, a, MavenConstants.SOURCES, MavenConstants.JAR, v ); + } + else if ( gav.getJavadocFilename().equals( fileName ) ) + { + artifact = new DefaultArtifact( g, a, MavenConstants.JAVADOC, MavenConstants.JAR, v ); + } + else if ( baseFileName.equals( fileName ) ) + { + artifact = new DefaultArtifact( g, a, extension, v ); + } + else + { + String classifier = + file.getName().substring( gav.getFilenameStart().length() + 1, + file.getName().length() - ( "." + extension ).length() ); + artifact = new DefaultArtifact( g, a, classifier, extension, v ); + } + return artifact; + } +} diff --git a/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/MavenRepositoryProvisioner.java b/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/MavenRepositoryProvisioner.java index b662204..74d7bd3 100644 --- a/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/MavenRepositoryProvisioner.java +++ b/maven-repository-provisioner/src/main/java/com/simpligility/maven/provisioner/MavenRepositoryProvisioner.java @@ -136,9 +136,8 @@ private static ArtifactRetriever retrieveArtifacts() private static MavenRepositoryDeployer deployArtifacts() { logger.info( "Artifact deployment starting." ); - MavenRepositoryDeployer helper = new MavenRepositoryDeployer( cacheDirectory ); - helper.deployToRemote( config.getTargetUrl(), config.getUsername(), config.getPassword(), - config.getCheckTarget(), config.getVerifyOnly() ); + MavenRepositoryDeployer helper = MavenRepositoryDeployer.getInstance( cacheDirectory, config ); + helper.run(); logger.info( "Artifact deployment completed." ); return helper; }