diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/dataobjects/PkgVersionUrl.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/dataobjects/PkgVersionUrl.java index 02cb88a7..9b3ab5d0 100644 --- a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/dataobjects/PkgVersionUrl.java +++ b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/dataobjects/PkgVersionUrl.java @@ -8,10 +8,7 @@ import org.apache.cayenne.validation.BeanValidationFailure; import org.apache.cayenne.validation.ValidationResult; import org.haiku.haikudepotserver.dataobjects.auto._PkgVersionUrl; -import org.haiku.haikudepotserver.support.URLHelper; - -import java.net.MalformedURLException; -import java.net.URL; +import org.haiku.haikudepotserver.support.URLHelperService; public class PkgVersionUrl extends _PkgVersionUrl { @@ -20,7 +17,7 @@ protected void validateForSave(ValidationResult validationResult) { super.validateForSave(validationResult); if(null != getUrl()) { - if (!URLHelper.isValidInfo(getUrl())) { + if (!URLHelperService.isValidInfo(getUrl())) { validationResult.addFailure(new BeanValidationFailure(this, URL.getName(), "malformed")); } } diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/PkgImportServiceImpl.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/PkgImportServiceImpl.java index 548e1b8b..23a88882 100644 --- a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/PkgImportServiceImpl.java +++ b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/PkgImportServiceImpl.java @@ -15,7 +15,7 @@ import org.haiku.haikudepotserver.pkg.model.PkgImportService; import org.haiku.haikudepotserver.pkg.model.PkgLocalizationService; import org.haiku.haikudepotserver.support.ExposureType; -import org.haiku.haikudepotserver.support.URLHelper; +import org.haiku.haikudepotserver.support.URLHelperService; import org.haiku.haikudepotserver.support.VersionCoordinates; import org.haiku.haikudepotserver.support.VersionCoordinatesComparator; import org.slf4j.Logger; @@ -36,12 +36,15 @@ public class PkgImportServiceImpl implements PkgImportService { private final PkgServiceImpl pkgServiceImpl; private final PkgLocalizationService pkgLocalizationService; + private final URLHelperService urlHelperService; public PkgImportServiceImpl( PkgServiceImpl pkgServiceImpl, - PkgLocalizationService pkgLocalizationService) { + PkgLocalizationService pkgLocalizationService, + URLHelperService urlHelperService) { this.pkgServiceImpl = Preconditions.checkNotNull(pkgServiceImpl); this.pkgLocalizationService = Preconditions.checkNotNull(pkgLocalizationService); + this.urlHelperService = Preconditions.checkNotNull(urlHelperService); } @Override @@ -252,20 +255,16 @@ private void importCopyrights(ObjectContext objectContext, org.haiku.pkg.model.P } private void populatePayloadLength(PkgVersion persistedPkgVersion) { - long length = -1; Optional pkgVersionHpkgURLOptional = persistedPkgVersion.tryGetHpkgURL(ExposureType.INTERNAL_FACING); if (pkgVersionHpkgURLOptional.isPresent()) { - try { - length = URLHelper.payloadLength(pkgVersionHpkgURLOptional.get()); + urlHelperService.tryGetPayloadLength(pkgVersionHpkgURLOptional.get()) + .filter(l -> l > 0L) + .ifPresent(persistedPkgVersion::setPayloadLength); } catch (IOException ioe) { LOGGER.error("unable to get the payload length for; " + persistedPkgVersion, ioe); } - - if (length > 0) { - persistedPkgVersion.setPayloadLength(length); - } } else { LOGGER.info("no package length recorded because there is no " + "hpkg url for [" + persistedPkgVersion + "]"); diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/PkgVersionPayloadLengthPopulationJobRunner.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/PkgVersionPayloadLengthPopulationJobRunner.java index 3c0a16f5..541fdee5 100644 --- a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/PkgVersionPayloadLengthPopulationJobRunner.java +++ b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/PkgVersionPayloadLengthPopulationJobRunner.java @@ -15,7 +15,7 @@ import org.haiku.haikudepotserver.job.model.JobService; import org.haiku.haikudepotserver.pkg.model.PkgVersionPayloadLengthPopulationJobSpecification; import org.haiku.haikudepotserver.support.ExposureType; -import org.haiku.haikudepotserver.support.URLHelper; +import org.haiku.haikudepotserver.support.URLHelperService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -37,11 +37,14 @@ public class PkgVersionPayloadLengthPopulationJobRunner private static Logger LOGGER = LoggerFactory.getLogger(PkgVersionPayloadLengthPopulationJobRunner.class); - private ServerRuntime serverRuntime; + private final ServerRuntime serverRuntime; + private final URLHelperService urlHelperService; public PkgVersionPayloadLengthPopulationJobRunner( - ServerRuntime serverRuntime) { + ServerRuntime serverRuntime, + URLHelperService urlHelperService) { this.serverRuntime = Preconditions.checkNotNull(serverRuntime); + this.urlHelperService = Preconditions.checkNotNull(urlHelperService); } @Override @@ -72,15 +75,13 @@ public void run( Optional urlOptional = pkgVersion.tryGetHpkgURL(ExposureType.INTERNAL_FACING); if (urlOptional.isPresent()) { - long len; - try { - len = URLHelper.payloadLength(urlOptional.get()); - - if(len > 0) { - pkgVersion.setPayloadLength(len); - context.commitChanges(); - } + urlHelperService.tryGetPayloadLength(urlOptional.get()) + .filter(l -> l > 0L) + .ifPresent(l -> { + pkgVersion.setPayloadLength(l); + context.commitChanges(); + }); } catch(IOException ioe) { LOGGER.error("unable to get the payload length for " + pkgVersion.toString(), ioe); diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/support/URLHelper.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/support/URLHelper.java deleted file mode 100644 index 983a9d11..00000000 --- a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/support/URLHelper.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2018-2019, Andrew Lindesay - * Distributed under the terms of the MIT License. - */ - -package org.haiku.haikudepotserver.support; - -import com.google.common.base.Preconditions; -import com.google.common.net.HttpHeaders; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; -import java.util.Optional; - -public class URLHelper { - - protected static Logger LOGGER = LoggerFactory.getLogger(URLHelper.class); - - private final static int PAYLOAD_LENGTH_CONNECT_TIMEOUT = 10 * 1000; - private final static int PAYLOAD_LENGTH_READ_TIMEOUT = 10 * 1000; - - public static boolean isValidInfo(String urlString) { - - if (StringUtils.isNotBlank(urlString)) { - try { - new URI(urlString); - return true; - } catch (URISyntaxException ignore) { - } - } - - return false; - } - - public static long payloadLength(URL url) throws IOException { - Preconditions.checkArgument(null != url, "the url must be supplied"); - - long result = -1; - - switch(url.getProtocol()) { - - case "http": - case "https": - try { - result = payloadLengthHttp(url.toURI()); - } catch (URISyntaxException use) { - throw new IllegalStateException("unable to obtain uri from [" + url + "]"); - } - break; - - case "file": - File file = new File(url.getPath()); - - if (file.exists() && file.isFile()) { - result = file.length(); - } else { - LOGGER.warn("unable to find the local file; {}", url.getPath()); - } - break; - - } - - LOGGER.info("did obtain length [{}b] for url [{}]", result, url); - - return result; - } - - private static long payloadLengthHttp(URI uri) throws IOException { - try { - HttpResponse response = HttpClient.newBuilder() - .connectTimeout(Duration.ofMillis(PAYLOAD_LENGTH_CONNECT_TIMEOUT)) - .followRedirects(HttpClient.Redirect.NORMAL) - .build() - .send(HttpRequest.newBuilder(uri) - .GET() - .timeout(Duration.ofMillis(PAYLOAD_LENGTH_READ_TIMEOUT)) - .build(), HttpResponse.BodyHandlers.ofByteArray()); - - if (200 == response.statusCode()) { - Optional lengthOptional = response.headers().firstValue(HttpHeaders.CONTENT_LENGTH) - .map(Long::parseLong); - - if (lengthOptional.isEmpty()) { - LOGGER.warn("missing or bad content-length header at [{}]", uri); - } - - return lengthOptional.get(); - } else { - LOGGER.warn("bad response from [" + uri + "] when getting the length"); - } - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - LOGGER.warn("interrupted when downloading url [" + uri + "] to file"); - } - - return -1; - } - -} diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/support/URLHelperService.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/support/URLHelperService.java new file mode 100644 index 00000000..315a2a86 --- /dev/null +++ b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/support/URLHelperService.java @@ -0,0 +1,122 @@ +/* + * Copyright 2018-2019, Andrew Lindesay + * Distributed under the terms of the MIT License. + */ + +package org.haiku.haikudepotserver.support; + +import com.google.common.base.Preconditions; +import com.google.common.net.HttpHeaders; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.Optional; + +@Service +public class URLHelperService { + + protected static Logger LOGGER = LoggerFactory.getLogger(URLHelperService.class); + + private final static int PAYLOAD_LENGTH_CONNECT_TIMEOUT = 10 * 1000; + private final static int PAYLOAD_LENGTH_READ_TIMEOUT = 10 * 1000; + + private final HttpClient httpClient; + + public URLHelperService() { + this.httpClient = HttpClient.newBuilder() + .connectTimeout(Duration.ofMillis(PAYLOAD_LENGTH_CONNECT_TIMEOUT)) + .followRedirects(HttpClient.Redirect.NORMAL) + .build(); + } + + public static boolean isValidInfo(String urlString) { + + if (StringUtils.isNotBlank(urlString)) { + try { + new URI(urlString); + return true; + } catch (URISyntaxException ignore) { + } + } + + return false; + } + + public Optional tryGetPayloadLength(URL url) throws IOException { + Preconditions.checkArgument(null != url, "the url must be supplied"); + + Optional result = Optional.empty(); + String protocol = StringUtils.trimToEmpty(url.getProtocol()); + + switch(protocol) { + + case "http": + case "https": + try { + result = tryGetPayloadLengthHttp(url.toURI()); + } catch (URISyntaxException use) { + throw new IllegalStateException("unable to obtain a uri from [" + url + "]"); + } + break; + + case "file": + File file = new File(url.getPath()); + + if (file.exists() && file.isFile()) { + result = Optional.of(file.length()) + .filter(l -> l > 0L); + } else { + LOGGER.warn("unable to find the local file [{}]", url.getPath()); + } + break; + default: + LOGGER.warn("unable to get the payload length for URL scheme [{}]", protocol); + break; + } + + result.ifPresent(l -> LOGGER.info("did obtain length [{}b] for url [{}]", l, url)); + return result; + } + + private Optional tryGetPayloadLengthHttp(URI uri) throws IOException { +// return Optional.empty(); +// } + try { + HttpResponse response = httpClient.send(HttpRequest.newBuilder(uri) + .method("HEAD", HttpRequest.BodyPublishers.noBody()) + .timeout(Duration.ofMillis(PAYLOAD_LENGTH_READ_TIMEOUT)) + .build(), HttpResponse.BodyHandlers.discarding()); + + switch (response.statusCode()) { + case 200: + case 204: + Optional lengthOptional = response.headers().firstValue(HttpHeaders.CONTENT_LENGTH) + .map(Long::parseLong); + if (lengthOptional.isEmpty()) { + LOGGER.warn("missing or bad content-length header at [{}]", uri); + } + return lengthOptional; + default: + LOGGER.warn("bad response from [" + uri + "] when getting the length"); + break; + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + LOGGER.warn("interrupted when downloading url [" + uri + "] to file"); + } + + return Optional.empty(); + } + +}