diff --git a/haikudepotserver-core-test/src/test/java/org/haiku/haikudepotserver/pkg/PkgImportServiceImplIT.java b/haikudepotserver-core-test/src/test/java/org/haiku/haikudepotserver/pkg/PkgImportServiceImplIT.java index 97b28158a..dd5bed374 100644 --- a/haikudepotserver-core-test/src/test/java/org/haiku/haikudepotserver/pkg/PkgImportServiceImplIT.java +++ b/haikudepotserver-core-test/src/test/java/org/haiku/haikudepotserver/pkg/PkgImportServiceImplIT.java @@ -1,10 +1,11 @@ /* - * Copyright 2018-2020, Andrew Lindesay + * Copyright 2018-2021, Andrew Lindesay * Distributed under the terms of the MIT License. */ package org.haiku.haikudepotserver.pkg; +import com.google.common.collect.Iterables; import com.google.common.io.Files; import com.google.common.io.Resources; import org.apache.cayenne.ObjectContext; @@ -29,6 +30,7 @@ import javax.annotation.Resource; import java.io.File; import java.io.InputStream; +import java.net.URL; import java.util.Collections; import java.util.List; import java.util.Random; @@ -36,6 +38,8 @@ @ContextConfiguration(classes = TestConfig.class) public class PkgImportServiceImplIT extends AbstractIntegrationTest { + private static final String RESOURCE_TEST = "tipster-1.1.1-1-x86_64.hpkg"; + @Resource private PkgImportService pkgImportService; @@ -51,19 +55,6 @@ public class PkgImportServiceImplIT extends AbstractIntegrationTest { @Resource private IntegrationTestSupportService integrationTestSupportService; - private Pkg createPkg(String minor) { - return new Pkg( - "testpkg", - new PkgVersion("1", minor, "3", "4", 5), - PkgArchitecture.X86_64, - null, - Collections.emptyList(), - Collections.emptyList(), - "test-summary-en", - "test-description-en", - null); - } - /** *

When a "_devel" package is imported there is a special behaviour that the localization and the * icons are copied from the main package over to the "_devel" package.

@@ -171,7 +162,7 @@ public void testImport_develPkgHandling() throws Exception { */ @Test - public void testImport_payloadLength() throws Exception { + public void testImport_payloadData() throws Exception { File repositoryDirectory = null; int expectedPayloadLength; @@ -193,11 +184,10 @@ public void testImport_payloadLength() throws Exception { throw new IllegalStateException("unable to create the on-disk repository"); } - Random random = new Random(System.currentTimeMillis()); File fileF = new File(repositoryDirectory, "testpkg-1.3.3~4-5-x86_64.hpkg"); - byte[] buffer = new byte[1000 + (Math.abs(random.nextInt()) % 10*1000)]; - Files.write(buffer,fileF); - expectedPayloadLength = buffer.length; + byte[] payload = Resources.toByteArray(Resources.getResource(RESOURCE_TEST)); + Files.write(payload, fileF); + expectedPayloadLength = payload.length; } // now load the next package version in @@ -219,7 +209,8 @@ public void testImport_payloadLength() throws Exception { context.commitChanges(); } - // check the length on that package is there and is correct. + // check the length on that package is there and is correct and that the + // package icon is loaded in. { ObjectContext context = serverRuntime.newContext(); @@ -233,10 +224,16 @@ public void testImport_payloadLength() throws Exception { )).get(); Assertions.assertThat(pkgVersion.getPayloadLength()).isEqualTo(expectedPayloadLength); + + List pkgIcons = pkg.getPkgSupplement().getPkgIcons(); + Assertions.assertThat(pkgIcons).hasSize(1); + PkgIcon pkgIcon = Iterables.getOnlyElement(pkgIcons); + byte[] actualIconData = pkgIcon.getPkgIconImage().getData(); + Assertions.assertThat(actualIconData).hasSize(544); } } finally { - if(null!=repositoryDirectory) { + if (null != repositoryDirectory) { FileHelper.delete(repositoryDirectory); } } @@ -344,4 +341,17 @@ public void testImport_versionRegressionDeactivatesNewerVersions() { } + private Pkg createPkg(String minor) { + return new Pkg( + "testpkg", + new PkgVersion("1", minor, "3", "4", 5), + PkgArchitecture.X86_64, + null, + Collections.emptyList(), + Collections.emptyList(), + "test-summary-en", + "test-description-en", + null); + } + } diff --git a/haikudepotserver-core-test/src/test/java/org/haiku/haikudepotserver/support/HpkgHelperTest.java b/haikudepotserver-core-test/src/test/java/org/haiku/haikudepotserver/support/HpkgHelperTest.java new file mode 100644 index 000000000..a47bace80 --- /dev/null +++ b/haikudepotserver-core-test/src/test/java/org/haiku/haikudepotserver/support/HpkgHelperTest.java @@ -0,0 +1,73 @@ +/* + * Copyright 2021, Andrew Lindesay + * Distributed under the terms of the MIT License. + */ +package org.haiku.haikudepotserver.support; + +import com.google.common.collect.Iterables; +import com.google.common.io.ByteSource; +import com.google.common.io.Files; +import com.google.common.io.Resources; +import junit.framework.AssertionFailedError; +import org.fest.assertions.Assertions; +import org.haiku.pkg.AttributeContext; +import org.haiku.pkg.HpkgFileExtractor; +import org.haiku.pkg.model.Attribute; +import org.haiku.pkg.model.AttributeId; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +public class HpkgHelperTest { + + private static final String RESOURCE_TEST = "tipster-1.1.1-1-x86_64.hpkg"; + + private static final int[] HVIF_MAGIC = { + 0x6e, 0x63, 0x69, 0x66 + }; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testFindIconAttributesFromAppExecutableDirEntries() throws Exception { + // GIVEN + File file = prepareTestFile(RESOURCE_TEST); + HpkgFileExtractor fileExtractor = new HpkgFileExtractor(file); + AttributeContext tocContext = fileExtractor.getTocContext(); + + // WHEN + List attributes = HpkgHelper.findIconAttributesFromExecutableDirEntries( + tocContext, fileExtractor.getToc()); + + // THEN + Assertions.assertThat(attributes).hasSize(1); + Attribute iconA = Iterables.getOnlyElement(attributes); + Attribute iconDataA = iconA.getChildAttribute(AttributeId.DATA); + ByteSource byteSource = (ByteSource) iconDataA.getValue(tocContext); + byte[] data = byteSource.read(); + Assertions.assertThat(data).hasSize(544); + assertIsHvif(data); + } + + File prepareTestFile(String resource) throws IOException { + byte[] payload = Resources.toByteArray(Resources.getResource(resource)); + File temporaryFile = temporaryFolder.newFile(resource); + Files.write(payload, temporaryFile); + return temporaryFile; + } + + private void assertIsHvif(byte[] payload) { + Assertions.assertThat(payload.length).isGreaterThan(HVIF_MAGIC.length); + for (int i = 0; i < HVIF_MAGIC.length; i++) { + if ((0xff & payload[i]) != HVIF_MAGIC[i]) { + throw new AssertionFailedError("mismatch on the magic in the data payload"); + } + } + } + +} diff --git a/haikudepotserver-core-test/src/test/resources/README.TXT b/haikudepotserver-core-test/src/test/resources/README.TXT index 8b3659f73..ec57bc2e7 100644 --- a/haikudepotserver-core-test/src/test/resources/README.TXT +++ b/haikudepotserver-core-test/src/test/resources/README.TXT @@ -2,4 +2,12 @@ The file "sample-repo.hpkr" was obtained from; http://haiku-files.org/files/repo/9818164862edcbf69404a90267090b9d595908a11941951e904dfc6244c3d566/repo -2013-09-30 \ No newline at end of file +2013-09-30 + +--- + +The file "tipster-1.1.1-1-x86_64.hpkg" was obtained from; + +https://eu.hpkg.haiku-os.org/haikuports/master/x86_64/current/packages/tipster-1.1.1-1-x86_64.hpkg + +2021-02-08 diff --git a/haikudepotserver-core-test/src/test/resources/tipster-1.1.1-1-x86_64.hpkg b/haikudepotserver-core-test/src/test/resources/tipster-1.1.1-1-x86_64.hpkg new file mode 100644 index 000000000..1cddad7cf Binary files /dev/null and b/haikudepotserver-core-test/src/test/resources/tipster-1.1.1-1-x86_64.hpkg differ diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/graphics/ImageHelper.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/graphics/ImageHelper.java index 005502820..f3ce87871 100644 --- a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/graphics/ImageHelper.java +++ b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/graphics/ImageHelper.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2016, Andrew Lindesay + * Copyright 2013-2021, Andrew Lindesay * Distributed under the terms of the MIT License. */ @@ -17,15 +17,15 @@ public class ImageHelper { protected static Logger LOGGER = LoggerFactory.getLogger(ImageHelper.class); - private int HVIF_MAGIC[] = { + private static final int[] HVIF_MAGIC = { 0x6e, 0x63, 0x69, 0x66 }; - private int PNG_MAGIC[] = { + private static final int[] PNG_MAGIC = { 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A }; - private int PNG_IHDR[] = { + private static final int[] PNG_IHDR = { 0x49, 0x48, 0x44, 0x52 }; diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/PkgIconServiceImpl.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/PkgIconServiceImpl.java index 86efd2f51..f96dd0a65 100644 --- a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/PkgIconServiceImpl.java +++ b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/PkgIconServiceImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019, Andrew Lindesay + * Copyright 2018-2021, Andrew Lindesay * Distributed under the terms of the MIT License. */ @@ -34,10 +34,10 @@ @Service public class PkgIconServiceImpl implements PkgIconService { - protected static Logger LOGGER = LoggerFactory.getLogger(PkgIconServiceImpl.class); + protected static final Logger LOGGER = LoggerFactory.getLogger(PkgIconServiceImpl.class); @SuppressWarnings("FieldCanBeLocal") - private static int ICON_SIZE_LIMIT = 100 * 1024; // 100k + private static final int ICON_SIZE_LIMIT = 100 * 1024; // 100k private final RenderedPkgIconRepository renderedPkgIconRepository; private final PngOptimizationService pngOptimizationService; @@ -157,16 +157,20 @@ public PkgIcon storePkgIconImage( pkgIconOptional = Optional.of(pkgIcon); } - pkgIconImage.setData(imageData); - pkgSupplement.setModifyTimestamp(); - pkgSupplement.setIconModifyTimestamp(new java.sql.Timestamp(Clock.systemUTC().millis())); - renderedPkgIconRepository.evict(context, pkgSupplement); + if (pkgIconImage.getData() == null || !Arrays.equals(pkgIconImage.getData(), imageData)) { + pkgIconImage.setData(imageData); + pkgSupplement.setModifyTimestamp(); + pkgSupplement.setIconModifyTimestamp(new java.sql.Timestamp(Clock.systemUTC().millis())); + renderedPkgIconRepository.evict(context, pkgSupplement); - if (null != size) { - LOGGER.info("the icon {}px for package {} has been updated", size, pkgSupplement.getBasePkgName()); + if (null != size) { + LOGGER.info("the icon {}px for package [{}] has been updated", size, pkgSupplement.getBasePkgName()); + } else { + LOGGER.info("the icon for package [{}] has been updated", pkgSupplement.getBasePkgName()); + } } else { - LOGGER.info("the icon for package {} has been updated", pkgSupplement.getBasePkgName()); + LOGGER.info("no change to package icon for [{}] ", pkgSupplement.getBasePkgName()); } return pkgIconOptional.orElseThrow(IllegalStateException::new); @@ -184,7 +188,7 @@ private List getInUsePkgIconMediaTypes(final ObjectContext context) { return codes .stream() - .map(c -> MediaType.tryGetByCode(context, c).get()) + .map(c -> MediaType.getByCode(context, c)) .collect(Collectors.toList()); } diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/PkgImportServiceImpl.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/PkgImportServiceImpl.java index 48d28f976..e3546744d 100644 --- a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/PkgImportServiceImpl.java +++ b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/PkgImportServiceImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2020, Andrew Lindesay + * Copyright 2018-2021, Andrew Lindesay * Distributed under the terms of the MIT License. */ @@ -8,41 +8,55 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.io.ByteSource; import org.apache.cayenne.ObjectContext; import org.apache.cayenne.ObjectId; +import org.apache.cayenne.configuration.server.ServerRuntime; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.haiku.haikudepotserver.dataobjects.*; -import org.haiku.haikudepotserver.pkg.model.PkgImportService; -import org.haiku.haikudepotserver.pkg.model.PkgLocalizationService; -import org.haiku.haikudepotserver.support.ExposureType; -import org.haiku.haikudepotserver.support.URLHelperService; -import org.haiku.haikudepotserver.support.VersionCoordinates; -import org.haiku.haikudepotserver.support.VersionCoordinatesComparator; +import org.haiku.haikudepotserver.pkg.model.*; +import org.haiku.haikudepotserver.support.*; +import org.haiku.pkg.AttributeContext; +import org.haiku.pkg.HpkgFileExtractor; +import org.haiku.pkg.model.Attribute; +import org.haiku.pkg.model.AttributeId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; import java.net.URL; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; @Service public class PkgImportServiceImpl implements PkgImportService { protected static Logger LOGGER = LoggerFactory.getLogger(PkgImportServiceImpl.class); + private final ServerRuntime serverRuntime; private final PkgServiceImpl pkgServiceImpl; + private final PkgIconService pkgIconService; private final PkgLocalizationService pkgLocalizationService; private final URLHelperService urlHelperService; public PkgImportServiceImpl( + ServerRuntime serverRuntime, PkgServiceImpl pkgServiceImpl, + PkgIconService pkgIconService, PkgLocalizationService pkgLocalizationService, URLHelperService urlHelperService) { + this.serverRuntime = Preconditions.checkNotNull(serverRuntime); this.pkgServiceImpl = Preconditions.checkNotNull(pkgServiceImpl); + this.pkgIconService = Preconditions.checkNotNull(pkgIconService); this.pkgLocalizationService = Preconditions.checkNotNull(pkgLocalizationService); this.urlHelperService = Preconditions.checkNotNull(urlHelperService); } @@ -52,7 +66,7 @@ public void importFrom( ObjectContext objectContext, ObjectId repositorySourceObjectId, org.haiku.pkg.model.Pkg pkg, - boolean shouldPopulatePayloadLength) { + boolean populateFromPayload) { Preconditions.checkArgument(null != pkg, "the package must be provided"); Preconditions.checkArgument(null != repositorySourceObjectId, "the repository source is must be provided"); @@ -152,14 +166,14 @@ public void importFrom( // [apl] // If this fails, we will let it go and it can be tried again a bit later on. The system can try to back-fill // those at some later date if any of the latest versions for packages are missing. This is better than - // failing the import at this stage since this is "just" meta data. + // failing the import at this stage since this is "just" meta data. The length of the payload is being used as + // a signal that the payload was downloaded and processed at some point. - if(shouldPopulatePayloadLength && null==persistedPkgVersion.getPayloadLength()) { - populatePayloadLength(persistedPkgVersion); + if(populateFromPayload && shouldPopulateFromPayload(persistedPkgVersion)) { + populateFromPayload(objectContext, persistedPkgVersion); } LOGGER.debug("have processed package {}", pkg.toString()); - } private Pkg createPkg(ObjectContext objectContext, String name) { @@ -254,20 +268,146 @@ private void importCopyrights(ObjectContext objectContext, org.haiku.pkg.model.P } } - private void populatePayloadLength(PkgVersion persistedPkgVersion) { - Optional pkgVersionHpkgURLOptional = persistedPkgVersion.tryGetHpkgURL(ExposureType.INTERNAL_FACING); + private boolean shouldPopulateFromPayload(PkgVersion persistedPkgVersion) { + return null == persistedPkgVersion.getPayloadLength() + && Stream.of( + PkgService.SUFFIX_PKG_DEBUGINFO, + PkgService.SUFFIX_PKG_DEVELOPMENT, + PkgService.SUFFIX_PKG_SOURCE).noneMatch(suffix -> persistedPkgVersion.getPkg().getName().endsWith(suffix)); + } + + /** + *

This will read in the payload into a temporary file. From there it will parse it + * and take up any data from it such as the icon and the length of the download in + * bytes.

+ */ + + private void populateFromPayload(ObjectContext objectContext, PkgVersion persistedPkgVersion) { + persistedPkgVersion.tryGetHpkgURL(ExposureType.INTERNAL_FACING) + .ifPresentOrElse( + u -> populateFromPayload(objectContext, persistedPkgVersion, u), + () -> LOGGER.info( + "no package payload data recorded because there is no " + + "hpkg url for pkg [{}] version [{}]", + persistedPkgVersion.getPkg(), persistedPkgVersion)); + } + + private void populateFromPayload( + ObjectContext objectContext, + PkgVersion persistedPkgVersion, URL url) { + File temporaryFile = null; + + try { + String prefix = persistedPkgVersion.getPkg().getName() + "_" + RandomStringUtils.randomAlphabetic(3) + "_"; + // ^ need to ensure minimum length of the prefix + temporaryFile = File.createTempFile(prefix, ".hpkg"); - if (pkgVersionHpkgURLOptional.isPresent()) { try { - urlHelperService.tryGetPayloadLength(pkgVersionHpkgURLOptional.get()) - .filter(l -> l > 0L) - .ifPresent(persistedPkgVersion::setPayloadLength); - } catch (IOException ioe) { - LOGGER.error("unable to get the payload length for; " + persistedPkgVersion, ioe); + urlHelperService.transferPayloadToFile(url, temporaryFile); } - } else { - LOGGER.info("no package length recorded because there is no " - + "hpkg url for [" + persistedPkgVersion + "]"); + catch (IOException ioe) { + // if we can't download then don't stop the entire import process - just log and carry on. + LOGGER.warn("unable to download from the url [{}] --> [{}]; will ignore", url, temporaryFile); + return; + } + + // the length of the payload is interesting and trivial to capture from + // the data downloaded. + + if (null == persistedPkgVersion.getPayloadLength() + || persistedPkgVersion.getPayloadLength() != temporaryFile.length()) { + persistedPkgVersion.setPayloadLength(temporaryFile.length()); + LOGGER.info("recording new length for [{}] version [{}] of {}bytes", + persistedPkgVersion.getPkg(), persistedPkgVersion, temporaryFile.length()); + } + + // more complex is the capture of the data in the parsed payload data. + + HpkgFileExtractor hpkgFileExtractor; + + try { + hpkgFileExtractor = new HpkgFileExtractor(temporaryFile); + } + catch (Throwable th) { + // if it is not possible to parse the HPKG then log and carry on. + LOGGER.warn("unable to parse the payload from [{}]", url, th); + return; + } + + populateIconFromPayload(objectContext, persistedPkgVersion, hpkgFileExtractor); + } + catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } + finally { + if (null != temporaryFile && temporaryFile.exists()) { + if (temporaryFile.delete()) { + LOGGER.debug("did delete the temporary file"); + } + else { + LOGGER.error("unable to delete the temporary file [{}]", temporaryFile); + } + } + } + } + + private void populateIconFromPayload( + ObjectContext objectContext, PkgVersion persistedPkgVersion, + HpkgFileExtractor hpkgFileExtractor) { + AttributeContext context = hpkgFileExtractor.getTocContext(); + List iconAttrs = HpkgHelper.findIconAttributesFromExecutableDirEntries( + context, hpkgFileExtractor.getToc()); + switch (iconAttrs.size()) { + case 0: + LOGGER.info("package [{}] version [{}] has no icons", + persistedPkgVersion.getPkg(), persistedPkgVersion); + break; + case 1: + populateIconFromPayload( + objectContext, persistedPkgVersion, context, + Iterables.getFirst(iconAttrs, null)); + break; + default: + LOGGER.info("package [{}] version [{}] has {} icons --> ambiguous so will not load any", + persistedPkgVersion.getPkg(), persistedPkgVersion, iconAttrs.size()); + break; + } + } + + private void populateIconFromPayload( + ObjectContext objectContext, PkgVersion persistedPkgVersion, + AttributeContext context, Attribute attribute) { + Attribute dataAttr = attribute.tryGetChildAttribute(AttributeId.DATA).orElse(null); + + if (null == dataAttr) { + LOGGER.warn("the icon [{}] found for package [{}] version [{}] does not have a data attribute", + AttributeId.FILE_ATTRIBUTE, persistedPkgVersion.getPkg(), persistedPkgVersion); + } + + ByteSource byteSource = (ByteSource) dataAttr.getValue(context); + + try { + LOGGER.info("did find {} bytes of icon data for package [{}] version [{}]", + byteSource.size(), persistedPkgVersion.getPkg(), persistedPkgVersion); + } + catch (IOException ignore) { + LOGGER.warn("cannot get the size of the icon payload for package [{}] version [{}]", + persistedPkgVersion.getPkg(), persistedPkgVersion); + } + + try (InputStream inputStream = byteSource.openStream()) { + pkgIconService.storePkgIconImage( + inputStream, + MediaType.getByCode(objectContext, MediaType.MEDIATYPE_HAIKUVECTORICONFILE), + null, + objectContext, + persistedPkgVersion.getPkg().getPkgSupplement()); + } + catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } + catch (BadPkgIconException e) { + LOGGER.info("a failure has arisen loading a package icon data", e); } } diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/model/PkgImportService.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/model/PkgImportService.java index 5e3032b5e..bf1f8edaf 100644 --- a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/model/PkgImportService.java +++ b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/model/PkgImportService.java @@ -1,5 +1,5 @@ /* - * Copyright 2016, Andrew Lindesay + * Copyright 2021, Andrew Lindesay * Distributed under the terms of the MIT License. */ @@ -16,7 +16,7 @@ public interface PkgImportService { * either creating it or updating it as necessary.

* @param pkg imports into the local database from this package model. * @param repositorySourceObjectId the {@link ObjectId} of the source of the package data. - * @param populatePayloadLength is able to signal to the import process that the length of the package should be + * @param populateFromPayload is able to signal to the import process that the length of the package should be * populated. */ @@ -24,6 +24,6 @@ void importFrom( ObjectContext objectContext, ObjectId repositorySourceObjectId, org.haiku.pkg.model.Pkg pkg, - boolean populatePayloadLength); + boolean populateFromPayload); } diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/repository/job/RepositoryHpkrIngressJobRunner.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/repository/job/RepositoryHpkrIngressJobRunner.java index f7f8db8ac..3fbc884a1 100644 --- a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/repository/job/RepositoryHpkrIngressJobRunner.java +++ b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/repository/job/RepositoryHpkrIngressJobRunner.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019, Andrew Lindesay + * Copyright 2018-2021, Andrew Lindesay * Distributed under the terms of the MIT License. */ @@ -63,17 +63,17 @@ public class RepositoryHpkrIngressJobRunner extends AbstractJobRunnerThis will find all of the directory entries. In these it will find all of the + * directory entries that are executable and then it will find those that have an icon. It will return + * a list of those attributes that are the icons.

+ */ + + public static List findIconAttributesFromExecutableDirEntries( + AttributeContext context, + List toc) { + return streamDirEntries(toc) + .flatMap(appA -> streamExecutableDirEntriesRecursively(context, appA)) + .flatMap(execA -> streamIconAttributesFromAppExecutableDirEntry(context, execA)) + .collect(Collectors.toList()); + } + + private static Stream streamDirEntries(List toc) { + return toc.stream().filter(a -> a.getAttributeId() == AttributeId.DIRECTORY_ENTRY); + } + + private static Stream streamExecutableDirEntriesRecursively( + AttributeContext context, + Attribute attribute) { + if (attribute.getAttributeId() != AttributeId.DIRECTORY_ENTRY) { + return Stream.of(); + } + + if (isExecutableDirEntry(context, attribute)) { + return Stream.of(attribute); + } + + return attribute.getChildAttributes() + .stream() + .flatMap(a -> streamExecutableDirEntriesRecursively(context, a)); + } + + private static boolean isExecutableDirEntry( + AttributeContext context, + Attribute attr) { + if (attr.getAttributeId() != AttributeId.DIRECTORY_ENTRY) { + return false; + } + + return attr.tryGetChildAttribute(AttributeId.FILE_PERMISSIONS) + .map(a -> ((Number) a.getValue(context)).intValue()) + .filter(permissions -> 0 != (0100 & permissions)) + // ^^ posix permissions checking to see that the user has the executable flag set + .isPresent(); + } + + private static Stream streamIconAttributesFromAppExecutableDirEntry( + AttributeContext context, + Attribute executableDirEntryAttribute) { + String desiredAttributeValue = FileAttributesValues.BEOS_ICON.getAttributeValue(); + return executableDirEntryAttribute.getChildAttributes(AttributeId.FILE_ATTRIBUTE) + .stream() + .filter(a -> a.getValue(context).toString().equals(desiredAttributeValue)); + } + +} diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/support/URLHelperService.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/support/URLHelperService.java index 381fc1d0b..ee40fef93 100644 --- a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/support/URLHelperService.java +++ b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/support/URLHelperService.java @@ -1,11 +1,12 @@ /* - * Copyright 2018-2019, Andrew Lindesay + * Copyright 2018-2021, Andrew Lindesay * Distributed under the terms of the MIT License. */ package org.haiku.haikudepotserver.support; import com.google.common.base.Preconditions; +import com.google.common.io.*; import com.google.common.net.HttpHeaders; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -53,6 +54,27 @@ public static boolean isValidInfo(String urlString) { return false; } + public void transferPayloadToFile(URL url, File targetFile) throws IOException { + LOGGER.info("will transfer [{}] --> [{}]", url, targetFile); + String protocol = StringUtils.trimToEmpty(url.getProtocol()); + + switch(protocol) { + case "http": + case "https": + FileHelper.streamUrlDataToFile(url, targetFile, PAYLOAD_LENGTH_READ_TIMEOUT); + LOGGER.info("copied [{}] to [{}]", url, targetFile); + break; + case "file": + File sourceFile = new File(url.getPath()); + Files.copy(sourceFile, targetFile); + LOGGER.info("copied [{}] to [{}]", sourceFile, targetFile); + break; + default: + LOGGER.warn("unable to transfer for URL scheme [{}]", protocol); + break; + } + } + public Optional tryGetPayloadLength(URL url) throws IOException { Preconditions.checkArgument(null != url, "the url must be supplied"); diff --git a/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/HpkgFileExtractor.java b/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/HpkgFileExtractor.java index bfb186ce3..5bccb45bb 100644 --- a/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/HpkgFileExtractor.java +++ b/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/HpkgFileExtractor.java @@ -1,14 +1,22 @@ +/* + * Copyright 2021, Andrew Lindesay + * Distributed under the terms of the MIT License. + */ package org.haiku.pkg; import com.google.common.base.Preconditions; import org.haiku.pkg.heap.HeapCompression; import org.haiku.pkg.heap.HpkHeapReader; +import org.haiku.pkg.model.Attribute; import org.haiku.pkg.model.FileType; import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; /** *

This object represents an object that can extract an Hpkg (Haiku Pkg) file. If you are wanting to @@ -104,6 +112,15 @@ public AttributeIterator getTocIterator() { return new AttributeIterator(getTocContext(), tocAttributeOffset); } + public List getToc() { + List assembly = new ArrayList<>(); + AttributeIterator attributeIterator = getTocIterator(); + while(attributeIterator.hasNext()) { + assembly.add(attributeIterator.next()); + } + return Collections.unmodifiableList(assembly); + } + private HpkgHeader readHeader() throws IOException { Preconditions.checkNotNull(file); FileHelper fileHelper = new FileHelper(); diff --git a/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/heap/HpkHeapReader.java b/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/heap/HpkHeapReader.java index 7df7391e1..2f264da72 100644 --- a/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/heap/HpkHeapReader.java +++ b/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/heap/HpkHeapReader.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019, Andrew Lindesay + * Copyright 2018-2021, Andrew Lindesay * Distributed under the terms of the MIT License. */ @@ -37,13 +37,13 @@ public class HpkHeapReader implements Closeable, HeapReader { private final long uncompressedSize; // excluding the shorts for the chunks' compressed sizes - private LoadingCache heapChunkUncompressedCache; + private final LoadingCache heapChunkUncompressedCache; - private int[] heapChunkCompressedLengths; + private final int[] heapChunkCompressedLengths; - private RandomAccessFile randomAccessFile; + private final RandomAccessFile randomAccessFile; - private FileHelper fileHelper = new FileHelper(); + private final FileHelper fileHelper = new FileHelper(); public HpkHeapReader( final File file, @@ -57,7 +57,7 @@ public HpkHeapReader( Preconditions.checkNotNull(file); Preconditions.checkNotNull(compression); - Preconditions.checkState(heapOffset > 0 &&heapOffset < Integer.MAX_VALUE); + Preconditions.checkState(heapOffset > 0 && heapOffset < Integer.MAX_VALUE); Preconditions.checkState(chunkSize > 0 && chunkSize < Integer.MAX_VALUE); Preconditions.checkState(compressedSize >= 0 && compressedSize < Integer.MAX_VALUE); Preconditions.checkState(uncompressedSize >= 0 && uncompressedSize < Integer.MAX_VALUE); diff --git a/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/model/FileAttributesValues.java b/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/model/FileAttributesValues.java new file mode 100644 index 000000000..506fdc529 --- /dev/null +++ b/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/model/FileAttributesValues.java @@ -0,0 +1,26 @@ +/* + * Copyright 2021, Andrew Lindesay + * Distributed under the terms of the MIT License. + */ +package org.haiku.pkg.model; + +/** + *

The attribute type {@link AttributeId#FILE_ATTRIBUTE} has a value + * which is a string that defines what sort of attribute it is.

+ */ + +public enum FileAttributesValues { + + BEOS_ICON("BEOS:ICON"); + + private String attributeValue; + + FileAttributesValues(String attributeValue) { + this.attributeValue = attributeValue; + } + + public String getAttributeValue() { + return attributeValue; + } + +} diff --git a/haikudepotserver-packagefile/src/test/java/org/haiku/pkg/HpkgFileExtractorAttributeTest.java b/haikudepotserver-packagefile/src/test/java/org/haiku/pkg/HpkgFileExtractorAttributeTest.java index c5503953a..4409c42a5 100644 --- a/haikudepotserver-packagefile/src/test/java/org/haiku/pkg/HpkgFileExtractorAttributeTest.java +++ b/haikudepotserver-packagefile/src/test/java/org/haiku/pkg/HpkgFileExtractorAttributeTest.java @@ -2,18 +2,15 @@ * Copyright 2021, Andrew Lindesay * Distributed under the terms of the MIT License. */ - package org.haiku.pkg; import com.google.common.base.Preconditions; import com.google.common.hash.HashCode; import com.google.common.hash.Hashing; import com.google.common.io.ByteSource; +import junit.framework.AssertionFailedError; import org.fest.assertions.Assertions; -import org.haiku.pkg.model.Attribute; -import org.haiku.pkg.model.AttributeId; -import org.haiku.pkg.model.AttributeType; -import org.haiku.pkg.model.IntAttribute; +import org.haiku.pkg.model.*; import org.junit.Test; import java.io.File; @@ -26,6 +23,10 @@ public class HpkgFileExtractorAttributeTest extends AbstractHpkTest { private static final String RESOURCE_TEST = "tipster-1.1.1-1-x86_64.hpkg"; + private static final int[] HVIF_MAGIC = { + 0x6e, 0x63, 0x69, 0x66 + }; + @Test public void testReadFile() throws Exception { @@ -49,12 +50,24 @@ public void testReadFile() throws Exception { // Pull out the actual binary to check. The expected data results were obtained // from a Haiku host with the package installed. - Attribute binaryDirectoryEntry = findByDirectoryEntries(tocAttributes, tocContext, List.of("apps", "Tipster")); - Attribute binaryData = binaryDirectoryEntry.getChildAttribute(AttributeId.DATA); + Attribute tipsterDirectoryEntry = findByDirectoryEntries(tocAttributes, tocContext, List.of("apps", "Tipster")); + + Attribute binaryData = tipsterDirectoryEntry.getChildAttribute(AttributeId.DATA); ByteSource binaryDataByteSource = (ByteSource) binaryData.getValue(tocContext); Assertions.assertThat(binaryDataByteSource.size()).isEqualTo(153840L); HashCode hashCode = binaryDataByteSource.hash(Hashing.md5()); Assertions.assertThat(hashCode.toString().toLowerCase(Locale.ROOT)).isEqualTo("13b16cd7d035ddda09a744c49a8ebdf2"); + + Attribute iconAttribute = tipsterDirectoryEntry.getChildAttributes(AttributeId.FILE_ATTRIBUTE) + .stream() + .map(a -> (StringAttribute) a) + .filter(a -> a.getValue(tocContext).equals("BEOS:ICON")) + .findFirst() + .orElseThrow(() -> new AssertionFailedError("could not find icon attribute")); + Attribute iconBinaryData = iconAttribute.getChildAttribute(AttributeId.DATA); + ByteSource iconDataByteSource = (ByteSource) iconBinaryData.getValue(tocContext); + byte[] iconBytes = iconDataByteSource.read(); + assertIsHvif(iconBytes); } } @@ -89,5 +102,13 @@ private List toList(AttributeIterator attributeIterator) { return assembly; } + private void assertIsHvif(byte[] payload) { + Assertions.assertThat(payload.length).isGreaterThan(HVIF_MAGIC.length); + for (int i = 0; i < HVIF_MAGIC.length; i++) { + if ((0xff & payload[i]) != HVIF_MAGIC[i]) { + throw new AssertionFailedError("mismatch on the magic in the data payload"); + } + } + } } diff --git a/support/deployment/config.properties b/support/deployment/config.properties index 32d3e1f70..747a6f5a6 100644 --- a/support/deployment/config.properties +++ b/support/deployment/config.properties @@ -24,7 +24,7 @@ optipng.path=/usr/bin/optipng # When set (either "true" or "false"), the repository import process will # obtain the data for the package and will thereby figure out the size of # the package. -repository.import.populatepayloadlength=true +repository.import.populatefrompayload=true # Configures a minimum version for the HaikiDepot desktop application. # Versions of HD desktop application less than this minimum are @@ -126,4 +126,4 @@ authentication.jws.issuer=prod.hds #smtp.starttls=false email.from=noreply@haiku-os.org -# ------------------------------------------- \ No newline at end of file +# -------------------------------------------