From e1d477042e1e1df2e75fe7b0833a8dcc1186d0f0 Mon Sep 17 00:00:00 2001 From: Andrew Lindesay Date: Tue, 9 Jun 2020 22:45:44 +1200 Subject: [PATCH] correct the bulk pkg and screenshot tar-ball jobs --- ...ractPkgResourceExportArchiveJobRunner.java | 17 ++--- .../job/PkgIconExportArchiveJobRunner.java | 56 ++++++++--------- .../PkgScreenshotExportArchiveJobRunner.java | 63 +++++++------------ .../src/main/resources/HaikuDepot.map.xml | 39 ++++++++++++ 4 files changed, 97 insertions(+), 78 deletions(-) diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/AbstractPkgResourceExportArchiveJobRunner.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/AbstractPkgResourceExportArchiveJobRunner.java index 03d5aa433..2036b73c5 100644 --- a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/AbstractPkgResourceExportArchiveJobRunner.java +++ b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/AbstractPkgResourceExportArchiveJobRunner.java @@ -1,5 +1,5 @@ /* - * Copyright 2018, Andrew Lindesay + * Copyright 2018-2020, Andrew Lindesay * Distributed under the terms of the MIT License. */ @@ -9,9 +9,10 @@ import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.google.common.net.MediaType; +import org.apache.cayenne.DataRow; import org.apache.cayenne.ObjectContext; import org.apache.cayenne.configuration.server.ServerRuntime; -import org.apache.cayenne.query.EJBQLQuery; +import org.apache.cayenne.query.SQLTemplate; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.haiku.haikudepotserver.job.AbstractJobRunner; @@ -74,13 +75,13 @@ public void run( State state = new State(); state.tarArchiveOutputStream = tarOutputStream; - EJBQLQuery query = createEjbqlQuery(specification); + SQLTemplate query = createQuery(specification); query.setFetchLimit(getBatchSize()); int countLastQuery; do { query.setFetchOffset(offset); - List queryResults = context.performQuery(query); + List queryResults = context.performQuery(query); countLastQuery = queryResults.size(); appendFromRawRows(state, queryResults); offset += countLastQuery; @@ -100,18 +101,18 @@ public void run( abstract int getBatchSize(); - abstract EJBQLQuery createEjbqlQuery(T specification); + abstract SQLTemplate createQuery(T specification); abstract String getPathComponentTop(); - private void appendFromRawRows(State state, List rows) + private void appendFromRawRows(State state, List rows) throws IOException { - for(Object[] row : rows) { + for(DataRow row : rows) { appendFromRawRow(state, row); } } - abstract void appendFromRawRow(State state, Object[] row) throws IOException; + abstract void appendFromRawRow(State state, DataRow row) throws IOException; /** *

Adds a little informational file into the tar-ball.

diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/PkgIconExportArchiveJobRunner.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/PkgIconExportArchiveJobRunner.java index a3a36aa53..e88aed859 100644 --- a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/PkgIconExportArchiveJobRunner.java +++ b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/PkgIconExportArchiveJobRunner.java @@ -1,17 +1,18 @@ /* - * Copyright 2018-2019, Andrew Lindesay + * Copyright 2018-2020, Andrew Lindesay * Distributed under the terms of the MIT License. */ package org.haiku.haikudepotserver.pkg.job; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import org.apache.cayenne.DataRow; import org.apache.cayenne.configuration.server.ServerRuntime; -import org.apache.cayenne.query.EJBQLQuery; +import org.apache.cayenne.query.SQLTemplate; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.haiku.haikudepotserver.dataobjects.Pkg; +import org.apache.commons.lang.StringUtils; import org.haiku.haikudepotserver.dataobjects.PkgIcon; -import org.haiku.haikudepotserver.dataobjects.PkgIconImage; import org.haiku.haikudepotserver.pkg.model.PkgIconExportArchiveJobSpecification; import org.haiku.haikudepotserver.support.RuntimeInformationService; import org.springframework.stereotype.Component; @@ -29,11 +30,11 @@ public class PkgIconExportArchiveJobRunner extends AbstractPkgResourceExportArch private final static int BATCH_SIZE = 25; - private final static int ROW_PKG_NAME = 0; - private final static int ROW_SIZE = 1; - private final static int ROW_MEDIA_TYPE_CODE = 2; - private final static int ROW_PAYLOAD = 3; - private final static int ROW_PKG_MODIFY_TIMESTAMP = 4; + private final static String COLUMN_PKG_NAME = "pkg_name"; + private final static String COLUMN_SIZE = "size"; + private final static String COLUMN_MEDIA_TYPE_CODE = "media_type_code"; + private final static String COLUMN_PAYLOAD = "payload"; + private final static String COLUMN_PKG_MODIFY_TIMESTAMP = "modify_timestamp"; final static String PATH_COMPONENT_TOP = "hicn"; @@ -54,15 +55,15 @@ String getPathComponentTop() { } @Override - void appendFromRawRow(State state, Object[] row) + void appendFromRawRow(State state, DataRow row) throws IOException { append( state, - String.class.cast(row[ROW_PKG_NAME]), - Number.class.cast(row[ROW_SIZE]), - String.class.cast(row[ROW_MEDIA_TYPE_CODE]), - byte[].class.cast(row[ROW_PAYLOAD]), - Date.class.cast(row[ROW_PKG_MODIFY_TIMESTAMP]) + String.class.cast(row.get(COLUMN_PKG_NAME)), + Number.class.cast(row.get(COLUMN_SIZE)), + String.class.cast(row.get(COLUMN_MEDIA_TYPE_CODE)), + byte[].class.cast(row.get(COLUMN_PAYLOAD)), + Date.class.cast(row.get(COLUMN_PKG_MODIFY_TIMESTAMP)) ); } @@ -74,6 +75,12 @@ private void append( byte[] payload, Date modifyTimestamp) throws IOException { + Preconditions.checkArgument(StringUtils.isNotBlank(pkgName)); + Preconditions.checkArgument(null == size || size.intValue() > 0); + Preconditions.checkArgument(StringUtils.isNotBlank(mediaTypeCode)); + Preconditions.checkArgument(null != payload && payload.length > 0); + Preconditions.checkArgument(null != modifyTimestamp); + String filename = String.join("/", PATH_COMPONENT_TOP, pkgName, PkgIcon.deriveFilename( mediaTypeCode, null==size ? null : size.intValue())); @@ -91,22 +98,9 @@ private void append( } @Override - EJBQLQuery createEjbqlQuery(PkgIconExportArchiveJobSpecification specification) { - return new EJBQLQuery(createEjbqlRawRowsExpression()); - } - - private String createEjbqlRawRowsExpression() { - return String.join("\n", - "SELECT pii.pkgIcon.pkgSupplement.basePkgName, pii.pkgIcon.size,", - "pii.pkgIcon.mediaType.code, pii.data, pii.pkgIcon.pkgSupplement.iconModifyTimestamp", - "FROM " + PkgIconImage.class.getSimpleName() + " pii", - "WHERE EXISTS", - "(SELECT p2.name FROM " + Pkg.class.getSimpleName() + " p2 WHERE", - "p2.pkgSupplement = pii.pkgIcon.pkgSupplement AND p2.active = true)", - "ORDER BY pii.pkgIcon.pkgSupplement.basePkgName ASC,", - "pii.pkgIcon.mediaType.code ASC,", - "pii.pkgIcon.size ASC" - ); + SQLTemplate createQuery(PkgIconExportArchiveJobSpecification specification) { + return (SQLTemplate) serverRuntime.newContext().getEntityResolver() + .getQueryDescriptor("AllPkgIcons").buildQuery(); } } diff --git a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/PkgScreenshotExportArchiveJobRunner.java b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/PkgScreenshotExportArchiveJobRunner.java index d9e942d96..48b358a11 100644 --- a/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/PkgScreenshotExportArchiveJobRunner.java +++ b/haikudepotserver-core/src/main/java/org/haiku/haikudepotserver/pkg/job/PkgScreenshotExportArchiveJobRunner.java @@ -1,23 +1,25 @@ /* - * Copyright 2018-2019, Andrew Lindesay + * Copyright 2018-2020, Andrew Lindesay * Distributed under the terms of the MIT License. */ package org.haiku.haikudepotserver.pkg.job; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import org.apache.cayenne.DataRow; import org.apache.cayenne.configuration.server.ServerRuntime; -import org.apache.cayenne.query.EJBQLQuery; +import org.apache.cayenne.query.SQLTemplate; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.haiku.haikudepotserver.dataobjects.Pkg; -import org.haiku.haikudepotserver.dataobjects.PkgScreenshotImage; +import org.apache.commons.lang3.StringUtils; import org.haiku.haikudepotserver.pkg.model.PkgScreenshotExportArchiveJobSpecification; import org.haiku.haikudepotserver.support.RuntimeInformationService; import org.springframework.stereotype.Component; import java.io.IOException; import java.util.Date; +import java.util.Map; /** *

Produces a dump of the screenshots; either for one specific package or for all of the packages.

@@ -29,10 +31,10 @@ public class PkgScreenshotExportArchiveJobRunner private static final int BATCH_SIZE = 4; // small because don't want to use too much memory! - private static final int ROW_PKG_NAME = 0; - private static final int ROW_PAYLOAD = 1; - private static final int ROW_PKG_MODIFY_TIMESTAMP = 2; - private static final int ROW_ORDERING = 3; + private static final String COLUMN_PKG_NAME = "pkg_name"; + private static final String COLUMN_PAYLOAD = "payload"; + private static final String COLUMN_PKG_MODIFY_TIMESTAMP = "modify_timestamp"; + private static final String COLUMN_ORDERING = "ordering"; public static final String PATH_COMPONENT_TOP = "hscr"; @@ -53,14 +55,14 @@ String getPathComponentTop() { } @Override - void appendFromRawRow(State state, Object[] row) + void appendFromRawRow(State state, DataRow row) throws IOException { append( state, - String.class.cast(row[ROW_PKG_NAME]), - byte[].class.cast(row[ROW_PAYLOAD]), - Date.class.cast(row[ROW_PKG_MODIFY_TIMESTAMP]), - Integer.class.cast(row[ROW_ORDERING]) + String.class.cast(row.get(COLUMN_PKG_NAME)), + byte[].class.cast(row.get(COLUMN_PAYLOAD)), + Date.class.cast(row.get(COLUMN_PKG_MODIFY_TIMESTAMP)), + Integer.class.cast(row.get(COLUMN_ORDERING)) ); } @@ -71,6 +73,11 @@ private void append( Date modifyTimestamp, Integer ordering) throws IOException { + Preconditions.checkArgument(StringUtils.isNotBlank(pkgName)); + Preconditions.checkArgument(null != payload && payload.length > 0); + Preconditions.checkArgument(null != modifyTimestamp); + Preconditions.checkArgument(null != ordering); + String filename = String.join("/", PATH_COMPONENT_TOP, pkgName, ordering.toString() + ".png"); TarArchiveEntry tarEntry = new TarArchiveEntry(filename); @@ -86,35 +93,13 @@ private void append( } @Override - EJBQLQuery createEjbqlQuery(PkgScreenshotExportArchiveJobSpecification specification) { - EJBQLQuery query = new EJBQLQuery(createEjbqlRawRowsExpression(specification)); - + SQLTemplate createQuery(PkgScreenshotExportArchiveJobSpecification specification) { + SQLTemplate query = (SQLTemplate) serverRuntime.newContext().getEntityResolver() + .getQueryDescriptor("AllPkgScreenshots").buildQuery(); if (!Strings.isNullOrEmpty(specification.getPkgName())) { - query.setParameter("pkgName", specification.getPkgName()); + query.setParams(Map.of("pkgName", specification.getPkgName())); } - return query; } - private String createEjbqlRawRowsExpression(PkgScreenshotExportArchiveJobSpecification specification) { - StringBuilder builder = new StringBuilder(); - - builder.append("SELECT psi.pkgScreenshot.pkgSupplement.basePkgName,\n"); - builder.append("psi.data, psi.pkgScreenshot.pkgSupplement.modifyTimestamp,\n"); - builder.append("psi.pkgScreenshot.ordering\n"); - builder.append("FROM " + PkgScreenshotImage.class.getSimpleName() + " psi\n"); - builder.append("WHERE EXISTS\n"); - builder.append("(SELECT p2.name FROM " + Pkg.class.getSimpleName() + " p2 WHERE\n"); - builder.append("p2.pkgSupplement = psi.pkgScreenshot.pkgSupplement AND p2.active = true)\n"); - - if (!Strings.isNullOrEmpty(specification.getPkgName())) { - builder.append("AND psi.pkgScreenshot.pkg.name = :pkgName\n"); - } - - builder.append("ORDER BY psi.pkgScreenshot.pkgSupplement.basePkgName ASC,\n"); - builder.append("psi.pkgScreenshot.ordering ASC\n"); - - return builder.toString(); - } - } diff --git a/haikudepotserver-core/src/main/resources/HaikuDepot.map.xml b/haikudepotserver-core/src/main/resources/HaikuDepot.map.xml index ac1d276ce..c663f7429 100644 --- a/haikudepotserver-core/src/main/resources/HaikuDepot.map.xml +++ b/haikudepotserver-core/src/main/resources/HaikuDepot.map.xml @@ -972,6 +972,45 @@ + + + + + + + +