Skip to content

Commit

Permalink
correct the bulk pkg and screenshot tar-ball jobs
Browse files Browse the repository at this point in the history
  • Loading branch information
@andponlin
andponlin committed Jun 9, 2020
1 parent 063d9ac commit e1d4770
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 78 deletions.
17 changes: 9 additions & 8 deletions ...in/java/org/haiku/haikudepotserver/pkg/job/AbstractPkgResourceExportArchiveJobRunner.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2018, Andrew Lindesay
* Copyright 2018-2020, Andrew Lindesay
* Distributed under the terms of the MIT License.
*/

Expand All @@ -9,9 +9,10 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
import com.google.common.net.MediaType;
import org.apache.cayenne.DataRow;
import org.apache.cayenne.ObjectContext;
import org.apache.cayenne.configuration.server.ServerRuntime;
import org.apache.cayenne.query.EJBQLQuery;
import org.apache.cayenne.query.SQLTemplate;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.haiku.haikudepotserver.job.AbstractJobRunner;
Expand Down Expand Up @@ -74,13 +75,13 @@ public void run(

State state = new State();
state.tarArchiveOutputStream = tarOutputStream;
EJBQLQuery query = createEjbqlQuery(specification);
SQLTemplate query = createQuery(specification);
query.setFetchLimit(getBatchSize());
int countLastQuery;

do {
query.setFetchOffset(offset);
List<Object[]> queryResults = context.performQuery(query);
List<DataRow> queryResults = context.performQuery(query);
countLastQuery = queryResults.size();
appendFromRawRows(state, queryResults);
offset += countLastQuery;
Expand All @@ -100,18 +101,18 @@ public void run(

abstract int getBatchSize();

abstract EJBQLQuery createEjbqlQuery(T specification);
abstract SQLTemplate createQuery(T specification);

abstract String getPathComponentTop();

private void appendFromRawRows(State state, List<Object[]> rows)
private void appendFromRawRows(State state, List<DataRow> rows)
throws IOException {
for(Object[] row : rows) {
for(DataRow row : rows) {
appendFromRawRow(state, row);
}
}

abstract void appendFromRawRow(State state, Object[] row) throws IOException;
abstract void appendFromRawRow(State state, DataRow row) throws IOException;

/**
* <p>Adds a little informational file into the tar-ball.</p>
Expand Down
56 changes: 25 additions & 31 deletions ...-core/src/main/java/org/haiku/haikudepotserver/pkg/job/PkgIconExportArchiveJobRunner.java
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
/*
* Copyright 2018-2019, Andrew Lindesay
* Copyright 2018-2020, Andrew Lindesay
* Distributed under the terms of the MIT License.
*/

package org.haiku.haikudepotserver.pkg.job;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import org.apache.cayenne.DataRow;
import org.apache.cayenne.configuration.server.ServerRuntime;
import org.apache.cayenne.query.EJBQLQuery;
import org.apache.cayenne.query.SQLTemplate;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.haiku.haikudepotserver.dataobjects.Pkg;
import org.apache.commons.lang.StringUtils;
import org.haiku.haikudepotserver.dataobjects.PkgIcon;
import org.haiku.haikudepotserver.dataobjects.PkgIconImage;
import org.haiku.haikudepotserver.pkg.model.PkgIconExportArchiveJobSpecification;
import org.haiku.haikudepotserver.support.RuntimeInformationService;
import org.springframework.stereotype.Component;
Expand All @@ -29,11 +30,11 @@ public class PkgIconExportArchiveJobRunner extends AbstractPkgResourceExportArch

private final static int BATCH_SIZE = 25;

private final static int ROW_PKG_NAME = 0;
private final static int ROW_SIZE = 1;
private final static int ROW_MEDIA_TYPE_CODE = 2;
private final static int ROW_PAYLOAD = 3;
private final static int ROW_PKG_MODIFY_TIMESTAMP = 4;
private final static String COLUMN_PKG_NAME = "pkg_name";
private final static String COLUMN_SIZE = "size";
private final static String COLUMN_MEDIA_TYPE_CODE = "media_type_code";
private final static String COLUMN_PAYLOAD = "payload";
private final static String COLUMN_PKG_MODIFY_TIMESTAMP = "modify_timestamp";

final static String PATH_COMPONENT_TOP = "hicn";

Expand All @@ -54,15 +55,15 @@ String getPathComponentTop() {
}

@Override
void appendFromRawRow(State state, Object[] row)
void appendFromRawRow(State state, DataRow row)
throws IOException {
append(
state,
String.class.cast(row[ROW_PKG_NAME]),
Number.class.cast(row[ROW_SIZE]),
String.class.cast(row[ROW_MEDIA_TYPE_CODE]),
byte[].class.cast(row[ROW_PAYLOAD]),
Date.class.cast(row[ROW_PKG_MODIFY_TIMESTAMP])
String.class.cast(row.get(COLUMN_PKG_NAME)),
Number.class.cast(row.get(COLUMN_SIZE)),
String.class.cast(row.get(COLUMN_MEDIA_TYPE_CODE)),
byte[].class.cast(row.get(COLUMN_PAYLOAD)),
Date.class.cast(row.get(COLUMN_PKG_MODIFY_TIMESTAMP))
);
}

Expand All @@ -74,6 +75,12 @@ private void append(
byte[] payload,
Date modifyTimestamp) throws IOException {

Preconditions.checkArgument(StringUtils.isNotBlank(pkgName));
Preconditions.checkArgument(null == size || size.intValue() > 0);
Preconditions.checkArgument(StringUtils.isNotBlank(mediaTypeCode));
Preconditions.checkArgument(null != payload && payload.length > 0);
Preconditions.checkArgument(null != modifyTimestamp);

String filename = String.join("/", PATH_COMPONENT_TOP,
pkgName, PkgIcon.deriveFilename(
mediaTypeCode, null==size ? null : size.intValue()));
Expand All @@ -91,22 +98,9 @@ private void append(
}

@Override
EJBQLQuery createEjbqlQuery(PkgIconExportArchiveJobSpecification specification) {
return new EJBQLQuery(createEjbqlRawRowsExpression());
}

private String createEjbqlRawRowsExpression() {
return String.join("\n",
"SELECT pii.pkgIcon.pkgSupplement.basePkgName, pii.pkgIcon.size,",
"pii.pkgIcon.mediaType.code, pii.data, pii.pkgIcon.pkgSupplement.iconModifyTimestamp",
"FROM " + PkgIconImage.class.getSimpleName() + " pii",
"WHERE EXISTS",
"(SELECT p2.name FROM " + Pkg.class.getSimpleName() + " p2 WHERE",
"p2.pkgSupplement = pii.pkgIcon.pkgSupplement AND p2.active = true)",
"ORDER BY pii.pkgIcon.pkgSupplement.basePkgName ASC,",
"pii.pkgIcon.mediaType.code ASC,",
"pii.pkgIcon.size ASC"
);
SQLTemplate createQuery(PkgIconExportArchiveJobSpecification specification) {
return (SQLTemplate) serverRuntime.newContext().getEntityResolver()
.getQueryDescriptor("AllPkgIcons").buildQuery();
}

}
63 changes: 24 additions & 39 deletions ...src/main/java/org/haiku/haikudepotserver/pkg/job/PkgScreenshotExportArchiveJobRunner.java
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,25 @@
/*
* Copyright 2018-2019, Andrew Lindesay
* Copyright 2018-2020, Andrew Lindesay
* Distributed under the terms of the MIT License.
*/

package org.haiku.haikudepotserver.pkg.job;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import org.apache.cayenne.DataRow;
import org.apache.cayenne.configuration.server.ServerRuntime;
import org.apache.cayenne.query.EJBQLQuery;
import org.apache.cayenne.query.SQLTemplate;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.haiku.haikudepotserver.dataobjects.Pkg;
import org.haiku.haikudepotserver.dataobjects.PkgScreenshotImage;
import org.apache.commons.lang3.StringUtils;
import org.haiku.haikudepotserver.pkg.model.PkgScreenshotExportArchiveJobSpecification;
import org.haiku.haikudepotserver.support.RuntimeInformationService;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.util.Date;
import java.util.Map;

/**
* <P>Produces a dump of the screenshots; either for one specific package or for all of the packages.</P>
Expand All @@ -29,10 +31,10 @@ public class PkgScreenshotExportArchiveJobRunner

private static final int BATCH_SIZE = 4; // small because don't want to use too much memory!

private static final int ROW_PKG_NAME = 0;
private static final int ROW_PAYLOAD = 1;
private static final int ROW_PKG_MODIFY_TIMESTAMP = 2;
private static final int ROW_ORDERING = 3;
private static final String COLUMN_PKG_NAME = "pkg_name";
private static final String COLUMN_PAYLOAD = "payload";
private static final String COLUMN_PKG_MODIFY_TIMESTAMP = "modify_timestamp";
private static final String COLUMN_ORDERING = "ordering";

public static final String PATH_COMPONENT_TOP = "hscr";

Expand All @@ -53,14 +55,14 @@ String getPathComponentTop() {
}

@Override
void appendFromRawRow(State state, Object[] row)
void appendFromRawRow(State state, DataRow row)
throws IOException {
append(
state,
String.class.cast(row[ROW_PKG_NAME]),
byte[].class.cast(row[ROW_PAYLOAD]),
Date.class.cast(row[ROW_PKG_MODIFY_TIMESTAMP]),
Integer.class.cast(row[ROW_ORDERING])
String.class.cast(row.get(COLUMN_PKG_NAME)),
byte[].class.cast(row.get(COLUMN_PAYLOAD)),
Date.class.cast(row.get(COLUMN_PKG_MODIFY_TIMESTAMP)),
Integer.class.cast(row.get(COLUMN_ORDERING))
);
}

Expand All @@ -71,6 +73,11 @@ private void append(
Date modifyTimestamp,
Integer ordering) throws IOException {

Preconditions.checkArgument(StringUtils.isNotBlank(pkgName));
Preconditions.checkArgument(null != payload && payload.length > 0);
Preconditions.checkArgument(null != modifyTimestamp);
Preconditions.checkArgument(null != ordering);

String filename = String.join("/", PATH_COMPONENT_TOP, pkgName, ordering.toString() + ".png");

TarArchiveEntry tarEntry = new TarArchiveEntry(filename);
Expand All @@ -86,35 +93,13 @@ private void append(
}

@Override
EJBQLQuery createEjbqlQuery(PkgScreenshotExportArchiveJobSpecification specification) {
EJBQLQuery query = new EJBQLQuery(createEjbqlRawRowsExpression(specification));

SQLTemplate createQuery(PkgScreenshotExportArchiveJobSpecification specification) {
SQLTemplate query = (SQLTemplate) serverRuntime.newContext().getEntityResolver()
.getQueryDescriptor("AllPkgScreenshots").buildQuery();
if (!Strings.isNullOrEmpty(specification.getPkgName())) {
query.setParameter("pkgName", specification.getPkgName());
query.setParams(Map.of("pkgName", specification.getPkgName()));
}

return query;
}

private String createEjbqlRawRowsExpression(PkgScreenshotExportArchiveJobSpecification specification) {
StringBuilder builder = new StringBuilder();

builder.append("SELECT psi.pkgScreenshot.pkgSupplement.basePkgName,\n");
builder.append("psi.data, psi.pkgScreenshot.pkgSupplement.modifyTimestamp,\n");
builder.append("psi.pkgScreenshot.ordering\n");
builder.append("FROM " + PkgScreenshotImage.class.getSimpleName() + " psi\n");
builder.append("WHERE EXISTS\n");
builder.append("(SELECT p2.name FROM " + Pkg.class.getSimpleName() + " p2 WHERE\n");
builder.append("p2.pkgSupplement = psi.pkgScreenshot.pkgSupplement AND p2.active = true)\n");

if (!Strings.isNullOrEmpty(specification.getPkgName())) {
builder.append("AND psi.pkgScreenshot.pkg.name = :pkgName\n");
}

builder.append("ORDER BY psi.pkgScreenshot.pkgSupplement.basePkgName ASC,\n");
builder.append("psi.pkgScreenshot.ordering ASC\n");

return builder.toString();
}

}
39 changes: 39 additions & 0 deletions haikudepotserver-core/src/main/resources/HaikuDepot.map.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -972,6 +972,45 @@
<obj-relationship name="userRatingStability" source="UserRating" target="UserRatingStability" deleteRule="Nullify" db-relationship-path="user_rating_stability"/>
<obj-relationship name="user" source="UserUsageConditionsAgreement" target="User" lock="true" deleteRule="Nullify" db-relationship-path="user"/>
<obj-relationship name="userUsageConditions" source="UserUsageConditionsAgreement" target="UserUsageConditions" deleteRule="Nullify" db-relationship-path="user_usage_conditions"/>
<query name="AllPkgIcons" type="SQLTemplate" root="data-map" root-name="HaikuDepot">
<property name="cayenne.GenericSelectQuery.fetchingDataRows" value="true"/>
<sql adapter-class="org.apache.cayenne.dba.postgres.PostgresAdapter"><![CDATA[SELECT
#result('p.name' 'String' 'pkg_name'),
#result('pi.size' 'Integer' 'size'),
#result('mt.code' 'String' 'media_type_code'),
#result('pii.data' 'byte[]' 'payload'),
#result('ps.icon_modify_timestamp' 'java.sql.Timestamp' 'modify_timestamp')
FROM
haikudepot.pkg_supplement ps
JOIN haikudepot.pkg p ON p.pkg_supplement_id = ps.id
JOIN haikudepot.pkg_icon pi ON pi.pkg_supplement_id = ps.id
JOIN haikudepot.pkg_icon_image pii ON pii.pkg_icon_id = pi.id
JOIN haikudepot.media_type mt ON mt.id = pi.media_type_id
ORDER BY
p.name ASC,
mt.code ASC,
pi.size ASC]]></sql>
</query>
<query name="AllPkgScreenshots" type="SQLTemplate" root="data-map" root-name="HaikuDepot">
<property name="cayenne.GenericSelectQuery.fetchingDataRows" value="true"/>
<sql adapter-class="org.apache.cayenne.dba.postgres.PostgresAdapter"><![CDATA[SELECT
#result('p.name' 'String' 'pkg_name'),
#result('psi.data' 'byte[]' 'payload'),
#result('psup.modify_timestamp' 'java.sql.Timestamp' 'modify_timestamp'),
#result('ps.ordering' 'Integer' 'ordering')
FROM
haikudepot.pkg_supplement psup
JOIN haikudepot.pkg p ON p.pkg_supplement_id = psup.id
JOIN haikudepot.pkg_screenshot ps ON ps.pkg_supplement_id = psup.id
JOIN haikudepot.pkg_screenshot_image psi ON psi.pkg_screenshot_id = ps.id
WHERE 1=1
#if($pkgName)
AND p.name = #bind($pkgName)
#end
ORDER BY
p.name ASC,
ps.ordering ASC]]></sql>
</query>
<query name="PkgNamesForRepositorySource" type="SQLTemplate" root="obj-entity" root-name="PkgVersion">
<property name="cayenne.GenericSelectQuery.fetchingDataRows" value="false"/>
<sql adapter-class="org.apache.cayenne.dba.postgres.PostgresAdapter"><![CDATA[SELECT
Expand Down

0 comments on commit e1d4770

Please sign in to comment.