Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enforce upload size limit on putResourceFromUrl api #8562

Open
wants to merge 17 commits into
base: main
Choose a base branch
from
+361 −39
Open

Enforce upload size limit on putResourceFromUrl api #8562

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
d4bcff3
Check content-length of resource at url
tylerjmchugh Dec 13, 2024
a0e2fdc
Remove unnecessary throw
tylerjmchugh Dec 13, 2024
b5b6dbf
Fallback to InputStream.available() if content length is not found or…
tylerjmchugh Dec 17, 2024
2b54b4d
Fallback maxUploadSize for unit tests
tylerjmchugh Dec 17, 2024
4d6a9e7
Fix default value always used
tylerjmchugh Dec 18, 2024
263d091
Remove content-length check as it cannot be trusted
tylerjmchugh Dec 18, 2024
00770fa
Download file to a temp file to check size
tylerjmchugh Dec 20, 2024
f4232fa
Add back the content-length check as a preliminary check
tylerjmchugh Dec 30, 2024
c5193fa
Stream file instead of using temp file
tylerjmchugh Dec 30, 2024
cd7418e
Rollback for JCloud
tylerjmchugh Dec 30, 2024
e73fb98
Fix rollback for JCloud
tylerjmchugh Dec 30, 2024
c0b921b
Fix abstract store and implement jcloud rollback
tylerjmchugh Jan 2, 2025
5878548
Fix typo and remove unused import
tylerjmchugh Jan 2, 2025
6e5d0d0
Fix unit tests
tylerjmchugh Jan 2, 2025
fba9139
Fix tests (mock response code)
tylerjmchugh Jan 2, 2025
afaef1b
Fix unit test, refactor, and add docs
tylerjmchugh Jan 2, 2025
bbc9526
Fix exception handling and use bounded input stream instead of custom…
tylerjmchugh Jan 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 19 additions & 3 deletions core/src/main/java/org/fao/geonet/api/records/attachments/AbstractStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import jeeves.server.context.ServiceContext;
import org.apache.commons.io.FilenameUtils;
import org.fao.geonet.ApplicationContextHolder;
import org.fao.geonet.api.exception.GeonetMaxUploadSizeExceededException;
import org.fao.geonet.api.exception.NotAllowedException;
import org.fao.geonet.api.exception.ResourceNotFoundException;
import org.fao.geonet.domain.AbstractMetadata;
Expand All @@ -35,13 +36,14 @@
import org.fao.geonet.kernel.AccessManager;
import org.fao.geonet.kernel.datamanager.IMetadataUtils;
import org.fao.geonet.repository.MetadataRepository;
import org.fao.geonet.util.FileUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.web.multipart.MultipartFile;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
Expand All @@ -59,6 +61,9 @@ public abstract class AbstractStore implements Store {
protected static final String RESOURCE_MANAGEMENT_EXTERNAL_PROPERTIES_ESCAPED_SEPARATOR = "\\:";
private static final Logger log = LoggerFactory.getLogger(AbstractStore.class);

@Value("${api.params.maxUploadSize}")
private int maxUploadSize;

@Override
public final List<MetadataResource> getResources(final ServiceContext context, final String metadataUuid, final Sort sort,
final String filter) throws Exception {
Expand Down Expand Up @@ -157,7 +162,7 @@ protected int canDownload(ServiceContext context, String metadataUuid, MetadataR
return metadataId;
}

protected String getFilenameFromHeader(final URL fileUrl) throws IOException {
protected String getFilenameFromHeader(final URL fileUrl) {
HttpURLConnection connection = null;
try {
connection = (HttpURLConnection) fileUrl.openConnection();
Expand Down Expand Up @@ -230,7 +235,18 @@ public final MetadataResource putResource(ServiceContext context, String metadat
if (filename == null) {
filename = getFilenameFromUrl(fileUrl);
}
return putResource(context, metadataUuid, filename, fileUrl.openStream(), null, visibility, approved);
try (InputStream is = fileUrl.openStream()) {
int availableBytes = is.available();
if (availableBytes > maxUploadSize) {
throw new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException")
.withMessageKey("exception.maxUploadSizeExceeded",
new String[]{FileUtil.humanizeFileSize(maxUploadSize)})
.withDescriptionKey("exception.maxUploadSizeExceeded.description",
new String[]{FileUtil.humanizeFileSize(availableBytes),
FileUtil.humanizeFileSize(maxUploadSize)});
}
return putResource(context, metadataUuid, filename, is, null, visibility, approved);
}
}

@Override
Expand Down
2 changes: 2 additions & 0 deletions core/src/test/resources/WEB-INF/config.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,7 @@ es.index.checker.interval=0/5 * * * * ?

thesaurus.cache.maxsize=400000

api.params.maxUploadSize=100000000

language.default=eng
language.forceDefault=false
Loading