Skip to content

Commit

Permalink
feat: ability to configure and use image thumbnails
Browse files Browse the repository at this point in the history
  • Loading branch information
guqing committed Aug 12, 2024
1 parent 97f653b commit 1250cc5
Show file tree
Hide file tree
Showing 8 changed files with 228 additions and 51 deletions.
3 changes: 2 additions & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,15 @@ java {
}

repositories {
mavenLocal()
mavenCentral()
maven { url 'https://s01.oss.sonatype.org/content/repositories/releases' }
maven { url 'https://s01.oss.sonatype.org/content/repositories/snapshots/' }
maven { url 'https://repo.spring.io/milestone' }
}

dependencies {
implementation platform('run.halo.tools.platform:plugin:2.17.0-SNAPSHOT')
implementation platform('run.halo.tools.platform:plugin:2.19.0-SNAPSHOT')
compileOnly 'run.halo.app:api'

implementation platform('software.amazon.awssdk:bom:2.19.8')
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/run/halo/s3os/S3LinkServiceImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public Mono<S3ListResult> listObjects(String policyName, String continuationToke
return client.fetch(ConfigMap.class, configMapName);
})
.flatMap((configMap) -> {
var properties = handler.getProperties(configMap);
var properties = S3OsProperties.convertFrom(configMap);
var finalLocation = FilePathUtils.getFilePathByPlaceholder(properties.getLocation());
return Mono.using(() -> handler.buildS3Client(properties),
// 执行 listObjects
Expand Down Expand Up @@ -231,7 +231,7 @@ public Mono<LinkResult.LinkResultItem> addAttachmentRecord(String policyName,
return client.fetch(ConfigMap.class, configMapName);
})
.flatMap(configMap -> {
var properties = handler.getProperties(configMap);
var properties = S3OsProperties.convertFrom(configMap);
return Mono.using(() -> handler.buildS3Client(properties),
(s3Client) -> Mono.fromCallable(
() -> s3Client.headObject(
Expand Down
49 changes: 8 additions & 41 deletions src/main/java/run/halo/s3os/S3OsAttachmentHandler.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileAlreadyExistsException;
import java.time.Duration;
import java.util.HashMap;
Expand All @@ -23,7 +22,6 @@
import org.springframework.lang.Nullable;
import org.springframework.web.server.ServerErrorException;
import org.springframework.web.server.ServerWebInputException;
import org.springframework.web.util.UriUtils;
import reactor.core.Exceptions;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
Expand All @@ -38,7 +36,6 @@
import run.halo.app.extension.ConfigMap;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.MetadataUtil;
import run.halo.app.infra.utils.JsonUtils;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.awscore.presigner.SdkPresigner;
import software.amazon.awssdk.core.SdkResponse;
Expand All @@ -47,16 +44,7 @@
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
import software.amazon.awssdk.services.s3.model.CompletedPart;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.services.s3.model.*;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
import software.amazon.awssdk.utils.SdkAutoCloseable;
Expand All @@ -79,7 +67,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
public Mono<Attachment> upload(UploadContext uploadContext) {
return Mono.just(uploadContext).filter(context -> this.shouldHandle(context.policy()))
.flatMap(context -> {
final var properties = getProperties(context.configMap());
final var properties = S3OsProperties.convertFrom(context.configMap());
return upload(context, properties)
.subscribeOn(Schedulers.boundedElastic())
.map(objectDetail -> this.buildAttachment(properties, objectDetail))
Expand All @@ -102,7 +90,7 @@ public Mono<Attachment> delete(DeleteContext deleteContext) {
log.info("Skip deleting object {} from S3.", objectKey);
return Mono.just(context);
}
var properties = getProperties(deleteContext.configMap());
var properties = S3OsProperties.convertFrom(deleteContext.configMap());
return Mono.using(() -> buildS3Client(properties),
client -> Mono.fromCallable(
() -> client.deleteObject(DeleteObjectRequest.builder()
Expand All @@ -123,7 +111,7 @@ public Mono<Attachment> delete(DeleteContext deleteContext) {

@Override
public Mono<URI> getSharedURL(Attachment attachment, Policy policy, ConfigMap configMap,
Duration ttl) {
Duration ttl) {
if (!this.shouldHandle(policy)) {
return Mono.empty();
}
Expand All @@ -132,7 +120,7 @@ public Mono<URI> getSharedURL(Attachment attachment, Policy policy, ConfigMap co
return Mono.error(new IllegalArgumentException(
"Cannot obtain object key from attachment " + attachment.getMetadata().getName()));
}
var properties = getProperties(configMap);
var properties = S3OsProperties.convertFrom(configMap);

return Mono.using(() -> buildS3Presigner(properties),
s3Presigner -> {
Expand Down Expand Up @@ -168,8 +156,8 @@ public Mono<URI> getPermalink(Attachment attachment, Policy policy, ConfigMap co
// fallback to default handler for backward compatibility
return Mono.empty();
}
var properties = getProperties(configMap);
var objectURL = getObjectURL(properties, objectKey);
var properties = S3OsProperties.convertFrom(configMap);
var objectURL = properties.toObjectURL(objectKey);
var urlSuffix = getUrlSuffixAnnotation(attachment);
if (StringUtils.isNotBlank(urlSuffix)) {
objectURL += urlSuffix;
Expand All @@ -195,13 +183,8 @@ private String getUrlSuffixAnnotation(Attachment attachment) {
return annotations.get(URL_SUFFIX_ANNO_KEY);
}

S3OsProperties getProperties(ConfigMap configMap) {
var settingJson = configMap.getData().getOrDefault("default", "{}");
return JsonUtils.jsonToObject(settingJson, S3OsProperties.class);
}

Attachment buildAttachment(S3OsProperties properties, ObjectDetail objectDetail) {
String externalLink = getObjectURL(properties, objectDetail.uploadState.objectKey);
String externalLink = properties.toObjectURL(objectDetail.uploadState.objectKey);
var urlSuffix = UrlUtils.findUrlSuffix(properties.getUrlSuffixes(),
objectDetail.uploadState.fileName);

Expand Down Expand Up @@ -229,22 +212,6 @@ Attachment buildAttachment(S3OsProperties properties, ObjectDetail objectDetail)
return attachment;
}

String getObjectURL(S3OsProperties properties, String objectKey) {
String objectURL;
if (StringUtils.isBlank(properties.getDomain())) {
String host;
if (properties.getEnablePathStyleAccess()) {
host = properties.getEndpoint() + "/" + properties.getBucket();
} else {
host = properties.getBucket() + "." + properties.getEndpoint();
}
objectURL = properties.getProtocol() + "://" + host + "/" + objectKey;
} else {
objectURL = properties.getProtocol() + "://" + properties.getDomain() + "/" + objectKey;
}
return UriUtils.encodePath(objectURL, StandardCharsets.UTF_8);
}

S3Client buildS3Client(S3OsProperties properties) {
return S3Client.builder()
.region(Region.of(properties.getRegion()))
Expand Down
1 change: 1 addition & 0 deletions src/main/java/run/halo/s3os/S3OsPlugin.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
*/
@Component
public class S3OsPlugin extends BasePlugin {
public static final String POLICY_SETTING_NAME = "s3os-policy-template-setting";

public S3OsPlugin(PluginContext pluginContext) {
super(pluginContext);
Expand Down
42 changes: 36 additions & 6 deletions src/main/java/run/halo/s3os/S3OsProperties.java
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
package run.halo.s3os;

import java.nio.charset.StandardCharsets;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor;
import org.springframework.util.StringUtils;

import java.time.LocalDate;
import org.springframework.web.util.UriUtils;
import run.halo.app.extension.ConfigMap;
import run.halo.app.infra.utils.JsonUtils;

@Data
class S3OsProperties {
public class S3OsProperties {

private String bucket;

Expand Down Expand Up @@ -49,6 +50,14 @@ class S3OsProperties {

private List<urlSuffixItem> urlSuffixes;

private ThumbnailParam thumbnailParam;

public record ThumbnailParam(String type, String pattern) {
public boolean hasPattern() {
return StringUtils.hasText(type) && StringUtils.hasText(pattern);
}
}

@Data
@AllArgsConstructor
@NoArgsConstructor
Expand Down Expand Up @@ -103,19 +112,40 @@ public void setRandomStringLength(String randomStringLength) { // if you use In
if (length >= 4 && length <= 16) {
this.randomStringLength = length;
}
} catch (NumberFormatException ignored) {
}
catch (NumberFormatException ignored) { }
}

public void setRegion(String region) {
if (!StringUtils.hasText(region)) {
this.region = "Auto";
}else {
} else {
this.region = region;
}
}

public void setEndpoint(String endpoint) {
this.endpoint = UrlUtils.removeHttpPrefix(endpoint);
}

public String toObjectURL(String objectKey) {
String objectURL;
if (!StringUtils.hasText(this.getDomain())) {
String host;
if (this.getEnablePathStyleAccess()) {
host = this.getEndpoint() + "/" + this.getBucket();
} else {
host = this.getBucket() + "." + this.getEndpoint();
}
objectURL = this.getProtocol() + "://" + host + "/" + objectKey;
} else {
objectURL = this.getProtocol() + "://" + this.getDomain() + "/" + objectKey;
}
return UriUtils.encodePath(objectURL, StandardCharsets.UTF_8);
}

public static S3OsProperties convertFrom(ConfigMap configMap) {
var settingJson = configMap.getData().getOrDefault("default", "{}");
return JsonUtils.jsonToObject(settingJson, S3OsProperties.class);
}
}
128 changes: 128 additions & 0 deletions src/main/java/run/halo/s3os/S3ThumbnailProvider.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
package run.halo.s3os;

import static run.halo.s3os.S3OsPlugin.POLICY_SETTING_NAME;

import java.net.URI;
import java.net.URL;
import java.util.List;
import java.util.Map;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import lombok.Builder;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.domain.Sort;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import org.springframework.web.util.UriComponentsBuilder;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import run.halo.app.core.attachment.ThumbnailProvider;
import run.halo.app.core.attachment.ThumbnailSize;
import run.halo.app.core.extension.attachment.Policy;
import run.halo.app.core.extension.attachment.PolicyTemplate;
import run.halo.app.extension.ConfigMap;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.ReactiveExtensionClient;

@Component
@RequiredArgsConstructor
public class S3ThumbnailProvider implements ThumbnailProvider {
static final String WIDTH_PLACEHOLDER = "{width}";
private final Cache<String, S3PropsCacheValue> s3PropsCache = CacheBuilder.newBuilder()
.maximumSize(50)
.build();

private final ReactiveExtensionClient client;

@Override
public Mono<URI> generate(ThumbnailContext thumbnailContext) {
var url = thumbnailContext.getImageUrl().toString();
var size = thumbnailContext.getSize();
return getCacheValue(url)
.mapNotNull(cacheValue -> placedPattern(cacheValue.pattern(), size))
.map(param -> {
if (param.startsWith("?")) {
UriComponentsBuilder.fromHttpUrl(url)
.queryParam(param.substring(1));
}
return url + param;
})
.map(URI::create);
}

private static String placedPattern(String pattern, ThumbnailSize size) {
return StringUtils.replace(pattern, WIDTH_PLACEHOLDER, String.valueOf(size.getWidth()));
}

@Override
public Mono<Void> delete(URL url) {
// do nothing for s3
return Mono.empty();
}

@Override
public Mono<Boolean> supports(ThumbnailContext thumbnailContext) {
var url = thumbnailContext.getImageUrl().toString();
return getCacheValue(url).hasElement();
}

private Mono<S3PropsCacheValue> getCacheValue(String imageUrl) {
return Flux.fromIterable(s3PropsCache.asMap().entrySet())
.filter(entry -> imageUrl.startsWith(entry.getKey()))
.next()
.map(Map.Entry::getValue)
.switchIfEmpty(Mono.defer(() -> listAllS3ObjectDomain()
.filter(entry -> imageUrl.startsWith(entry.getKey()))
.map(Map.Entry::getValue)
.next()
));
}

@Builder
record S3PropsCacheValue(String pattern, String configMapName) {
}

private Flux<Map.Entry<String, S3PropsCacheValue>> listAllS3ObjectDomain() {
return listS3PolicyTemplateNames()
.collectList()
.flatMapMany(this::listAllS3Policy)
.flatMap(s3Policy -> {
var s3ConfigMapName = s3Policy.getSpec().getConfigMapName();
return fetchS3PropsByConfigMapName(s3ConfigMapName)
.mapNotNull(properties -> {
var thumbnailParam = properties.getThumbnailParam();
if (thumbnailParam == null || !thumbnailParam.hasPattern()) {
return null;
}
var objectDomain = properties.toObjectURL("");
var cacheValue = S3PropsCacheValue.builder()
.pattern(thumbnailParam.pattern())
.configMapName(s3ConfigMapName)
.build();
return Map.entry(objectDomain, cacheValue);
});
})
.doOnNext(cache -> s3PropsCache.put(cache.getKey(), cache.getValue()));
}

private Flux<Policy> listAllS3Policy(List<String> s3PolicyTemplateNames) {
Assert.notNull(s3PolicyTemplateNames, "The s3PolicyTemplateNames must not be null.");
return client.listAll(Policy.class, new ListOptions(), Sort.unsorted())
.filter(policy -> {
var templateName = policy.getSpec().getTemplateName();
return s3PolicyTemplateNames.contains(templateName);
});
}

private Mono<S3OsProperties> fetchS3PropsByConfigMapName(String name) {
return client.fetch(ConfigMap.class, name)
.map(S3OsProperties::convertFrom);
}

private Flux<String> listS3PolicyTemplateNames() {
return client.listAll(PolicyTemplate.class, new ListOptions(), Sort.unsorted())
.filter(policyTemplate -> POLICY_SETTING_NAME.equals(policyTemplate.getSpec().getSettingName()))
.map(template -> template.getMetadata().getName());
}
}
9 changes: 9 additions & 0 deletions src/main/resources/extensions/ext-definitions.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
apiVersion: plugin.halo.run/v1alpha1
kind: ExtensionDefinition
metadata:
name: s3os-thumbnail-provider
spec:
className: run.halo.s3os.S3ThumbnailProvider
extensionPointName: thumbnail-provider
displayName: "S3 协议 OSS 缩略图生成"
description: "为上传到支持 S3 协议的 OSS 的图片生成缩略图"
Loading

0 comments on commit 1250cc5

Please sign in to comment.