Skip to content

Commit

Permalink
feat: ability to configure and use image thumbnails (#167)
Browse files Browse the repository at this point in the history
### What this PR does?
适配 Halo 2.19(WIP) 的缩略图机制支持 S3 配置并使用缩略图

缩略图规则参考:
- [腾讯云 OSS 图片缩放](https://cloud.tencent.com/document/product/1246/109210)
- [百度 OSS 图片缩放](https://cloud.baidu.com/doc/BOS/s/gkbisf3l4)
- [阿里云 OSS 图片缩放](https://help.aliyun.com/zh/oss/user-guide/resize-images-4)
- [七牛云 OSS 图片缩放](https://developer.qiniu.com/dora/api/basic-processing-images-imageview2)
- [青云 OSS 图片缩放](https://docsv3.qingcloud.com/storage/object-storage/api/object/image_process/resize/)
- [京东云 OSS](https://docs.jdcloud.com/cn/object-storage-service/resize-images)
- [又拍云图片缩放](https://docs.upyun.com/cloud/image/#_11)


```release-note
支持配置并使用图片缩略图机制
```
  • Loading branch information
guqing authored Sep 26, 2024
1 parent 97f653b commit 68b0bee
Show file tree
Hide file tree
Showing 8 changed files with 183 additions and 53 deletions.
4 changes: 2 additions & 2 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ repositories {
}

dependencies {
implementation platform('run.halo.tools.platform:plugin:2.17.0-SNAPSHOT')
implementation platform('run.halo.tools.platform:plugin:2.19.0-SNAPSHOT')
compileOnly 'run.halo.app:api'

implementation platform('software.amazon.awssdk:bom:2.19.8')
Expand All @@ -40,7 +40,7 @@ configurations.runtimeClasspath {


halo {
version = '2.17.0'
version = '2.19'
}

haloPlugin {
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/run/halo/s3os/S3LinkServiceImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public Mono<S3ListResult> listObjects(String policyName, String continuationToke
return client.fetch(ConfigMap.class, configMapName);
})
.flatMap((configMap) -> {
var properties = handler.getProperties(configMap);
var properties = S3OsProperties.convertFrom(configMap);
var finalLocation = FilePathUtils.getFilePathByPlaceholder(properties.getLocation());
return Mono.using(() -> handler.buildS3Client(properties),
// 执行 listObjects
Expand Down Expand Up @@ -231,7 +231,7 @@ public Mono<LinkResult.LinkResultItem> addAttachmentRecord(String policyName,
return client.fetch(ConfigMap.class, configMapName);
})
.flatMap(configMap -> {
var properties = handler.getProperties(configMap);
var properties = S3OsProperties.convertFrom(configMap);
return Mono.using(() -> handler.buildS3Client(properties),
(s3Client) -> Mono.fromCallable(
() -> s3Client.headObject(
Expand Down
49 changes: 8 additions & 41 deletions src/main/java/run/halo/s3os/S3OsAttachmentHandler.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileAlreadyExistsException;
import java.time.Duration;
import java.util.HashMap;
Expand All @@ -23,7 +22,6 @@
import org.springframework.lang.Nullable;
import org.springframework.web.server.ServerErrorException;
import org.springframework.web.server.ServerWebInputException;
import org.springframework.web.util.UriUtils;
import reactor.core.Exceptions;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
Expand All @@ -38,7 +36,6 @@
import run.halo.app.extension.ConfigMap;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.MetadataUtil;
import run.halo.app.infra.utils.JsonUtils;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.awscore.presigner.SdkPresigner;
import software.amazon.awssdk.core.SdkResponse;
Expand All @@ -47,16 +44,7 @@
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
import software.amazon.awssdk.services.s3.model.CompletedPart;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.services.s3.model.*;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
import software.amazon.awssdk.utils.SdkAutoCloseable;
Expand All @@ -79,7 +67,7 @@ public class S3OsAttachmentHandler implements AttachmentHandler {
public Mono<Attachment> upload(UploadContext uploadContext) {
return Mono.just(uploadContext).filter(context -> this.shouldHandle(context.policy()))
.flatMap(context -> {
final var properties = getProperties(context.configMap());
final var properties = S3OsProperties.convertFrom(context.configMap());
return upload(context, properties)
.subscribeOn(Schedulers.boundedElastic())
.map(objectDetail -> this.buildAttachment(properties, objectDetail))
Expand All @@ -102,7 +90,7 @@ public Mono<Attachment> delete(DeleteContext deleteContext) {
log.info("Skip deleting object {} from S3.", objectKey);
return Mono.just(context);
}
var properties = getProperties(deleteContext.configMap());
var properties = S3OsProperties.convertFrom(deleteContext.configMap());
return Mono.using(() -> buildS3Client(properties),
client -> Mono.fromCallable(
() -> client.deleteObject(DeleteObjectRequest.builder()
Expand All @@ -123,7 +111,7 @@ public Mono<Attachment> delete(DeleteContext deleteContext) {

@Override
public Mono<URI> getSharedURL(Attachment attachment, Policy policy, ConfigMap configMap,
Duration ttl) {
Duration ttl) {
if (!this.shouldHandle(policy)) {
return Mono.empty();
}
Expand All @@ -132,7 +120,7 @@ public Mono<URI> getSharedURL(Attachment attachment, Policy policy, ConfigMap co
return Mono.error(new IllegalArgumentException(
"Cannot obtain object key from attachment " + attachment.getMetadata().getName()));
}
var properties = getProperties(configMap);
var properties = S3OsProperties.convertFrom(configMap);

return Mono.using(() -> buildS3Presigner(properties),
s3Presigner -> {
Expand Down Expand Up @@ -168,8 +156,8 @@ public Mono<URI> getPermalink(Attachment attachment, Policy policy, ConfigMap co
// fallback to default handler for backward compatibility
return Mono.empty();
}
var properties = getProperties(configMap);
var objectURL = getObjectURL(properties, objectKey);
var properties = S3OsProperties.convertFrom(configMap);
var objectURL = properties.toObjectURL(objectKey);
var urlSuffix = getUrlSuffixAnnotation(attachment);
if (StringUtils.isNotBlank(urlSuffix)) {
objectURL += urlSuffix;
Expand All @@ -195,13 +183,8 @@ private String getUrlSuffixAnnotation(Attachment attachment) {
return annotations.get(URL_SUFFIX_ANNO_KEY);
}

S3OsProperties getProperties(ConfigMap configMap) {
var settingJson = configMap.getData().getOrDefault("default", "{}");
return JsonUtils.jsonToObject(settingJson, S3OsProperties.class);
}

Attachment buildAttachment(S3OsProperties properties, ObjectDetail objectDetail) {
String externalLink = getObjectURL(properties, objectDetail.uploadState.objectKey);
String externalLink = properties.toObjectURL(objectDetail.uploadState.objectKey);
var urlSuffix = UrlUtils.findUrlSuffix(properties.getUrlSuffixes(),
objectDetail.uploadState.fileName);

Expand Down Expand Up @@ -229,22 +212,6 @@ Attachment buildAttachment(S3OsProperties properties, ObjectDetail objectDetail)
return attachment;
}

String getObjectURL(S3OsProperties properties, String objectKey) {
String objectURL;
if (StringUtils.isBlank(properties.getDomain())) {
String host;
if (properties.getEnablePathStyleAccess()) {
host = properties.getEndpoint() + "/" + properties.getBucket();
} else {
host = properties.getBucket() + "." + properties.getEndpoint();
}
objectURL = properties.getProtocol() + "://" + host + "/" + objectKey;
} else {
objectURL = properties.getProtocol() + "://" + properties.getDomain() + "/" + objectKey;
}
return UriUtils.encodePath(objectURL, StandardCharsets.UTF_8);
}

S3Client buildS3Client(S3OsProperties properties) {
return S3Client.builder()
.region(Region.of(properties.getRegion()))
Expand Down
36 changes: 30 additions & 6 deletions src/main/java/run/halo/s3os/S3OsProperties.java
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
package run.halo.s3os;

import java.nio.charset.StandardCharsets;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor;
import org.springframework.util.StringUtils;

import java.time.LocalDate;
import org.springframework.web.util.UriUtils;
import run.halo.app.extension.ConfigMap;
import run.halo.app.infra.utils.JsonUtils;

@Data
class S3OsProperties {
public class S3OsProperties {

private String bucket;

Expand Down Expand Up @@ -49,6 +50,8 @@ class S3OsProperties {

private List<urlSuffixItem> urlSuffixes;

private String thumbnailParamPattern;

@Data
@AllArgsConstructor
@NoArgsConstructor
Expand Down Expand Up @@ -103,19 +106,40 @@ public void setRandomStringLength(String randomStringLength) { // if you use In
if (length >= 4 && length <= 16) {
this.randomStringLength = length;
}
} catch (NumberFormatException ignored) {
}
catch (NumberFormatException ignored) { }
}

public void setRegion(String region) {
if (!StringUtils.hasText(region)) {
this.region = "Auto";
}else {
} else {
this.region = region;
}
}

public void setEndpoint(String endpoint) {
this.endpoint = UrlUtils.removeHttpPrefix(endpoint);
}

public String toObjectURL(String objectKey) {
String objectURL;
if (!StringUtils.hasText(this.getDomain())) {
String host;
if (this.getEnablePathStyleAccess()) {
host = this.getEndpoint() + "/" + this.getBucket();
} else {
host = this.getBucket() + "." + this.getEndpoint();
}
objectURL = this.getProtocol() + "://" + host + "/" + objectKey;
} else {
objectURL = this.getProtocol() + "://" + this.getDomain() + "/" + objectKey;
}
return UriUtils.encodePath(objectURL, StandardCharsets.UTF_8);
}

public static S3OsProperties convertFrom(ConfigMap configMap) {
var settingJson = configMap.getData().getOrDefault("default", "{}");
return JsonUtils.jsonToObject(settingJson, S3OsProperties.class);
}
}
106 changes: 106 additions & 0 deletions src/main/java/run/halo/s3os/S3ThumbnailProvider.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
package run.halo.s3os;

import java.net.URI;
import java.net.URL;
import java.util.Map;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import lombok.Builder;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.web.util.UriComponentsBuilder;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import run.halo.app.core.attachment.ThumbnailProvider;
import run.halo.app.core.attachment.ThumbnailSize;
import run.halo.app.extension.ConfigMap;
import run.halo.app.extension.ReactiveExtensionClient;

@Component
@RequiredArgsConstructor
public class S3ThumbnailProvider implements ThumbnailProvider {
static final String WIDTH_PLACEHOLDER = "{width}";
private final Cache<String, S3PropsCacheValue> s3PropsCache = CacheBuilder.newBuilder()
.maximumSize(50)
.build();

private final ReactiveExtensionClient client;
private final S3LinkService s3LinkService;

@Override
public Mono<URI> generate(ThumbnailContext thumbnailContext) {
var url = thumbnailContext.getImageUrl().toString();
var size = thumbnailContext.getSize();
return getCacheValue(url)
.mapNotNull(cacheValue -> placedPattern(cacheValue.pattern(), size))
.map(param -> {
if (param.startsWith("?")) {
return UriComponentsBuilder.fromHttpUrl(url)
.queryParam(param.substring(1))
.build()
.toString();
}
return url + param;
})
.map(URI::create);
}

private static String placedPattern(String pattern, ThumbnailSize size) {
return StringUtils.replace(pattern, WIDTH_PLACEHOLDER, String.valueOf(size.getWidth()));
}

@Override
public Mono<Void> delete(URL url) {
// do nothing for s3
return Mono.empty();
}

@Override
public Mono<Boolean> supports(ThumbnailContext thumbnailContext) {
var url = thumbnailContext.getImageUrl().toString();
return getCacheValue(url).hasElement();
}

private Mono<S3PropsCacheValue> getCacheValue(String imageUrl) {
return Flux.fromIterable(s3PropsCache.asMap().entrySet())
.filter(entry -> imageUrl.startsWith(entry.getKey()))
.next()
.map(Map.Entry::getValue)
.switchIfEmpty(Mono.defer(() -> listAllS3ObjectDomain()
.filter(entry -> imageUrl.startsWith(entry.getKey()))
.map(Map.Entry::getValue)
.next()
));
}

@Builder
record S3PropsCacheValue(String pattern, String configMapName) {
}

private Flux<Map.Entry<String, S3PropsCacheValue>> listAllS3ObjectDomain() {
return s3LinkService.listS3Policies()
.flatMap(s3Policy -> {
var s3ConfigMapName = s3Policy.getSpec().getConfigMapName();
return fetchS3PropsByConfigMapName(s3ConfigMapName)
.mapNotNull(properties -> {
var thumbnailParam = properties.getThumbnailParamPattern();
if (StringUtils.isBlank(thumbnailParam)) {
return null;
}
var objectDomain = properties.toObjectURL("");
var cacheValue = S3PropsCacheValue.builder()
.pattern(thumbnailParam)
.configMapName(s3ConfigMapName)
.build();
return Map.entry(objectDomain, cacheValue);
});
})
.doOnNext(cache -> s3PropsCache.put(cache.getKey(), cache.getValue()));
}

private Mono<S3OsProperties> fetchS3PropsByConfigMapName(String name) {
return client.fetch(ConfigMap.class, name)
.map(S3OsProperties::convertFrom);
}
}
9 changes: 9 additions & 0 deletions src/main/resources/extensions/ext-definitions.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
apiVersion: plugin.halo.run/v1alpha1
kind: ExtensionDefinition
metadata:
name: s3os-thumbnail-provider
spec:
className: run.halo.s3os.S3ThumbnailProvider
extensionPointName: thumbnail-provider
displayName: "S3 协议 OSS 缩略图生成"
description: "为上传到支持 S3 协议的 OSS 的图片生成缩略图"
26 changes: 25 additions & 1 deletion src/main/resources/extensions/policy-template-s3os.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -144,4 +144,28 @@ spec:
name: urlSuffix
label: 网址后缀
placeholder: 例如:?imageMogr2/format/webp
validation: required
validation: required
- $formkit: select
name: thumbnailParamPattern
label: 缩略图参数
allowCreate: true
searchable: true
value: ""
help: |
请根据您的对象存储服务商选择对应的缩略图参数或自定义参数,{width} 为宽度占位符将被替换为所需缩略图宽度值,
如: 400,参数需要以 ? 开头,间隔符除外
options:
- label:
value: ""
- label: 腾讯云 COS / 七牛云 KODO
value: "?imageView2/0/w/{width}"
- label: 阿里云 OSS
value: "?x-oss-process=image/resize,w_{width},m_lfit"
- label: 百度云 BOS
value: "?x-bce-process=image/resize,m_lfit,w_{width}"
- label: 青云 OSS
value: "?image&action=resize:w_{width},m_2"
- label: 京东云
value: "?x-oss-process=img/sw/{width}"
- label: 又拍云
value: "!/fw/{width}"
2 changes: 1 addition & 1 deletion src/main/resources/plugin.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ metadata:
name: PluginS3ObjectStorage
spec:
enabled: true
requires: ">=2.17.0"
requires: ">=2.19.0"
author:
name: Halo
website: https://github.com/halo-dev
Expand Down

0 comments on commit 68b0bee

Please sign in to comment.