From b5a5558921764e04131102d795cb5b6a92f2ec09 Mon Sep 17 00:00:00 2001 From: yangsen Date: Tue, 18 May 2021 17:22:36 +0800 Subject: [PATCH 01/16] uc query api to v3 --- .../java/com/qiniu/storage/AutoRegion.java | 334 ++++++++++-------- src/main/java/com/qiniu/storage/Region.java | 31 ++ .../java/com/qiniu/storage/RegionGroup.java | 104 ++++++ src/test/java/test/com/qiniu/TestConfig.java | 13 +- 4 files changed, 329 insertions(+), 153 deletions(-) create mode 100644 src/main/java/com/qiniu/storage/RegionGroup.java diff --git a/src/main/java/com/qiniu/storage/AutoRegion.java b/src/main/java/com/qiniu/storage/AutoRegion.java index 649c0935d..1d66e315d 100644 --- a/src/main/java/com/qiniu/storage/AutoRegion.java +++ b/src/main/java/com/qiniu/storage/AutoRegion.java @@ -19,7 +19,7 @@ class AutoRegion extends Region { /** * 空间机房，域名信息缓存 */ - private Map regions; + private Map regions; /** * 根据API返回的上传域名推导出其他资源管理域名 @@ -35,17 +35,17 @@ class AutoRegion extends Region { this.ucServer = ucServer; this.client = new Client(); this.regions = new ConcurrentHashMap<>(); - this.inferDomainsMap = new ConcurrentHashMap<>(); - this.inferDomainsMap.put("up.qiniup.com", region0()); - this.inferDomainsMap.put("up-jjh.qiniup.com", region0()); - this.inferDomainsMap.put("up-xs.qiniup.com", region0()); - this.inferDomainsMap.put("up-z1.qiniup.com", region1()); - this.inferDomainsMap.put("up-z2.qiniup.com", region2()); - this.inferDomainsMap.put("up-dg.qiniup.com", region2()); - this.inferDomainsMap.put("up-fs.qiniup.com", region2()); - this.inferDomainsMap.put("up-na0.qiniup.com", regionNa0()); - this.inferDomainsMap.put("up-as0.qiniup.com", regionAs0()); - this.inferDomainsMap.put("up-fog-cn-east-1.qiniup.com", regionFogCnEast1()); +// this.inferDomainsMap = new ConcurrentHashMap<>(); +// this.inferDomainsMap.put("up.qiniup.com", region0()); +// this.inferDomainsMap.put("up-jjh.qiniup.com", region0()); +// this.inferDomainsMap.put("up-xs.qiniup.com", region0()); +// this.inferDomainsMap.put("up-z1.qiniup.com", region1()); +// this.inferDomainsMap.put("up-z2.qiniup.com", region2()); +// this.inferDomainsMap.put("up-dg.qiniup.com", region2()); +// this.inferDomainsMap.put("up-fs.qiniup.com", region2()); +// this.inferDomainsMap.put("up-na0.qiniup.com", regionNa0()); +// this.inferDomainsMap.put("up-as0.qiniup.com", regionAs0()); +// this.inferDomainsMap.put("up-fog-cn-east-1.qiniup.com", regionFogCnEast1()); } /** @@ -57,12 +57,71 @@ class AutoRegion extends Region { * na0: http://uc.qbox.me/v2/query?ak=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5&bucket=sdk-na0 */ private UCRet getRegionJson(RegionIndex index) throws QiniuException { - String address = ucServer + "/v2/query?ak=" + index.accessKey + "&bucket=" + index.bucket; + String address = ucServer + "/v3/query?ak=" + index.accessKey + "&bucket=" + index.bucket; Response r = client.get(address); return r.jsonToObject(UCRet.class); } + static RegionGroup regionGroup(UCRet ret) { + if (ret == null || ret.hosts == null || ret.hosts.length == 0) { + return null; + } + + RegionGroup group = new RegionGroup(); + for (HostRet host : ret.hosts) { + long timestamp = host.ttl + System.currentTimeMillis() / 1000; + List srcUpHosts = new ArrayList<>(); + List accUpHosts = new ArrayList<>(); + if (host.up != null) { + srcUpHosts = host.up.allSrcHosts(); + accUpHosts = host.up.allAccHosts(); + } + + String iovipHost = null; + if (host.io != null) { + iovipHost = host.io.getOneHost(); + } + + String rsHost = null; + if (host.rs != null) { + rsHost = host.rs.getOneHost(); + } + + String rsfHost = null; + if (host.rsf != null) { + rsfHost = host.rsf.getOneHost(); + } + + String apiHost = null; + if (host.api != null) { + apiHost = host.api.getOneHost(); + } + + String ucHost = null; + if (host.uc != null) { + ucHost = host.uc.getOneHost(); + } + + // 根据 iovipHost 反推 regionId + String regionId = iovipHost; + if (iovipHost != null) { + String[] regionIdList = new String[]{"z0", "z1", "z2", "na0", "as0", "fog-cn-east-1"}; + for (String regionIdP : regionIdList) { + if (iovipHost.contains("-" + regionIdP)) { + regionId = regionIdP; + break; + } + } + } + + Region region = new Region(timestamp, regionId, srcUpHosts, accUpHosts, iovipHost, rsHost, rsfHost, apiHost, ucHost); + group.addRegion(region); + } + + return group; + } + /** * 首先从缓存读取Region信息，如果没有则发送请求从接口查询 * @@ -70,55 +129,35 @@ private UCRet getRegionJson(RegionIndex index) throws QiniuException { * @param bucket 空间名 * @return 机房域名信息 */ - private RegionInfo queryRegionInfo(String accessKey, String bucket) throws QiniuException { + private RegionGroup queryRegionInfo(String accessKey, String bucket) throws QiniuException { RegionIndex index = new RegionIndex(accessKey, bucket); - RegionInfo info = regions.get(index); + RegionGroup regionGroup = regions.get(index); Exception ex = null; - // 隔一段时间重新获取 uc 信息 // - if (info == null || info.createTime < System.currentTimeMillis() - 1000 * 3600 * 8) { - try { - // 1 - UCRet ret = getRegionJson(index); - RegionInfo info2 = RegionInfo.buildFromUcRet(ret); - // 初次获取报错，info == null ，响应 null // - // 后续重新获取，正常获取则替换以前的 // - if (info2 != null) { - regions.put(index, info2); - info = info2; - } - } catch (Exception e) { - ex = e; - if (info == null) { - try { - Thread.sleep(100); - } catch (Exception e1) { - // do nothing - } - try { - // 2 - UCRet ret = getRegionJson(index); - RegionInfo info2 = RegionInfo.buildFromUcRet(ret); - // 初次获取报错，info == null ，响应 null // - // 后续重新获取，正常获取则替换以前的 // - if (info2 != null) { - regions.put(index, info2); - info = info2; - } - } catch (Exception e1) { - ex = e1; + // 隔一段时间重新获取 uc 信息 // || regionGroup.createTime < System.currentTimeMillis() - 1000 * 3600 * 8 + if (regionGroup == null) { + for (int i = 0; i < 2; i++) { + try { + UCRet ret = getRegionJson(index); + regionGroup = AutoRegion.regionGroup(ret); + if (regionGroup != null) { + regions.put(index, regionGroup); + break; } + } catch (Exception e) { + ex = e; } } } + // info 不能为 null // - if (info == null) { + if (regionGroup == null) { if (ex instanceof QiniuException) { throw (QiniuException) ex; } throw new QiniuException(ex, "auto region get region info from uc failed."); } - return info; + return regionGroup; } /** @@ -127,13 +166,27 @@ private RegionInfo queryRegionInfo(String accessKey, String bucket) throws Qiniu * @param regionReqInfo 封装了 accessKey 和 bucket 的对象 * @return 机房域名信息 */ - private RegionInfo queryRegionInfo(RegionReqInfo regionReqInfo) throws QiniuException { + private RegionGroup queryRegionInfo(RegionReqInfo regionReqInfo) throws QiniuException { return queryRegionInfo(regionReqInfo.getAccessKey(), regionReqInfo.getBucket()); } @Override String getRegion(RegionReqInfo regionReqInfo) { - return ""; + Region currentRegion = getCurrentRegion(regionReqInfo); + if (currentRegion == null) { + return ""; + } else { + return currentRegion.getRegion(regionReqInfo); + } + } + + Region getCurrentRegion(RegionReqInfo regionReqInfo) { + try { + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getCurrentRegion(regionReqInfo); + } catch (QiniuException e) { + return null; + } } /** @@ -141,8 +194,8 @@ String getRegion(RegionReqInfo regionReqInfo) { */ @Override List getSrcUpHost(RegionReqInfo regionReqInfo) throws QiniuException { - RegionInfo info = queryRegionInfo(regionReqInfo); - return info.srcUpHosts; + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getSrcUpHost(regionReqInfo); } /** @@ -150,8 +203,8 @@ List getSrcUpHost(RegionReqInfo regionReqInfo) throws QiniuException { */ @Override List getAccUpHost(RegionReqInfo regionReqInfo) throws QiniuException { - RegionInfo info = queryRegionInfo(regionReqInfo); - return info.accUpHosts; + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getAccUpHost(regionReqInfo); } /** @@ -159,8 +212,8 @@ List getAccUpHost(RegionReqInfo regionReqInfo) throws QiniuException { */ @Override String getIovipHost(RegionReqInfo regionReqInfo) throws QiniuException { - RegionInfo info = queryRegionInfo(regionReqInfo); - return info.iovipHost; + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getIovipHost(regionReqInfo); } /** @@ -168,18 +221,8 @@ String getIovipHost(RegionReqInfo regionReqInfo) throws QiniuException { */ @Override String getRsHost(RegionReqInfo regionReqInfo) throws QiniuException { - RegionInfo info; - try { - info = queryRegionInfo(regionReqInfo); - } catch (QiniuException e) { - return super.getRsHost(regionReqInfo); - } - Region region = this.inferDomainsMap.get(info.srcUpHosts.get(0)); - if (region != null) { - return region.getRsHost(regionReqInfo); - } else { - return super.getRsHost(regionReqInfo); - } + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getRsHost(regionReqInfo); } /** @@ -187,18 +230,8 @@ String getRsHost(RegionReqInfo regionReqInfo) throws QiniuException { */ @Override String getRsfHost(RegionReqInfo regionReqInfo) throws QiniuException { - RegionInfo info; - try { - info = queryRegionInfo(regionReqInfo); - } catch (QiniuException e) { - return super.getRsfHost(regionReqInfo); - } - Region region = this.inferDomainsMap.get(info.srcUpHosts.get(0)); - if (region != null) { - return region.getRsfHost(regionReqInfo); - } else { - return super.getRsfHost(regionReqInfo); - } + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getRsfHost(regionReqInfo); } /** @@ -206,70 +239,8 @@ String getRsfHost(RegionReqInfo regionReqInfo) throws QiniuException { */ @Override String getApiHost(RegionReqInfo regionReqInfo) throws QiniuException { - RegionInfo info; - try { - info = queryRegionInfo(regionReqInfo); - } catch (QiniuException e) { - return super.getApiHost(regionReqInfo); - } - Region region = this.inferDomainsMap.get(info.srcUpHosts.get(0)); - if (region != null) { - return region.getApiHost(regionReqInfo); - } else { - return super.getApiHost(regionReqInfo); - } - } - - /** - * 从接口获取的域名信息 - */ - static class RegionInfo { - final List srcUpHosts; - final List accUpHosts; - final String iovipHost; - final long createTime; - - protected RegionInfo(List srcUpHosts, List accUpHosts, String iovipHost) { - this.srcUpHosts = srcUpHosts; - this.accUpHosts = accUpHosts; - this.iovipHost = iovipHost; - createTime = System.currentTimeMillis(); - } - - /** - * { - * "io": {"src": {"main": ["iovip.qbox.me"]}}, - * "up": { - * "acc": { - * "main": ["upload.qiniup.com"], - * "backup": ["upload-jjh.qiniup.com", "upload-xs.qiniup.com"] - * }, - * "src": { - * "main": ["up.qiniup.com"], - * "backup": ["up-jjh.qiniup.com", "up-xs.qiniup.com"] - * } - * } - * } - * - * @param ret - * @return - */ - static RegionInfo buildFromUcRet(UCRet ret) { - List srcUpHosts = new ArrayList<>(); - addAll(srcUpHosts, ret.up.src.get("main")); - addAll(srcUpHosts, ret.up.src.get("backup")); - List accUpHosts = new ArrayList<>(); - addAll(accUpHosts, ret.up.acc.get("main")); - addAll(accUpHosts, ret.up.acc.get("backup")); - String iovipHost = ret.io.src.get("main").get(0); - return new RegionInfo(srcUpHosts, accUpHosts, iovipHost); - } - - static void addAll(List s, List p) { - if (p != null) { - s.addAll(p); - } - } + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getApiHost(regionReqInfo); } private static class RegionIndex { @@ -292,17 +263,76 @@ public boolean equals(Object obj) { } private class UCRet { - UPRet up; - IORet io; + HostRet[] hosts; } - private class UPRet { - Map> acc; - Map> src; + private class HostRet { + long ttl; + HostInfoRet up; + HostInfoRet rs; + HostInfoRet rsf; + HostInfoRet uc; + HostInfoRet api; + HostInfoRet io; } - private class IORet { + private class HostInfoRet { + Map> acc; Map> src; - } + private String getOneHost() { + List hosts = allHosts(); + if (hosts.size() > 0) { + return hosts.get(0); + } else { + return null; + } + } + + private List allHosts() { + List hosts = new ArrayList<>(); + List srcHosts = allSrcHosts(); + if (srcHosts.size() > 0) { + hosts.addAll(srcHosts); + } + + List accHosts = allAccHosts(); + if (accHosts.size() > 0) { + hosts.addAll(accHosts); + } + return hosts; + } + + private List allSrcHosts() { + List hosts = new ArrayList<>(); + if (acc != null) { + List mainHosts = acc.get("main"); + if (mainHosts != null && mainHosts.size() > 0) { + hosts.addAll(mainHosts); + } + + List backupHosts = acc.get("backup"); + if (backupHosts != null && backupHosts.size() > 0) { + hosts.addAll(backupHosts); + } + } + return hosts; + } + + private List allAccHosts() { + List hosts = new ArrayList<>(); + if (src != null) { + List mainHosts = src.get("main"); + if (mainHosts != null && mainHosts.size() > 0) { + hosts.addAll(mainHosts); + } + + List backupHosts = src.get("backup"); + if (backupHosts != null && backupHosts.size() > 0) { + hosts.addAll(backupHosts); + } + } + return hosts; + } + } } diff --git a/src/main/java/com/qiniu/storage/Region.java b/src/main/java/com/qiniu/storage/Region.java index 8b5f44016..c7136bd00 100644 --- a/src/main/java/com/qiniu/storage/Region.java +++ b/src/main/java/com/qiniu/storage/Region.java @@ -7,6 +7,8 @@ public class Region { + // 有效期，过了有效期，region会无效，此处只在取时缓存判断； -1 为无限期 + private long timestamp = -1; // 区域名称：z0 华东 z1 华北 z2 华南 na0 北美 as0 东南亚 private String region = "z0"; @@ -25,6 +27,23 @@ public class Region { private String rsHost = "rs.qbox.me"; private String rsfHost = "rsf.qbox.me"; private String apiHost = "api.qiniu.com"; + private String ucHost = "uc.qbox.me"; + + Region() { + } + + Region(long timestamp, String region, List srcUpHosts, List accUpHosts, String iovipHost, + String rsHost, String rsfHost, String apiHost, String ucHost) { + this.timestamp = timestamp; + this.region = region; + this.srcUpHosts = srcUpHosts; + this.accUpHosts = accUpHosts; + this.iovipHost = iovipHost; + this.rsHost = rsHost; + this.rsfHost = rsfHost; + this.apiHost = apiHost; + this.ucHost = ucHost; + } /** * 华东机房相关域名 @@ -215,6 +234,10 @@ String getRegion(RegionReqInfo regionReqInfo) { return this.region; } + Region getCurrentRegion(RegionReqInfo regionReqInfo) { + return this; + } + List getSrcUpHost(RegionReqInfo regionReqInfo) throws QiniuException { return this.srcUpHosts; } @@ -239,6 +262,14 @@ String getApiHost(RegionReqInfo regionReqInfo) throws QiniuException { return apiHost; } + boolean isValid() { + if (timestamp < 0) { + return true; + } else { + return System.currentTimeMillis() < timestamp * 1000; + } + } + /** * 域名构造器 */ diff --git a/src/main/java/com/qiniu/storage/RegionGroup.java b/src/main/java/com/qiniu/storage/RegionGroup.java new file mode 100644 index 000000000..3a6b6bdd6 --- /dev/null +++ b/src/main/java/com/qiniu/storage/RegionGroup.java @@ -0,0 +1,104 @@ +package com.qiniu.storage; + +import com.qiniu.common.QiniuException; + +import java.util.ArrayList; +import java.util.List; + +public class RegionGroup extends Region { + + private Region currentRegion = null; + private int currentRegionIndex = 0; + private final List regionList = new ArrayList<>(); + + + public boolean addRegion(Region region) { + if (region == null) { + return false; + } + + regionList.add(region); + + if (currentRegion == null) { + updateCurrentRegion(); + } + + return true; + } + + String getRegion(RegionReqInfo regionReqInfo) { + if (currentRegion == null) { + return ""; + } else { + return currentRegion.getRegion(regionReqInfo); + } + } + + List getSrcUpHost(RegionReqInfo regionReqInfo) throws QiniuException { + if (currentRegion == null) { + return null; + } else { + return currentRegion.getSrcUpHost(regionReqInfo); + } + } + + List getAccUpHost(RegionReqInfo regionReqInfo) throws QiniuException { + if (currentRegion == null) { + return null; + } else { + return currentRegion.getAccUpHost(regionReqInfo); + } + } + + String getIovipHost(RegionReqInfo regionReqInfo) throws QiniuException { + if (currentRegion == null) { + return null; + } else { + return currentRegion.getIovipHost(regionReqInfo); + } + } + + String getRsHost(RegionReqInfo regionReqInfo) throws QiniuException { + if (currentRegion == null) { + return null; + } else { + return currentRegion.getRsHost(regionReqInfo); + } + } + + String getRsfHost(RegionReqInfo regionReqInfo) throws QiniuException { + if (currentRegion == null) { + return null; + } else { + return currentRegion.getRsfHost(regionReqInfo); + } + } + + String getApiHost(RegionReqInfo regionReqInfo) throws QiniuException { + if (currentRegion == null) { + return null; + } else { + return currentRegion.getApiHost(regionReqInfo); + } + } + + Region getCurrentRegion(RegionReqInfo regionReqInfo) { + if (currentRegion == null) { + return null; + } else if (currentRegion instanceof AutoRegion || currentRegion instanceof RegionGroup) { + return currentRegion.getCurrentRegion(regionReqInfo); + } else { + return currentRegion; + } + } + + private void updateCurrentRegion() { + if (regionList.size() == 0) { + return; + } + + if (currentRegionIndex < regionList.size()) { + currentRegion = regionList.get(currentRegionIndex); + } + } +} diff --git a/src/test/java/test/com/qiniu/TestConfig.java b/src/test/java/test/com/qiniu/TestConfig.java index 73857842f..97db3b786 100644 --- a/src/test/java/test/com/qiniu/TestConfig.java +++ b/src/test/java/test/com/qiniu/TestConfig.java @@ -98,6 +98,17 @@ public static TestFile[] getTestFileArray(String fileSaveKey, String fileMimeTyp z0.regionId = "z0"; z0.region = Region.region0(); + TestFile z0_auto = new TestFile(); + z0_auto.key = fileSaveKey; + z0_auto.mimeType = fileMimeType; + z0_auto.bucketName = testBucket_z0; + z0_auto.testDomain = testDomain_z0; + z0_auto.testUrl = "http://" + testDomain_z0 + "/" + fileSaveKey; + z0_auto.testDomainTimeStamp = testDomain_z0_timeStamp; + z0_auto.testUrlTimeStamp = "http://" + testDomain_z0_timeStamp + "/" + fileSaveKey; + z0_auto.regionId = "z0"; + z0_auto.region = Region.region0(); + TestFile fog = new TestFile(); fog.key = fileSaveKey; fog.mimeType = fileMimeType; @@ -124,7 +135,7 @@ public static TestFile[] getTestFileArray(String fileSaveKey, String fileMimeTyp return new TestFile[]{na0}; } else { // return new TestFile[]{fog, fog1, z0, na0}; - return new TestFile[]{z0}; + return new TestFile[]{z0_auto}; } } From 06fcf379efa6acd9d6933342585112b362925408 Mon Sep 17 00:00:00 2001 From: yangsen Date: Tue, 18 May 2021 18:55:56 +0800 Subject: [PATCH 02/16] upload support switch region --- .../java/com/qiniu/storage/AutoRegion.java | 10 +++ .../java/com/qiniu/storage/BaseUploader.java | 83 +++++++++++++++++++ .../com/qiniu/storage/FixBlockUploader.java | 2 + .../java/com/qiniu/storage/FormUploader.java | 32 ++++--- src/main/java/com/qiniu/storage/Region.java | 4 + .../java/com/qiniu/storage/RegionGroup.java | 15 ++++ .../com/qiniu/storage/ResumeUploader.java | 17 ++-- 7 files changed, 133 insertions(+), 30 deletions(-) create mode 100644 src/main/java/com/qiniu/storage/BaseUploader.java diff --git a/src/main/java/com/qiniu/storage/AutoRegion.java b/src/main/java/com/qiniu/storage/AutoRegion.java index 1d66e315d..a4cf00f88 100644 --- a/src/main/java/com/qiniu/storage/AutoRegion.java +++ b/src/main/java/com/qiniu/storage/AutoRegion.java @@ -170,6 +170,16 @@ private RegionGroup queryRegionInfo(RegionReqInfo regionReqInfo) throws QiniuExc return queryRegionInfo(regionReqInfo.getAccessKey(), regionReqInfo.getBucket()); } + @Override + boolean switchRegion(RegionReqInfo regionReqInfo) { + Region currentRegion = getCurrentRegion(regionReqInfo); + if (currentRegion == null) { + return false; + } else { + return currentRegion.switchRegion(regionReqInfo); + } + } + @Override String getRegion(RegionReqInfo regionReqInfo) { Region currentRegion = getCurrentRegion(regionReqInfo); diff --git a/src/main/java/com/qiniu/storage/BaseUploader.java b/src/main/java/com/qiniu/storage/BaseUploader.java new file mode 100644 index 000000000..0cf0938d3 --- /dev/null +++ b/src/main/java/com/qiniu/storage/BaseUploader.java @@ -0,0 +1,83 @@ +package com.qiniu.storage; + +import com.qiniu.common.QiniuException; +import com.qiniu.http.Client; +import com.qiniu.http.Response; + +import javax.net.ssl.SSLException; +import java.net.ProtocolException; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; + +public abstract class BaseUploader { + + protected final Client client; + protected final String key; + protected final String upToken; + protected final Configuration config; + + BaseUploader(Client client, String upToken, String key, Configuration config) { + this.client = client; + this.key = key; + this.upToken = upToken; + this.config = config; + } + + public Response upload() throws QiniuException { + return uploadWithRegionRetry(); + } + + private Response uploadWithRegionRetry() throws QiniuException { + Response response = null; + while (true) { + try { + response = uploadFlows(); + if (!couldSwitchRegionAndRetry(response, null) || config.region == null || !config.region.switchRegion(new UploadToken(upToken))) { + break; + } + } catch (QiniuException e) { + if (!couldSwitchRegionAndRetry(null, e) || config.region == null || !config.region.switchRegion(new UploadToken(upToken))) { + throw e; + } + } + } + return response; + } + + abstract Response uploadFlows() throws QiniuException; + + private boolean couldSwitchRegionAndRetry(Response response, QiniuException exception) { + Response checkResponse = response; + if (checkResponse == null && exception != null) { + checkResponse = exception.response; + } + + if (checkResponse != null) { + int statusCode = checkResponse.statusCode; + if ((statusCode > 300 && statusCode < 400) + || (statusCode >= 400 && statusCode < 500 && statusCode != 400 && statusCode != 406) + || statusCode == 501 || statusCode == 573 + || statusCode == 608 || statusCode == 612 || statusCode == 614 || statusCode == 616 + || statusCode == 619 || statusCode == 630 || statusCode == 631 || statusCode == 640 + || statusCode == 701 + || statusCode < 100) { + return false; + } else { + return true; + } + } + + if (exception != null && exception.getCause() != null) { + Throwable e = exception.getCause(); + String msg = e.getMessage(); + return e instanceof UnknownHostException + || (msg != null && msg.indexOf("Broken pipe") == 0) + || e instanceof SocketTimeoutException + || e instanceof java.net.ConnectException + || e instanceof ProtocolException + || e instanceof SSLException; + } + + return false; + } +} diff --git a/src/main/java/com/qiniu/storage/FixBlockUploader.java b/src/main/java/com/qiniu/storage/FixBlockUploader.java index 0cfb9ef42..d08abc3ab 100644 --- a/src/main/java/com/qiniu/storage/FixBlockUploader.java +++ b/src/main/java/com/qiniu/storage/FixBlockUploader.java @@ -18,6 +18,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +@Deprecated public class FixBlockUploader { private final int blockSize; private final ConfigHelper configHelper; @@ -883,6 +884,7 @@ class Record { long size; long blockSize; List etagIdxes; + // 用于区分记录是 V1 还是 V2 boolean isValid() { return uploadId != null && etagIdxes != null && etagIdxes.size() > 0; diff --git a/src/main/java/com/qiniu/storage/FormUploader.java b/src/main/java/com/qiniu/storage/FormUploader.java index a7093dec4..180f5a7b9 100644 --- a/src/main/java/com/qiniu/storage/FormUploader.java +++ b/src/main/java/com/qiniu/storage/FormUploader.java @@ -14,10 +14,8 @@ * 该类封装了七牛提供的表单上传机制 * 参考文档：表单上传 */ -public final class FormUploader { +public final class FormUploader extends BaseUploader { - private final String token; - private final String key; private final File file; private final byte[] data; private final String mime; @@ -45,9 +43,9 @@ public FormUploader(Client client, String upToken, String key, File file, String private FormUploader(Client client, String upToken, String key, byte[] data, File file, StringMap params, String mime, boolean checkCrc, Configuration configuration) { + super(client, upToken, key, configuration); + this.client = client; - token = upToken; - this.key = key; this.file = file; this.data = data; this.params = params; @@ -56,23 +54,21 @@ private FormUploader(Client client, String upToken, String key, byte[] data, Fil this.configHelper = new ConfigHelper(configuration); } - /** - * 同步上传文件 - */ - public Response upload() throws QiniuException { + @Override + Response uploadFlows() throws QiniuException { buildParams(); - String host = configHelper.upHost(token); + String host = configHelper.upHost(upToken); try { if (data != null) { - return client.multipartPost(configHelper.upHost(token), params, "file", filename, data, + return client.multipartPost(configHelper.upHost(upToken), params, "file", filename, data, mime, new StringMap()); } else { - return client.multipartPost(configHelper.upHost(token), params, "file", filename, file, + return client.multipartPost(configHelper.upHost(upToken), params, "file", filename, file, mime, new StringMap()); } } catch (QiniuException e) { if (e.response == null || e.response.needSwitchServer()) { - changeHost(token, host); + changeHost(upToken, host); } throw e; } @@ -83,26 +79,26 @@ public Response upload() throws QiniuException { */ public void asyncUpload(final UpCompletionHandler handler) throws IOException { buildParams(); - final String host = configHelper.upHost(token); + final String host = configHelper.upHost(upToken); if (data != null) { client.asyncMultipartPost(host, params, "file", filename, data, mime, new StringMap(), new AsyncCallback() { @Override public void complete(Response res) { if (res != null && res.needSwitchServer()) { - changeHost(token, host); + changeHost(upToken, host); } handler.complete(key, res); } }); return; } - client.asyncMultipartPost(configHelper.upHost(token), params, "file", filename, + client.asyncMultipartPost(configHelper.upHost(upToken), params, "file", filename, file, mime, new StringMap(), new AsyncCallback() { @Override public void complete(Response res) { if (res != null && res.needSwitchServer()) { - changeHost(token, host); + changeHost(upToken, host); } handler.complete(key, res); } @@ -120,7 +116,7 @@ private void changeHost(String upToken, String host) { private void buildParams() throws QiniuException { if (params == null) return; - params.put("token", token); + params.put("token", upToken); if (key != null) { params.put("key", key); diff --git a/src/main/java/com/qiniu/storage/Region.java b/src/main/java/com/qiniu/storage/Region.java index c7136bd00..11679af55 100644 --- a/src/main/java/com/qiniu/storage/Region.java +++ b/src/main/java/com/qiniu/storage/Region.java @@ -230,6 +230,10 @@ public static Region autoRegion(String ucServer) { return new Builder().autoRegion(ucServer); } + boolean switchRegion(RegionReqInfo regionReqInfo) { + return false; + } + String getRegion(RegionReqInfo regionReqInfo) { return this.region; } diff --git a/src/main/java/com/qiniu/storage/RegionGroup.java b/src/main/java/com/qiniu/storage/RegionGroup.java index 3a6b6bdd6..b20d701f4 100644 --- a/src/main/java/com/qiniu/storage/RegionGroup.java +++ b/src/main/java/com/qiniu/storage/RegionGroup.java @@ -26,6 +26,21 @@ public boolean addRegion(Region region) { return true; } + @Override + boolean switchRegion(RegionReqInfo regionReqInfo) { + if (currentRegion != null && currentRegion.switchRegion(regionReqInfo)) { + return true; + } + + if ((currentRegionIndex + 1) < regionList.size()) { + currentRegionIndex += 1; + updateCurrentRegion(); + return true; + } else { + return false; + } + } + String getRegion(RegionReqInfo regionReqInfo) { if (currentRegion == null) { return ""; diff --git a/src/main/java/com/qiniu/storage/ResumeUploader.java b/src/main/java/com/qiniu/storage/ResumeUploader.java index 75a3872b6..2bf4e3e84 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploader.java +++ b/src/main/java/com/qiniu/storage/ResumeUploader.java @@ -31,16 +31,11 @@ * 服务端网络较稳定，较大文件（如500M以上）才需要将块记录保存下来。 * 小文件没有必要，可以有效地实现大文件的上传。 */ -public class ResumeUploader { - private final Client client; - private final String key; - private final String upToken; +public class ResumeUploader extends BaseUploader { private final ResumeUploadSource source; private final Recorder recorder; private final UploadOptions options; - final Configuration config; - ResumeUploadPerformer uploadPerformer; /** @@ -123,14 +118,11 @@ public ResumeUploader(Client client, String upToken, String key, InputStream str private ResumeUploader(Client client, String key, String upToken, ResumeUploadSource source, Recorder recorder, UploadOptions options, Configuration configuration) { + super(client, upToken, key, configuration); - this.client = client; - this.key = key; - this.upToken = upToken; this.source = source; this.recorder = recorder; this.options = options == null ? UploadOptions.defaultOptions() : options; - this.config = configuration; } /** @@ -138,13 +130,14 @@ private ResumeUploader(Client client, String key, String upToken, ResumeUploadSo */ public Response upload() throws QiniuException { try { - return uploadFlows(); + return super.upload(); } finally { close(); } } - private Response uploadFlows() throws QiniuException { + @Override + Response uploadFlows() throws QiniuException { // 检查参数 checkParam(); From 4837522bd5c71537d9efe6ad9542fbb577342c43 Mon Sep 17 00:00:00 2001 From: yangsen Date: Wed, 19 May 2021 17:27:13 +0800 Subject: [PATCH 03/16] upload performer support switch region --- .../java/com/qiniu/storage/BaseUploader.java | 36 +-- .../qiniu/storage/ResumeUploadPerformer.java | 60 +---- .../com/qiniu/storage/ResumeUploadSource.java | 21 +- .../qiniu/storage/ResumeUploadSourceFile.java | 12 +- .../storage/ResumeUploadSourceStream.java | 4 +- .../com/qiniu/storage/ResumeUploader.java | 106 ++++++-- .../com/qiniu/storage/RecordUploadTest.java | 4 +- .../com/qiniu/storage/SwitchRegionTest.java | 239 ++++++++++++++++++ 8 files changed, 360 insertions(+), 122 deletions(-) create mode 100644 src/test/java/test/com/qiniu/storage/SwitchRegionTest.java diff --git a/src/main/java/com/qiniu/storage/BaseUploader.java b/src/main/java/com/qiniu/storage/BaseUploader.java index 0cf0938d3..91431b145 100644 --- a/src/main/java/com/qiniu/storage/BaseUploader.java +++ b/src/main/java/com/qiniu/storage/BaseUploader.java @@ -4,16 +4,12 @@ import com.qiniu.http.Client; import com.qiniu.http.Response; -import javax.net.ssl.SSLException; -import java.net.ProtocolException; -import java.net.SocketTimeoutException; -import java.net.UnknownHostException; - public abstract class BaseUploader { protected final Client client; protected final String key; protected final String upToken; + protected final ConfigHelper configHelper; protected final Configuration config; BaseUploader(Client client, String upToken, String key, Configuration config) { @@ -21,6 +17,7 @@ public abstract class BaseUploader { this.key = key; this.upToken = upToken; this.config = config; + this.configHelper = new ConfigHelper(config); } public Response upload() throws QiniuException { @@ -54,30 +51,15 @@ private boolean couldSwitchRegionAndRetry(Response response, QiniuException exce if (checkResponse != null) { int statusCode = checkResponse.statusCode; - if ((statusCode > 300 && statusCode < 400) - || (statusCode >= 400 && statusCode < 500 && statusCode != 400 && statusCode != 406) - || statusCode == 501 || statusCode == 573 - || statusCode == 608 || statusCode == 612 || statusCode == 614 || statusCode == 616 - || statusCode == 619 || statusCode == 630 || statusCode == 631 || statusCode == 640 - || statusCode == 701 - || statusCode < 100) { - return false; - } else { - return true; - } + return (statusCode < 200 || statusCode > 299) && statusCode > -2 + && statusCode != 401 && statusCode != 413 && statusCode != 419 + && statusCode != 608 && statusCode != 614 && statusCode != 630; } - if (exception != null && exception.getCause() != null) { - Throwable e = exception.getCause(); - String msg = e.getMessage(); - return e instanceof UnknownHostException - || (msg != null && msg.indexOf("Broken pipe") == 0) - || e instanceof SocketTimeoutException - || e instanceof java.net.ConnectException - || e instanceof ProtocolException - || e instanceof SSLException; + if (exception == null || !exception.isUnrecoverable()) { + return true; + } else { + return false; } - - return false; } } diff --git a/src/main/java/com/qiniu/storage/ResumeUploadPerformer.java b/src/main/java/com/qiniu/storage/ResumeUploadPerformer.java index 3af831421..0793c55c7 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploadPerformer.java +++ b/src/main/java/com/qiniu/storage/ResumeUploadPerformer.java @@ -1,12 +1,8 @@ package com.qiniu.storage; -import com.google.gson.Gson; -import com.qiniu.common.Constants; import com.qiniu.common.QiniuException; import com.qiniu.http.Client; import com.qiniu.http.Response; -import com.qiniu.util.StringMap; -import com.qiniu.util.StringUtils; import java.io.IOException; @@ -77,51 +73,11 @@ private ResumeUploadSource.Block getNextUploadingBlock() throws QiniuException { try { block = uploadSource.getNextUploadingBlock(); } catch (IOException e) { - throw new QiniuException(e); + throw QiniuException.unrecoverable(e); } return block; } - void recoverUploadProgressFromLocal() { - if (recorder == null || StringUtils.isNullOrEmpty(uploadSource.recordKey)) { - return; - } - - byte[] data = recorder.get(uploadSource.recordKey); - if (data == null) { - return; - } - String jsonString = new String(data, Constants.UTF_8); - ResumeUploadSource source = null; - try { - source = new Gson().fromJson(jsonString, uploadSource.getClass()); - } catch (Exception ignored) { - } - if (source == null) { - return; - } - - boolean isCopy = uploadSource.recoverFromRecordInfo(source); - if (!isCopy) { - removeUploadProgressFromLocal(); - } - } - - void saveUploadProgressToLocal() { - if (recorder == null || StringUtils.isNullOrEmpty(uploadSource.recordKey)) { - return; - } - String dataString = new Gson().toJson(uploadSource); - recorder.set(uploadSource.recordKey, dataString.getBytes(Constants.UTF_8)); - } - - void removeUploadProgressFromLocal() { - if (recorder == null || StringUtils.isNullOrEmpty(uploadSource.recordKey)) { - return; - } - recorder.del(uploadSource.recordKey); - } - private String getUploadHost() throws QiniuException { return configHelper.upHost(token.getToken()); } @@ -133,18 +89,6 @@ private void changeHost(String host) { } } - Response post(String url, byte[] data, int offset, int size) throws QiniuException { - StringMap header = new StringMap(); - header.put("Authorization", "UpToken " + token.getToken()); - return client.post(url, data, offset, size, header, options.mimeType); - } - - Response put(String url, byte[] data, int offset, int size) throws QiniuException { - StringMap header = new StringMap(); - header.put("Authorization", "UpToken " + token.getToken()); - return client.put(url, data, offset, size, header, options.mimeType); - } - Response retryUploadAction(UploadAction action) throws QiniuException { Response response = null; int retryCount = 0; @@ -183,7 +127,7 @@ Response retryUploadAction(UploadAction action) throws QiniuException { retryCount++; if (retryCount >= config.retryMax) { - throw QiniuException.unrecoverable("failed after retry times"); + throw new QiniuException(null, "failed after retry times"); } if (shouldSwitchHost) { diff --git a/src/main/java/com/qiniu/storage/ResumeUploadSource.java b/src/main/java/com/qiniu/storage/ResumeUploadSource.java index 76e18e96d..ee3efc50d 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploadSource.java +++ b/src/main/java/com/qiniu/storage/ResumeUploadSource.java @@ -10,7 +10,6 @@ abstract class ResumeUploadSource { final String recordKey; final int blockSize; - final String targetRegionId; final Configuration.ResumableUploadAPIVersion resumableUploadAPIVersion; transient Configuration config; @@ -22,17 +21,15 @@ abstract class ResumeUploadSource { Long expireAt; ResumeUploadSource() { - this.targetRegionId = null; this.blockSize = 0; this.recordKey = null; this.resumableUploadAPIVersion = Configuration.ResumableUploadAPIVersion.V1; } - ResumeUploadSource(Configuration config, String recordKey, String targetRegionId) { + ResumeUploadSource(Configuration config, String recordKey) { this.config = config; this.blockSize = getBlockSize(config); this.recordKey = recordKey; - this.targetRegionId = targetRegionId; this.resumableUploadAPIVersion = config.resumableUploadAPIVersion; } @@ -91,6 +88,22 @@ ResumeUploadSource.Block getNextUploadingBlock() throws IOException { // 获取文件名 abstract String getFileName(); + // 是否有已上传的数据 + boolean hasUploadData() { + if (blockList == null || blockList.size() == 0) { + return false; + } + + boolean hasUploadData = false; + for (ResumeUploadSource.Block block : blockList) { + if (block.isUploaded()) { + hasUploadData = true; + break; + } + } + return hasUploadData; + } + boolean recoverFromRecordInfo(ResumeUploadSource source) { return false; } diff --git a/src/main/java/com/qiniu/storage/ResumeUploadSourceFile.java b/src/main/java/com/qiniu/storage/ResumeUploadSourceFile.java index 8de5e3164..26e4bce62 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploadSourceFile.java +++ b/src/main/java/com/qiniu/storage/ResumeUploadSourceFile.java @@ -15,8 +15,8 @@ class ResumeUploadSourceFile extends ResumeUploadSource { private final transient File file; private transient RandomAccessFile randomAccessFile; - ResumeUploadSourceFile(File file, Configuration config, String recordKey, String targetRegionId) { - super(config, recordKey, targetRegionId); + ResumeUploadSourceFile(File file, Configuration config, String recordKey) { + super(config, recordKey); this.file = file; this.fileName = file.getName(); this.size = file.length(); @@ -99,12 +99,12 @@ boolean recoverFromRecordInfo(ResumeUploadSource source) { boolean needRecovered = true; if (source.resumableUploadAPIVersion == Configuration.ResumableUploadAPIVersion.V2) { - if (StringUtils.isNullOrEmpty(uploadId)) { + if (StringUtils.isNullOrEmpty(source.uploadId)) { return false; } // 服务端是 7 天，此处有效期少 1 天，为 6 天 long currentTimestamp = new Date().getTime() / 1000; - long expireAtTimestamp = expireAt - 24 * 3600; + long expireAtTimestamp = source.expireAt - 24 * 3600; needRecovered = expireAtTimestamp > currentTimestamp; } @@ -138,10 +138,6 @@ private boolean isSameResource(ResumeUploadSource source) { return false; } - if (sourceFile.targetRegionId == null || !sourceFile.targetRegionId.equals(targetRegionId)) { - return false; - } - return sourceFile.resumableUploadAPIVersion == resumableUploadAPIVersion; } diff --git a/src/main/java/com/qiniu/storage/ResumeUploadSourceStream.java b/src/main/java/com/qiniu/storage/ResumeUploadSourceStream.java index 33fa0f2a2..f173bbac1 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploadSourceStream.java +++ b/src/main/java/com/qiniu/storage/ResumeUploadSourceStream.java @@ -11,8 +11,8 @@ public class ResumeUploadSourceStream extends ResumeUploadSource { private final InputStream inputStream; private final String fileName; - ResumeUploadSourceStream(InputStream inputStream, Configuration config, String recordKey, String targetRegionId, String fileName) { - super(config, recordKey, targetRegionId); + ResumeUploadSourceStream(InputStream inputStream, Configuration config, String recordKey, String fileName) { + super(config, recordKey); this.inputStream = inputStream; this.fileName = fileName; this.blockList = new LinkedList<>(); diff --git a/src/main/java/com/qiniu/storage/ResumeUploader.java b/src/main/java/com/qiniu/storage/ResumeUploader.java index 2bf4e3e84..a2327d938 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploader.java +++ b/src/main/java/com/qiniu/storage/ResumeUploader.java @@ -1,9 +1,15 @@ package com.qiniu.storage; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.qiniu.common.Constants; import com.qiniu.common.QiniuException; import com.qiniu.http.Client; import com.qiniu.http.Response; import com.qiniu.util.StringMap; +import com.qiniu.util.StringUtils; import java.io.File; import java.io.InputStream; @@ -61,7 +67,7 @@ public class ResumeUploader extends BaseUploader { public ResumeUploader(Client client, String upToken, String key, File file, StringMap params, String mime, Recorder recorder, Configuration configuration) { this(client, key, upToken, - new ResumeUploadSourceFile(file, configuration, getRecorderKey(key, file, recorder), getRegionTargetId(upToken, configuration)), + new ResumeUploadSourceFile(file, configuration, getRecorderKey(key, file, recorder)), recorder, new UploadOptions.Builder().params(params).metaData(params).mimeType(mime).build(), configuration); } @@ -112,7 +118,7 @@ public ResumeUploader(Client client, String upToken, String key, InputStream str public ResumeUploader(Client client, String upToken, String key, InputStream stream, String fileName, StringMap params, String mime, Configuration configuration) { this(client, key, upToken, - new ResumeUploadSourceStream(stream, configuration, null, getRegionTargetId(upToken, configuration), fileName), + new ResumeUploadSourceStream(stream, configuration, null, fileName), null, new UploadOptions.Builder().params(params).metaData(params).mimeType(mime).build(), configuration); } @@ -130,7 +136,15 @@ private ResumeUploader(Client client, String key, String upToken, ResumeUploadSo */ public Response upload() throws QiniuException { try { - return super.upload(); + recoverUploadProgressFromLocal(); + Response response = super.upload(); + if (response != null && response.isOK()) { + removeUploadProgressFromLocal(); + } + return response; + } catch (QiniuException e) { + saveUploadProgressToLocal(); + throw e; } finally { close(); } @@ -138,6 +152,7 @@ public Response upload() throws QiniuException { @Override Response uploadFlows() throws QiniuException { + // 检查参数 checkParam(); @@ -149,9 +164,6 @@ Response uploadFlows() throws QiniuException { uploadPerformer = new ResumeUploadPerformerV1(client, key, token, source, recorder, options, config); } - // 恢复本地断点续传数据 - uploadPerformer.recoverUploadProgressFromLocal(); - // 上传数据至服务 - 步骤1 Response response = null; if (uploadPerformer.shouldUploadInit()) { @@ -171,9 +183,6 @@ Response uploadFlows() throws QiniuException { // 上传数据至服务 - 步骤3 response = uploadPerformer.completeUpload(); - if (response.isOK()) { - uploadPerformer.removeUploadProgressFromLocal(); - } return response; } @@ -182,9 +191,6 @@ Response uploadData() throws QiniuException { Response response = null; do { response = uploadPerformer.uploadNextData(); - if (response != null && response.isOK()) { - uploadPerformer.saveUploadProgressToLocal(); - } } while (!uploadPerformer.isAllBlocksUploadingOrUploaded()); return response; } @@ -227,20 +233,78 @@ private static String getRecorderKey(String key, File file, Recorder recorder) { return recorder.recorderKeyGenerate(key, file); } - private static String getRegionTargetId(String upToken, Configuration config) { - if (config == null || upToken == null) { - return null; + // recorder + void recoverUploadProgressFromLocal() { + if (recorder == null || source == null || StringUtils.isNullOrEmpty(source.recordKey)) { + return; } - UploadToken token = null; + byte[] data = recorder.get(source.recordKey); + if (data == null) { + return; + } + + String jsonString = new String(data, Constants.UTF_8); + Region region = null; + ResumeUploadSource uploadSource = null; try { - token = new UploadToken(upToken); - } catch (QiniuException ignored) { + JsonObject jsonObject = (JsonObject) new JsonParser().parse(jsonString); + JsonObject sourceJson = jsonObject.getAsJsonObject("source"); + uploadSource = new Gson().fromJson(sourceJson, source.getClass()); + + JsonObject regionJson = jsonObject.getAsJsonObject("region"); + region = new Gson().fromJson(regionJson, Region.class); + } catch (Exception e) { + e.printStackTrace(); } - if (token == null || !token.isValid()) { - return null; + if (uploadSource == null || region == null) { + removeUploadProgressFromLocal(); + return; + } + + boolean isCopy = source.recoverFromRecordInfo(uploadSource); + if (!isCopy) { + removeUploadProgressFromLocal(); + return; + } + + if (config.region == null) { + config.region = region; + } else { + RegionGroup regionGroup = new RegionGroup(); + regionGroup.addRegion(region); + regionGroup.addRegion(config.region); + config.region = regionGroup; + } + } + + void saveUploadProgressToLocal() { + if (recorder == null || source == null || !source.hasUploadData() || StringUtils.isNullOrEmpty(source.recordKey)) { + return; + } + try { + JsonObject jsonObject = new JsonObject(); + JsonElement sourceJson = new Gson().toJsonTree(source); + if (sourceJson == null) { + return; + } + jsonObject.add("source", sourceJson); + JsonElement regionJson = new Gson().toJsonTree(config.region.getCurrentRegion(new UploadToken(upToken))); + if (regionJson == null) { + return; + } + jsonObject.add("region", regionJson); + String dataString = jsonObject.toString(); + recorder.set(source.recordKey, dataString.getBytes(Constants.UTF_8)); + } catch (Exception ignored) { + } + } + + void removeUploadProgressFromLocal() { + if (recorder == null || source == null || StringUtils.isNullOrEmpty(source.recordKey)) { + return; } - return config.region.getRegion(token); + recorder.del(source.recordKey); } } diff --git a/src/test/java/test/com/qiniu/storage/RecordUploadTest.java b/src/test/java/test/com/qiniu/storage/RecordUploadTest.java index 07dc7e5cd..eff21b298 100644 --- a/src/test/java/test/com/qiniu/storage/RecordUploadTest.java +++ b/src/test/java/test/com/qiniu/storage/RecordUploadTest.java @@ -74,7 +74,7 @@ private void template(final int size, boolean isResumeV2, boolean isConcurrent) final Complete complete = new Complete(); try { - final String token = TestConfig.testAuth.uploadToken(bucket, expectKey); + final String token = TestConfig.testAuth.uploadToken(bucket, expectKey, 3600, null); final String recordKey = recorder.recorderKeyGenerate(expectKey, f); // 开始第一部分上传 @@ -110,7 +110,7 @@ public void run() { if (f.length() > Constants.BLOCK_SIZE) { // 终止第一部分上传,期望其部分成功 - for (int i = 150; i > 0; --i) { + for (int i = 15; i > 0; --i) { byte[] data = getRecord(recorder, recordKey); if (data != null) { doSleep(1000); diff --git a/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java b/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java new file mode 100644 index 000000000..2c98deb44 --- /dev/null +++ b/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java @@ -0,0 +1,239 @@ +package test.com.qiniu.storage; + +import com.qiniu.common.Constants; +import com.qiniu.common.QiniuException; +import com.qiniu.http.Client; +import com.qiniu.http.Response; +import com.qiniu.storage.*; +import com.qiniu.storage.model.FileInfo; +import com.qiniu.util.Etag; +import com.qiniu.util.Md5; +import com.qiniu.util.StringMap; +import org.junit.Test; +import test.com.qiniu.TempFile; +import test.com.qiniu.TestConfig; + +import java.io.File; +import java.io.IOException; +import java.net.URLEncoder; +import java.util.Date; + +import static org.junit.Assert.*; + +public class SwitchRegionTest { + + private static boolean[][] testConfigList = { + // isHttps, isResumeV2, isConcurrent +// {false, true, false}, +// {false, true, true}, +// {false, false, false}, + {false, false, true}, + {true, true, false}, + {true, true, true}, + {true, false, false}, + {true, false, true}, + }; + + /** + * 分片上传 + * 检测key、hash、fszie、fname是否符合预期 + * + * @param size 文件大小 + * @param isHttps 是否采用 https 方式, 反之为 http + * @param isResumeV2 是否使用分片上传 api v2, 反之为 v1 + * @param isConcurrent 是否采用并发方式上传 + * @throws IOException + */ + private void template(int size, boolean isHttps, boolean isResumeV2, boolean isConcurrent) throws IOException { + TestConfig.TestFile[] files = TestConfig.getTestFileArray(); + for (TestConfig.TestFile file : files) { + // 雾存储不支持 v1 + if (file.isFog() && !isResumeV2) { + continue; + } + String bucket = file.getBucketName(); + + Region mockRegion = new Region.Builder() + .region("custom") + .srcUpHost("mock.src.host.com") + .accUpHost("mock.acc.host.com") + .build(); + Region region = file.getRegion(); + RegionGroup regionGroup = new RegionGroup(); + regionGroup.addRegion(mockRegion); + regionGroup.addRegion(region); + + Configuration config = new Configuration(regionGroup); + if (isResumeV2) { + config.resumableUploadAPIVersion = Configuration.ResumableUploadAPIVersion.V2; + config.resumableUploadAPIV2BlockSize = 2 * 1024 * 1024; + } + + config.useHttpsDomains = isHttps; + String key = "switch_region_"; + key += isHttps ? "_https" : "_http"; + key += isResumeV2 ? "_resumeV2" : "_resumeV1"; + key += isConcurrent ? "_concurrent" : "_serial"; + key += "_" + new Date().getTime(); + final String expectKey = "\r\n?&r=" + size + "k" + key; + final File f = TempFile.createFile(size); + final String fooKey = "foo"; + final String fooValue = "fooValue"; + final String metaDataKey = "metaDataKey"; + final String metaDataValue = "metaDataValue"; + final String returnBody = "{\"key\":\"$(key)\",\"hash\":\"$(etag)\",\"fsize\":\"$(fsize)\"" + + ",\"fname\":\"$(fname)\",\"mimeType\":\"$(mimeType)\",\"foo\":\"$(x:foo)\"}"; + String token = TestConfig.testAuth.uploadToken(bucket, expectKey, 3600, + new StringMap().put("returnBody", returnBody)); + + System.out.printf("\r\nkey:%s zone:%s\n", expectKey, regionGroup); + + + StringMap param = new StringMap(); + param.put("x:" + fooKey, fooValue); + param.put("x-qn-meta-" + metaDataKey, metaDataValue); + try { + ResumeUploader up = null; + if (!isConcurrent) { + up = new ResumeUploader(new Client(), token, expectKey, f, param, null, null, config); + } else { + config.resumableUploadMaxConcurrentTaskCount = 3; + up = new ConcurrentResumeUploader(new Client(), token, expectKey, f, param, null, null, config); + } + Response r = up.upload(); + assertTrue(r + "", r.isOK()); + + StringMap ret = r.jsonToMap(); + assertEquals(expectKey, ret.get("key")); + assertEquals(f.getName(), ret.get("fname")); + assertEquals(String.valueOf(f.length()), ret.get("fsize").toString()); + assertEquals(fooValue, ret.get(fooKey).toString()); + + boolean checkMd5 = false; + if (config.resumableUploadAPIVersion == Configuration.ResumableUploadAPIVersion.V2 + && config.resumableUploadAPIV2BlockSize != Constants.BLOCK_SIZE) { + checkMd5 = true; + } + if (checkMd5) { + if (file.isFog()) { + String md5 = Md5.md5(f); + String serverMd5 = getFileMD5(file.getTestDomain(), expectKey); + System.out.println(" md5:" + md5); + System.out.println("serverMd5:" + serverMd5); + assertNotNull(serverMd5); + assertEquals(md5, serverMd5); + } + } else { + final String etag = Etag.file(f); + System.out.println(" etag:" + etag); + System.out.println("serverEtag:" + ret.get("hash")); + assertNotNull(ret.get("hash")); + assertEquals(etag, ret.get("hash")); + } + + FileInfo fileInfo = getFileInfo(config, bucket, expectKey); + assertEquals(fileInfo + "", metaDataValue, fileInfo.meta.get(metaDataKey).toString()); + } catch (QiniuException e) { + assertEquals("", e.response == null ? e + "e.response is null" : e.response.bodyString()); + fail(); + } + TempFile.remove(f); + } + } + + private String getFileMD5(String bucketDomain, String key) { + String url = "http://" + bucketDomain + "/" + URLEncoder.encode(key) + "?qhash/md5"; + Client client = new Client(); + + String md5 = null; + try { + Response response = client.get(url); + StringMap data = response.jsonToMap(); + md5 = data.get("hash").toString(); + } catch (QiniuException e) { + e.printStackTrace(); + } + return md5; + } + + private FileInfo getFileInfo(Configuration cfg, String bucket, String key) { + BucketManager manager = new BucketManager(TestConfig.testAuth, cfg); + try { + return manager.stat(bucket, key); + } catch (QiniuException e) { + e.printStackTrace(); + } + return null; + } + + @Test + public void test600k() throws Throwable { + for (boolean[] config : testConfigList) { + template(600, config[0], config[1], config[2]); + } + } + + @Test + public void test3M() throws Throwable { + if (TestConfig.isTravis()) { + return; + } + for (boolean[] config : testConfigList) { + template(1024 * 3, config[0], config[1], config[2]); + } + } + + @Test + public void test5M() throws Throwable { + if (TestConfig.isTravis()) { + return; + } + for (boolean[] config : testConfigList) { + template(1024 * 4, config[0], config[1], config[2]); + } + } + + @Test + public void test8M() throws Throwable { + for (boolean[] config : testConfigList) { + template(1024 * 8, config[0], config[1], config[2]); + } + } + + @Test + public void test8M1k() throws Throwable { + for (boolean[] config : testConfigList) { + template(1024 * 8 + 1, config[0], config[1], config[2]); + } + } + + @Test + public void test10M() throws Throwable { + for (boolean[] config : testConfigList) { + template(1024 * 10, config[0], config[1], config[2]); + } + } + + @Test + public void test20M() throws Throwable { + for (boolean[] config : testConfigList) { + template(1024 * 20, config[0], config[1], config[2]); + } + } + + @Test + public void test20M1K() throws Throwable { + for (boolean[] config : testConfigList) { + template(1024 * 20 + 1, config[0], config[1], config[2]); + } + } + + + class MyRet { + public String hash; + public String key; + public String fsize; + public String fname; + public String mimeType; + } +} From 30b59f89b4f865f1bbae4ee73bb0efb31de39f60 Mon Sep 17 00:00:00 2001 From: yangsen Date: Wed, 19 May 2021 17:56:21 +0800 Subject: [PATCH 04/16] switch region case add form --- .../com/qiniu/storage/SwitchRegionTest.java | 89 +++++++++++++------ 1 file changed, 60 insertions(+), 29 deletions(-) diff --git a/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java b/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java index 2c98deb44..e1abbb88b 100644 --- a/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java +++ b/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java @@ -22,33 +22,48 @@ public class SwitchRegionTest { - private static boolean[][] testConfigList = { - // isHttps, isResumeV2, isConcurrent -// {false, true, false}, -// {false, true, true}, -// {false, false, false}, - {false, false, true}, - {true, true, false}, - {true, true, true}, - {true, false, false}, - {true, false, true}, + private static final int httpType = 0; + private static final int httpsType = 1; + + private static final int resumableV1Type = 0; + private static final int resumableV2Type = 1; + + private static final int formType = 0; + private static final int serialType = 1; + private static final int concurrentType = 2; + + private static int[][] testConfigList = { + {httpType, -1, formType}, + {httpType, resumableV1Type, serialType}, + {httpType, resumableV1Type, concurrentType}, + {httpType, resumableV2Type, serialType}, + {httpType, resumableV2Type, concurrentType}, + + {httpsType, -1, formType}, + {httpsType, resumableV1Type, serialType}, + {httpsType, resumableV1Type, concurrentType}, + {httpsType, resumableV2Type, serialType}, + {httpsType, resumableV2Type, concurrentType}, }; /** * 分片上传 * 检测key、hash、fszie、fname是否符合预期 * - * @param size 文件大小 - * @param isHttps 是否采用 https 方式, 反之为 http - * @param isResumeV2 是否使用分片上传 api v2, 反之为 v1 - * @param isConcurrent 是否采用并发方式上传 + * @param size 文件大小 + * @param httpType 采用 http / https 方式 + * @param uploadType 使用 form / 分片上传 api v1/v2 + * @param uploadStyle 采用串行或者并发方式上传 * @throws IOException */ - private void template(int size, boolean isHttps, boolean isResumeV2, boolean isConcurrent) throws IOException { + private void template(int size, int httpType, int uploadType, int uploadStyle) throws IOException { + boolean isHttps = httpType == httpsType; + boolean isConcurrent = uploadStyle == concurrentType; + TestConfig.TestFile[] files = TestConfig.getTestFileArray(); for (TestConfig.TestFile file : files) { // 雾存储不支持 v1 - if (file.isFog() && !isResumeV2) { + if (file.isFog() && uploadType == resumableV1Type) { continue; } String bucket = file.getBucketName(); @@ -64,7 +79,7 @@ private void template(int size, boolean isHttps, boolean isResumeV2, boolean isC regionGroup.addRegion(region); Configuration config = new Configuration(regionGroup); - if (isResumeV2) { + if (uploadType == resumableV2Type) { config.resumableUploadAPIVersion = Configuration.ResumableUploadAPIVersion.V2; config.resumableUploadAPIV2BlockSize = 2 * 1024 * 1024; } @@ -72,8 +87,21 @@ private void template(int size, boolean isHttps, boolean isResumeV2, boolean isC config.useHttpsDomains = isHttps; String key = "switch_region_"; key += isHttps ? "_https" : "_http"; - key += isResumeV2 ? "_resumeV2" : "_resumeV1"; - key += isConcurrent ? "_concurrent" : "_serial"; + + if (uploadType == resumableV1Type) { + key += "_resumeV1"; + } else if (uploadType == resumableV2Type) { + key += "_resumeV2"; + } + + if (uploadStyle == formType) { + key += "_form"; + } else if (uploadStyle == serialType) { + key += "_serial"; + } else if (uploadStyle == concurrentType) { + key += "_concurrent"; + } + key += "_" + new Date().getTime(); final String expectKey = "\r\n?&r=" + size + "k" + key; final File f = TempFile.createFile(size); @@ -93,13 +121,16 @@ private void template(int size, boolean isHttps, boolean isResumeV2, boolean isC param.put("x:" + fooKey, fooValue); param.put("x-qn-meta-" + metaDataKey, metaDataValue); try { - ResumeUploader up = null; - if (!isConcurrent) { + BaseUploader up = null; + if (uploadStyle == formType) { + up = new FormUploader(new Client(), token, expectKey, f, param, Client.DefaultMime, false, config); + } else if (uploadStyle == serialType) { up = new ResumeUploader(new Client(), token, expectKey, f, param, null, null, config); } else { config.resumableUploadMaxConcurrentTaskCount = 3; up = new ConcurrentResumeUploader(new Client(), token, expectKey, f, param, null, null, config); } + Response r = up.upload(); assertTrue(r + "", r.isOK()); @@ -168,7 +199,7 @@ private FileInfo getFileInfo(Configuration cfg, String bucket, String key) { @Test public void test600k() throws Throwable { - for (boolean[] config : testConfigList) { + for (int[] config : testConfigList) { template(600, config[0], config[1], config[2]); } } @@ -178,7 +209,7 @@ public void test3M() throws Throwable { if (TestConfig.isTravis()) { return; } - for (boolean[] config : testConfigList) { + for (int[] config : testConfigList) { template(1024 * 3, config[0], config[1], config[2]); } } @@ -188,42 +219,42 @@ public void test5M() throws Throwable { if (TestConfig.isTravis()) { return; } - for (boolean[] config : testConfigList) { + for (int[] config : testConfigList) { template(1024 * 4, config[0], config[1], config[2]); } } @Test public void test8M() throws Throwable { - for (boolean[] config : testConfigList) { + for (int[] config : testConfigList) { template(1024 * 8, config[0], config[1], config[2]); } } @Test public void test8M1k() throws Throwable { - for (boolean[] config : testConfigList) { + for (int[] config : testConfigList) { template(1024 * 8 + 1, config[0], config[1], config[2]); } } @Test public void test10M() throws Throwable { - for (boolean[] config : testConfigList) { + for (int[] config : testConfigList) { template(1024 * 10, config[0], config[1], config[2]); } } @Test public void test20M() throws Throwable { - for (boolean[] config : testConfigList) { + for (int[] config : testConfigList) { template(1024 * 20, config[0], config[1], config[2]); } } @Test public void test20M1K() throws Throwable { - for (boolean[] config : testConfigList) { + for (int[] config : testConfigList) { template(1024 * 20 + 1, config[0], config[1], config[2]); } } From 7130d6c6f24cfaa6de2f1d1d19ceb592f6a749ae Mon Sep 17 00:00:00 2001 From: yangsen Date: Wed, 19 May 2021 19:03:17 +0800 Subject: [PATCH 05/16] check region valid --- src/main/java/com/qiniu/storage/AutoRegion.java | 7 ++++++- src/main/java/com/qiniu/storage/BaseUploader.java | 6 +----- src/main/java/com/qiniu/storage/RegionGroup.java | 9 +++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/qiniu/storage/AutoRegion.java b/src/main/java/com/qiniu/storage/AutoRegion.java index a4cf00f88..93075bc3d 100644 --- a/src/main/java/com/qiniu/storage/AutoRegion.java +++ b/src/main/java/com/qiniu/storage/AutoRegion.java @@ -135,7 +135,7 @@ private RegionGroup queryRegionInfo(String accessKey, String bucket) throws Qini Exception ex = null; // 隔一段时间重新获取 uc 信息 // || regionGroup.createTime < System.currentTimeMillis() - 1000 * 3600 * 8 - if (regionGroup == null) { + if (regionGroup == null || !regionGroup.isValid()) { for (int i = 0; i < 2; i++) { try { UCRet ret = getRegionJson(index); @@ -190,6 +190,11 @@ String getRegion(RegionReqInfo regionReqInfo) { } } + @Override + boolean isValid() { + return true; + } + Region getCurrentRegion(RegionReqInfo regionReqInfo) { try { RegionGroup regionGroup = queryRegionInfo(regionReqInfo); diff --git a/src/main/java/com/qiniu/storage/BaseUploader.java b/src/main/java/com/qiniu/storage/BaseUploader.java index 91431b145..9b4c19fca 100644 --- a/src/main/java/com/qiniu/storage/BaseUploader.java +++ b/src/main/java/com/qiniu/storage/BaseUploader.java @@ -56,10 +56,6 @@ private boolean couldSwitchRegionAndRetry(Response response, QiniuException exce && statusCode != 608 && statusCode != 614 && statusCode != 630; } - if (exception == null || !exception.isUnrecoverable()) { - return true; - } else { - return false; - } + return exception == null || !exception.isUnrecoverable(); } } diff --git a/src/main/java/com/qiniu/storage/RegionGroup.java b/src/main/java/com/qiniu/storage/RegionGroup.java index b20d701f4..c66091d07 100644 --- a/src/main/java/com/qiniu/storage/RegionGroup.java +++ b/src/main/java/com/qiniu/storage/RegionGroup.java @@ -107,6 +107,15 @@ Region getCurrentRegion(RegionReqInfo regionReqInfo) { } } + @Override + boolean isValid() { + if (currentRegion == null) { + return false; + } + // 只判断当前的 + return currentRegion.isValid(); + } + private void updateCurrentRegion() { if (regionList.size() == 0) { return; From 4ae3983db13f1d947b0d183dae5706762cf9db9b Mon Sep 17 00:00:00 2001 From: yangsen Date: Wed, 19 May 2021 19:07:24 +0800 Subject: [PATCH 06/16] switch region check region valid --- src/main/java/com/qiniu/storage/RegionGroup.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/qiniu/storage/RegionGroup.java b/src/main/java/com/qiniu/storage/RegionGroup.java index c66091d07..8d7d65301 100644 --- a/src/main/java/com/qiniu/storage/RegionGroup.java +++ b/src/main/java/com/qiniu/storage/RegionGroup.java @@ -28,7 +28,7 @@ public boolean addRegion(Region region) { @Override boolean switchRegion(RegionReqInfo regionReqInfo) { - if (currentRegion != null && currentRegion.switchRegion(regionReqInfo)) { + if (currentRegion != null && currentRegion.isValid() && currentRegion.switchRegion(regionReqInfo)) { return true; } From 25b65a2fd622ae3b90211e245ab2a466702074b6 Mon Sep 17 00:00:00 2001 From: yangsen Date: Mon, 24 May 2021 16:14:04 +0800 Subject: [PATCH 07/16] delete useless code --- src/main/java/com/qiniu/storage/AutoRegion.java | 11 ----------- src/test/java/test/com/qiniu/TestConfig.java | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/main/java/com/qiniu/storage/AutoRegion.java b/src/main/java/com/qiniu/storage/AutoRegion.java index 93075bc3d..202dae156 100644 --- a/src/main/java/com/qiniu/storage/AutoRegion.java +++ b/src/main/java/com/qiniu/storage/AutoRegion.java @@ -35,17 +35,6 @@ class AutoRegion extends Region { this.ucServer = ucServer; this.client = new Client(); this.regions = new ConcurrentHashMap<>(); -// this.inferDomainsMap = new ConcurrentHashMap<>(); -// this.inferDomainsMap.put("up.qiniup.com", region0()); -// this.inferDomainsMap.put("up-jjh.qiniup.com", region0()); -// this.inferDomainsMap.put("up-xs.qiniup.com", region0()); -// this.inferDomainsMap.put("up-z1.qiniup.com", region1()); -// this.inferDomainsMap.put("up-z2.qiniup.com", region2()); -// this.inferDomainsMap.put("up-dg.qiniup.com", region2()); -// this.inferDomainsMap.put("up-fs.qiniup.com", region2()); -// this.inferDomainsMap.put("up-na0.qiniup.com", regionNa0()); -// this.inferDomainsMap.put("up-as0.qiniup.com", regionAs0()); -// this.inferDomainsMap.put("up-fog-cn-east-1.qiniup.com", regionFogCnEast1()); } /** diff --git a/src/test/java/test/com/qiniu/TestConfig.java b/src/test/java/test/com/qiniu/TestConfig.java index 97db3b786..e26cc40b3 100644 --- a/src/test/java/test/com/qiniu/TestConfig.java +++ b/src/test/java/test/com/qiniu/TestConfig.java @@ -135,7 +135,7 @@ public static TestFile[] getTestFileArray(String fileSaveKey, String fileMimeTyp return new TestFile[]{na0}; } else { // return new TestFile[]{fog, fog1, z0, na0}; - return new TestFile[]{z0_auto}; + return new TestFile[]{na0, z0_auto}; } } From e7dd7462ec6621fca2f07919c77ae45789f16d71 Mon Sep 17 00:00:00 2001 From: yangsen Date: Mon, 24 May 2021 16:16:45 +0800 Subject: [PATCH 08/16] static region id list --- src/main/java/com/qiniu/storage/AutoRegion.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/qiniu/storage/AutoRegion.java b/src/main/java/com/qiniu/storage/AutoRegion.java index 202dae156..8d19ebf47 100644 --- a/src/main/java/com/qiniu/storage/AutoRegion.java +++ b/src/main/java/com/qiniu/storage/AutoRegion.java @@ -16,6 +16,11 @@ class AutoRegion extends Region { */ private final String ucServer; + /** + * region id 列表 + */ + private final static String[] regionIdList = new String[]{"z0", "z1", "z2", "na0", "as0", "fog-cn-east-1"}; + /** * 空间机房，域名信息缓存 */ @@ -31,6 +36,7 @@ class AutoRegion extends Region { */ private Client client; + AutoRegion(String ucServer) { this.ucServer = ucServer; this.client = new Client(); @@ -95,7 +101,6 @@ static RegionGroup regionGroup(UCRet ret) { // 根据 iovipHost 反推 regionId String regionId = iovipHost; if (iovipHost != null) { - String[] regionIdList = new String[]{"z0", "z1", "z2", "na0", "as0", "fog-cn-east-1"}; for (String regionIdP : regionIdList) { if (iovipHost.contains("-" + regionIdP)) { regionId = regionIdP; From c7dd361d5112680d778bb85c7b508d19f8c07e37 Mon Sep 17 00:00:00 2001 From: yangsen Date: Mon, 24 May 2021 16:33:18 +0800 Subject: [PATCH 09/16] base uploader check config --- src/main/java/com/qiniu/storage/BaseUploader.java | 11 +++++++++-- src/main/java/com/qiniu/storage/Region.java | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/qiniu/storage/BaseUploader.java b/src/main/java/com/qiniu/storage/BaseUploader.java index 9b4c19fca..e566524c1 100644 --- a/src/main/java/com/qiniu/storage/BaseUploader.java +++ b/src/main/java/com/qiniu/storage/BaseUploader.java @@ -16,11 +16,18 @@ public abstract class BaseUploader { this.client = client; this.key = key; this.upToken = upToken; - this.config = config; - this.configHelper = new ConfigHelper(config); + if (config == null) { + this.config = new Configuration(); + } else { + this.config = config.clone(); + } + this.configHelper = new ConfigHelper(this.config); } public Response upload() throws QiniuException { + if (this.config == null) { + throw QiniuException.unrecoverable("config can't be empty"); + } return uploadWithRegionRetry(); } diff --git a/src/main/java/com/qiniu/storage/Region.java b/src/main/java/com/qiniu/storage/Region.java index 11679af55..f5d55cd06 100644 --- a/src/main/java/com/qiniu/storage/Region.java +++ b/src/main/java/com/qiniu/storage/Region.java @@ -7,7 +7,7 @@ public class Region { - // 有效期，过了有效期，region会无效，此处只在取时缓存判断； -1 为无限期 + // 有效时间戳，过了有效期，region会无效，此处只在取时缓存判断； -1 为无限期 private long timestamp = -1; // 区域名称：z0 华东 z1 华北 z2 华南 na0 北美 as0 东南亚 private String region = "z0"; From 824f619bfa568d52dafbc32797cee29e371075dd Mon Sep 17 00:00:00 2001 From: yangsen Date: Mon, 24 May 2021 16:40:48 +0800 Subject: [PATCH 10/16] static before final --- src/main/java/com/qiniu/storage/AutoRegion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/qiniu/storage/AutoRegion.java b/src/main/java/com/qiniu/storage/AutoRegion.java index 8d19ebf47..7f24020e9 100644 --- a/src/main/java/com/qiniu/storage/AutoRegion.java +++ b/src/main/java/com/qiniu/storage/AutoRegion.java @@ -19,7 +19,7 @@ class AutoRegion extends Region { /** * region id 列表 */ - private final static String[] regionIdList = new String[]{"z0", "z1", "z2", "na0", "as0", "fog-cn-east-1"}; + private static final String[] regionIdList = new String[]{"z0", "z1", "z2", "na0", "as0", "fog-cn-east-1"}; /** * 空间机房，域名信息缓存 From 4a4c19809c136521dc9e630da52d422220b22cb1 Mon Sep 17 00:00:00 2001 From: yangsen Date: Tue, 25 May 2021 12:02:23 +0800 Subject: [PATCH 11/16] reload source --- .../java/com/qiniu/storage/BaseUploader.java | 12 ++++++++-- .../java/com/qiniu/storage/FormUploader.java | 10 ++++++++ .../com/qiniu/storage/ResumeUploadSource.java | 24 ++++++++++++++++--- .../qiniu/storage/ResumeUploadSourceFile.java | 10 ++++++++ .../com/qiniu/storage/ResumeUploader.java | 15 ++++++++++++ 5 files changed, 66 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/qiniu/storage/BaseUploader.java b/src/main/java/com/qiniu/storage/BaseUploader.java index e566524c1..e49946588 100644 --- a/src/main/java/com/qiniu/storage/BaseUploader.java +++ b/src/main/java/com/qiniu/storage/BaseUploader.java @@ -36,11 +36,15 @@ private Response uploadWithRegionRetry() throws QiniuException { while (true) { try { response = uploadFlows(); - if (!couldSwitchRegionAndRetry(response, null) || config.region == null || !config.region.switchRegion(new UploadToken(upToken))) { + if (!couldSwitchRegionAndRetry(response, null) || + !couldReloadSource() || !reloadSource() || + config.region == null || !config.region.switchRegion(new UploadToken(upToken))) { break; } } catch (QiniuException e) { - if (!couldSwitchRegionAndRetry(null, e) || config.region == null || !config.region.switchRegion(new UploadToken(upToken))) { + if (!couldSwitchRegionAndRetry(null, e) || + !couldReloadSource() || !reloadSource() || + config.region == null || !config.region.switchRegion(new UploadToken(upToken))) { throw e; } } @@ -50,6 +54,10 @@ private Response uploadWithRegionRetry() throws QiniuException { abstract Response uploadFlows() throws QiniuException; + abstract boolean couldReloadSource(); + + abstract boolean reloadSource(); + private boolean couldSwitchRegionAndRetry(Response response, QiniuException exception) { Response checkResponse = response; if (checkResponse == null && exception != null) { diff --git a/src/main/java/com/qiniu/storage/FormUploader.java b/src/main/java/com/qiniu/storage/FormUploader.java index 180f5a7b9..d0c7fbf38 100644 --- a/src/main/java/com/qiniu/storage/FormUploader.java +++ b/src/main/java/com/qiniu/storage/FormUploader.java @@ -105,6 +105,16 @@ public void complete(Response res) { }); } + @Override + boolean couldReloadSource() { + return true; + } + + @Override + boolean reloadSource() { + return true; + } + private void changeHost(String upToken, String host) { try { configHelper.tryChangeUpHost(upToken, host); diff --git a/src/main/java/com/qiniu/storage/ResumeUploadSource.java b/src/main/java/com/qiniu/storage/ResumeUploadSource.java index ee3efc50d..8fe667b6d 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploadSource.java +++ b/src/main/java/com/qiniu/storage/ResumeUploadSource.java @@ -64,6 +64,20 @@ boolean isAllBlocksUploaded() { return isAllBlockUploaded; } + boolean couldReload() { + return false; + } + + boolean reload() { + return false; + } + + void clearState() { + for (ResumeUploadSource.Block block : blockList) { + block.clearState(); + } + } + // 获取下一个需要上传的块 ResumeUploadSource.Block getNextUploadingBlock() throws IOException { ResumeUploadSource.Block block = null; @@ -178,9 +192,7 @@ static class Block { this.offset = offset; this.size = blockSize; this.index = index; - this.isUploading = false; - this.etag = null; - this.context = null; + this.clearState(); } boolean isUploaded() { @@ -196,5 +208,11 @@ boolean isUploaded() { } return isUploaded; } + + void clearState() { + isUploading = false; + this.etag = null; + this.context = null; + } } } diff --git a/src/main/java/com/qiniu/storage/ResumeUploadSourceFile.java b/src/main/java/com/qiniu/storage/ResumeUploadSourceFile.java index 26e4bce62..884f7ffcd 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploadSourceFile.java +++ b/src/main/java/com/qiniu/storage/ResumeUploadSourceFile.java @@ -52,6 +52,16 @@ String getFileName() { return fileName; } + @Override + boolean couldReload() { + return true; + } + + @Override + boolean reload() { + return true; + } + @Override ResumeUploadSource.Block getNextUploadingBlock() throws IOException { ResumeUploadSource.Block block = super.getNextUploadingBlock(); diff --git a/src/main/java/com/qiniu/storage/ResumeUploader.java b/src/main/java/com/qiniu/storage/ResumeUploader.java index a2327d938..354ab43de 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploader.java +++ b/src/main/java/com/qiniu/storage/ResumeUploader.java @@ -195,6 +195,21 @@ Response uploadData() throws QiniuException { return response; } + @Override + boolean couldReloadSource() { + return source.couldReload(); + } + + @Override + boolean reloadSource() { + if (source.reload()) { + source.clearState(); + return true; + } else { + return false; + } + } + private void close() { try { source.close(); From 678bc3e81897aa19e7d0923c933404cfc1631622 Mon Sep 17 00:00:00 2001 From: yangsen Date: Tue, 25 May 2021 14:05:05 +0800 Subject: [PATCH 12/16] code style --- src/main/java/com/qiniu/storage/BaseUploader.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/qiniu/storage/BaseUploader.java b/src/main/java/com/qiniu/storage/BaseUploader.java index e49946588..8d7c10d2f 100644 --- a/src/main/java/com/qiniu/storage/BaseUploader.java +++ b/src/main/java/com/qiniu/storage/BaseUploader.java @@ -36,15 +36,15 @@ private Response uploadWithRegionRetry() throws QiniuException { while (true) { try { response = uploadFlows(); - if (!couldSwitchRegionAndRetry(response, null) || - !couldReloadSource() || !reloadSource() || - config.region == null || !config.region.switchRegion(new UploadToken(upToken))) { + if (!couldSwitchRegionAndRetry(response, null) + || !couldReloadSource() || !reloadSource() + || config.region == null || !config.region.switchRegion(new UploadToken(upToken))) { break; } } catch (QiniuException e) { - if (!couldSwitchRegionAndRetry(null, e) || - !couldReloadSource() || !reloadSource() || - config.region == null || !config.region.switchRegion(new UploadToken(upToken))) { + if (!couldSwitchRegionAndRetry(null, e) + || !couldReloadSource() || !reloadSource() + || config.region == null || !config.region.switchRegion(new UploadToken(upToken))) { throw e; } } From 984275d9ee34c47124d4d3b725e31948cca43057 Mon Sep 17 00:00:00 2001 From: yangsen Date: Tue, 25 May 2021 14:50:09 +0800 Subject: [PATCH 13/16] optimize switch region logic --- src/main/java/com/qiniu/storage/BaseUploader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/qiniu/storage/BaseUploader.java b/src/main/java/com/qiniu/storage/BaseUploader.java index 8d7c10d2f..cf9e407d1 100644 --- a/src/main/java/com/qiniu/storage/BaseUploader.java +++ b/src/main/java/com/qiniu/storage/BaseUploader.java @@ -66,9 +66,9 @@ private boolean couldSwitchRegionAndRetry(Response response, QiniuException exce if (checkResponse != null) { int statusCode = checkResponse.statusCode; - return (statusCode < 200 || statusCode > 299) && statusCode > -2 + return (statusCode > -2 && statusCode < 200) || (statusCode > 299 && statusCode != 401 && statusCode != 413 && statusCode != 419 - && statusCode != 608 && statusCode != 614 && statusCode != 630; + && statusCode != 608 && statusCode != 614 && statusCode != 630); } return exception == null || !exception.isUnrecoverable(); From e42bfe9ebfaa1f557cc8a253163fa42f34a9f0a6 Mon Sep 17 00:00:00 2001 From: yangsen Date: Fri, 28 May 2021 10:38:32 +0800 Subject: [PATCH 14/16] change get regionid logic & add switch region test --- .../java/com/qiniu/storage/AutoRegion.java | 22 +--- .../com/qiniu/storage/ResumeUploadSource.java | 5 +- src/test/java/test/com/qiniu/TestConfig.java | 4 + .../com/qiniu/storage/SwitchRegionTest.java | 114 ++++++++++++++++++ 4 files changed, 126 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/qiniu/storage/AutoRegion.java b/src/main/java/com/qiniu/storage/AutoRegion.java index 7f24020e9..54524ca9e 100644 --- a/src/main/java/com/qiniu/storage/AutoRegion.java +++ b/src/main/java/com/qiniu/storage/AutoRegion.java @@ -16,21 +16,11 @@ class AutoRegion extends Region { */ private final String ucServer; - /** - * region id 列表 - */ - private static final String[] regionIdList = new String[]{"z0", "z1", "z2", "na0", "as0", "fog-cn-east-1"}; - /** * 空间机房，域名信息缓存 */ private Map regions; - /** - * 根据API返回的上传域名推导出其他资源管理域名 - */ - private Map inferDomainsMap; - /** * 定义HTTP请求管理相关方法 */ @@ -99,14 +89,9 @@ static RegionGroup regionGroup(UCRet ret) { } // 根据 iovipHost 反推 regionId - String regionId = iovipHost; - if (iovipHost != null) { - for (String regionIdP : regionIdList) { - if (iovipHost.contains("-" + regionIdP)) { - regionId = regionIdP; - break; - } - } + String regionId = host.region; + if (regionId == null) { + regionId = ""; } Region region = new Region(timestamp, regionId, srcUpHosts, accUpHosts, iovipHost, rsHost, rsfHost, apiHost, ucHost); @@ -277,6 +262,7 @@ private class UCRet { private class HostRet { long ttl; + String region; HostInfoRet up; HostInfoRet rs; HostInfoRet rsf; diff --git a/src/main/java/com/qiniu/storage/ResumeUploadSource.java b/src/main/java/com/qiniu/storage/ResumeUploadSource.java index 8fe667b6d..33fb96323 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploadSource.java +++ b/src/main/java/com/qiniu/storage/ResumeUploadSource.java @@ -76,6 +76,8 @@ void clearState() { for (ResumeUploadSource.Block block : blockList) { block.clearState(); } + uploadId = null; + expireAt = null; } // 获取下一个需要上传的块 @@ -210,9 +212,10 @@ boolean isUploaded() { } void clearState() { - isUploading = false; + this.isUploading = false; this.etag = null; this.context = null; + this.data = null; } } } diff --git a/src/test/java/test/com/qiniu/TestConfig.java b/src/test/java/test/com/qiniu/TestConfig.java index e26cc40b3..95dc30fac 100644 --- a/src/test/java/test/com/qiniu/TestConfig.java +++ b/src/test/java/test/com/qiniu/TestConfig.java @@ -21,6 +21,10 @@ public final class TestConfig { //test: ak, sk, auth public static final String testAccessKey = System.getenv("QINIU_ACCESS_KEY"); public static final String testSecretKey = System.getenv("QINIU_SECRET_KEY"); + // 内部测试环境 AK/SK + public static final String innerAccessKey = System.getenv("testAK"); + public static final String innerSecretKey = System.getenv("testSK"); + //sms: ak, sk, auth public static final String smsAccessKey = "test"; public static final String smsSecretKey = "test"; diff --git a/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java b/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java index e1abbb88b..e02cb7550 100644 --- a/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java +++ b/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java @@ -6,9 +6,11 @@ import com.qiniu.http.Response; import com.qiniu.storage.*; import com.qiniu.storage.model.FileInfo; +import com.qiniu.util.Auth; import com.qiniu.util.Etag; import com.qiniu.util.Md5; import com.qiniu.util.StringMap; +import org.junit.Assert; import org.junit.Test; import test.com.qiniu.TempFile; import test.com.qiniu.TestConfig; @@ -259,6 +261,118 @@ public void test20M1K() throws Throwable { } } + // 内部环境测试 + @Test + public void notestInnerEnvSwitchRegion() { + try { + long s = new Date().getTime(); + uploadByInnerEnvSwitchRegion(1024 * 500 + 1, httpType, resumableV2Type, concurrentType); + long e = new Date().getTime(); + System.out.println("耗时：" + (e - s)); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail("testInnerEnvSwitchRegion:" + e); + } + } + + public void uploadByInnerEnvSwitchRegion(int size, int httpType, int uploadType, int uploadStyle) throws Exception { + + boolean isHttps = httpType == httpsType; + boolean isConcurrent = uploadStyle == concurrentType; + String bucket = "aaaabbbbb"; + + Region region0 = new Region.Builder() + .srcUpHost("10.200.20.23:5010") + .accUpHost("10.200.20.23:5010") + .build(); + + Region region1 = new Region.Builder() + .srcUpHost("10.200.20.24:5010") + .accUpHost("10.200.20.24:5010") + .build(); + + RegionGroup regionGroup = new RegionGroup(); + regionGroup.addRegion(region0); + regionGroup.addRegion(region1); + + Configuration config = new Configuration(regionGroup); + if (uploadType == resumableV2Type) { + config.resumableUploadAPIVersion = Configuration.ResumableUploadAPIVersion.V2; + config.resumableUploadAPIV2BlockSize = 4 * 1024 * 1024; + } + + config.useHttpsDomains = isHttps; + String key = "switch_region_"; + key += isHttps ? "_https" : "_http"; + + if (uploadType == resumableV1Type) { + key += "_resumeV1"; + } else if (uploadType == resumableV2Type) { + key += "_resumeV2"; + } + + if (uploadStyle == formType) { + key += "_form"; + } else if (uploadStyle == serialType) { + key += "_serial"; + } else if (uploadStyle == concurrentType) { + key += "_concurrent"; + } + + key += "_" + new Date().getTime(); + final String expectKey = "\r\n?&r=" + size + "k" + key; + final File f = TempFile.createFile(size); + + final String fooKey = "foo"; + final String fooValue = "fooValue"; + final String metaDataKey = "metaDataKey"; + final String metaDataValue = "metaDataValue"; + final String returnBody = "{\"key\":\"$(key)\",\"hash\":\"$(etag)\",\"fsize\":\"$(fsize)\"" + + ",\"fname\":\"$(fname)\",\"mimeType\":\"$(mimeType)\",\"foo\":\"$(x:foo)\"}"; + + Auth auth = Auth.create(TestConfig.innerAccessKey, TestConfig.innerSecretKey); + ; + String token = auth.uploadToken(bucket, expectKey, 3600, + new StringMap().put("returnBody", returnBody)); + + System.out.printf("\r\nkey:%s zone:%s\n", expectKey, regionGroup); + + + StringMap param = new StringMap(); + param.put("x:" + fooKey, fooValue); + param.put("x-qn-meta-" + metaDataKey, metaDataValue); + try { + BaseUploader up = null; + if (uploadStyle == formType) { + up = new FormUploader(new Client(), token, expectKey, f, param, Client.DefaultMime, false, config); + } else if (uploadStyle == serialType) { + up = new ResumeUploader(new Client(), token, expectKey, f, param, null, null, config); + } else { + config.resumableUploadMaxConcurrentTaskCount = 3; + up = new ConcurrentResumeUploader(new Client(), token, expectKey, f, param, null, null, config); + } + + Response r = up.upload(); + assertTrue(r + "", r.isOK()); + + StringMap ret = r.jsonToMap(); + assertEquals(expectKey, ret.get("key")); + assertEquals(f.getName(), ret.get("fname")); + assertEquals(String.valueOf(f.length()), ret.get("fsize").toString()); + assertEquals(fooValue, ret.get(fooKey).toString()); + + final String etag = Etag.file(f); + System.out.println(" etag:" + etag); + System.out.println("serverEtag:" + ret.get("hash")); + assertNotNull(ret.get("hash")); + assertEquals(etag, ret.get("hash")); + } catch (QiniuException e) { + assertEquals("", e.response == null ? e + "e.response is null" : e.response.bodyString()); + fail(); + } + TempFile.remove(f); + } + class MyRet { public String hash; From 47ea1abb07ce5798e932d16afd6184792ffad10f Mon Sep 17 00:00:00 2001 From: yangsen Date: Fri, 28 May 2021 10:49:48 +0800 Subject: [PATCH 15/16] code style --- src/test/java/test/com/qiniu/storage/SwitchRegionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java b/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java index e02cb7550..b9543c83f 100644 --- a/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java +++ b/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java @@ -331,7 +331,7 @@ public void uploadByInnerEnvSwitchRegion(int size, int httpType, int uploadType, + ",\"fname\":\"$(fname)\",\"mimeType\":\"$(mimeType)\",\"foo\":\"$(x:foo)\"}"; Auth auth = Auth.create(TestConfig.innerAccessKey, TestConfig.innerSecretKey); - ; + String token = auth.uploadToken(bucket, expectKey, 3600, new StringMap().put("returnBody", returnBody)); From 4f5f6069db5df548cc3914f29ae0af709d85ee17 Mon Sep 17 00:00:00 2001 From: yangsen Date: Fri, 28 May 2021 14:02:20 +0800 Subject: [PATCH 16/16] close private clound test --- src/test/java/test/com/qiniu/storage/SwitchRegionTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java b/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java index b9543c83f..d43dc7f28 100644 --- a/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java +++ b/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java @@ -262,8 +262,8 @@ public void test20M1K() throws Throwable { } // 内部环境测试 - @Test - public void notestInnerEnvSwitchRegion() { +// @Test + public void testInnerEnvSwitchRegion() { try { long s = new Date().getTime(); uploadByInnerEnvSwitchRegion(1024 * 500 + 1, httpType, resumableV2Type, concurrentType);