diff --git a/src/main/java/com/qiniu/storage/AutoRegion.java b/src/main/java/com/qiniu/storage/AutoRegion.java index 649c0935..54524ca9 100644 --- a/src/main/java/com/qiniu/storage/AutoRegion.java +++ b/src/main/java/com/qiniu/storage/AutoRegion.java @@ -19,33 +19,18 @@ class AutoRegion extends Region { /** * 空间机房，域名信息缓存 */ - private Map regions; - - /** - * 根据API返回的上传域名推导出其他资源管理域名 - */ - private Map inferDomainsMap; + private Map regions; /** * 定义HTTP请求管理相关方法 */ private Client client; + AutoRegion(String ucServer) { this.ucServer = ucServer; this.client = new Client(); this.regions = new ConcurrentHashMap<>(); - this.inferDomainsMap = new ConcurrentHashMap<>(); - this.inferDomainsMap.put("up.qiniup.com", region0()); - this.inferDomainsMap.put("up-jjh.qiniup.com", region0()); - this.inferDomainsMap.put("up-xs.qiniup.com", region0()); - this.inferDomainsMap.put("up-z1.qiniup.com", region1()); - this.inferDomainsMap.put("up-z2.qiniup.com", region2()); - this.inferDomainsMap.put("up-dg.qiniup.com", region2()); - this.inferDomainsMap.put("up-fs.qiniup.com", region2()); - this.inferDomainsMap.put("up-na0.qiniup.com", regionNa0()); - this.inferDomainsMap.put("up-as0.qiniup.com", regionAs0()); - this.inferDomainsMap.put("up-fog-cn-east-1.qiniup.com", regionFogCnEast1()); } /** @@ -57,12 +42,65 @@ class AutoRegion extends Region { * na0: http://uc.qbox.me/v2/query?ak=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5&bucket=sdk-na0 */ private UCRet getRegionJson(RegionIndex index) throws QiniuException { - String address = ucServer + "/v2/query?ak=" + index.accessKey + "&bucket=" + index.bucket; + String address = ucServer + "/v3/query?ak=" + index.accessKey + "&bucket=" + index.bucket; Response r = client.get(address); return r.jsonToObject(UCRet.class); } + static RegionGroup regionGroup(UCRet ret) { + if (ret == null || ret.hosts == null || ret.hosts.length == 0) { + return null; + } + + RegionGroup group = new RegionGroup(); + for (HostRet host : ret.hosts) { + long timestamp = host.ttl + System.currentTimeMillis() / 1000; + List srcUpHosts = new ArrayList<>(); + List accUpHosts = new ArrayList<>(); + if (host.up != null) { + srcUpHosts = host.up.allSrcHosts(); + accUpHosts = host.up.allAccHosts(); + } + + String iovipHost = null; + if (host.io != null) { + iovipHost = host.io.getOneHost(); + } + + String rsHost = null; + if (host.rs != null) { + rsHost = host.rs.getOneHost(); + } + + String rsfHost = null; + if (host.rsf != null) { + rsfHost = host.rsf.getOneHost(); + } + + String apiHost = null; + if (host.api != null) { + apiHost = host.api.getOneHost(); + } + + String ucHost = null; + if (host.uc != null) { + ucHost = host.uc.getOneHost(); + } + + // 根据 iovipHost 反推 regionId + String regionId = host.region; + if (regionId == null) { + regionId = ""; + } + + Region region = new Region(timestamp, regionId, srcUpHosts, accUpHosts, iovipHost, rsHost, rsfHost, apiHost, ucHost); + group.addRegion(region); + } + + return group; + } + /** * 首先从缓存读取Region信息，如果没有则发送请求从接口查询 * @@ -70,55 +108,35 @@ private UCRet getRegionJson(RegionIndex index) throws QiniuException { * @param bucket 空间名 * @return 机房域名信息 */ - private RegionInfo queryRegionInfo(String accessKey, String bucket) throws QiniuException { + private RegionGroup queryRegionInfo(String accessKey, String bucket) throws QiniuException { RegionIndex index = new RegionIndex(accessKey, bucket); - RegionInfo info = regions.get(index); + RegionGroup regionGroup = regions.get(index); Exception ex = null; - // 隔一段时间重新获取 uc 信息 // - if (info == null || info.createTime < System.currentTimeMillis() - 1000 * 3600 * 8) { - try { - // 1 - UCRet ret = getRegionJson(index); - RegionInfo info2 = RegionInfo.buildFromUcRet(ret); - // 初次获取报错，info == null ，响应 null // - // 后续重新获取，正常获取则替换以前的 // - if (info2 != null) { - regions.put(index, info2); - info = info2; - } - } catch (Exception e) { - ex = e; - if (info == null) { - try { - Thread.sleep(100); - } catch (Exception e1) { - // do nothing - } - try { - // 2 - UCRet ret = getRegionJson(index); - RegionInfo info2 = RegionInfo.buildFromUcRet(ret); - // 初次获取报错，info == null ，响应 null // - // 后续重新获取，正常获取则替换以前的 // - if (info2 != null) { - regions.put(index, info2); - info = info2; - } - } catch (Exception e1) { - ex = e1; + // 隔一段时间重新获取 uc 信息 // || regionGroup.createTime < System.currentTimeMillis() - 1000 * 3600 * 8 + if (regionGroup == null || !regionGroup.isValid()) { + for (int i = 0; i < 2; i++) { + try { + UCRet ret = getRegionJson(index); + regionGroup = AutoRegion.regionGroup(ret); + if (regionGroup != null) { + regions.put(index, regionGroup); + break; } + } catch (Exception e) { + ex = e; } } } + // info 不能为 null // - if (info == null) { + if (regionGroup == null) { if (ex instanceof QiniuException) { throw (QiniuException) ex; } throw new QiniuException(ex, "auto region get region info from uc failed."); } - return info; + return regionGroup; } /** @@ -127,13 +145,42 @@ private RegionInfo queryRegionInfo(String accessKey, String bucket) throws Qiniu * @param regionReqInfo 封装了 accessKey 和 bucket 的对象 * @return 机房域名信息 */ - private RegionInfo queryRegionInfo(RegionReqInfo regionReqInfo) throws QiniuException { + private RegionGroup queryRegionInfo(RegionReqInfo regionReqInfo) throws QiniuException { return queryRegionInfo(regionReqInfo.getAccessKey(), regionReqInfo.getBucket()); } + @Override + boolean switchRegion(RegionReqInfo regionReqInfo) { + Region currentRegion = getCurrentRegion(regionReqInfo); + if (currentRegion == null) { + return false; + } else { + return currentRegion.switchRegion(regionReqInfo); + } + } + @Override String getRegion(RegionReqInfo regionReqInfo) { - return ""; + Region currentRegion = getCurrentRegion(regionReqInfo); + if (currentRegion == null) { + return ""; + } else { + return currentRegion.getRegion(regionReqInfo); + } + } + + @Override + boolean isValid() { + return true; + } + + Region getCurrentRegion(RegionReqInfo regionReqInfo) { + try { + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getCurrentRegion(regionReqInfo); + } catch (QiniuException e) { + return null; + } } /** @@ -141,8 +188,8 @@ String getRegion(RegionReqInfo regionReqInfo) { */ @Override List getSrcUpHost(RegionReqInfo regionReqInfo) throws QiniuException { - RegionInfo info = queryRegionInfo(regionReqInfo); - return info.srcUpHosts; + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getSrcUpHost(regionReqInfo); } /** @@ -150,8 +197,8 @@ List getSrcUpHost(RegionReqInfo regionReqInfo) throws QiniuException { */ @Override List getAccUpHost(RegionReqInfo regionReqInfo) throws QiniuException { - RegionInfo info = queryRegionInfo(regionReqInfo); - return info.accUpHosts; + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getAccUpHost(regionReqInfo); } /** @@ -159,8 +206,8 @@ List getAccUpHost(RegionReqInfo regionReqInfo) throws QiniuException { */ @Override String getIovipHost(RegionReqInfo regionReqInfo) throws QiniuException { - RegionInfo info = queryRegionInfo(regionReqInfo); - return info.iovipHost; + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getIovipHost(regionReqInfo); } /** @@ -168,18 +215,8 @@ String getIovipHost(RegionReqInfo regionReqInfo) throws QiniuException { */ @Override String getRsHost(RegionReqInfo regionReqInfo) throws QiniuException { - RegionInfo info; - try { - info = queryRegionInfo(regionReqInfo); - } catch (QiniuException e) { - return super.getRsHost(regionReqInfo); - } - Region region = this.inferDomainsMap.get(info.srcUpHosts.get(0)); - if (region != null) { - return region.getRsHost(regionReqInfo); - } else { - return super.getRsHost(regionReqInfo); - } + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getRsHost(regionReqInfo); } /** @@ -187,18 +224,8 @@ String getRsHost(RegionReqInfo regionReqInfo) throws QiniuException { */ @Override String getRsfHost(RegionReqInfo regionReqInfo) throws QiniuException { - RegionInfo info; - try { - info = queryRegionInfo(regionReqInfo); - } catch (QiniuException e) { - return super.getRsfHost(regionReqInfo); - } - Region region = this.inferDomainsMap.get(info.srcUpHosts.get(0)); - if (region != null) { - return region.getRsfHost(regionReqInfo); - } else { - return super.getRsfHost(regionReqInfo); - } + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getRsfHost(regionReqInfo); } /** @@ -206,70 +233,8 @@ String getRsfHost(RegionReqInfo regionReqInfo) throws QiniuException { */ @Override String getApiHost(RegionReqInfo regionReqInfo) throws QiniuException { - RegionInfo info; - try { - info = queryRegionInfo(regionReqInfo); - } catch (QiniuException e) { - return super.getApiHost(regionReqInfo); - } - Region region = this.inferDomainsMap.get(info.srcUpHosts.get(0)); - if (region != null) { - return region.getApiHost(regionReqInfo); - } else { - return super.getApiHost(regionReqInfo); - } - } - - /** - * 从接口获取的域名信息 - */ - static class RegionInfo { - final List srcUpHosts; - final List accUpHosts; - final String iovipHost; - final long createTime; - - protected RegionInfo(List srcUpHosts, List accUpHosts, String iovipHost) { - this.srcUpHosts = srcUpHosts; - this.accUpHosts = accUpHosts; - this.iovipHost = iovipHost; - createTime = System.currentTimeMillis(); - } - - /** - * { - * "io": {"src": {"main": ["iovip.qbox.me"]}}, - * "up": { - * "acc": { - * "main": ["upload.qiniup.com"], - * "backup": ["upload-jjh.qiniup.com", "upload-xs.qiniup.com"] - * }, - * "src": { - * "main": ["up.qiniup.com"], - * "backup": ["up-jjh.qiniup.com", "up-xs.qiniup.com"] - * } - * } - * } - * - * @param ret - * @return - */ - static RegionInfo buildFromUcRet(UCRet ret) { - List srcUpHosts = new ArrayList<>(); - addAll(srcUpHosts, ret.up.src.get("main")); - addAll(srcUpHosts, ret.up.src.get("backup")); - List accUpHosts = new ArrayList<>(); - addAll(accUpHosts, ret.up.acc.get("main")); - addAll(accUpHosts, ret.up.acc.get("backup")); - String iovipHost = ret.io.src.get("main").get(0); - return new RegionInfo(srcUpHosts, accUpHosts, iovipHost); - } - - static void addAll(List s, List p) { - if (p != null) { - s.addAll(p); - } - } + RegionGroup regionGroup = queryRegionInfo(regionReqInfo); + return regionGroup.getApiHost(regionReqInfo); } private static class RegionIndex { @@ -292,17 +257,77 @@ public boolean equals(Object obj) { } private class UCRet { - UPRet up; - IORet io; + HostRet[] hosts; } - private class UPRet { - Map> acc; - Map> src; + private class HostRet { + long ttl; + String region; + HostInfoRet up; + HostInfoRet rs; + HostInfoRet rsf; + HostInfoRet uc; + HostInfoRet api; + HostInfoRet io; } - private class IORet { + private class HostInfoRet { + Map> acc; Map> src; - } + private String getOneHost() { + List hosts = allHosts(); + if (hosts.size() > 0) { + return hosts.get(0); + } else { + return null; + } + } + + private List allHosts() { + List hosts = new ArrayList<>(); + List srcHosts = allSrcHosts(); + if (srcHosts.size() > 0) { + hosts.addAll(srcHosts); + } + + List accHosts = allAccHosts(); + if (accHosts.size() > 0) { + hosts.addAll(accHosts); + } + return hosts; + } + + private List allSrcHosts() { + List hosts = new ArrayList<>(); + if (acc != null) { + List mainHosts = acc.get("main"); + if (mainHosts != null && mainHosts.size() > 0) { + hosts.addAll(mainHosts); + } + + List backupHosts = acc.get("backup"); + if (backupHosts != null && backupHosts.size() > 0) { + hosts.addAll(backupHosts); + } + } + return hosts; + } + + private List allAccHosts() { + List hosts = new ArrayList<>(); + if (src != null) { + List mainHosts = src.get("main"); + if (mainHosts != null && mainHosts.size() > 0) { + hosts.addAll(mainHosts); + } + + List backupHosts = src.get("backup"); + if (backupHosts != null && backupHosts.size() > 0) { + hosts.addAll(backupHosts); + } + } + return hosts; + } + } } diff --git a/src/main/java/com/qiniu/storage/BaseUploader.java b/src/main/java/com/qiniu/storage/BaseUploader.java new file mode 100644 index 00000000..cf9e407d --- /dev/null +++ b/src/main/java/com/qiniu/storage/BaseUploader.java @@ -0,0 +1,76 @@ +package com.qiniu.storage; + +import com.qiniu.common.QiniuException; +import com.qiniu.http.Client; +import com.qiniu.http.Response; + +public abstract class BaseUploader { + + protected final Client client; + protected final String key; + protected final String upToken; + protected final ConfigHelper configHelper; + protected final Configuration config; + + BaseUploader(Client client, String upToken, String key, Configuration config) { + this.client = client; + this.key = key; + this.upToken = upToken; + if (config == null) { + this.config = new Configuration(); + } else { + this.config = config.clone(); + } + this.configHelper = new ConfigHelper(this.config); + } + + public Response upload() throws QiniuException { + if (this.config == null) { + throw QiniuException.unrecoverable("config can't be empty"); + } + return uploadWithRegionRetry(); + } + + private Response uploadWithRegionRetry() throws QiniuException { + Response response = null; + while (true) { + try { + response = uploadFlows(); + if (!couldSwitchRegionAndRetry(response, null) + || !couldReloadSource() || !reloadSource() + || config.region == null || !config.region.switchRegion(new UploadToken(upToken))) { + break; + } + } catch (QiniuException e) { + if (!couldSwitchRegionAndRetry(null, e) + || !couldReloadSource() || !reloadSource() + || config.region == null || !config.region.switchRegion(new UploadToken(upToken))) { + throw e; + } + } + } + return response; + } + + abstract Response uploadFlows() throws QiniuException; + + abstract boolean couldReloadSource(); + + abstract boolean reloadSource(); + + private boolean couldSwitchRegionAndRetry(Response response, QiniuException exception) { + Response checkResponse = response; + if (checkResponse == null && exception != null) { + checkResponse = exception.response; + } + + if (checkResponse != null) { + int statusCode = checkResponse.statusCode; + return (statusCode > -2 && statusCode < 200) || (statusCode > 299 + && statusCode != 401 && statusCode != 413 && statusCode != 419 + && statusCode != 608 && statusCode != 614 && statusCode != 630); + } + + return exception == null || !exception.isUnrecoverable(); + } +} diff --git a/src/main/java/com/qiniu/storage/FixBlockUploader.java b/src/main/java/com/qiniu/storage/FixBlockUploader.java index 0cfb9ef4..d08abc3a 100644 --- a/src/main/java/com/qiniu/storage/FixBlockUploader.java +++ b/src/main/java/com/qiniu/storage/FixBlockUploader.java @@ -18,6 +18,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +@Deprecated public class FixBlockUploader { private final int blockSize; private final ConfigHelper configHelper; @@ -883,6 +884,7 @@ class Record { long size; long blockSize; List etagIdxes; + // 用于区分记录是 V1 还是 V2 boolean isValid() { return uploadId != null && etagIdxes != null && etagIdxes.size() > 0; diff --git a/src/main/java/com/qiniu/storage/FormUploader.java b/src/main/java/com/qiniu/storage/FormUploader.java index a7093dec..d0c7fbf3 100644 --- a/src/main/java/com/qiniu/storage/FormUploader.java +++ b/src/main/java/com/qiniu/storage/FormUploader.java @@ -14,10 +14,8 @@ * 该类封装了七牛提供的表单上传机制 * 参考文档：表单上传 */ -public final class FormUploader { +public final class FormUploader extends BaseUploader { - private final String token; - private final String key; private final File file; private final byte[] data; private final String mime; @@ -45,9 +43,9 @@ public FormUploader(Client client, String upToken, String key, File file, String private FormUploader(Client client, String upToken, String key, byte[] data, File file, StringMap params, String mime, boolean checkCrc, Configuration configuration) { + super(client, upToken, key, configuration); + this.client = client; - token = upToken; - this.key = key; this.file = file; this.data = data; this.params = params; @@ -56,23 +54,21 @@ private FormUploader(Client client, String upToken, String key, byte[] data, Fil this.configHelper = new ConfigHelper(configuration); } - /** - * 同步上传文件 - */ - public Response upload() throws QiniuException { + @Override + Response uploadFlows() throws QiniuException { buildParams(); - String host = configHelper.upHost(token); + String host = configHelper.upHost(upToken); try { if (data != null) { - return client.multipartPost(configHelper.upHost(token), params, "file", filename, data, + return client.multipartPost(configHelper.upHost(upToken), params, "file", filename, data, mime, new StringMap()); } else { - return client.multipartPost(configHelper.upHost(token), params, "file", filename, file, + return client.multipartPost(configHelper.upHost(upToken), params, "file", filename, file, mime, new StringMap()); } } catch (QiniuException e) { if (e.response == null || e.response.needSwitchServer()) { - changeHost(token, host); + changeHost(upToken, host); } throw e; } @@ -83,32 +79,42 @@ public Response upload() throws QiniuException { */ public void asyncUpload(final UpCompletionHandler handler) throws IOException { buildParams(); - final String host = configHelper.upHost(token); + final String host = configHelper.upHost(upToken); if (data != null) { client.asyncMultipartPost(host, params, "file", filename, data, mime, new StringMap(), new AsyncCallback() { @Override public void complete(Response res) { if (res != null && res.needSwitchServer()) { - changeHost(token, host); + changeHost(upToken, host); } handler.complete(key, res); } }); return; } - client.asyncMultipartPost(configHelper.upHost(token), params, "file", filename, + client.asyncMultipartPost(configHelper.upHost(upToken), params, "file", filename, file, mime, new StringMap(), new AsyncCallback() { @Override public void complete(Response res) { if (res != null && res.needSwitchServer()) { - changeHost(token, host); + changeHost(upToken, host); } handler.complete(key, res); } }); } + @Override + boolean couldReloadSource() { + return true; + } + + @Override + boolean reloadSource() { + return true; + } + private void changeHost(String upToken, String host) { try { configHelper.tryChangeUpHost(upToken, host); @@ -120,7 +126,7 @@ private void changeHost(String upToken, String host) { private void buildParams() throws QiniuException { if (params == null) return; - params.put("token", token); + params.put("token", upToken); if (key != null) { params.put("key", key); diff --git a/src/main/java/com/qiniu/storage/Region.java b/src/main/java/com/qiniu/storage/Region.java index 8b5f4401..f5d55cd0 100644 --- a/src/main/java/com/qiniu/storage/Region.java +++ b/src/main/java/com/qiniu/storage/Region.java @@ -7,6 +7,8 @@ public class Region { + // 有效时间戳，过了有效期，region会无效，此处只在取时缓存判断； -1 为无限期 + private long timestamp = -1; // 区域名称：z0 华东 z1 华北 z2 华南 na0 北美 as0 东南亚 private String region = "z0"; @@ -25,6 +27,23 @@ public class Region { private String rsHost = "rs.qbox.me"; private String rsfHost = "rsf.qbox.me"; private String apiHost = "api.qiniu.com"; + private String ucHost = "uc.qbox.me"; + + Region() { + } + + Region(long timestamp, String region, List srcUpHosts, List accUpHosts, String iovipHost, + String rsHost, String rsfHost, String apiHost, String ucHost) { + this.timestamp = timestamp; + this.region = region; + this.srcUpHosts = srcUpHosts; + this.accUpHosts = accUpHosts; + this.iovipHost = iovipHost; + this.rsHost = rsHost; + this.rsfHost = rsfHost; + this.apiHost = apiHost; + this.ucHost = ucHost; + } /** * 华东机房相关域名 @@ -211,10 +230,18 @@ public static Region autoRegion(String ucServer) { return new Builder().autoRegion(ucServer); } + boolean switchRegion(RegionReqInfo regionReqInfo) { + return false; + } + String getRegion(RegionReqInfo regionReqInfo) { return this.region; } + Region getCurrentRegion(RegionReqInfo regionReqInfo) { + return this; + } + List getSrcUpHost(RegionReqInfo regionReqInfo) throws QiniuException { return this.srcUpHosts; } @@ -239,6 +266,14 @@ String getApiHost(RegionReqInfo regionReqInfo) throws QiniuException { return apiHost; } + boolean isValid() { + if (timestamp < 0) { + return true; + } else { + return System.currentTimeMillis() < timestamp * 1000; + } + } + /** * 域名构造器 */ diff --git a/src/main/java/com/qiniu/storage/RegionGroup.java b/src/main/java/com/qiniu/storage/RegionGroup.java new file mode 100644 index 00000000..8d7d6530 --- /dev/null +++ b/src/main/java/com/qiniu/storage/RegionGroup.java @@ -0,0 +1,128 @@ +package com.qiniu.storage; + +import com.qiniu.common.QiniuException; + +import java.util.ArrayList; +import java.util.List; + +public class RegionGroup extends Region { + + private Region currentRegion = null; + private int currentRegionIndex = 0; + private final List regionList = new ArrayList<>(); + + + public boolean addRegion(Region region) { + if (region == null) { + return false; + } + + regionList.add(region); + + if (currentRegion == null) { + updateCurrentRegion(); + } + + return true; + } + + @Override + boolean switchRegion(RegionReqInfo regionReqInfo) { + if (currentRegion != null && currentRegion.isValid() && currentRegion.switchRegion(regionReqInfo)) { + return true; + } + + if ((currentRegionIndex + 1) < regionList.size()) { + currentRegionIndex += 1; + updateCurrentRegion(); + return true; + } else { + return false; + } + } + + String getRegion(RegionReqInfo regionReqInfo) { + if (currentRegion == null) { + return ""; + } else { + return currentRegion.getRegion(regionReqInfo); + } + } + + List getSrcUpHost(RegionReqInfo regionReqInfo) throws QiniuException { + if (currentRegion == null) { + return null; + } else { + return currentRegion.getSrcUpHost(regionReqInfo); + } + } + + List getAccUpHost(RegionReqInfo regionReqInfo) throws QiniuException { + if (currentRegion == null) { + return null; + } else { + return currentRegion.getAccUpHost(regionReqInfo); + } + } + + String getIovipHost(RegionReqInfo regionReqInfo) throws QiniuException { + if (currentRegion == null) { + return null; + } else { + return currentRegion.getIovipHost(regionReqInfo); + } + } + + String getRsHost(RegionReqInfo regionReqInfo) throws QiniuException { + if (currentRegion == null) { + return null; + } else { + return currentRegion.getRsHost(regionReqInfo); + } + } + + String getRsfHost(RegionReqInfo regionReqInfo) throws QiniuException { + if (currentRegion == null) { + return null; + } else { + return currentRegion.getRsfHost(regionReqInfo); + } + } + + String getApiHost(RegionReqInfo regionReqInfo) throws QiniuException { + if (currentRegion == null) { + return null; + } else { + return currentRegion.getApiHost(regionReqInfo); + } + } + + Region getCurrentRegion(RegionReqInfo regionReqInfo) { + if (currentRegion == null) { + return null; + } else if (currentRegion instanceof AutoRegion || currentRegion instanceof RegionGroup) { + return currentRegion.getCurrentRegion(regionReqInfo); + } else { + return currentRegion; + } + } + + @Override + boolean isValid() { + if (currentRegion == null) { + return false; + } + // 只判断当前的 + return currentRegion.isValid(); + } + + private void updateCurrentRegion() { + if (regionList.size() == 0) { + return; + } + + if (currentRegionIndex < regionList.size()) { + currentRegion = regionList.get(currentRegionIndex); + } + } +} diff --git a/src/main/java/com/qiniu/storage/ResumeUploadPerformer.java b/src/main/java/com/qiniu/storage/ResumeUploadPerformer.java index 3af83142..0793c55c 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploadPerformer.java +++ b/src/main/java/com/qiniu/storage/ResumeUploadPerformer.java @@ -1,12 +1,8 @@ package com.qiniu.storage; -import com.google.gson.Gson; -import com.qiniu.common.Constants; import com.qiniu.common.QiniuException; import com.qiniu.http.Client; import com.qiniu.http.Response; -import com.qiniu.util.StringMap; -import com.qiniu.util.StringUtils; import java.io.IOException; @@ -77,51 +73,11 @@ private ResumeUploadSource.Block getNextUploadingBlock() throws QiniuException { try { block = uploadSource.getNextUploadingBlock(); } catch (IOException e) { - throw new QiniuException(e); + throw QiniuException.unrecoverable(e); } return block; } - void recoverUploadProgressFromLocal() { - if (recorder == null || StringUtils.isNullOrEmpty(uploadSource.recordKey)) { - return; - } - - byte[] data = recorder.get(uploadSource.recordKey); - if (data == null) { - return; - } - String jsonString = new String(data, Constants.UTF_8); - ResumeUploadSource source = null; - try { - source = new Gson().fromJson(jsonString, uploadSource.getClass()); - } catch (Exception ignored) { - } - if (source == null) { - return; - } - - boolean isCopy = uploadSource.recoverFromRecordInfo(source); - if (!isCopy) { - removeUploadProgressFromLocal(); - } - } - - void saveUploadProgressToLocal() { - if (recorder == null || StringUtils.isNullOrEmpty(uploadSource.recordKey)) { - return; - } - String dataString = new Gson().toJson(uploadSource); - recorder.set(uploadSource.recordKey, dataString.getBytes(Constants.UTF_8)); - } - - void removeUploadProgressFromLocal() { - if (recorder == null || StringUtils.isNullOrEmpty(uploadSource.recordKey)) { - return; - } - recorder.del(uploadSource.recordKey); - } - private String getUploadHost() throws QiniuException { return configHelper.upHost(token.getToken()); } @@ -133,18 +89,6 @@ private void changeHost(String host) { } } - Response post(String url, byte[] data, int offset, int size) throws QiniuException { - StringMap header = new StringMap(); - header.put("Authorization", "UpToken " + token.getToken()); - return client.post(url, data, offset, size, header, options.mimeType); - } - - Response put(String url, byte[] data, int offset, int size) throws QiniuException { - StringMap header = new StringMap(); - header.put("Authorization", "UpToken " + token.getToken()); - return client.put(url, data, offset, size, header, options.mimeType); - } - Response retryUploadAction(UploadAction action) throws QiniuException { Response response = null; int retryCount = 0; @@ -183,7 +127,7 @@ Response retryUploadAction(UploadAction action) throws QiniuException { retryCount++; if (retryCount >= config.retryMax) { - throw QiniuException.unrecoverable("failed after retry times"); + throw new QiniuException(null, "failed after retry times"); } if (shouldSwitchHost) { diff --git a/src/main/java/com/qiniu/storage/ResumeUploadSource.java b/src/main/java/com/qiniu/storage/ResumeUploadSource.java index 76e18e96..33fb9632 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploadSource.java +++ b/src/main/java/com/qiniu/storage/ResumeUploadSource.java @@ -10,7 +10,6 @@ abstract class ResumeUploadSource { final String recordKey; final int blockSize; - final String targetRegionId; final Configuration.ResumableUploadAPIVersion resumableUploadAPIVersion; transient Configuration config; @@ -22,17 +21,15 @@ abstract class ResumeUploadSource { Long expireAt; ResumeUploadSource() { - this.targetRegionId = null; this.blockSize = 0; this.recordKey = null; this.resumableUploadAPIVersion = Configuration.ResumableUploadAPIVersion.V1; } - ResumeUploadSource(Configuration config, String recordKey, String targetRegionId) { + ResumeUploadSource(Configuration config, String recordKey) { this.config = config; this.blockSize = getBlockSize(config); this.recordKey = recordKey; - this.targetRegionId = targetRegionId; this.resumableUploadAPIVersion = config.resumableUploadAPIVersion; } @@ -67,6 +64,22 @@ boolean isAllBlocksUploaded() { return isAllBlockUploaded; } + boolean couldReload() { + return false; + } + + boolean reload() { + return false; + } + + void clearState() { + for (ResumeUploadSource.Block block : blockList) { + block.clearState(); + } + uploadId = null; + expireAt = null; + } + // 获取下一个需要上传的块 ResumeUploadSource.Block getNextUploadingBlock() throws IOException { ResumeUploadSource.Block block = null; @@ -91,6 +104,22 @@ ResumeUploadSource.Block getNextUploadingBlock() throws IOException { // 获取文件名 abstract String getFileName(); + // 是否有已上传的数据 + boolean hasUploadData() { + if (blockList == null || blockList.size() == 0) { + return false; + } + + boolean hasUploadData = false; + for (ResumeUploadSource.Block block : blockList) { + if (block.isUploaded()) { + hasUploadData = true; + break; + } + } + return hasUploadData; + } + boolean recoverFromRecordInfo(ResumeUploadSource source) { return false; } @@ -165,9 +194,7 @@ static class Block { this.offset = offset; this.size = blockSize; this.index = index; - this.isUploading = false; - this.etag = null; - this.context = null; + this.clearState(); } boolean isUploaded() { @@ -183,5 +210,12 @@ boolean isUploaded() { } return isUploaded; } + + void clearState() { + this.isUploading = false; + this.etag = null; + this.context = null; + this.data = null; + } } } diff --git a/src/main/java/com/qiniu/storage/ResumeUploadSourceFile.java b/src/main/java/com/qiniu/storage/ResumeUploadSourceFile.java index 8de5e316..884f7ffc 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploadSourceFile.java +++ b/src/main/java/com/qiniu/storage/ResumeUploadSourceFile.java @@ -15,8 +15,8 @@ class ResumeUploadSourceFile extends ResumeUploadSource { private final transient File file; private transient RandomAccessFile randomAccessFile; - ResumeUploadSourceFile(File file, Configuration config, String recordKey, String targetRegionId) { - super(config, recordKey, targetRegionId); + ResumeUploadSourceFile(File file, Configuration config, String recordKey) { + super(config, recordKey); this.file = file; this.fileName = file.getName(); this.size = file.length(); @@ -52,6 +52,16 @@ String getFileName() { return fileName; } + @Override + boolean couldReload() { + return true; + } + + @Override + boolean reload() { + return true; + } + @Override ResumeUploadSource.Block getNextUploadingBlock() throws IOException { ResumeUploadSource.Block block = super.getNextUploadingBlock(); @@ -99,12 +109,12 @@ boolean recoverFromRecordInfo(ResumeUploadSource source) { boolean needRecovered = true; if (source.resumableUploadAPIVersion == Configuration.ResumableUploadAPIVersion.V2) { - if (StringUtils.isNullOrEmpty(uploadId)) { + if (StringUtils.isNullOrEmpty(source.uploadId)) { return false; } // 服务端是 7 天，此处有效期少 1 天，为 6 天 long currentTimestamp = new Date().getTime() / 1000; - long expireAtTimestamp = expireAt - 24 * 3600; + long expireAtTimestamp = source.expireAt - 24 * 3600; needRecovered = expireAtTimestamp > currentTimestamp; } @@ -138,10 +148,6 @@ private boolean isSameResource(ResumeUploadSource source) { return false; } - if (sourceFile.targetRegionId == null || !sourceFile.targetRegionId.equals(targetRegionId)) { - return false; - } - return sourceFile.resumableUploadAPIVersion == resumableUploadAPIVersion; } diff --git a/src/main/java/com/qiniu/storage/ResumeUploadSourceStream.java b/src/main/java/com/qiniu/storage/ResumeUploadSourceStream.java index 33fa0f2a..f173bbac 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploadSourceStream.java +++ b/src/main/java/com/qiniu/storage/ResumeUploadSourceStream.java @@ -11,8 +11,8 @@ public class ResumeUploadSourceStream extends ResumeUploadSource { private final InputStream inputStream; private final String fileName; - ResumeUploadSourceStream(InputStream inputStream, Configuration config, String recordKey, String targetRegionId, String fileName) { - super(config, recordKey, targetRegionId); + ResumeUploadSourceStream(InputStream inputStream, Configuration config, String recordKey, String fileName) { + super(config, recordKey); this.inputStream = inputStream; this.fileName = fileName; this.blockList = new LinkedList<>(); diff --git a/src/main/java/com/qiniu/storage/ResumeUploader.java b/src/main/java/com/qiniu/storage/ResumeUploader.java index 75a3872b..354ab43d 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploader.java +++ b/src/main/java/com/qiniu/storage/ResumeUploader.java @@ -1,9 +1,15 @@ package com.qiniu.storage; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.qiniu.common.Constants; import com.qiniu.common.QiniuException; import com.qiniu.http.Client; import com.qiniu.http.Response; import com.qiniu.util.StringMap; +import com.qiniu.util.StringUtils; import java.io.File; import java.io.InputStream; @@ -31,16 +37,11 @@ * 服务端网络较稳定，较大文件（如500M以上）才需要将块记录保存下来。 * 小文件没有必要，可以有效地实现大文件的上传。 */ -public class ResumeUploader { - private final Client client; - private final String key; - private final String upToken; +public class ResumeUploader extends BaseUploader { private final ResumeUploadSource source; private final Recorder recorder; private final UploadOptions options; - final Configuration config; - ResumeUploadPerformer uploadPerformer; /** @@ -66,7 +67,7 @@ public class ResumeUploader { public ResumeUploader(Client client, String upToken, String key, File file, StringMap params, String mime, Recorder recorder, Configuration configuration) { this(client, key, upToken, - new ResumeUploadSourceFile(file, configuration, getRecorderKey(key, file, recorder), getRegionTargetId(upToken, configuration)), + new ResumeUploadSourceFile(file, configuration, getRecorderKey(key, file, recorder)), recorder, new UploadOptions.Builder().params(params).metaData(params).mimeType(mime).build(), configuration); } @@ -117,20 +118,17 @@ public ResumeUploader(Client client, String upToken, String key, InputStream str public ResumeUploader(Client client, String upToken, String key, InputStream stream, String fileName, StringMap params, String mime, Configuration configuration) { this(client, key, upToken, - new ResumeUploadSourceStream(stream, configuration, null, getRegionTargetId(upToken, configuration), fileName), + new ResumeUploadSourceStream(stream, configuration, null, fileName), null, new UploadOptions.Builder().params(params).metaData(params).mimeType(mime).build(), configuration); } private ResumeUploader(Client client, String key, String upToken, ResumeUploadSource source, Recorder recorder, UploadOptions options, Configuration configuration) { + super(client, upToken, key, configuration); - this.client = client; - this.key = key; - this.upToken = upToken; this.source = source; this.recorder = recorder; this.options = options == null ? UploadOptions.defaultOptions() : options; - this.config = configuration; } /** @@ -138,13 +136,23 @@ private ResumeUploader(Client client, String key, String upToken, ResumeUploadSo */ public Response upload() throws QiniuException { try { - return uploadFlows(); + recoverUploadProgressFromLocal(); + Response response = super.upload(); + if (response != null && response.isOK()) { + removeUploadProgressFromLocal(); + } + return response; + } catch (QiniuException e) { + saveUploadProgressToLocal(); + throw e; } finally { close(); } } - private Response uploadFlows() throws QiniuException { + @Override + Response uploadFlows() throws QiniuException { + // 检查参数 checkParam(); @@ -156,9 +164,6 @@ private Response uploadFlows() throws QiniuException { uploadPerformer = new ResumeUploadPerformerV1(client, key, token, source, recorder, options, config); } - // 恢复本地断点续传数据 - uploadPerformer.recoverUploadProgressFromLocal(); - // 上传数据至服务 - 步骤1 Response response = null; if (uploadPerformer.shouldUploadInit()) { @@ -178,9 +183,6 @@ private Response uploadFlows() throws QiniuException { // 上传数据至服务 - 步骤3 response = uploadPerformer.completeUpload(); - if (response.isOK()) { - uploadPerformer.removeUploadProgressFromLocal(); - } return response; } @@ -189,13 +191,25 @@ Response uploadData() throws QiniuException { Response response = null; do { response = uploadPerformer.uploadNextData(); - if (response != null && response.isOK()) { - uploadPerformer.saveUploadProgressToLocal(); - } } while (!uploadPerformer.isAllBlocksUploadingOrUploaded()); return response; } + @Override + boolean couldReloadSource() { + return source.couldReload(); + } + + @Override + boolean reloadSource() { + if (source.reload()) { + source.clearState(); + return true; + } else { + return false; + } + } + private void close() { try { source.close(); @@ -234,20 +248,78 @@ private static String getRecorderKey(String key, File file, Recorder recorder) { return recorder.recorderKeyGenerate(key, file); } - private static String getRegionTargetId(String upToken, Configuration config) { - if (config == null || upToken == null) { - return null; + // recorder + void recoverUploadProgressFromLocal() { + if (recorder == null || source == null || StringUtils.isNullOrEmpty(source.recordKey)) { + return; + } + + byte[] data = recorder.get(source.recordKey); + if (data == null) { + return; + } + + String jsonString = new String(data, Constants.UTF_8); + Region region = null; + ResumeUploadSource uploadSource = null; + try { + JsonObject jsonObject = (JsonObject) new JsonParser().parse(jsonString); + JsonObject sourceJson = jsonObject.getAsJsonObject("source"); + uploadSource = new Gson().fromJson(sourceJson, source.getClass()); + + JsonObject regionJson = jsonObject.getAsJsonObject("region"); + region = new Gson().fromJson(regionJson, Region.class); + } catch (Exception e) { + e.printStackTrace(); + } + + if (uploadSource == null || region == null) { + removeUploadProgressFromLocal(); + return; + } + + boolean isCopy = source.recoverFromRecordInfo(uploadSource); + if (!isCopy) { + removeUploadProgressFromLocal(); + return; } - UploadToken token = null; + if (config.region == null) { + config.region = region; + } else { + RegionGroup regionGroup = new RegionGroup(); + regionGroup.addRegion(region); + regionGroup.addRegion(config.region); + config.region = regionGroup; + } + } + + void saveUploadProgressToLocal() { + if (recorder == null || source == null || !source.hasUploadData() || StringUtils.isNullOrEmpty(source.recordKey)) { + return; + } try { - token = new UploadToken(upToken); - } catch (QiniuException ignored) { + JsonObject jsonObject = new JsonObject(); + JsonElement sourceJson = new Gson().toJsonTree(source); + if (sourceJson == null) { + return; + } + jsonObject.add("source", sourceJson); + JsonElement regionJson = new Gson().toJsonTree(config.region.getCurrentRegion(new UploadToken(upToken))); + if (regionJson == null) { + return; + } + jsonObject.add("region", regionJson); + String dataString = jsonObject.toString(); + recorder.set(source.recordKey, dataString.getBytes(Constants.UTF_8)); + } catch (Exception ignored) { } + } - if (token == null || !token.isValid()) { - return null; + void removeUploadProgressFromLocal() { + if (recorder == null || source == null || StringUtils.isNullOrEmpty(source.recordKey)) { + return; } - return config.region.getRegion(token); + recorder.del(source.recordKey); } } diff --git a/src/test/java/test/com/qiniu/TestConfig.java b/src/test/java/test/com/qiniu/TestConfig.java index 73857842..95dc30fa 100644 --- a/src/test/java/test/com/qiniu/TestConfig.java +++ b/src/test/java/test/com/qiniu/TestConfig.java @@ -21,6 +21,10 @@ public final class TestConfig { //test: ak, sk, auth public static final String testAccessKey = System.getenv("QINIU_ACCESS_KEY"); public static final String testSecretKey = System.getenv("QINIU_SECRET_KEY"); + // 内部测试环境 AK/SK + public static final String innerAccessKey = System.getenv("testAK"); + public static final String innerSecretKey = System.getenv("testSK"); + //sms: ak, sk, auth public static final String smsAccessKey = "test"; public static final String smsSecretKey = "test"; @@ -98,6 +102,17 @@ public static TestFile[] getTestFileArray(String fileSaveKey, String fileMimeTyp z0.regionId = "z0"; z0.region = Region.region0(); + TestFile z0_auto = new TestFile(); + z0_auto.key = fileSaveKey; + z0_auto.mimeType = fileMimeType; + z0_auto.bucketName = testBucket_z0; + z0_auto.testDomain = testDomain_z0; + z0_auto.testUrl = "http://" + testDomain_z0 + "/" + fileSaveKey; + z0_auto.testDomainTimeStamp = testDomain_z0_timeStamp; + z0_auto.testUrlTimeStamp = "http://" + testDomain_z0_timeStamp + "/" + fileSaveKey; + z0_auto.regionId = "z0"; + z0_auto.region = Region.region0(); + TestFile fog = new TestFile(); fog.key = fileSaveKey; fog.mimeType = fileMimeType; @@ -124,7 +139,7 @@ public static TestFile[] getTestFileArray(String fileSaveKey, String fileMimeTyp return new TestFile[]{na0}; } else { // return new TestFile[]{fog, fog1, z0, na0}; - return new TestFile[]{z0}; + return new TestFile[]{na0, z0_auto}; } } diff --git a/src/test/java/test/com/qiniu/storage/RecordUploadTest.java b/src/test/java/test/com/qiniu/storage/RecordUploadTest.java index 07dc7e5c..eff21b29 100644 --- a/src/test/java/test/com/qiniu/storage/RecordUploadTest.java +++ b/src/test/java/test/com/qiniu/storage/RecordUploadTest.java @@ -74,7 +74,7 @@ private void template(final int size, boolean isResumeV2, boolean isConcurrent) final Complete complete = new Complete(); try { - final String token = TestConfig.testAuth.uploadToken(bucket, expectKey); + final String token = TestConfig.testAuth.uploadToken(bucket, expectKey, 3600, null); final String recordKey = recorder.recorderKeyGenerate(expectKey, f); // 开始第一部分上传 @@ -110,7 +110,7 @@ public void run() { if (f.length() > Constants.BLOCK_SIZE) { // 终止第一部分上传,期望其部分成功 - for (int i = 150; i > 0; --i) { + for (int i = 15; i > 0; --i) { byte[] data = getRecord(recorder, recordKey); if (data != null) { doSleep(1000); diff --git a/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java b/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java new file mode 100644 index 00000000..d43dc7f2 --- /dev/null +++ b/src/test/java/test/com/qiniu/storage/SwitchRegionTest.java @@ -0,0 +1,384 @@ +package test.com.qiniu.storage; + +import com.qiniu.common.Constants; +import com.qiniu.common.QiniuException; +import com.qiniu.http.Client; +import com.qiniu.http.Response; +import com.qiniu.storage.*; +import com.qiniu.storage.model.FileInfo; +import com.qiniu.util.Auth; +import com.qiniu.util.Etag; +import com.qiniu.util.Md5; +import com.qiniu.util.StringMap; +import org.junit.Assert; +import org.junit.Test; +import test.com.qiniu.TempFile; +import test.com.qiniu.TestConfig; + +import java.io.File; +import java.io.IOException; +import java.net.URLEncoder; +import java.util.Date; + +import static org.junit.Assert.*; + +public class SwitchRegionTest { + + private static final int httpType = 0; + private static final int httpsType = 1; + + private static final int resumableV1Type = 0; + private static final int resumableV2Type = 1; + + private static final int formType = 0; + private static final int serialType = 1; + private static final int concurrentType = 2; + + private static int[][] testConfigList = { + {httpType, -1, formType}, + {httpType, resumableV1Type, serialType}, + {httpType, resumableV1Type, concurrentType}, + {httpType, resumableV2Type, serialType}, + {httpType, resumableV2Type, concurrentType}, + + {httpsType, -1, formType}, + {httpsType, resumableV1Type, serialType}, + {httpsType, resumableV1Type, concurrentType}, + {httpsType, resumableV2Type, serialType}, + {httpsType, resumableV2Type, concurrentType}, + }; + + /** + * 分片上传 + * 检测key、hash、fszie、fname是否符合预期 + * + * @param size 文件大小 + * @param httpType 采用 http / https 方式 + * @param uploadType 使用 form / 分片上传 api v1/v2 + * @param uploadStyle 采用串行或者并发方式上传 + * @throws IOException + */ + private void template(int size, int httpType, int uploadType, int uploadStyle) throws IOException { + boolean isHttps = httpType == httpsType; + boolean isConcurrent = uploadStyle == concurrentType; + + TestConfig.TestFile[] files = TestConfig.getTestFileArray(); + for (TestConfig.TestFile file : files) { + // 雾存储不支持 v1 + if (file.isFog() && uploadType == resumableV1Type) { + continue; + } + String bucket = file.getBucketName(); + + Region mockRegion = new Region.Builder() + .region("custom") + .srcUpHost("mock.src.host.com") + .accUpHost("mock.acc.host.com") + .build(); + Region region = file.getRegion(); + RegionGroup regionGroup = new RegionGroup(); + regionGroup.addRegion(mockRegion); + regionGroup.addRegion(region); + + Configuration config = new Configuration(regionGroup); + if (uploadType == resumableV2Type) { + config.resumableUploadAPIVersion = Configuration.ResumableUploadAPIVersion.V2; + config.resumableUploadAPIV2BlockSize = 2 * 1024 * 1024; + } + + config.useHttpsDomains = isHttps; + String key = "switch_region_"; + key += isHttps ? "_https" : "_http"; + + if (uploadType == resumableV1Type) { + key += "_resumeV1"; + } else if (uploadType == resumableV2Type) { + key += "_resumeV2"; + } + + if (uploadStyle == formType) { + key += "_form"; + } else if (uploadStyle == serialType) { + key += "_serial"; + } else if (uploadStyle == concurrentType) { + key += "_concurrent"; + } + + key += "_" + new Date().getTime(); + final String expectKey = "\r\n?&r=" + size + "k" + key; + final File f = TempFile.createFile(size); + final String fooKey = "foo"; + final String fooValue = "fooValue"; + final String metaDataKey = "metaDataKey"; + final String metaDataValue = "metaDataValue"; + final String returnBody = "{\"key\":\"$(key)\",\"hash\":\"$(etag)\",\"fsize\":\"$(fsize)\"" + + ",\"fname\":\"$(fname)\",\"mimeType\":\"$(mimeType)\",\"foo\":\"$(x:foo)\"}"; + String token = TestConfig.testAuth.uploadToken(bucket, expectKey, 3600, + new StringMap().put("returnBody", returnBody)); + + System.out.printf("\r\nkey:%s zone:%s\n", expectKey, regionGroup); + + + StringMap param = new StringMap(); + param.put("x:" + fooKey, fooValue); + param.put("x-qn-meta-" + metaDataKey, metaDataValue); + try { + BaseUploader up = null; + if (uploadStyle == formType) { + up = new FormUploader(new Client(), token, expectKey, f, param, Client.DefaultMime, false, config); + } else if (uploadStyle == serialType) { + up = new ResumeUploader(new Client(), token, expectKey, f, param, null, null, config); + } else { + config.resumableUploadMaxConcurrentTaskCount = 3; + up = new ConcurrentResumeUploader(new Client(), token, expectKey, f, param, null, null, config); + } + + Response r = up.upload(); + assertTrue(r + "", r.isOK()); + + StringMap ret = r.jsonToMap(); + assertEquals(expectKey, ret.get("key")); + assertEquals(f.getName(), ret.get("fname")); + assertEquals(String.valueOf(f.length()), ret.get("fsize").toString()); + assertEquals(fooValue, ret.get(fooKey).toString()); + + boolean checkMd5 = false; + if (config.resumableUploadAPIVersion == Configuration.ResumableUploadAPIVersion.V2 + && config.resumableUploadAPIV2BlockSize != Constants.BLOCK_SIZE) { + checkMd5 = true; + } + if (checkMd5) { + if (file.isFog()) { + String md5 = Md5.md5(f); + String serverMd5 = getFileMD5(file.getTestDomain(), expectKey); + System.out.println(" md5:" + md5); + System.out.println("serverMd5:" + serverMd5); + assertNotNull(serverMd5); + assertEquals(md5, serverMd5); + } + } else { + final String etag = Etag.file(f); + System.out.println(" etag:" + etag); + System.out.println("serverEtag:" + ret.get("hash")); + assertNotNull(ret.get("hash")); + assertEquals(etag, ret.get("hash")); + } + + FileInfo fileInfo = getFileInfo(config, bucket, expectKey); + assertEquals(fileInfo + "", metaDataValue, fileInfo.meta.get(metaDataKey).toString()); + } catch (QiniuException e) { + assertEquals("", e.response == null ? e + "e.response is null" : e.response.bodyString()); + fail(); + } + TempFile.remove(f); + } + } + + private String getFileMD5(String bucketDomain, String key) { + String url = "http://" + bucketDomain + "/" + URLEncoder.encode(key) + "?qhash/md5"; + Client client = new Client(); + + String md5 = null; + try { + Response response = client.get(url); + StringMap data = response.jsonToMap(); + md5 = data.get("hash").toString(); + } catch (QiniuException e) { + e.printStackTrace(); + } + return md5; + } + + private FileInfo getFileInfo(Configuration cfg, String bucket, String key) { + BucketManager manager = new BucketManager(TestConfig.testAuth, cfg); + try { + return manager.stat(bucket, key); + } catch (QiniuException e) { + e.printStackTrace(); + } + return null; + } + + @Test + public void test600k() throws Throwable { + for (int[] config : testConfigList) { + template(600, config[0], config[1], config[2]); + } + } + + @Test + public void test3M() throws Throwable { + if (TestConfig.isTravis()) { + return; + } + for (int[] config : testConfigList) { + template(1024 * 3, config[0], config[1], config[2]); + } + } + + @Test + public void test5M() throws Throwable { + if (TestConfig.isTravis()) { + return; + } + for (int[] config : testConfigList) { + template(1024 * 4, config[0], config[1], config[2]); + } + } + + @Test + public void test8M() throws Throwable { + for (int[] config : testConfigList) { + template(1024 * 8, config[0], config[1], config[2]); + } + } + + @Test + public void test8M1k() throws Throwable { + for (int[] config : testConfigList) { + template(1024 * 8 + 1, config[0], config[1], config[2]); + } + } + + @Test + public void test10M() throws Throwable { + for (int[] config : testConfigList) { + template(1024 * 10, config[0], config[1], config[2]); + } + } + + @Test + public void test20M() throws Throwable { + for (int[] config : testConfigList) { + template(1024 * 20, config[0], config[1], config[2]); + } + } + + @Test + public void test20M1K() throws Throwable { + for (int[] config : testConfigList) { + template(1024 * 20 + 1, config[0], config[1], config[2]); + } + } + + // 内部环境测试 +// @Test + public void testInnerEnvSwitchRegion() { + try { + long s = new Date().getTime(); + uploadByInnerEnvSwitchRegion(1024 * 500 + 1, httpType, resumableV2Type, concurrentType); + long e = new Date().getTime(); + System.out.println("耗时：" + (e - s)); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail("testInnerEnvSwitchRegion:" + e); + } + } + + public void uploadByInnerEnvSwitchRegion(int size, int httpType, int uploadType, int uploadStyle) throws Exception { + + boolean isHttps = httpType == httpsType; + boolean isConcurrent = uploadStyle == concurrentType; + String bucket = "aaaabbbbb"; + + Region region0 = new Region.Builder() + .srcUpHost("10.200.20.23:5010") + .accUpHost("10.200.20.23:5010") + .build(); + + Region region1 = new Region.Builder() + .srcUpHost("10.200.20.24:5010") + .accUpHost("10.200.20.24:5010") + .build(); + + RegionGroup regionGroup = new RegionGroup(); + regionGroup.addRegion(region0); + regionGroup.addRegion(region1); + + Configuration config = new Configuration(regionGroup); + if (uploadType == resumableV2Type) { + config.resumableUploadAPIVersion = Configuration.ResumableUploadAPIVersion.V2; + config.resumableUploadAPIV2BlockSize = 4 * 1024 * 1024; + } + + config.useHttpsDomains = isHttps; + String key = "switch_region_"; + key += isHttps ? "_https" : "_http"; + + if (uploadType == resumableV1Type) { + key += "_resumeV1"; + } else if (uploadType == resumableV2Type) { + key += "_resumeV2"; + } + + if (uploadStyle == formType) { + key += "_form"; + } else if (uploadStyle == serialType) { + key += "_serial"; + } else if (uploadStyle == concurrentType) { + key += "_concurrent"; + } + + key += "_" + new Date().getTime(); + final String expectKey = "\r\n?&r=" + size + "k" + key; + final File f = TempFile.createFile(size); + + final String fooKey = "foo"; + final String fooValue = "fooValue"; + final String metaDataKey = "metaDataKey"; + final String metaDataValue = "metaDataValue"; + final String returnBody = "{\"key\":\"$(key)\",\"hash\":\"$(etag)\",\"fsize\":\"$(fsize)\"" + + ",\"fname\":\"$(fname)\",\"mimeType\":\"$(mimeType)\",\"foo\":\"$(x:foo)\"}"; + + Auth auth = Auth.create(TestConfig.innerAccessKey, TestConfig.innerSecretKey); + + String token = auth.uploadToken(bucket, expectKey, 3600, + new StringMap().put("returnBody", returnBody)); + + System.out.printf("\r\nkey:%s zone:%s\n", expectKey, regionGroup); + + + StringMap param = new StringMap(); + param.put("x:" + fooKey, fooValue); + param.put("x-qn-meta-" + metaDataKey, metaDataValue); + try { + BaseUploader up = null; + if (uploadStyle == formType) { + up = new FormUploader(new Client(), token, expectKey, f, param, Client.DefaultMime, false, config); + } else if (uploadStyle == serialType) { + up = new ResumeUploader(new Client(), token, expectKey, f, param, null, null, config); + } else { + config.resumableUploadMaxConcurrentTaskCount = 3; + up = new ConcurrentResumeUploader(new Client(), token, expectKey, f, param, null, null, config); + } + + Response r = up.upload(); + assertTrue(r + "", r.isOK()); + + StringMap ret = r.jsonToMap(); + assertEquals(expectKey, ret.get("key")); + assertEquals(f.getName(), ret.get("fname")); + assertEquals(String.valueOf(f.length()), ret.get("fsize").toString()); + assertEquals(fooValue, ret.get(fooKey).toString()); + + final String etag = Etag.file(f); + System.out.println(" etag:" + etag); + System.out.println("serverEtag:" + ret.get("hash")); + assertNotNull(ret.get("hash")); + assertEquals(etag, ret.get("hash")); + } catch (QiniuException e) { + assertEquals("", e.response == null ? e + "e.response is null" : e.response.bodyString()); + fail(); + } + TempFile.remove(f); + } + + + class MyRet { + public String hash; + public String key; + public String fsize; + public String fname; + public String mimeType; + } +}