From efd6c326a025ab3388946069066613ff62b5ff96 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 21 Sep 2016 18:18:56 +0800 Subject: [PATCH 01/10] use autozone --- .xmake/cache/history | 9 + build.gradle | 2 +- src/main/java/com/qiniu/common/AutoZone.java | 154 ++++++++++ src/main/java/com/qiniu/common/Config.java | 88 ------ src/main/java/com/qiniu/common/Constants.java | 25 ++ src/main/java/com/qiniu/common/FixedZone.java | 72 +++++ src/main/java/com/qiniu/common/Zone.java | 86 ++++-- src/main/java/com/qiniu/http/Client.java | 32 +- .../qiniu/processing/OperationManager.java | 16 +- .../java/com/qiniu/storage/BucketManager.java | 15 +- .../java/com/qiniu/storage/Configuration.java | 62 ++++ .../java/com/qiniu/storage/FormUploader.java | 21 +- src/main/java/com/qiniu/storage/Recorder.java | 4 + .../com/qiniu/storage/ResumeUploader.java | 29 +- .../java/com/qiniu/storage/UploadManager.java | 48 +-- .../java/com/qiniu/storage/model/Policy.java | 7 + .../storage/persistent/FileRecorder.java | 5 + src/main/java/com/qiniu/util/Auth.java | 2 +- src/main/java/com/qiniu/util/Etag.java | 8 +- src/main/java/com/qiniu/util/StringUtils.java | 6 +- src/main/java/com/qiniu/util/UC.java | 203 ------------- .../java/com/qiniu/util/UrlSafeBase64.java | 4 +- src/test/java/com/qiniu/HttpTest.java | 16 +- .../java/com/qiniu/common/AutoZoneTest.java | 69 +++++ .../java/com/qiniu/processing/PfopTest.java | 4 +- .../java/com/qiniu/storage/BucketTest.java | 5 +- .../com/qiniu/storage/FormUploadTest.java | 39 ++- .../com/qiniu/storage/RecordUploadTest.java | 7 +- .../com/qiniu/storage/ResumeUploadTest.java | 56 ++-- src/test/java/com/qiniu/util/EtagTest.java | 4 +- src/test/java/com/qiniu/util/UCTest.java | 285 ------------------ 31 files changed, 610 insertions(+), 773 deletions(-) create mode 100644 .xmake/cache/history create mode 100644 src/main/java/com/qiniu/common/AutoZone.java delete mode 100644 src/main/java/com/qiniu/common/Config.java create mode 100644 src/main/java/com/qiniu/common/Constants.java create mode 100644 src/main/java/com/qiniu/common/FixedZone.java create mode 100644 src/main/java/com/qiniu/storage/Configuration.java create mode 100644 src/main/java/com/qiniu/storage/model/Policy.java delete mode 100644 src/main/java/com/qiniu/util/UC.java create mode 100644 src/test/java/com/qiniu/common/AutoZoneTest.java delete mode 100644 src/test/java/com/qiniu/util/UCTest.java diff --git a/.xmake/cache/history b/.xmake/cache/history new file mode 100644 index 000000000..e9e5ae188 --- /dev/null +++ b/.xmake/cache/history @@ -0,0 +1,9 @@ + +{ + ["__version"] = "XMake v2.0.5" +, ["cmdlines"] = + { + "xmake" + } + +} diff --git a/build.gradle b/build.gradle index eac14912c..4cce28edb 100755 --- a/build.gradle +++ b/build.gradle @@ -34,7 +34,7 @@ apply plugin: 'checkstyle' def versionName() { - String config = 'src/main/java/com/qiniu/common/Config.java' + String config = 'src/main/java/com/qiniu/common/Constants.java' String fileContents = new File(config).text Matcher myMatcher = fileContents =~ /VERSION = "(.+)";/ String version = myMatcher[0][1] diff --git a/src/main/java/com/qiniu/common/AutoZone.java b/src/main/java/com/qiniu/common/AutoZone.java new file mode 100644 index 000000000..0a45c244f --- /dev/null +++ b/src/main/java/com/qiniu/common/AutoZone.java @@ -0,0 +1,154 @@ +package com.qiniu.common; + +import com.qiniu.http.Client; +import com.qiniu.http.Response; +import com.qiniu.util.Json; +import com.qiniu.util.UrlSafeBase64; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Created by bailong on 16/9/15. + */ +final class AutoZone extends Zone { + static AutoZone instance = new AutoZone(); + private final String ucServer; + private Map zones = new ConcurrentHashMap<>(); + private Client client; + + AutoZone() { + this("https://uc.qbox.me"); + } + + AutoZone(String ucServer) { + this.ucServer = ucServer; + client = new Client(); + } + + private UCRet getZoneJson(ZoneIndex index) throws QiniuException { + String address = ucServer + "/v1/query?ak=" + index.accessKey + "&bucket=" + index.bucket; + + Response r = client.get(address); + return r.jsonToObject(UCRet.class); + } + + // only for test public + ZoneInfo zoneInfo(String ak, String bucket) throws QiniuException { + ZoneIndex index = new ZoneIndex(ak, bucket); + ZoneInfo info = zones.get(index); + if (info == null) { + UCRet ret = getZoneJson(index); + try { + info = ZoneInfo.buildFromUcRet(ret); + } catch (Exception e) { + e.printStackTrace(); + } + + if (info != null) { + zones.put(index, info); + } + } + return info; + } + + // only for test public + ZoneInfo queryByToken(String token) { + try { + // http://developer.qiniu.com/article/developer/security/upload-token.html + // http://developer.qiniu.com/article/developer/security/put-policy.html + String[] strings = token.split(":"); + String ak = strings[0]; + String policy = new String(UrlSafeBase64.decode(strings[2]), Constants.UTF_8); + String bkt = Json.decode(policy).get("scope").toString().split(":")[0]; + return zoneInfo(ak, bkt); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public String upHost(String token) { + ZoneInfo info = queryByToken(token); + if (info == null) { + return ""; + } + return info.upHost; + } + + public String upHostBackup(String token) { + ZoneInfo info = queryByToken(token); + if (info == null) { + return ""; + } + return info.upBackup; + } + + public String upIpBackup(String token) { + ZoneInfo info = queryByToken(token); + if (info == null) { + return ""; + } + return info.upIp; + } + + public String upHostHttps(String token) { + ZoneInfo info = queryByToken(token); + if (info == null) { + return ""; + } + return info.upHttps; + } + + static class ZoneInfo { + final String ioHost; + final String upHost; + final String upIp; + final String upBackup; + final String upHttps; + + private ZoneInfo(String ioHost, String upHost, String upIp, String upBackup, String upHttps) { + this.ioHost = ioHost; + this.upHost = upHost; + this.upIp = upIp; + this.upBackup = upBackup; + this.upHttps = upHttps; + } + + static ZoneInfo buildFromUcRet(UCRet ret) { + String ioHost = ret.http.get("io").get(0); + List up = ret.http.get("up"); + String upHost = up.get(0); + String upBackup = up.get(1); + String upIp = up.get(2).split(" ")[2].split("//")[1]; + String upHttps = ret.https.get("up").get(0); + + return new ZoneInfo(ioHost, upHost, upIp, upBackup, upHttps); + } + } + + private static class ZoneIndex { + private final String accessKey; + private final String bucket; + + ZoneIndex(String accessKey, String bucket) { + this.accessKey = accessKey; + this.bucket = bucket; + } + + public int hashCode() { + return accessKey.hashCode() * 37 + bucket.hashCode(); + } + + public boolean equals(Object obj) { + return obj == this || !(obj == null || !(obj instanceof ZoneIndex)) + && ((ZoneIndex) obj).accessKey.equals(accessKey) && ((ZoneIndex) obj).bucket.equals(bucket); + } + } + + private class UCRet { + Map> http; + Map> https; + } +} diff --git a/src/main/java/com/qiniu/common/Config.java b/src/main/java/com/qiniu/common/Config.java deleted file mode 100644 index e0f9da287..000000000 --- a/src/main/java/com/qiniu/common/Config.java +++ /dev/null @@ -1,88 +0,0 @@ -package com.qiniu.common; - -import com.qiniu.http.ProxyConfiguration; -import qiniu.happydns.DnsClient; - -import java.nio.charset.Charset; - -// CHECKSTYLE:OFF - -public final class Config { - - public static final String VERSION = "7.1.3"; - /** - * 断点上传时的分块大小(默认的分块大小, 不允许改变) - */ - public static final int BLOCK_SIZE = 4 * 1024 * 1024; - - public static final Charset UTF_8 = Charset.forName("UTF-8"); - /** - * 默认API服务器 - */ - public static String API_HOST = "http://api.qiniu.com"; - /** - * 默认文件列表服务器 - */ - public static String RSF_HOST = "http://rsf.qbox.me"; - /** - * 默认文件管理服务器 - */ - public static String RS_HOST = "http://rs.qbox.me"; - /** - * 默认文件服务器 - */ - public static String IO_HOST = "http://iovip.qbox.me"; - - /** - * 获取上传地址服务器 - */ - public static String UC_HOST = "https://uc.qbox.me"; - - /** - * 使用的Zone, 若不指定,通过七牛服务自动判断 - */ - public static Zone zone = null; - - /** - * 上传是否使用 https , 默认否 - */ - public static boolean UPLOAD_BY_HTTPS = false; - - /** - * 如果文件大小大于此值则使用断点上传, 否则使用Form上传 - */ - public static int PUT_THRESHOLD = BLOCK_SIZE; - /** - * 连接超时时间 单位秒(默认10s) - */ - public static int CONNECT_TIMEOUT = 10; - /** - * 写超时时间 单位秒(默认 0 , 不超时) - */ - public static int WRITE_TIMEOUT = 0; - /** - * 回复超时时间 单位秒(默认30s) - */ - public static int RESPONSE_TIMEOUT = 30; - /** - * 上传失败重试次数 - */ - public static int RETRY_MAX = 5; - /** - * 外部dns - */ - public static DnsClient dns = null; - /* - * 解析域名时,优先使用host配置,主要针对内部局域网配置 - */ - public static boolean dnsHostFirst = false; - /** - * proxy - */ - public static ProxyConfiguration proxy = null; - - private Config() { - } - -} -// CHECKSTYLE:ON diff --git a/src/main/java/com/qiniu/common/Constants.java b/src/main/java/com/qiniu/common/Constants.java new file mode 100644 index 000000000..3615e8d6c --- /dev/null +++ b/src/main/java/com/qiniu/common/Constants.java @@ -0,0 +1,25 @@ +package com.qiniu.common; + +import java.nio.charset.Charset; + +/** + * Created by bailong on 16/9/14. + */ +public interface Constants { + String VERSION = "8.0.0"; + int BLOCK_SIZE = 4 * 1024 * 1024; + Charset UTF_8 = Charset.forName("UTF-8"); + /** + * 连接超时时间 单位秒(默认10s) + */ + public int CONNECT_TIMEOUT = 10; + /** + * 写超时时间 单位秒(默认 0 , 不超时) + */ + public int WRITE_TIMEOUT = 0; + /** + * 回复超时时间 单位秒(默认30s) + */ + public int RESPONSE_TIMEOUT = 30; +} + diff --git a/src/main/java/com/qiniu/common/FixedZone.java b/src/main/java/com/qiniu/common/FixedZone.java new file mode 100644 index 000000000..4c2f518d3 --- /dev/null +++ b/src/main/java/com/qiniu/common/FixedZone.java @@ -0,0 +1,72 @@ +package com.qiniu.common; + +/** + * Created by bailong on 16/9/15. + */ +public final class FixedZone extends Zone { + /** + * 默认上传服务器 + */ + private final String upHost; + /** + * 备用上传服务器，当默认服务器网络链接失败时使用 + */ + private final String upHostBackup; + + private final String upHostHttps; + + private final String upHttpIp; + + private final String rsHost; + + private final String rsfHost; + + private final String ioHost; + + private final String apiHost; + + + public FixedZone(String upHost, String upHostBackup, String upHttpIp, + String rsHost, String rsfHost, String ioHost, String upHostHttps, String apiHost) { + this.upHost = upHost; + this.upHostBackup = upHostBackup; + this.upHttpIp = upHttpIp; + this.rsHost = rsHost; + this.rsfHost = rsfHost; + this.ioHost = ioHost; + this.upHostHttps = upHostHttps; + this.apiHost = apiHost; + } + + public String upHost(String token) { + return upHost; + } + + public String upHostBackup(String token) { + return upHostBackup; + } + + public String upIpBackup(String token) { + return upHttpIp; + } + + public String upHostHttps(String token) { + return upHostHttps; + } + + public String rsHost(String ak, String bucket) { + return rsHost; + } + + public String rsfHost(String ak, String bucket) { + return rsfHost; + } + + public String ioHost(String ak, String bucket) { + return ioHost; + } + + public String apiHost(String ak, String bucket) { + return apiHost; + } +} diff --git a/src/main/java/com/qiniu/common/Zone.java b/src/main/java/com/qiniu/common/Zone.java index 39d98b5b6..bbb5489bf 100644 --- a/src/main/java/com/qiniu/common/Zone.java +++ b/src/main/java/com/qiniu/common/Zone.java @@ -3,45 +3,73 @@ /** * 上传多区域 */ -public final class Zone { - /** - * 默认上传服务器 - */ - public final String upHost; - /** - * 备用上传服务器，当默认服务器网络链接失败时使用 - */ - public final String upHostBackup; - - public Zone(String upHost, String upHostBackup) { - this.upHost = upHost.trim(); - this.upHostBackup = upHostBackup.trim(); - } +public abstract class Zone { public static Zone zone0() { - return new Zone("http://up.qiniu.com", "http://upload.qiniu.com"); + return new FixedZone("http://up.qiniu.com", "http://upload.qiniu.com", + "", "http://rs.qbox.me", "http://rsf.qbox.me", "http://iovip.qbox.me", + "https://up.qbox.me", "http://api.qiniu.com"); } public static Zone zone1() { - return new Zone("http://up-z1.qiniu.com", "http://upload-z1.qiniu.com"); + return new FixedZone("http://up-z1.qiniu.com", "http://upload-z1.qiniu.com", + "", "http://rs-z1.qbox.me", "http://rsf-z1.qbox.me", "http://iovip-z1.qbox.me", + "https://up-z1.qbox.me", "http://api-z1.qiniu.com"); + } + + public static Zone autoZone() { + return AutoZone.instance; + } + + protected static String upHostFromPolicy(String token) { + return null; + } + + public String upHost(String token) { + throw new UnsupportedOperationException(); + } + + public String upHostBackup(String token) { + throw new UnsupportedOperationException(); + } + + public String upIpBackup(String token) { + throw new UnsupportedOperationException(); + } + + public String upHostHttps(String token) { + throw new UnsupportedOperationException(); + } + + public String rsHost(String ak, String bucket) { + throw new UnsupportedOperationException(); + } + + public String rsHost() { + return rsHost("", ""); + } + + public String rsfHost(String ak, String bucket) { + throw new UnsupportedOperationException(); + } + + public String rsfHost() { + return rsfHost("", ""); + } + + public String ioHost(String ak, String bucket) { + throw new UnsupportedOperationException(); } - @Override - public int hashCode() { - return upHost.hashCode() * upHostBackup.hashCode(); + public String ioHost() { + return ioHost("", ""); } - @Override - public boolean equals(Object obj) { - if (obj instanceof Zone) { - Zone that = (Zone) obj; - return that.upHost.equals(this.upHost) && that.upHostBackup.equals(this.upHostBackup); - } - return super.equals(obj); + public String apiHost(String ak, String bucket) { + throw new UnsupportedOperationException(); } - @Override - public String toString() { - return super.toString() + ", upHost: " + this.upHost + ", upHostBackup: " + this.upHostBackup; + public String apiHost() { + return apiHost("", ""); } } diff --git a/src/main/java/com/qiniu/http/Client.java b/src/main/java/com/qiniu/http/Client.java index 60a816a8f..e0e0962db 100644 --- a/src/main/java/com/qiniu/http/Client.java +++ b/src/main/java/com/qiniu/http/Client.java @@ -1,6 +1,6 @@ package com.qiniu.http; -import com.qiniu.common.Config; +import com.qiniu.common.Constants; import com.qiniu.common.QiniuException; import com.qiniu.util.StringMap; import com.qiniu.util.StringUtils; @@ -29,6 +29,12 @@ public final class Client { private final OkHttpClient httpClient; public Client() { + this(null, false, null, + Constants.CONNECT_TIMEOUT, Constants.RESPONSE_TIMEOUT, Constants.WRITE_TIMEOUT); + } + + public Client(final DnsClient dns, final boolean hostFirst, final ProxyConfiguration proxy, + int connTimeout, int readTimeout, int writeTimeout) { Dispatcher dispatcher = new Dispatcher(); dispatcher.setMaxRequests(64); dispatcher.setMaxRequestsPerHost(16); @@ -53,15 +59,14 @@ public okhttp3.Response intercept(Chain chain) throws IOException { return response; } }); - if (Config.dns != null) { - final DnsClient d = Config.dns; + if (dns != null) { builder.dns(new Dns() { @Override public List lookup(String hostname) throws UnknownHostException { InetAddress[] ips; - Domain domain = new Domain(hostname, false, Config.dnsHostFirst); + Domain domain = new Domain(hostname, false, hostFirst); try { - ips = d.queryInetAddress(domain); + ips = dns.queryInetAddress(domain); } catch (IOException e) { e.printStackTrace(); throw new UnknownHostException(e.getMessage()); @@ -75,16 +80,15 @@ public List lookup(String hostname) throws UnknownHostException { } }); } - final ProxyConfiguration p = Config.proxy; - if (p != null) { - builder.proxy(p.proxy()); - if (p.user != null && p.password != null) { - builder.proxyAuthenticator(p.authenticator()); + if (proxy != null) { + builder.proxy(proxy.proxy()); + if (proxy.user != null && proxy.password != null) { + builder.proxyAuthenticator(proxy.authenticator()); } } - builder.connectTimeout(Config.CONNECT_TIMEOUT, TimeUnit.SECONDS); - builder.readTimeout(Config.RESPONSE_TIMEOUT, TimeUnit.SECONDS); - builder.writeTimeout(Config.WRITE_TIMEOUT, TimeUnit.SECONDS); + builder.connectTimeout(connTimeout, TimeUnit.SECONDS); + builder.readTimeout(readTimeout, TimeUnit.SECONDS); + builder.writeTimeout(writeTimeout, TimeUnit.SECONDS); httpClient = builder.build(); } @@ -93,7 +97,7 @@ private static String userAgent() { String javaVersion = "Java/" + System.getProperty("java.version"); String os = System.getProperty("os.name") + " " + System.getProperty("os.arch") + " " + System.getProperty("os.version"); - String sdk = "QiniuJava/" + Config.VERSION; + String sdk = "QiniuJava/" + Constants.VERSION; return sdk + " (" + os + ") " + javaVersion; } diff --git a/src/main/java/com/qiniu/processing/OperationManager.java b/src/main/java/com/qiniu/processing/OperationManager.java index 448bf7916..0c3af839d 100644 --- a/src/main/java/com/qiniu/processing/OperationManager.java +++ b/src/main/java/com/qiniu/processing/OperationManager.java @@ -1,9 +1,9 @@ package com.qiniu.processing; -import com.qiniu.common.Config; import com.qiniu.common.QiniuException; import com.qiniu.http.Client; import com.qiniu.http.Response; +import com.qiniu.storage.Configuration; import com.qiniu.util.Auth; import com.qiniu.util.StringMap; import com.qiniu.util.StringUtils; @@ -15,10 +15,13 @@ public final class OperationManager { private final Client client; private final Auth auth; + private final Configuration configuration; - public OperationManager(Auth auth) { + public OperationManager(Auth auth, Configuration c) { this.auth = auth; - this.client = new Client(); + configuration = c.clone(); + this.client = new Client(configuration.dns, configuration.dnsHostFirst, configuration.proxy, + configuration.connectTimeout, configuration.responseTimeout, configuration.writeTimeout); } /** @@ -50,11 +53,14 @@ public String pfop(String bucket, String key, String fops, StringMap params) thr params = params == null ? new StringMap() : params; params.put("bucket", bucket).put("key", key).put("fops", fops); byte[] data = StringUtils.utf8Bytes(params.formString()); - String url = Config.API_HOST + "/pfop/"; + String url = configuration.zone.apiHost(auth.accessKey, bucket) + "/pfop/"; StringMap headers = auth.authorization(url, data, Client.FormMime); Response response = client.post(url, data, headers, Client.FormMime); PfopStatus status = response.jsonToObject(PfopStatus.class); - return status.persistentId; + if (status != null) { + return status.persistentId; + } + return null; } private class PfopStatus { diff --git a/src/main/java/com/qiniu/storage/BucketManager.java b/src/main/java/com/qiniu/storage/BucketManager.java index 3c0bad223..760df9018 100644 --- a/src/main/java/com/qiniu/storage/BucketManager.java +++ b/src/main/java/com/qiniu/storage/BucketManager.java @@ -1,6 +1,5 @@ package com.qiniu.storage; -import com.qiniu.common.Config; import com.qiniu.common.QiniuException; import com.qiniu.http.Client; import com.qiniu.http.Response; @@ -22,11 +21,13 @@ */ public final class BucketManager { private final Auth auth; + private final Configuration configuration; private final Client client; - public BucketManager(Auth auth) { + public BucketManager(Auth auth, Configuration c) { this.auth = auth; - client = new Client(); + this.configuration = c.clone(); + client = new Client(c.dns, c.dnsHostFirst, c.proxy, c.connectTimeout, c.responseTimeout, c.writeTimeout); } /** @@ -109,7 +110,7 @@ public FileListing listFiles(String bucket, String prefix, String marker, int li StringMap map = new StringMap().put("bucket", bucket).putNotEmpty("marker", marker) .putNotEmpty("prefix", prefix).putNotEmpty("delimiter", delimiter).putWhen("limit", limit, limit > 0); - String url = Config.RSF_HOST + "/list?" + map.formString(); + String url = configuration.zone.rsfHost(auth.accessKey, bucket) + "/list?" + map.formString(); Response r = get(url); return r.jsonToObject(FileListing.class); } @@ -291,7 +292,7 @@ public Response batch(Batch operations) throws QiniuException { } private Response rsPost(String path, byte[] body) throws QiniuException { - String url = Config.RS_HOST + path; + String url = configuration.zone.rsHost() + path; return post(url, body); } @@ -300,12 +301,12 @@ private Response rsPost(String path) throws QiniuException { } private Response rsGet(String path) throws QiniuException { - String url = Config.RS_HOST + path; + String url = configuration.zone.rsHost() + path; return get(url); } private Response ioPost(String path) throws QiniuException { - String url = Config.IO_HOST + path; + String url = configuration.zone.ioHost() + path; return post(url, null); } diff --git a/src/main/java/com/qiniu/storage/Configuration.java b/src/main/java/com/qiniu/storage/Configuration.java new file mode 100644 index 000000000..ad7bf2701 --- /dev/null +++ b/src/main/java/com/qiniu/storage/Configuration.java @@ -0,0 +1,62 @@ +package com.qiniu.storage; + +import com.qiniu.common.Constants; +import com.qiniu.common.Zone; +import com.qiniu.http.ProxyConfiguration; +import qiniu.happydns.DnsClient; + +/** + * Created by bailong on 16/9/21. + */ +public final class Configuration { + + /** + * 使用的Zone + */ + public final Zone zone; + /** + * 上传是否使用 https , 默认否 + */ + public boolean uploadByHttps = false; + /** + * 如果文件大小大于此值则使用断点上传, 否则使用Form上传 + */ + public int putThreshold = Constants.BLOCK_SIZE; + /** + * 连接超时时间 单位秒(默认10s) + */ + public int connectTimeout = Constants.CONNECT_TIMEOUT; + /** + * 写超时时间 单位秒(默认 0 , 不超时) + */ + public int writeTimeout = Constants.WRITE_TIMEOUT; + /** + * 回复超时时间 单位秒(默认30s) + */ + public int responseTimeout = Constants.RESPONSE_TIMEOUT; + /** + * 上传失败重试次数 + */ + public int RETRY_MAX = 5; + /** + * 外部dns + */ + public DnsClient dns = null; + /* + * 解析域名时,优先使用host配置,主要针对内部局域网配置 + */ + public boolean dnsHostFirst = false; + /** + * proxy + */ + public ProxyConfiguration proxy = null; + + public Configuration(Zone zone) { + this.zone = zone; + } + + public Configuration clone() { + return this; + } + +} diff --git a/src/main/java/com/qiniu/storage/FormUploader.java b/src/main/java/com/qiniu/storage/FormUploader.java index 7b7ac39cc..ec96f9706 100644 --- a/src/main/java/com/qiniu/storage/FormUploader.java +++ b/src/main/java/com/qiniu/storage/FormUploader.java @@ -6,7 +6,6 @@ import com.qiniu.http.Response; import com.qiniu.util.Crc32; import com.qiniu.util.StringMap; -import com.qiniu.util.UC; import java.io.File; import java.io.IOException; @@ -19,22 +18,23 @@ public final class FormUploader { private final byte[] data; private final String mime; private final boolean checkCrc; + private final Configuration configuration; private StringMap params; private Client client; private String fileName; FormUploader(Client client, String upToken, String key, byte[] data, StringMap params, - String mime, boolean checkCrc) { - this(client, upToken, key, data, null, params, mime, checkCrc); + String mime, boolean checkCrc, Configuration configuration) { + this(client, upToken, key, data, null, params, mime, checkCrc, configuration); } FormUploader(Client client, String upToken, String key, File file, StringMap params, - String mime, boolean checkCrc) { - this(client, upToken, key, null, file, params, mime, checkCrc); + String mime, boolean checkCrc, Configuration configuration) { + this(client, upToken, key, null, file, params, mime, checkCrc, configuration); } private FormUploader(Client client, String upToken, String key, byte[] data, File file, StringMap params, - String mime, boolean checkCrc) { + String mime, boolean checkCrc, Configuration configuration) { this.client = client; token = upToken; this.key = key; @@ -43,21 +43,22 @@ private FormUploader(Client client, String upToken, String key, byte[] data, Fil this.params = params; this.mime = mime; this.checkCrc = checkCrc; + this.configuration = configuration; } Response upload() throws QiniuException { buildParams(); if (data != null) { - return client.multipartPost(UC.zone(token).upHost, params, "file", fileName, data, mime, new StringMap()); + return client.multipartPost(configuration.zone.upHost(token), params, "file", fileName, data, mime, new StringMap()); } - return client.multipartPost(UC.zone(token).upHost, params, "file", fileName, file, mime, new StringMap()); + return client.multipartPost(configuration.zone.upHost(token), params, "file", fileName, file, mime, new StringMap()); } void asyncUpload(final UpCompletionHandler handler) throws IOException { buildParams(); if (data != null) { - client.asyncMultipartPost(UC.zone(token).upHost, params, "file", fileName, + client.asyncMultipartPost(configuration.zone.upHost(token), params, "file", fileName, data, mime, new StringMap(), new AsyncCallback() { @Override public void complete(Response r) { @@ -66,7 +67,7 @@ public void complete(Response r) { }); return; } - client.asyncMultipartPost(UC.zone(token).upHost, params, "file", fileName, + client.asyncMultipartPost(configuration.zone.upHost(token), params, "file", fileName, file, mime, new StringMap(), new AsyncCallback() { @Override public void complete(Response r) { diff --git a/src/main/java/com/qiniu/storage/Recorder.java b/src/main/java/com/qiniu/storage/Recorder.java index 952c3ebe2..d4bae3e29 100644 --- a/src/main/java/com/qiniu/storage/Recorder.java +++ b/src/main/java/com/qiniu/storage/Recorder.java @@ -1,5 +1,7 @@ package com.qiniu.storage; +import java.io.File; + /** * 定义分片上传时纪录上传进度的接口 */ @@ -27,4 +29,6 @@ public interface Recorder { * @param key 持久化的键 */ void del(String key); + + String recorderKeyGenerate(String key, File file); } diff --git a/src/main/java/com/qiniu/storage/ResumeUploader.java b/src/main/java/com/qiniu/storage/ResumeUploader.java index b519cf971..b3d398f23 100644 --- a/src/main/java/com/qiniu/storage/ResumeUploader.java +++ b/src/main/java/com/qiniu/storage/ResumeUploader.java @@ -1,14 +1,13 @@ package com.qiniu.storage; import com.google.gson.Gson; -import com.qiniu.common.Config; +import com.qiniu.common.Constants; import com.qiniu.common.QiniuException; import com.qiniu.http.Client; import com.qiniu.http.Response; import com.qiniu.storage.model.ResumeBlockInfo; import com.qiniu.util.StringMap; import com.qiniu.util.StringUtils; -import com.qiniu.util.UC; import com.qiniu.util.UrlSafeBase64; import java.io.File; @@ -38,17 +37,18 @@ public final class ResumeUploader { private final StringMap params; private final String mime; private final String[] contexts; + private final Configuration configuration; private final Client client; private final byte[] blockBuffer; private final Recorder recorder; - private final String recorderKey; private final long modifyTime; private final RecordHelper helper; private FileInputStream file; private String host; ResumeUploader(Client client, String upToken, String key, File file, - StringMap params, String mime, Recorder recorder, String recorderKey) { + StringMap params, String mime, Recorder recorder, Configuration configuration) { + this.configuration = configuration; this.client = client; this.upToken = upToken; this.key = key; @@ -56,18 +56,17 @@ public final class ResumeUploader { this.size = file.length(); this.params = params; this.mime = mime == null ? Client.DefaultMime : mime; - long count = (size + Config.BLOCK_SIZE - 1) / Config.BLOCK_SIZE; + long count = (size + Constants.BLOCK_SIZE - 1) / Constants.BLOCK_SIZE; this.contexts = new String[(int) count]; - this.blockBuffer = new byte[Config.BLOCK_SIZE]; + this.blockBuffer = new byte[Constants.BLOCK_SIZE]; this.recorder = recorder; - this.recorderKey = recorderKey; this.modifyTime = f.lastModified(); helper = new RecordHelper(); } public Response upload() throws QiniuException { if (host == null) { - this.host = UC.zone(upToken).upHost; + this.host = configuration.zone.upHost(upToken); } long uploaded = helper.recoveryFromRecord(); try { @@ -98,7 +97,7 @@ public Response upload() throws QiniuException { response = makeBlock(blockBuffer, blockSize); } catch (QiniuException e) { if (e.code() < 0) { - host = UC.zone(upToken).upHostBackup; + host = configuration.zone.upHostBackup(upToken); } if (e.response == null || e.response.needRetry()) { retry = true; @@ -191,17 +190,18 @@ private Response post(String url, byte[] data, int offset, int size) throws Qini } private int nextBlockSize(long uploaded) { - if (size < uploaded + Config.BLOCK_SIZE) { + if (size < uploaded + Constants.BLOCK_SIZE) { return (int) (size - uploaded); } - return Config.BLOCK_SIZE; + return Constants.BLOCK_SIZE; } private int blockIdx(long offset) { - return (int) (offset / Config.BLOCK_SIZE); + return (int) (offset / Constants.BLOCK_SIZE); } private class RecordHelper { + long recoveryFromRecord() { try { return recoveryFromRecord0(); @@ -217,6 +217,9 @@ long recoveryFromRecord0() { if (recorder == null) { return 0; } + + String recorderKey = recorder.recorderKeyGenerate(key, f); + byte[] data = recorder.get(recorderKey); if (data == null) { return 0; @@ -237,6 +240,7 @@ long recoveryFromRecord0() { void removeRecord() { try { if (recorder != null) { + String recorderKey = recorder.recorderKeyGenerate(key, f); recorder.del(recorderKey); } } catch (Exception e) { @@ -257,6 +261,7 @@ void record(long offset) { if (recorder == null || offset == 0) { return; } + String recorderKey = recorder.recorderKeyGenerate(key, f); String data = new Gson().toJson(new Record(size, offset, modifyTime, contexts)); recorder.set(recorderKey, data.getBytes()); } catch (Exception e) { diff --git a/src/main/java/com/qiniu/storage/UploadManager.java b/src/main/java/com/qiniu/storage/UploadManager.java index 5f55e6cda..2491d4ba9 100644 --- a/src/main/java/com/qiniu/storage/UploadManager.java +++ b/src/main/java/com/qiniu/storage/UploadManager.java @@ -1,6 +1,5 @@ package com.qiniu.storage; -import com.qiniu.common.Config; import com.qiniu.common.QiniuException; import com.qiniu.http.Client; import com.qiniu.http.Response; @@ -13,15 +12,15 @@ * 七牛文件上传管理器 *

* 一般默认可以使用这个类的方法来上传数据和文件。这个类自动检测文件的大小， - * 只要超过了{@link com.qiniu.common.Config#PUT_THRESHOLD} + * 只要超过了{@link Configuration#putThreshold} */ public final class UploadManager { private final Client client; private final Recorder recorder; - private final RecordKeyGenerator keyGen; + private final Configuration configuration; - public UploadManager() { - this(null, null); + public UploadManager(Configuration c) { + this(c, null); } /** @@ -30,27 +29,12 @@ public UploadManager() { * * @param recorder 断点记录者 */ - public UploadManager(Recorder recorder) { - this(recorder, new RecordKeyGenerator() { - - @Override - public String gen(String key, File file) { - return key + "_._" + file.getAbsolutePath(); - } - }); - } - - /** - * 断点上传记录。只针对 文件分块上传。 - * 分块上传中，将每一块上传的记录保存下来。上传中断后可在上一次断点记录基础上上传剩余部分。 - * - * @param recorder 断点记录者 - * @param keyGen 生成文件的断点记录标示，根据生成的标示，可找到断点记录的内容 - */ - public UploadManager(Recorder recorder, RecordKeyGenerator keyGen) { - client = new Client(); + public UploadManager(Configuration c, Recorder recorder) { + configuration = c.clone(); + client = new Client(configuration.dns, configuration.dnsHostFirst, configuration.proxy, + configuration.connectTimeout, configuration.responseTimeout, configuration.writeTimeout); this.recorder = recorder; - this.keyGen = keyGen; + } private static void checkArgs(final String key, byte[] data, File f, String token) { @@ -123,7 +107,7 @@ public Response put(final byte[] data, final String key, final String token, Str mime = Client.DefaultMime; } params = filterParam(params); - return new FormUploader(client, token, key, data, params, mime, checkCrc).upload(); + return new FormUploader(client, token, key, data, params, mime, checkCrc, configuration).upload(); } /** @@ -180,16 +164,12 @@ public Response put(File file, String key, String token, StringMap params, } params = filterParam(params); long size = file.length(); - if (size <= Config.PUT_THRESHOLD) { - return new FormUploader(client, token, key, file, params, mime, checkCrc).upload(); + if (size <= configuration.putThreshold) { + return new FormUploader(client, token, key, file, params, mime, checkCrc, configuration).upload(); } - String recorderKey = key; - if (keyGen != null) { - recorderKey = keyGen.gen(key, file); - } ResumeUploader uploader = new ResumeUploader(client, token, key, file, - params, mime, recorder, recorderKey); + params, mime, recorder, configuration); return uploader.upload(); } @@ -201,6 +181,6 @@ public void asyncPut(final byte[] data, final String key, final String token, St mime = Client.DefaultMime; } params = filterParam(params); - new FormUploader(client, token, key, data, params, mime, checkCrc).asyncUpload(handler); + new FormUploader(client, token, key, data, params, mime, checkCrc, configuration).asyncUpload(handler); } } diff --git a/src/main/java/com/qiniu/storage/model/Policy.java b/src/main/java/com/qiniu/storage/model/Policy.java new file mode 100644 index 000000000..39458b0fa --- /dev/null +++ b/src/main/java/com/qiniu/storage/model/Policy.java @@ -0,0 +1,7 @@ +package com.qiniu.storage.model; + +/** + * Created by bailong on 16/9/21. + */ +public class Policy { +} diff --git a/src/main/java/com/qiniu/storage/persistent/FileRecorder.java b/src/main/java/com/qiniu/storage/persistent/FileRecorder.java index 9e924a23c..dd312dbbe 100644 --- a/src/main/java/com/qiniu/storage/persistent/FileRecorder.java +++ b/src/main/java/com/qiniu/storage/persistent/FileRecorder.java @@ -122,4 +122,9 @@ public void del(String key) { File f = new File(directory, UrlSafeBase64.encodeToString(key)); f.delete(); } + + @Override + public String recorderKeyGenerate(String key, File file) { + return key + "_._" + file.getName(); + } } diff --git a/src/main/java/com/qiniu/util/Auth.java b/src/main/java/com/qiniu/util/Auth.java index f713c7ba8..4d53f2c71 100644 --- a/src/main/java/com/qiniu/util/Auth.java +++ b/src/main/java/com/qiniu/util/Auth.java @@ -42,7 +42,7 @@ public final class Auth { private static final String[] deprecatedPolicyFields = new String[]{ "asyncOps", }; - private final String accessKey; + public final String accessKey; private final SecretKeySpec secretKey; private Auth(String accessKey, SecretKeySpec secretKeySpec) { diff --git a/src/main/java/com/qiniu/util/Etag.java b/src/main/java/com/qiniu/util/Etag.java index b821c73ac..0c6958fe7 100644 --- a/src/main/java/com/qiniu/util/Etag.java +++ b/src/main/java/com/qiniu/util/Etag.java @@ -1,6 +1,6 @@ package com.qiniu.util; -import com.qiniu.common.Config; +import com.qiniu.common.Constants; import java.io.*; import java.security.MessageDigest; @@ -78,10 +78,10 @@ public static String stream(InputStream in, long len) throws IOException { return "Fto5o-5ea0sNMlW_75VgGJCv2AcJ"; } byte[] buffer = new byte[64 * 1024]; - byte[][] blocks = new byte[(int) (len + Config.BLOCK_SIZE - 1) / Config.BLOCK_SIZE][]; + byte[][] blocks = new byte[(int) (len + Constants.BLOCK_SIZE - 1) / Constants.BLOCK_SIZE][]; for (int i = 0; i < blocks.length; i++) { - long left = len - (long) Config.BLOCK_SIZE * i; - long read = left > Config.BLOCK_SIZE ? Config.BLOCK_SIZE : left; + long left = len - (long) Constants.BLOCK_SIZE * i; + long read = left > Constants.BLOCK_SIZE ? Constants.BLOCK_SIZE : left; blocks[i] = oneBlock(buffer, in, (int) read); } return resultEncode(blocks); diff --git a/src/main/java/com/qiniu/util/StringUtils.java b/src/main/java/com/qiniu/util/StringUtils.java index 11a0f4c0f..399f66ff8 100644 --- a/src/main/java/com/qiniu/util/StringUtils.java +++ b/src/main/java/com/qiniu/util/StringUtils.java @@ -1,6 +1,6 @@ package com.qiniu.util; -import com.qiniu.common.Config; +import com.qiniu.common.Constants; import java.util.Collection; @@ -127,11 +127,11 @@ public static boolean inStringArray(String s, String[] array) { } public static byte[] utf8Bytes(String data) { - return data.getBytes(Config.UTF_8); + return data.getBytes(Constants.UTF_8); } public static String utf8String(byte[] data) { - return new String(data, Config.UTF_8); + return new String(data, Constants.UTF_8); } } diff --git a/src/main/java/com/qiniu/util/UC.java b/src/main/java/com/qiniu/util/UC.java deleted file mode 100644 index 7d2e66b7a..000000000 --- a/src/main/java/com/qiniu/util/UC.java +++ /dev/null @@ -1,203 +0,0 @@ -package com.qiniu.util; - -import com.qiniu.common.Config; -import com.qiniu.common.QiniuException; -import com.qiniu.common.Zone; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; - -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -/** - * Created by Simon on 6/22/16. - */ -public class UC { - private static UC uc = new UC(); - private Map zones = new ConcurrentHashMap(); - private OkHttpClient client = new OkHttpClient.Builder() - .connectTimeout(10000, TimeUnit.SECONDS) - .readTimeout(10000, TimeUnit.SECONDS) - .writeTimeout(10000, TimeUnit.SECONDS).build(); - - private UC() { - - } - - public static Zone zone(String ak, String bkt) throws QiniuException { - return uc.getZone(Config.zone, ak, bkt, Config.UPLOAD_BY_HTTPS); - } - - public static Zone zone(String uptoken) throws QiniuException { - try { - // http://developer.qiniu.com/article/developer/security/upload-token.html - // http://developer.qiniu.com/article/developer/security/put-policy.html - String[] strings = uptoken.split(":"); - String ak = strings[0]; - String policy = new String(UrlSafeBase64.decode(strings[2]), Config.UTF_8); - String bkt = Json.decode(policy).get("scope").toString().split(":")[0]; - return zone(ak, bkt); - } catch (QiniuException e) { - throw e; - } catch (Exception e) { - throw new QiniuException(e); - } - } - - - public static String ucVal(String ak, String bkt) throws QiniuException { - // 不解析返回的值,此时 isHttps (Config.UPLOAD_BY_HTTPS) 无实际意义 - return uc.getUCVal(ak, bkt, Config.UPLOAD_BY_HTTPS); - } - - /** - * 清除缓存。比如空间迁移到不同机房后调用此方法 - */ - public static void clear() { - uc.zones.clear(); - } - - /** - * 返回 java-sdk 中需要的 zone 对象 - */ - private Zone getZone(final Zone userSetZone, final String ak, final String bkt, boolean isHttps) - throws QiniuException { - if (userSetZone != null) { - return userSetZone; - } - ZoneInfo zoneInfo = getZoneInfo(ak, bkt, isHttps); - return zoneInfo.zone; - } - - /** - * 返回 uc.qbox.me 的原始字符串 - */ - public String getUCVal(final String ak, final String bkt, boolean isHttps) throws QiniuException { - ZoneInfo zoneInfo = getZoneInfo(ak, bkt, isHttps); - return zoneInfo.ucjson; - } - - - ZoneInfo getZoneInfo(final String ak, final String bkt, boolean isHttps) throws QiniuException { - final AKBKT akbkt = new AKBKT(ak, bkt, isHttps); - ZoneInfo zoneInfo = zones.get(akbkt); - - if (zoneInfo != null) { - return zoneInfo; - } else { - build(akbkt); - zoneInfo = zones.get(akbkt); - return zoneInfo; - } - } - - private void build(AKBKT akbkt) throws QiniuException { - try { - String address = Config.UC_HOST + "/v1/query?ak=" + akbkt.ak + "&bucket=" + akbkt.bkt; - long start = System.currentTimeMillis(); - Response res = client.newCall(new Request.Builder() - .url(address) - .build()).execute(); - double duration = (System.currentTimeMillis() - start) / 1000.0; - build(akbkt, res, address, duration); - } catch (QiniuException e) { - throw e; - } catch (Exception e) { - throw new QiniuException(e); - } - } - - private void build(AKBKT akbkt, Response res, String address, double duration) throws QiniuException { - com.qiniu.http.Response qnRes = com.qiniu.http.Response.create(res, address, duration); - if (!qnRes.isOK()) { - throw new QiniuException(qnRes); - } - try { - String ucVal = qnRes.bodyString(); - UCRet ret = qnRes.jsonToObject(UCRet.class); - - List args = null; - if (akbkt.isHttps) { - args = ret.https.get("up"); - } else { - args = ret.http.get("up"); - } - - String[] zoneArgs = new String[2]; - zoneArgs[0] = getZoneHost(args.get(0)); - if (args.size() > 1) { - zoneArgs[1] = getZoneHost(args.get(1)); - } - if (zoneArgs[1] == null) { - zoneArgs[1] = zoneArgs[0]; - } - - Zone new_zone = new Zone(zoneArgs[0], zoneArgs[1]); - - if (zoneArgs[0] == null || zoneArgs[0].trim().length() == 0 - || new_zone == null || ucVal == null) { - throw new QiniuException(qnRes); - } - - zones.put(akbkt, new ZoneInfo(new_zone, ucVal)); - } catch (QiniuException e) { - throw e; - } catch (Exception e) { - throw new QiniuException(e); - } - } - - - private String getZoneHost(String p) { - if (p.startsWith("http")) { - return p; - } else { - return null; - } - } - - private class AKBKT { - String ak; - String bkt; - boolean isHttps; - - AKBKT(String ak, String bkt, boolean isHttps) { - this.ak = ak.trim(); - this.bkt = bkt.trim(); - this.isHttps = isHttps; - } - - @Override - public int hashCode() { - return ak.hashCode() * bkt.hashCode() * (isHttps ? 5 : 1); - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof AKBKT) { - AKBKT that = (AKBKT) obj; - return this.bkt.equals(that.bkt) && this.ak.equals(that.ak) && (this.isHttps == that.isHttps); - } - return false; - } - } - - private class ZoneInfo { - public final String ucjson; - public final Zone zone; - - ZoneInfo(Zone zone, String ucjson) { - this.zone = zone; - this.ucjson = ucjson; - } - } - - private class UCRet { - Map> http; - Map> https; - } - -} diff --git a/src/main/java/com/qiniu/util/UrlSafeBase64.java b/src/main/java/com/qiniu/util/UrlSafeBase64.java index 0b13f59ff..820de701f 100644 --- a/src/main/java/com/qiniu/util/UrlSafeBase64.java +++ b/src/main/java/com/qiniu/util/UrlSafeBase64.java @@ -1,6 +1,6 @@ package com.qiniu.util; -import com.qiniu.common.Config; +import com.qiniu.common.Constants; /** * URL安全的Base64编码和解码 @@ -18,7 +18,7 @@ private UrlSafeBase64() { * @return 结果字符串 */ public static String encodeToString(String data) { - return encodeToString(data.getBytes(Config.UTF_8)); + return encodeToString(data.getBytes(Constants.UTF_8)); } /** diff --git a/src/test/java/com/qiniu/HttpTest.java b/src/test/java/com/qiniu/HttpTest.java index 8664e86c4..e0518b50b 100644 --- a/src/test/java/com/qiniu/HttpTest.java +++ b/src/test/java/com/qiniu/HttpTest.java @@ -1,6 +1,6 @@ package com.qiniu; -import com.qiniu.common.Config; +import com.qiniu.common.Constants; import com.qiniu.common.QiniuException; import com.qiniu.http.Client; import com.qiniu.http.ProxyConfiguration; @@ -53,16 +53,17 @@ public void testDns() { } Hosts h = new Hosts(); h.put("upnonodns.qiniu.com", "115.231.183.168"); - Config.dns = new DnsClient(new IResolver[]{r1, r2}, h); + DnsClient dns = new DnsClient(new IResolver[]{r1, r2}, h); + Client c = new Client(dns, false, null, + Constants.CONNECT_TIMEOUT, Constants.RESPONSE_TIMEOUT, Constants.WRITE_TIMEOUT); Response r = null; try { - r = new Client().post("http://upnonodns.qiniu.com", "hello", null); + r = c.post("http://upnonodns.qiniu.com", "hello", null); Assert.fail(); } catch (QiniuException e) { Assert.assertNotNull(e.response.reqId); Assert.assertEquals(e.response.statusCode, 400); } - Config.dns = null; } @Test @@ -112,15 +113,16 @@ public void testPost5() { @Test public void testProxy() { - Config.proxy = new ProxyConfiguration("115.231.183.168", 80); + ProxyConfiguration proxy = new ProxyConfiguration("115.231.183.168", 80); + Client c = new Client(null, false, proxy, + Constants.CONNECT_TIMEOUT, Constants.RESPONSE_TIMEOUT, Constants.WRITE_TIMEOUT); Response r = null; try { - r = new Client().post("http://upproxy1.qiniu.com", "hello", null); + r = c.post("http://upproxy1.qiniu.com", "hello", null); Assert.fail(); } catch (QiniuException e) { Assert.assertNotNull(e.response.reqId); Assert.assertEquals(e.response.statusCode, 400); } - Config.proxy = null; } } diff --git a/src/test/java/com/qiniu/common/AutoZoneTest.java b/src/test/java/com/qiniu/common/AutoZoneTest.java new file mode 100644 index 000000000..17c25afc5 --- /dev/null +++ b/src/test/java/com/qiniu/common/AutoZoneTest.java @@ -0,0 +1,69 @@ +package com.qiniu.common; + +import com.qiniu.TestConfig; +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Created by Simon on 6/22/16. + */ +public class AutoZoneTest { + private String ak = TestConfig.ak; + private String bkt = TestConfig.bucket; + + @Test + public void testHttp() { + try { + AutoZone zone = AutoZone.instance; + AutoZone.ZoneInfo zoneInfo = zone.zoneInfo(ak, bkt); + assertEquals(zoneInfo.upHost, "http://up.qiniu.com"); + assertEquals(zoneInfo.upBackup, "http://upload.qiniu.com"); + assertEquals(zoneInfo.upHttps, "https://up.qbox.me"); + } catch (QiniuException e) { + e.printStackTrace(); + System.out.println(e.response.url()); + System.out.println(e.response.toString()); + Assert.fail(); + } + } + + @Test + public void testHttpFail() { + try { + AutoZone zone = AutoZone.instance; + AutoZone.ZoneInfo zoneInfo = zone.zoneInfo(ak + "_not_be_ak", bkt); + Assert.fail(); + } catch (QiniuException e) { + e.printStackTrace(); + System.out.println(e.response.url()); + System.out.println(e.response.toString()); + Assert.assertEquals(e.code(), 612); + } + } + + @Test + public void testSplitE() { + String s1 = "bkt:key"; + String s2 = "bkt"; + Assert.assertEquals(s1.split(":")[0], s2.split(":")[0]); + } + + @Test + public void testC1() { + try { + AutoZone.ZoneInfo info = AutoZone.instance.zoneInfo(ak, bkt); + System.out.println("zone0: " + info.toString()); + + AutoZone.ZoneInfo info2 = AutoZone.instance.zoneInfo(ak, bkt); + Assert.assertSame(info, info2); + + } catch (QiniuException e) { + e.printStackTrace(); + System.out.println(e.response.url()); + System.out.println(e.response.toString()); + Assert.fail(); + } + } +} diff --git a/src/test/java/com/qiniu/processing/PfopTest.java b/src/test/java/com/qiniu/processing/PfopTest.java index f300def0c..7866bb170 100644 --- a/src/test/java/com/qiniu/processing/PfopTest.java +++ b/src/test/java/com/qiniu/processing/PfopTest.java @@ -2,7 +2,9 @@ import com.qiniu.TestConfig; import com.qiniu.common.QiniuException; +import com.qiniu.common.Zone; import com.qiniu.http.Response; +import com.qiniu.storage.Configuration; import com.qiniu.util.Auth; import com.qiniu.util.StringMap; import com.qiniu.util.UrlSafeBase64; @@ -14,7 +16,7 @@ public class PfopTest { private Auth auth = TestConfig.testAuth; - private OperationManager operater = new OperationManager(auth); + private OperationManager operater = new OperationManager(auth, new Configuration(Zone.zone0())); @Test public void testAvthumb() { diff --git a/src/test/java/com/qiniu/storage/BucketTest.java b/src/test/java/com/qiniu/storage/BucketTest.java index 2725dced3..20cf4290a 100644 --- a/src/test/java/com/qiniu/storage/BucketTest.java +++ b/src/test/java/com/qiniu/storage/BucketTest.java @@ -2,6 +2,7 @@ import com.qiniu.TestConfig; import com.qiniu.common.QiniuException; +import com.qiniu.common.Zone; import com.qiniu.http.Response; import com.qiniu.storage.model.BatchStatus; import com.qiniu.storage.model.FileInfo; @@ -14,8 +15,8 @@ @SuppressWarnings("ConstantConditions") public class BucketTest { - private BucketManager bucketManager = new BucketManager(TestConfig.testAuth); - private BucketManager dummyBucketManager = new BucketManager(TestConfig.dummyAuth); + private BucketManager bucketManager = new BucketManager(TestConfig.testAuth, new Configuration(Zone.zone0())); + private BucketManager dummyBucketManager = new BucketManager(TestConfig.dummyAuth, new Configuration(Zone.zone0())); @Test public void testBuckets() { diff --git a/src/test/java/com/qiniu/storage/FormUploadTest.java b/src/test/java/com/qiniu/storage/FormUploadTest.java index 01dc8264d..4eaeed350 100644 --- a/src/test/java/com/qiniu/storage/FormUploadTest.java +++ b/src/test/java/com/qiniu/storage/FormUploadTest.java @@ -2,8 +2,8 @@ import com.qiniu.TempFile; import com.qiniu.TestConfig; -import com.qiniu.common.Config; import com.qiniu.common.QiniuException; +import com.qiniu.common.Zone; import com.qiniu.http.Response; import com.qiniu.util.StringMap; import org.junit.Test; @@ -17,39 +17,30 @@ import static org.junit.Assert.*; public class FormUploadTest { - private UploadManager uploadManager = new UploadManager(); + UploadManager uploadManager = new UploadManager(new Configuration(Zone.zone0())); @Test public void testHello1() { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = false; - hello(); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } + hello(uploadManager); } @Test public void testHello2() { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = true; - hello(); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } + Configuration c = new Configuration(Zone.zone0()); + c.uploadByHttps = true; + UploadManager uploadManager = new UploadManager(c); + hello(uploadManager); } - public void hello() { + public void hello(UploadManager up) { final String expectKey = "你好?&=\r\n"; StringMap params = new StringMap().put("x:foo", "foo_val"); String token = TestConfig.testAuth.uploadToken(TestConfig.bucket, expectKey); Response r = null; try { - r = uploadManager.put("hello".getBytes(), expectKey, token, params, null, false); + r = up.put("hello".getBytes(), expectKey, token, params, null, false); } catch (QiniuException e) { fail(); } @@ -299,12 +290,14 @@ public void testSizeMin2() { // @Test public void testFormLargeSize() { - Config.PUT_THRESHOLD = 25 * 1024 * 1024; + Configuration c = new Configuration(Zone.zone0()); + c.putThreshold = 25 * 1024 * 1024; + UploadManager uploadManager = new UploadManager(new Configuration(Zone.zone0())); final String expectKey = "yyyyyy"; File f = null; try { - f = TempFile.createFile(Config.PUT_THRESHOLD / 1024 - 1); + f = TempFile.createFile(c.putThreshold / 1024 - 1); } catch (IOException e) { e.printStackTrace(); } @@ -324,13 +317,15 @@ public void testFormLargeSize() { // @Test public void testFormLargeSize2() { - Config.PUT_THRESHOLD = 25 * 1024 * 1024; + Configuration c = new Configuration(Zone.zone0()); + c.putThreshold = 25 * 1024 * 1024; + UploadManager uploadManager = new UploadManager(new Configuration(Zone.zone0())); final String expectKey = "xxxxxxx"; byte[] bb = null; File f = null; try { - f = TempFile.createFile(Config.PUT_THRESHOLD / 1024 - 1); + f = TempFile.createFile(c.putThreshold / 1024 - 1); bb = new byte[(int) (f.length())]; FileInputStream fis = new FileInputStream(f); fis.read(bb, 0, (int) (f.length())); diff --git a/src/test/java/com/qiniu/storage/RecordUploadTest.java b/src/test/java/com/qiniu/storage/RecordUploadTest.java index 9677bcd56..6c571ca99 100644 --- a/src/test/java/com/qiniu/storage/RecordUploadTest.java +++ b/src/test/java/com/qiniu/storage/RecordUploadTest.java @@ -2,7 +2,8 @@ import com.qiniu.TempFile; import com.qiniu.TestConfig; -import com.qiniu.common.Config; +import com.qiniu.common.Constants; +import com.qiniu.common.Zone; import com.qiniu.http.Client; import com.qiniu.http.Response; import com.qiniu.storage.persistent.FileRecorder; @@ -73,7 +74,7 @@ public void run() { showRecord.setDaemon(true); showRecord.start(); - if (f.length() > Config.BLOCK_SIZE) { + if (f.length() > Constants.BLOCK_SIZE) { // 终止第一部分上传,期望其部分成功 for (int i = 150; i > 0; --i) { byte[] data = getRecord(recorder, recordKey); @@ -280,7 +281,7 @@ public Response up() throws Exception { recorder = new FileRecorder(file.getParentFile()); } uploader = new ResumeUploader(client, token, key, file, - null, Client.DefaultMime, recorder, recorderKey); + null, Client.DefaultMime, recorder, new Configuration(Zone.zone0())); Response res = uploader.upload(); System.out.println("UP: " + i + ", left up"); return res; diff --git a/src/test/java/com/qiniu/storage/ResumeUploadTest.java b/src/test/java/com/qiniu/storage/ResumeUploadTest.java index 0ff581888..7809f4e1b 100644 --- a/src/test/java/com/qiniu/storage/ResumeUploadTest.java +++ b/src/test/java/com/qiniu/storage/ResumeUploadTest.java @@ -2,8 +2,8 @@ import com.qiniu.TempFile; import com.qiniu.TestConfig; -import com.qiniu.common.Config; import com.qiniu.common.QiniuException; +import com.qiniu.common.Zone; import com.qiniu.http.Client; import com.qiniu.http.Response; import com.qiniu.util.StringMap; @@ -34,7 +34,7 @@ public void testXVar() throws IOException { new StringMap().put("returnBody", returnBody)); try { - UploadManager uploadManager = new UploadManager(); + UploadManager uploadManager = new UploadManager(new Configuration(Zone.zone0())); Response res = uploadManager.put(f, expectKey, token, params, null, true); StringMap m = res.jsonToMap(); assertEquals("foo_val", m.get("foo")); @@ -46,7 +46,10 @@ public void testXVar() throws IOException { } } - private void template(int size) throws IOException { + private void template(int size, boolean https) throws IOException { + Configuration c = new Configuration(Zone.zone0()); + c.uploadByHttps = https; + UploadManager uploadManager = new UploadManager(c); final String expectKey = "\r\n?&r=" + size + "k"; final File f = TempFile.createFile(size); final String returnBody = "{\"key\":\"$(key)\",\"hash\":\"$(etag)\",\"fsize\":\"$(fsize)\"" @@ -55,7 +58,8 @@ private void template(int size) throws IOException { new StringMap().put("returnBody", returnBody)); try { - ResumeUploader up = new ResumeUploader(new Client(), token, expectKey, f, null, null, null, null); + ResumeUploader up = new ResumeUploader(new Client(), token, expectKey, f, null, null, null, + new Configuration(Zone.zone0())); Response r = up.upload(); MyRet ret = r.jsonToObject(MyRet.class); assertEquals(expectKey, ret.key); @@ -69,29 +73,17 @@ private void template(int size) throws IOException { @Test public void test1K() throws Throwable { - template(1); + template(1, false); } @Test public void test600k() throws Throwable { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = true; - template(600); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } + template(600, true); } @Test public void test600k2() throws IOException { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = false; - template(600); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } + template(600, false); } @Test @@ -99,35 +91,23 @@ public void test4M() throws Throwable { if (TestConfig.isTravis()) { return; } - template(1024 * 4); + template(1024 * 4, false); } @Test public void test8M1k() throws Throwable { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = false; - if (TestConfig.isTravis()) { - return; - } - template(1024 * 8 + 1); - } finally { - Config.UPLOAD_BY_HTTPS = h; + if (TestConfig.isTravis()) { + return; } + template(1024 * 8 + 1, false); } @Test public void test8M1k2() throws Throwable { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = true; - if (TestConfig.isTravis()) { - return; - } - template(1024 * 8 + 1); - } finally { - Config.UPLOAD_BY_HTTPS = h; + if (TestConfig.isTravis()) { + return; } + template(1024 * 8 + 1, true); } class MyRet { diff --git a/src/test/java/com/qiniu/util/EtagTest.java b/src/test/java/com/qiniu/util/EtagTest.java index cc600493b..f5aa23614 100644 --- a/src/test/java/com/qiniu/util/EtagTest.java +++ b/src/test/java/com/qiniu/util/EtagTest.java @@ -1,7 +1,7 @@ package com.qiniu.util; import com.qiniu.TempFile; -import com.qiniu.common.Config; +import com.qiniu.common.Constants; import org.junit.Test; import java.io.File; @@ -15,7 +15,7 @@ public void testData() { String m = Etag.data(new byte[0]); assertEquals("Fto5o-5ea0sNMlW_75VgGJCv2AcJ", m); - String etag = Etag.data("etag".getBytes(Config.UTF_8)); + String etag = Etag.data("etag".getBytes(Constants.UTF_8)); assertEquals("FpLiADEaVoALPkdb8tJEJyRTXoe_", etag); } diff --git a/src/test/java/com/qiniu/util/UCTest.java b/src/test/java/com/qiniu/util/UCTest.java deleted file mode 100644 index 84d3dba8f..000000000 --- a/src/test/java/com/qiniu/util/UCTest.java +++ /dev/null @@ -1,285 +0,0 @@ -package com.qiniu.util; - -import com.qiniu.TestConfig; -import com.qiniu.common.Config; -import com.qiniu.common.QiniuException; -import com.qiniu.common.Zone; -import org.junit.Assert; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * Created by Simon on 6/22/16. - */ -public class UCTest { - private String ak = TestConfig.ak; - private String bkt = TestConfig.bucket; - - @Test - public void testHttp() { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = false; - Zone zone = UC.zone(ak, bkt); - assertEquals(zone.upHost, "http://up.qiniu.com"); - assertEquals(zone.upHostBackup, "http://upload.qiniu.com"); - } catch (QiniuException e) { - e.printStackTrace(); - System.out.println(e.response.url()); - System.out.println(e.response.toString()); - Assert.fail(); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } - } - - @Test - public void testHttps() { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = true; - Zone zone = UC.zone(ak, bkt); - assertEquals(zone.upHost, "https://up.qbox.me"); - assertEquals(zone.upHostBackup, "https://up.qbox.me"); - } catch (QiniuException e) { - e.printStackTrace(); - System.out.println(e.response.url()); - System.out.println(e.response.toString()); - Assert.fail(); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } - } - - @Test - public void testHttpFail() { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = false; - Zone zone = UC.zone(ak + "_not_be_ak", bkt); - Assert.fail(); - } catch (QiniuException e) { - e.printStackTrace(); - System.out.println(e.response.url()); - System.out.println(e.response.toString()); - Assert.assertEquals(e.code(), 612); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } - } - - @Test - public void testE() { - Assert.assertEquals(new Zone("upHost", "upHostBackup"), new Zone("upH" + "ost", "upHost" + "Backup")); - } - - @Test - public void testSplitE() { - String s1 = "bkt:key"; - String s2 = "bkt"; - Assert.assertEquals(s1.split(":")[0], s2.split(":")[0]); - } - - @Test - public void testC1() { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = false; - final Zone zone0 = UC.zone(ak, bkt); - System.out.println("zone0: " + zone0.toString()); - - - new Thread() { - public void run() { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = false; - Zone zone1 = UC.zone(ak, bkt); - System.out.println("zone1: " + zone1.toString()); - Assert.assertEquals(zone1, zone0); - } catch (QiniuException e) { - e.printStackTrace(); - System.out.println(e.response.url()); - System.out.println(e.response.toString()); - Assert.fail(); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } - } - }.start(); - - new Thread() { - public void run() { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = false; - Zone zone2 = UC.zone(ak, bkt); - System.out.println("zone2: " + zone2.toString()); - Assert.assertEquals(zone2, zone0); - } catch (QiniuException e) { - e.printStackTrace(); - System.out.println(e.response.url()); - System.out.println(e.response.toString()); - Assert.fail(); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } - } - }.start(); - - try { - Thread.sleep(200); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - new Thread() { - public void run() { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = false; - Zone zone3 = UC.zone(ak, bkt); - System.out.println("zone3: " + zone3.toString()); - Assert.assertEquals(zone3, zone0); - } catch (QiniuException e) { - e.printStackTrace(); - System.out.println(e.response.url()); - System.out.println(e.response.toString()); - Assert.fail(); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } - } - }.start(); - - try { - Thread.sleep(20); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - } catch (QiniuException e) { - e.printStackTrace(); - System.out.println(e.response.url()); - System.out.println(e.response.toString()); - Assert.fail(); - } finally { - Config.UPLOAD_BY_HTTPS = h; - UC.clear(); - } - } - - @Test - public void testC2() { - final Zone[] zones = new Zone[5]; - boolean h = Config.UPLOAD_BY_HTTPS; - new Thread() { - public void run() { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = false; - Zone zone0 = UC.zone(ak, bkt); - System.out.println("zone0: " + zone0.toString()); - zones[0] = zone0; - } catch (QiniuException e) { - e.printStackTrace(); - System.out.println(e.response.url()); - System.out.println(e.response.toString()); - Assert.fail(); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } - } - }.start(); - - new Thread() { - public void run() { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = false; - Zone zone1 = UC.zone(ak, bkt); - System.out.println("zone1: " + zone1.toString()); - zones[1] = zone1; - } catch (QiniuException e) { - e.printStackTrace(); - System.out.println(e.response.url()); - System.out.println(e.response.toString()); - Assert.fail(); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } - } - }.start(); - - try { - Thread.sleep(600); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - new Thread() { - public void run() { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = false; - Zone zone2 = UC.zone(ak, bkt); - System.out.println("zone2: " + zone2.toString()); - zones[2] = zone2; - } catch (QiniuException e) { - e.printStackTrace(); - System.out.println(e.response.url()); - System.out.println(e.response.toString()); - Assert.fail(); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } - } - }.start(); - - new Thread() { - public void run() { - boolean h = Config.UPLOAD_BY_HTTPS; - try { - Config.UPLOAD_BY_HTTPS = false; - Zone zone3 = UC.zone(ak, bkt); - System.out.println("zone3: " + zone3.toString()); - zones[3] = zone3; - } catch (QiniuException e) { - e.printStackTrace(); - System.out.println(e.response.url()); - System.out.println(e.response.toString()); - Assert.fail(); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } - } - }.start(); - - try { - Config.UPLOAD_BY_HTTPS = false; - Zone zone4 = UC.zone(ak, bkt); - System.out.println("zone4: " + zone4.toString()); - zones[4] = zone4; - } catch (QiniuException e) { - e.printStackTrace(); - System.out.println(e.response.url()); - System.out.println(e.response.toString()); - Assert.fail(); - } finally { - Config.UPLOAD_BY_HTTPS = h; - } - - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - for (Zone z : zones) { - assertEquals(zones[0], z); - } - UC.clear(); - } - -} From 0fa904587e8df374e7e810e5f9e4abf882fbddf1 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 21 Sep 2016 18:20:14 +0800 Subject: [PATCH 02/10] remove useless --- .xmake/cache/history | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 .xmake/cache/history diff --git a/.xmake/cache/history b/.xmake/cache/history deleted file mode 100644 index e9e5ae188..000000000 --- a/.xmake/cache/history +++ /dev/null @@ -1,9 +0,0 @@ - -{ - ["__version"] = "XMake v2.0.5" -, ["cmdlines"] = - { - "xmake" - } - -} From d051cf92df45ac82b0bebac33181638a77aebe87 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 21 Sep 2016 18:28:59 +0800 Subject: [PATCH 03/10] add zone2 --- src/main/java/com/qiniu/common/Zone.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/qiniu/common/Zone.java b/src/main/java/com/qiniu/common/Zone.java index bbb5489bf..8baa1588a 100644 --- a/src/main/java/com/qiniu/common/Zone.java +++ b/src/main/java/com/qiniu/common/Zone.java @@ -17,6 +17,12 @@ public static Zone zone1() { "https://up-z1.qbox.me", "http://api-z1.qiniu.com"); } + public static Zone zone2() { + return new FixedZone("http://up-z2.qiniu.com", "http://upload-z2.qiniu.com", + "", "http://rs-z2.qbox.me", "http://rsf-z2.qbox.me", "http://iovip-z2.qbox.me", + "https://up-z2.qbox.me", "http://api-z2.qiniu.com"); + } + public static Zone autoZone() { return AutoZone.instance; } From 8780815e933c840eea3222d2dc5381d3551b898e Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 21 Sep 2016 19:32:16 +0800 Subject: [PATCH 04/10] add policy auto --- .../java/com/qiniu/storage/model/Policy.java | 7 --- .../com/qiniu/storage/model/UploadPolicy.java | 43 +++++++++++++++++++ src/main/java/com/qiniu/util/Auth.java | 5 +++ src/main/java/com/qiniu/util/Json.java | 7 +++ src/test/java/com/qiniu/util/AuthTest.java | 20 +++++++++ 5 files changed, 75 insertions(+), 7 deletions(-) delete mode 100644 src/main/java/com/qiniu/storage/model/Policy.java create mode 100644 src/main/java/com/qiniu/storage/model/UploadPolicy.java diff --git a/src/main/java/com/qiniu/storage/model/Policy.java b/src/main/java/com/qiniu/storage/model/Policy.java deleted file mode 100644 index 39458b0fa..000000000 --- a/src/main/java/com/qiniu/storage/model/Policy.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.qiniu.storage.model; - -/** - * Created by bailong on 16/9/21. - */ -public class Policy { -} diff --git a/src/main/java/com/qiniu/storage/model/UploadPolicy.java b/src/main/java/com/qiniu/storage/model/UploadPolicy.java new file mode 100644 index 000000000..360a2bdd2 --- /dev/null +++ b/src/main/java/com/qiniu/storage/model/UploadPolicy.java @@ -0,0 +1,43 @@ +package com.qiniu.storage.model; + +/** + * Created by bailong on 16/9/21. + */ + +// 这只是个例子，可以自己创建policy 类使用 +public final class UploadPolicy { + + public final String scope; + public final long deadline; + public int insertOnly; + + public String endUser; + + public String returnUrl; + public String returnBody; + + public String callbackUrl; + public String callbackHost; + public String callbackBody; + public String callbackBodyType; + public int callbackFetchKey; + + public String persistentOps; + public String persistentNotifyUrl; + public String persistentPipeline; + public String saveKey; + + public long fsizeMin; + public long fsizeLimit; + + public int detectMime; + + public String mimeLimit; + + public int deleteAfterDays; + + public UploadPolicy(String bucket, String key, long expired) { + this.scope = key == null ? bucket : bucket + "" + key; + this.deadline = System.currentTimeMillis() / 1000 + expired; + } +} diff --git a/src/main/java/com/qiniu/util/Auth.java b/src/main/java/com/qiniu/util/Auth.java index 4d53f2c71..f5d9619dc 100644 --- a/src/main/java/com/qiniu/util/Auth.java +++ b/src/main/java/com/qiniu/util/Auth.java @@ -263,6 +263,11 @@ public String uploadTokenWithDeadline(String bucket, String key, long deadline, return signWithData(StringUtils.utf8Bytes(s)); } + public String uploadTokenWithPolicy(Object obj) { + String s = Json.encode(obj); + return signWithData(StringUtils.utf8Bytes(s)); + } + public StringMap authorization(String url, byte[] body, String contentType) { String authorization = "QBox " + signRequest(url, body, contentType); return new StringMap().put("Authorization", authorization); diff --git a/src/main/java/com/qiniu/util/Json.java b/src/main/java/com/qiniu/util/Json.java index ed7dc437e..9e93b52e9 100644 --- a/src/main/java/com/qiniu/util/Json.java +++ b/src/main/java/com/qiniu/util/Json.java @@ -1,6 +1,9 @@ package com.qiniu.util; +import com.google.gson.ExclusionStrategy; +import com.google.gson.FieldAttributes; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; import java.lang.reflect.Type; @@ -15,6 +18,10 @@ public static String encode(StringMap map) { return new Gson().toJson(map.map()); } + public static String encode(Object obj) { + return new GsonBuilder().serializeNulls().create().toJson(obj); + } + public static T decode(String json, Class classOfT) { return new Gson().fromJson(json, classOfT); } diff --git a/src/test/java/com/qiniu/util/AuthTest.java b/src/test/java/com/qiniu/util/AuthTest.java index 218416fa2..69e07e7ab 100644 --- a/src/test/java/com/qiniu/util/AuthTest.java +++ b/src/test/java/com/qiniu/util/AuthTest.java @@ -2,6 +2,7 @@ import com.qiniu.TestConfig; import com.qiniu.http.Client; +import com.qiniu.storage.model.UploadPolicy; import org.junit.Test; import static org.junit.Assert.*; @@ -56,4 +57,23 @@ public void testUploadToken() { // CHECKSTYLE:ON assertEquals(exp, token); } + + static class Policy{ + String scope; + long deadline; + String endUser; + } + + @Test + public void testUploadToken2() { + Policy p = new Policy(); + p.endUser = "y"; + p.scope = "1:2"; + p.deadline = 1234567890L + 3600; + String token = TestConfig.dummyAuth.uploadTokenWithPolicy(p); + // CHECKSTYLE:OFF + String exp = "abcdefghklmnopq:zx3NdMGffQ0JhUlgGSU5oeTx9Nk=:eyJzY29wZSI6IjE6MiIsImRlYWRsaW5lIjoxMjM0NTcxNDkwLCJlbmRVc2VyIjoieSJ9"; + // CHECKSTYLE:ON + assertEquals(exp, token); + } } From 414c087249588ecc14c0ef97bbed6b71bbc66692 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 21 Sep 2016 19:48:25 +0800 Subject: [PATCH 05/10] recorder --- .../com/qiniu/storage/RecordKeyGenerator.java | 17 ----------------- src/main/java/com/qiniu/storage/Recorder.java | 7 +++++++ .../com/qiniu/storage/RecordUploadTest.java | 13 +++---------- 3 files changed, 10 insertions(+), 27 deletions(-) delete mode 100644 src/main/java/com/qiniu/storage/RecordKeyGenerator.java diff --git a/src/main/java/com/qiniu/storage/RecordKeyGenerator.java b/src/main/java/com/qiniu/storage/RecordKeyGenerator.java deleted file mode 100644 index 90b875b17..000000000 --- a/src/main/java/com/qiniu/storage/RecordKeyGenerator.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.qiniu.storage; - -import java.io.File; - -/** - * Created by Simon on 2015/3/30. - */ -public interface RecordKeyGenerator { - /** - * 根据服务器的key和本地文件名生成持久化纪录的key - * - * @param key 服务器的key - * @param file 本地文件名 - * @return 持久化上传纪录的key - */ - String gen(String key, File file); -} diff --git a/src/main/java/com/qiniu/storage/Recorder.java b/src/main/java/com/qiniu/storage/Recorder.java index d4bae3e29..6589b252d 100644 --- a/src/main/java/com/qiniu/storage/Recorder.java +++ b/src/main/java/com/qiniu/storage/Recorder.java @@ -30,5 +30,12 @@ public interface Recorder { */ void del(String key); + /** + * 根据服务器的key和本地文件名生成持久化纪录的key + * + * @param key 服务器的key + * @param file 本地文件名 + * @return 持久化上传纪录的key + */ String recorderKeyGenerate(String key, File file); } diff --git a/src/test/java/com/qiniu/storage/RecordUploadTest.java b/src/test/java/com/qiniu/storage/RecordUploadTest.java index 6c571ca99..0b9bd3af4 100644 --- a/src/test/java/com/qiniu/storage/RecordUploadTest.java +++ b/src/test/java/com/qiniu/storage/RecordUploadTest.java @@ -25,12 +25,7 @@ */ public class RecordUploadTest { final Random r = new Random(); - final RecordKeyGenerator keyGen = new RecordKeyGenerator() { - @Override - public String gen(String key, File file) { - return key + "_._" + file.getAbsolutePath(); - } - }; + final Client client = new Client(); FileRecorder recorder = null; private Response response = null; @@ -42,7 +37,7 @@ private void template(final int size) throws IOException { recorder = new FileRecorder(f.getParentFile()); try { final String token = TestConfig.testAuth.uploadToken(TestConfig.bucket, expectKey); - final String recordKey = keyGen.gen(expectKey, f); + final String recordKey = recorder.recorderKeyGenerate(expectKey, f); // 开始第一部分上传 final Up up = new Up(f, expectKey, token); @@ -273,13 +268,11 @@ public Response up() throws Exception { System.out.println("UP: " + i + ", enter up"); String recorderKey = key; - if (keyGen != null) { - recorderKey = keyGen.gen(key, file); - } if (recorder == null) { recorder = new FileRecorder(file.getParentFile()); } + recorderKey = recorder.recorderKeyGenerate(key, file); uploader = new ResumeUploader(client, token, key, file, null, Client.DefaultMime, recorder, new Configuration(Zone.zone0())); Response res = uploader.upload(); From 5c22cc49c0a654dabe5e9f7e2420f53341c6c072 Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 23 Sep 2016 21:39:25 +0800 Subject: [PATCH 06/10] pili integrate done --- src/main/java/com/qiniu/cdn/CdnManager.java | 53 +++++ .../java/com/qiniu/common/QiniuException.java | 19 ++ src/main/java/com/qiniu/http/Error.java | 8 + .../java/com/qiniu/storage/Configuration.java | 9 +- .../com/qiniu/streaming/StreamingManager.java | 191 ++++++++++++++++++ .../java/com/qiniu/streaming/UrlFactory.java | 79 ++++++++ .../streaming/model/ActivityRecords.java | 13 ++ .../streaming/model/StreamAttribute.java | 11 + .../qiniu/streaming/model/StreamListing.java | 28 +++ .../qiniu/streaming/model/StreamStatus.java | 18 ++ src/main/java/com/qiniu/util/Auth.java | 55 +++++ src/main/java/com/qiniu/util/Json.java | 2 - src/test/java/com/qiniu/CdnTest.java | 26 +++ .../com/qiniu/streaming/StreamingTest.java | 113 +++++++++++ .../java/com/qiniu/streaming/UrlTest.java | 45 +++++ src/test/java/com/qiniu/util/AuthTest.java | 13 +- 16 files changed, 672 insertions(+), 11 deletions(-) create mode 100644 src/main/java/com/qiniu/cdn/CdnManager.java create mode 100644 src/main/java/com/qiniu/http/Error.java create mode 100644 src/main/java/com/qiniu/streaming/StreamingManager.java create mode 100644 src/main/java/com/qiniu/streaming/UrlFactory.java create mode 100644 src/main/java/com/qiniu/streaming/model/ActivityRecords.java create mode 100644 src/main/java/com/qiniu/streaming/model/StreamAttribute.java create mode 100644 src/main/java/com/qiniu/streaming/model/StreamListing.java create mode 100644 src/main/java/com/qiniu/streaming/model/StreamStatus.java create mode 100644 src/test/java/com/qiniu/CdnTest.java create mode 100644 src/test/java/com/qiniu/streaming/StreamingTest.java create mode 100644 src/test/java/com/qiniu/streaming/UrlTest.java diff --git a/src/main/java/com/qiniu/cdn/CdnManager.java b/src/main/java/com/qiniu/cdn/CdnManager.java new file mode 100644 index 000000000..994494826 --- /dev/null +++ b/src/main/java/com/qiniu/cdn/CdnManager.java @@ -0,0 +1,53 @@ +package com.qiniu.cdn; + +import com.qiniu.common.Constants; +import com.qiniu.common.QiniuException; +import com.qiniu.http.Client; +import com.qiniu.http.Response; +import com.qiniu.util.Auth; +import com.qiniu.util.Json; +import com.qiniu.util.StringMap; + +import java.util.HashMap; + +/** + * Created by bailong on 16/9/21. + */ +public final class CdnManager { + private final Auth auth; + private final String server; + private final Client client; + + public CdnManager(Auth auth) { + this(auth, "http://fusion.qiniuapi.com"); + } + + private CdnManager(Auth auth, String server) { + this.auth = auth; + this.server = server; + this.client = new Client(null, false, null, + Constants.CONNECT_TIMEOUT, Constants.RESPONSE_TIMEOUT, Constants.WRITE_TIMEOUT); + } + + public Response refreshUrls(String[] urls) throws QiniuException { + return refreshUrlsAndDirs(urls, null); + } + + public Response refreshDirs(String[] dirs) throws QiniuException { + return refreshUrlsAndDirs(null, dirs); + } + + public Response refreshUrlsAndDirs(String[] urls, String[] dirs) throws QiniuException { + HashMap req = new HashMap<>(); + if (urls != null) { + req.put("urls", urls); + } + if (dirs != null) { + req.put("dirs", dirs); + } + byte[] body = Json.encode(req).getBytes(Constants.UTF_8); + String url = server + "/v2/tune/refresh"; + StringMap headers = auth.authorizationV2(url, "POST", body, Client.JsonMime); + return client.post(url, body, headers, Client.JsonMime); + } +} diff --git a/src/main/java/com/qiniu/common/QiniuException.java b/src/main/java/com/qiniu/common/QiniuException.java index 7ff0bb81e..e4e70693f 100644 --- a/src/main/java/com/qiniu/common/QiniuException.java +++ b/src/main/java/com/qiniu/common/QiniuException.java @@ -1,5 +1,6 @@ package com.qiniu.common; +import com.qiniu.http.Error; import com.qiniu.http.Response; import java.io.IOException; @@ -7,6 +8,7 @@ public final class QiniuException extends IOException { public final Response response; + private String error; public QiniuException(Response response) { @@ -25,4 +27,21 @@ public String url() { public int code() { return response == null ? -1 : response.statusCode; } + + public String error() { + if (error != null) { + return error; + } + if (response == null || response.statusCode / 100 == 2 || !response.isJson()) { + return null; + } + Error e = null; + try { + e = response.jsonToObject(Error.class); + } catch (QiniuException e1) { + e1.printStackTrace(); + } + error = e == null ? "" : e.error; + return error; + } } diff --git a/src/main/java/com/qiniu/http/Error.java b/src/main/java/com/qiniu/http/Error.java new file mode 100644 index 000000000..a3af93a80 --- /dev/null +++ b/src/main/java/com/qiniu/http/Error.java @@ -0,0 +1,8 @@ +package com.qiniu.http; + +/** + * Created by bailong on 16/9/23. + */ +public final class Error { + public String error; +} diff --git a/src/main/java/com/qiniu/storage/Configuration.java b/src/main/java/com/qiniu/storage/Configuration.java index ad7bf2701..9e22f87b1 100644 --- a/src/main/java/com/qiniu/storage/Configuration.java +++ b/src/main/java/com/qiniu/storage/Configuration.java @@ -8,7 +8,7 @@ /** * Created by bailong on 16/9/21. */ -public final class Configuration { +public final class Configuration implements Cloneable { /** * 使用的Zone @@ -56,7 +56,12 @@ public Configuration(Zone zone) { } public Configuration clone() { - return this; + try { + return (Configuration) super.clone(); + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + } + return null; } } diff --git a/src/main/java/com/qiniu/streaming/StreamingManager.java b/src/main/java/com/qiniu/streaming/StreamingManager.java new file mode 100644 index 000000000..51951c4cc --- /dev/null +++ b/src/main/java/com/qiniu/streaming/StreamingManager.java @@ -0,0 +1,191 @@ +package com.qiniu.streaming; + +import com.qiniu.common.Constants; +import com.qiniu.common.QiniuException; +import com.qiniu.http.Client; +import com.qiniu.http.Response; +import com.qiniu.streaming.model.ActivityRecords; +import com.qiniu.streaming.model.StreamAttribute; +import com.qiniu.streaming.model.StreamListing; +import com.qiniu.streaming.model.StreamStatus; +import com.qiniu.util.Auth; +import com.qiniu.util.StringMap; +import com.qiniu.util.StringUtils; +import com.qiniu.util.UrlSafeBase64; + +import java.util.Iterator; + +/** + * Created by bailong on 16/9/22. + */ +public final class StreamingManager { + private final String apiServer; + private final String hub; + private final Client client; + private final Auth auth; + + public StreamingManager(Auth auth, String hub) { + this(auth, hub, "http://pili.qiniuapi.com"); + } + + StreamingManager(Auth auth, String hub, String server) { + apiServer = server; + this.hub = hub; + this.auth = auth; + client = new Client(null, false, null, + Constants.CONNECT_TIMEOUT, Constants.RESPONSE_TIMEOUT, Constants.WRITE_TIMEOUT); + } + + public void create(String key) throws QiniuException { + String path = ""; + String body = String.format("{\"key\":\"%s\"}", key); + post(path, body, null); + } + + public StreamAttribute attribute(String key) throws QiniuException { + String path = encodeKey(key); + return get(path, StreamAttribute.class); + } + + /** + * 根据前缀获取流列表的迭代器 + * + * @param live 是否在推流 + * @param prefix 文件名前缀 + * @return Stream迭代器 + */ + public ListIterator createStreamListIterator(boolean live, String prefix) { + return new ListIterator(live, prefix); + } + + public StreamListing listStreams(boolean live, String prefix, String marker) throws QiniuException { + StringMap map = new StringMap(); + if (live) { + map.put("liveonly", live); + } + if (!StringUtils.isNullOrEmpty(prefix)) { + map.put("prefix", prefix); + } + if (!StringUtils.isNullOrEmpty(marker)) { + map.put("marker", marker); + } + String path = ""; + + if (map.size() != 0) { + path += "?" + map.formString(); + } + return get(path, StreamListing.class); + } + + public void disableTill(String key, long epoch) throws QiniuException { + String path = encodeKey(key) + "/disabled"; + String body = String.format("{\"disabledTill\":%d}", epoch); + post(path, body, null); + } + + public void enable(String key) throws QiniuException { + disableTill(key, 0); + } + + public StreamStatus status(String key) throws QiniuException { + String path = encodeKey(key) + "/live"; + return get(path, StreamStatus.class); + } + + public String saveAs(String key, String fileName) throws QiniuException { + return saveAs(key, fileName, 0, 0); + } + + public String saveAs(String key, String fileName, long start, long end) throws QiniuException { + String path = encodeKey(key) + "/saveas"; + String body; + if (fileName == null) { + body = String.format("{\"start\": %d,\"end\": %d}", start, end); + } else { + body = String.format("{\"fname\": %s,\"start\": %d,\"end\": %d}", fileName, start, end); + } + SaveRet r = post(path, body, SaveRet.class); + return r.fname; + } + + public ActivityRecords history(String key, long start, long end) throws QiniuException { + if (start <= 0 || end < 0 || (start >= end && end != 0)) { + throw new QiniuException(new IllegalArgumentException("bad argument" + start + "," + end)); + } + String path = encodeKey(key) + "/historyactivity?" + start; + if (end != 0) { + path += "&end=" + end; + } + return get(path, ActivityRecords.class); + } + + private String encodeKey(String key) { + return "/" + UrlSafeBase64.encodeToString(key); + } + + private T get(String path, Class classOfT) throws QiniuException { + String url = apiServer + "/v2/hubs/" + hub + "/streams" + path; + StringMap headers = auth.authorizationV2(url); + Response r = client.get(url, headers); + if (classOfT != null) { + return r.jsonToObject(classOfT); + } + return null; + } + + private T post(String path, String body, Class classOfT) throws QiniuException { + String url = apiServer + "/v2/hubs/" + hub + "/streams" + path; + byte[] b = body.getBytes(); + StringMap headers = auth.authorizationV2(url, "POST", b, Client.JsonMime); + Response r = client.post(url, b, headers, Client.JsonMime); + if (classOfT != null) { + return r.jsonToObject(classOfT); + } + return null; + } + + private static class SaveRet { + public String fname; + } + + /** + * 获取文件列表迭代器 + */ + public class ListIterator implements Iterator { + private final boolean live; + private String marker = null; + private String prefix; + private QiniuException exception = null; + + public ListIterator(boolean live, String prefix) { + this.live = live; + this.prefix = prefix; + } + + public QiniuException exception() { + return exception; + } + + @Override + public boolean hasNext() { + return exception == null && !"".equals(marker); + } + + @Override + public String[] next() { + try { + StreamListing l = listStreams(live, prefix, marker); + this.marker = l.marker == null ? "" : l.marker; + return l.keys(); + } catch (QiniuException e) { + this.exception = e; + return null; + } + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove"); + } + } +} diff --git a/src/main/java/com/qiniu/streaming/UrlFactory.java b/src/main/java/com/qiniu/streaming/UrlFactory.java new file mode 100644 index 000000000..c28e7d61b --- /dev/null +++ b/src/main/java/com/qiniu/streaming/UrlFactory.java @@ -0,0 +1,79 @@ +package com.qiniu.streaming; + +import com.qiniu.util.Auth; + +import java.util.Date; + +/** + * Created by bailong on 16/9/22. + */ +public final class UrlFactory { + + private final String hub; + private final Auth auth; + private final String rtmpPubDomain; + private final String rtmpPlayDomain; + private final String hlsDomain; + private final String hdlDomain; + private final String snapDomain; + + public UrlFactory(String hub, Auth auth, String rtmpPubDomain, String rtmpPlayDomain) { + this(hub, auth, rtmpPubDomain, rtmpPlayDomain, null, null, null); + } + + public UrlFactory(String hub, Auth auth, String rtmpPubDomain, + String rtmpPlayDomain, String hlsDomain, String hdlDomain, String snapDomain) { + this.hub = hub; + this.auth = auth; + this.rtmpPubDomain = rtmpPubDomain; + this.rtmpPlayDomain = rtmpPlayDomain; + this.hlsDomain = hlsDomain; + this.hdlDomain = hdlDomain; + this.snapDomain = snapDomain; + } + + public String rtmpPublishUrl(String streamKey) { + return String.format("rtmp://%s/%s/%s", rtmpPubDomain, hub, streamKey); + } + + public String rtmpPublishUrl(String streamKey, int expireAfterSeconds) { + long expire = new Date().getTime() + expireAfterSeconds; + String path = String.format("/%s/%s?e=%d", hub, streamKey, expire); + String token; + try { + token = auth.sign(path); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + return String.format("rtmp://%s%s&token=%s", rtmpPubDomain, path, token); + } + + /* + RTMPPlayURL generates RTMP play URL + */ + public String rtmpPlayUrl(String streamKey) { + return String.format("rtmp://%s/%s/%s", rtmpPlayDomain, hub, streamKey); + } + + /* + HLSPlayURL generates HLS play URL + */ + public String hlsPlayUrl(String streamKey) { + return String.format("http://%s/%s/%s.m3u8", hlsDomain, hub, streamKey); + } + + /* + HDLPlayURL generates HDL play URL + */ + public String hdlPlayUrl(String streamKey) { + return String.format("http://%s/%s/%s.flv", hdlDomain, hub, streamKey); + } + + /* + SnapshotPlayURL generates snapshot URL + */ + public String snapshotUrl(String streamKey) { + return String.format("http://%s/%s/%s.jpg", snapDomain, hub, streamKey); + } +} diff --git a/src/main/java/com/qiniu/streaming/model/ActivityRecords.java b/src/main/java/com/qiniu/streaming/model/ActivityRecords.java new file mode 100644 index 000000000..ed9db76fd --- /dev/null +++ b/src/main/java/com/qiniu/streaming/model/ActivityRecords.java @@ -0,0 +1,13 @@ +package com.qiniu.streaming.model; + +/** + * Created by bailong on 16/9/22. + */ +public final class ActivityRecords { + public Item[] items; + + public static class Item { + public long start; + public long end; + } +} diff --git a/src/main/java/com/qiniu/streaming/model/StreamAttribute.java b/src/main/java/com/qiniu/streaming/model/StreamAttribute.java new file mode 100644 index 000000000..d5d433352 --- /dev/null +++ b/src/main/java/com/qiniu/streaming/model/StreamAttribute.java @@ -0,0 +1,11 @@ +package com.qiniu.streaming.model; + +/** + * Created by bailong on 16/9/22. + */ +public final class StreamAttribute { + public long createdAt; + public long updatedAt; + public long expireAt; + public long disabledTill; +} diff --git a/src/main/java/com/qiniu/streaming/model/StreamListing.java b/src/main/java/com/qiniu/streaming/model/StreamListing.java new file mode 100644 index 000000000..45b2902a1 --- /dev/null +++ b/src/main/java/com/qiniu/streaming/model/StreamListing.java @@ -0,0 +1,28 @@ +package com.qiniu.streaming.model; + +import com.qiniu.util.StringUtils; + +/** + * Created by bailong on 16/9/23. + */ +public final class StreamListing { + public Item[] items; + public String marker; + + public boolean isEOF() { + return StringUtils.isNullOrEmpty(marker); + } + + public String[] keys() { + String[] keys = new String[items.length]; + int i = 0; + for (Item item : items) { + keys[i++] = item.key; + } + return keys; + } + + private static class Item { + public String key; + } +} diff --git a/src/main/java/com/qiniu/streaming/model/StreamStatus.java b/src/main/java/com/qiniu/streaming/model/StreamStatus.java new file mode 100644 index 000000000..90142e5dc --- /dev/null +++ b/src/main/java/com/qiniu/streaming/model/StreamStatus.java @@ -0,0 +1,18 @@ +package com.qiniu.streaming.model; + +/** + * Created by bailong on 16/9/22. + */ +public final class StreamStatus { + public long startAt; + public String clientIP; + public long bps; + + public Fps fps; + + public static class Fps { + public int audio; + public int video; + public int data; + } +} diff --git a/src/main/java/com/qiniu/util/Auth.java b/src/main/java/com/qiniu/util/Auth.java index f5d9619dc..7e1a7302f 100644 --- a/src/main/java/com/qiniu/util/Auth.java +++ b/src/main/java/com/qiniu/util/Auth.java @@ -276,4 +276,59 @@ public StringMap authorization(String url, byte[] body, String contentType) { public StringMap authorization(String url) { return authorization(url, null, null); } + + /** + * 生成HTTP请求签名字符串 + * + * @param urlString + * @param body + * @param contentType + * @return + */ + public String signRequestV2(String urlString, String method, byte[] body, String contentType) { + URI uri = URI.create(urlString); + + Mac mac = createMac(); + + + // + StringBuilder sb = new StringBuilder(); + + sb.append(String.format("%s %s", method, uri.getPath())); + if (uri.getQuery() != null) { + sb.append(String.format("?%s", uri.getQuery())); + } + + sb.append(String.format("\nHost: %s", uri.getHost())); + if (uri.getPort() > 0) { + sb.append(String.format(":%d", uri.getPort())); + } + + if (contentType != null) { + sb.append(String.format("\nContent-Type: %s", contentType)); + } + + // body + sb.append("\n\n"); + if (body != null && body.length > 0 && !StringUtils.isNullOrEmpty(contentType)) { + if (contentType.equals(Client.FormMime) + || contentType.equals(Client.JsonMime)) { + sb.append(new String(body)); + } + } + mac.update(StringUtils.utf8Bytes(sb.toString())); + + String digest = UrlSafeBase64.encodeToString(mac.doFinal()); + + return this.accessKey + ":" + digest; + } + + public StringMap authorizationV2(String url, String method, byte[] body, String contentType) { + String authorization = "Qiniu " + signRequestV2(url, method, body, contentType); + return new StringMap().put("Authorization", authorization); + } + + public StringMap authorizationV2(String url) { + return authorizationV2(url, "GET", null, null); + } } diff --git a/src/main/java/com/qiniu/util/Json.java b/src/main/java/com/qiniu/util/Json.java index 9e93b52e9..478dfdffc 100644 --- a/src/main/java/com/qiniu/util/Json.java +++ b/src/main/java/com/qiniu/util/Json.java @@ -1,7 +1,5 @@ package com.qiniu.util; -import com.google.gson.ExclusionStrategy; -import com.google.gson.FieldAttributes; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; diff --git a/src/test/java/com/qiniu/CdnTest.java b/src/test/java/com/qiniu/CdnTest.java new file mode 100644 index 000000000..833a6d13d --- /dev/null +++ b/src/test/java/com/qiniu/CdnTest.java @@ -0,0 +1,26 @@ +package com.qiniu; + +import com.qiniu.cdn.CdnManager; +import com.qiniu.common.QiniuException; +import com.qiniu.http.Response; +import org.junit.Assert; +import org.junit.Test; + +/** + * Created by bailong on 16/9/21. + */ +public class CdnTest { + @Test + public void testRefresh() { + CdnManager c = new CdnManager(TestConfig.testAuth); + Response r = null; + try { + r = c.refreshUrls(new String[]{"http://javasdk.qiniudn.com/gopher.jpg"}); + Assert.assertEquals(200, r.statusCode); + } catch (QiniuException e) { + e.printStackTrace(); + Assert.fail(); + } + + } +} diff --git a/src/test/java/com/qiniu/streaming/StreamingTest.java b/src/test/java/com/qiniu/streaming/StreamingTest.java new file mode 100644 index 000000000..d10ca4144 --- /dev/null +++ b/src/test/java/com/qiniu/streaming/StreamingTest.java @@ -0,0 +1,113 @@ +package com.qiniu.streaming; + +import com.qiniu.common.QiniuException; +import com.qiniu.streaming.model.ActivityRecords; +import com.qiniu.streaming.model.StreamAttribute; +import com.qiniu.streaming.model.StreamListing; +import com.qiniu.streaming.model.StreamStatus; +import com.qiniu.util.Auth; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Created by bailong on 16/9/22. + */ +public class StreamingTest { + private Auth auth = Auth.create(System.getenv("ak"), System.getenv("sk")); + private String hub = "pilisdktest"; + private String streamKeyPrefix = "pilijava" + System.currentTimeMillis(); + private StreamingManager manager = new StreamingManager(auth, hub); + + + @Test + public void testGetNoExistStream() { + try { + manager.attribute("nnnoexist"); + fail("should not exist"); + } catch (QiniuException e) { + e.printStackTrace(); + assertEquals(612, e.code()); + } + } + + @Test + public void testStreamOperation() throws QiniuException { + String streamKey = streamKeyPrefix + "-a"; + + manager.create(streamKey); + + StreamAttribute attr = manager.attribute(streamKey); + assertEquals(0, attr.disabledTill); + assertNotEquals(0, attr.createdAt); + + try { + manager.create(streamKey); + fail("has already existed"); + } catch (QiniuException e) { + assertEquals(614, e.code()); + } + + manager.disableTill(streamKey, -1); + + attr = manager.attribute(streamKey); + assertEquals(-1, attr.disabledTill); + assertNotEquals(0, attr.updatedAt); + + manager.enable(streamKey); + attr = manager.attribute(streamKey); + assertEquals(0, attr.disabledTill); + assertNotEquals(0, attr.updatedAt); + + long t = System.currentTimeMillis() / 1000 + 3600; + manager.disableTill(streamKey, t); + attr = manager.attribute(streamKey); + assertEquals(t, attr.disabledTill); + assertNotEquals(0, attr.updatedAt); + + manager.enable(streamKey); + attr = manager.attribute(streamKey); + assertEquals(0, attr.disabledTill); + assertNotEquals(0, attr.updatedAt); + + try { + StreamStatus status = manager.status(streamKey); + fail(); + } catch (QiniuException e) { + assertEquals(619, e.code()); + } + + try { + manager.saveAs(streamKey, null, 0, 0); + fail(); + } catch (QiniuException e) { + assertEquals(619, e.code()); + } + + ActivityRecords records = manager.history(streamKey, System.currentTimeMillis() / 1000 - 1000, 0); + assertEquals(0, records.items.length); + + StreamListing l = manager.listStreams(false, streamKeyPrefix, null); + String[] keys = l.keys(); + assertEquals(1, keys.length); + assertEquals("", l.marker); + + l = manager.listStreams(true, streamKeyPrefix, null); + keys = l.keys(); + assertEquals(0, keys.length); + + StreamingManager.ListIterator it = manager.createStreamListIterator(false, streamKeyPrefix); + assertTrue(it.hasNext()); + keys = it.next(); + assertEquals(1, keys.length); + + assertFalse(it.hasNext()); + + it = manager.createStreamListIterator(true, streamKeyPrefix); + assertTrue(it.hasNext()); + keys = it.next(); + assertEquals(0, keys.length); + + assertFalse(it.hasNext()); + } +} diff --git a/src/test/java/com/qiniu/streaming/UrlTest.java b/src/test/java/com/qiniu/streaming/UrlTest.java new file mode 100644 index 000000000..5b740b9a6 --- /dev/null +++ b/src/test/java/com/qiniu/streaming/UrlTest.java @@ -0,0 +1,45 @@ +package com.qiniu.streaming; + +import com.qiniu.TestConfig; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + + +/** + * Created by bailong on 16/9/22. + */ +public class UrlTest { + @Test + public void testUrl() { + String hubName = "test"; + String pubDomain = "publish-rtmp.test.com"; + String rtmpDomain = "live-rtmp.test.com"; + String hlsDomain = "live-hls.test.com"; + String hdlDomain = "live-hdl.test.com"; + String snapDomain = "live-snapshot.test.com"; + + UrlFactory uf = new UrlFactory(hubName, TestConfig.dummyAuth, + pubDomain, rtmpDomain, hlsDomain, hdlDomain, snapDomain); + String expect = "rtmp://publish-rtmp.test.com/" + hubName + "/key?e="; + String url = uf.rtmpPublishUrl("key", 3600); + System.out.println(url); + assertTrue(url.startsWith(expect)); + + expect = "rtmp://live-rtmp.test.com/" + hubName + "/key"; + url = uf.rtmpPlayUrl("key"); + assertTrue(url.startsWith(expect)); + + expect = "http://live-hls.test.com/" + hubName + "/key.m3u8"; + url = uf.hlsPlayUrl("key"); + assertTrue(url.startsWith(expect)); + + expect = "http://live-hdl.test.com/" + hubName + "/key.flv"; + url = uf.hdlPlayUrl("key"); + assertTrue(url.startsWith(expect)); + + expect = "http://live-snapshot.test.com/" + hubName + "/key.jpg"; + url = uf.snapshotUrl("key"); + assertTrue(url.startsWith(expect)); + } +} diff --git a/src/test/java/com/qiniu/util/AuthTest.java b/src/test/java/com/qiniu/util/AuthTest.java index 69e07e7ab..c6c000337 100644 --- a/src/test/java/com/qiniu/util/AuthTest.java +++ b/src/test/java/com/qiniu/util/AuthTest.java @@ -2,7 +2,6 @@ import com.qiniu.TestConfig; import com.qiniu.http.Client; -import com.qiniu.storage.model.UploadPolicy; import org.junit.Test; import static org.junit.Assert.*; @@ -58,12 +57,6 @@ public void testUploadToken() { assertEquals(exp, token); } - static class Policy{ - String scope; - long deadline; - String endUser; - } - @Test public void testUploadToken2() { Policy p = new Policy(); @@ -76,4 +69,10 @@ public void testUploadToken2() { // CHECKSTYLE:ON assertEquals(exp, token); } + + static class Policy { + String scope; + long deadline; + String endUser; + } } From 78fd83619cfcdbdeb9fc12940522b05317832568 Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 23 Sep 2016 21:48:23 +0800 Subject: [PATCH 07/10] format --- src/main/java/com/qiniu/common/Constants.java | 23 +++++++++++++------ .../java/com/qiniu/storage/Configuration.java | 2 +- .../java/com/qiniu/storage/FormUploader.java | 6 +++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/qiniu/common/Constants.java b/src/main/java/com/qiniu/common/Constants.java index 3615e8d6c..4d589395b 100644 --- a/src/main/java/com/qiniu/common/Constants.java +++ b/src/main/java/com/qiniu/common/Constants.java @@ -5,21 +5,30 @@ /** * Created by bailong on 16/9/14. */ -public interface Constants { - String VERSION = "8.0.0"; - int BLOCK_SIZE = 4 * 1024 * 1024; - Charset UTF_8 = Charset.forName("UTF-8"); +public final class Constants { + /** + * 版本号 + */ + public static final String VERSION = "8.0.0"; + /** + * 块大小，不能改变 + */ + public static final int BLOCK_SIZE = 4 * 1024 * 1024; + /** + * 所有都是UTF-8编码 + */ + public static final Charset UTF_8 = Charset.forName("UTF-8"); /** * 连接超时时间 单位秒(默认10s) */ - public int CONNECT_TIMEOUT = 10; + public static final int CONNECT_TIMEOUT = 10; /** * 写超时时间 单位秒(默认 0 , 不超时) */ - public int WRITE_TIMEOUT = 0; + public static final int WRITE_TIMEOUT = 0; /** * 回复超时时间 单位秒(默认30s) */ - public int RESPONSE_TIMEOUT = 30; + public static final int RESPONSE_TIMEOUT = 30; } diff --git a/src/main/java/com/qiniu/storage/Configuration.java b/src/main/java/com/qiniu/storage/Configuration.java index 9e22f87b1..ccb28ecd5 100644 --- a/src/main/java/com/qiniu/storage/Configuration.java +++ b/src/main/java/com/qiniu/storage/Configuration.java @@ -37,7 +37,7 @@ public final class Configuration implements Cloneable { /** * 上传失败重试次数 */ - public int RETRY_MAX = 5; + public int retryMax = 5; /** * 外部dns */ diff --git a/src/main/java/com/qiniu/storage/FormUploader.java b/src/main/java/com/qiniu/storage/FormUploader.java index ec96f9706..155a8192e 100644 --- a/src/main/java/com/qiniu/storage/FormUploader.java +++ b/src/main/java/com/qiniu/storage/FormUploader.java @@ -50,9 +50,11 @@ private FormUploader(Client client, String upToken, String key, byte[] data, Fil Response upload() throws QiniuException { buildParams(); if (data != null) { - return client.multipartPost(configuration.zone.upHost(token), params, "file", fileName, data, mime, new StringMap()); + return client.multipartPost(configuration.zone.upHost(token), params, "file", fileName, data, + mime, new StringMap()); } - return client.multipartPost(configuration.zone.upHost(token), params, "file", fileName, file, mime, new StringMap()); + return client.multipartPost(configuration.zone.upHost(token), params, "file", fileName, file, + mime, new StringMap()); } void asyncUpload(final UpCompletionHandler handler) throws IOException { From 85bde6ce494163cbebb6f4f76af9818aa7679e22 Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 23 Sep 2016 21:54:58 +0800 Subject: [PATCH 08/10] fixed style --- src/main/java/com/qiniu/common/Constants.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/qiniu/common/Constants.java b/src/main/java/com/qiniu/common/Constants.java index 4d589395b..39544734a 100644 --- a/src/main/java/com/qiniu/common/Constants.java +++ b/src/main/java/com/qiniu/common/Constants.java @@ -30,5 +30,8 @@ public final class Constants { * 回复超时时间 单位秒(默认30s) */ public static final int RESPONSE_TIMEOUT = 30; + + private Constants(){ + } } From 8c6c1941b42c5dffeadc4b8232673acc1c88525c Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 23 Sep 2016 21:58:48 +0800 Subject: [PATCH 09/10] format --- src/main/java/com/qiniu/common/Constants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/qiniu/common/Constants.java b/src/main/java/com/qiniu/common/Constants.java index 39544734a..e2a23c23a 100644 --- a/src/main/java/com/qiniu/common/Constants.java +++ b/src/main/java/com/qiniu/common/Constants.java @@ -31,7 +31,7 @@ public final class Constants { */ public static final int RESPONSE_TIMEOUT = 30; - private Constants(){ + private Constants() { } } From d53f3f560d89be544aca6bca6ae6973588eabe6b Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 23 Sep 2016 22:29:12 +0800 Subject: [PATCH 10/10] check style fixed --- src/main/java/com/qiniu/common/QiniuException.java | 4 +++- src/test/java/com/qiniu/streaming/StreamingTest.java | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/qiniu/common/QiniuException.java b/src/main/java/com/qiniu/common/QiniuException.java index e4e70693f..35eccbf07 100644 --- a/src/main/java/com/qiniu/common/QiniuException.java +++ b/src/main/java/com/qiniu/common/QiniuException.java @@ -5,7 +5,9 @@ import java.io.IOException; - +/** + * 异常，封装了http响应数据 + */ public final class QiniuException extends IOException { public final Response response; private String error; diff --git a/src/test/java/com/qiniu/streaming/StreamingTest.java b/src/test/java/com/qiniu/streaming/StreamingTest.java index d10ca4144..5cee6dc1b 100644 --- a/src/test/java/com/qiniu/streaming/StreamingTest.java +++ b/src/test/java/com/qiniu/streaming/StreamingTest.java @@ -31,8 +31,10 @@ public void testGetNoExistStream() { } } + // CHECKSTYLE:OFF @Test public void testStreamOperation() throws QiniuException { + // CHECKSTYLE:ON String streamKey = streamKeyPrefix + "-a"; manager.create(streamKey);