Merge pull request #101 from SonicCloudOrg/scrcpy_dev

Scrcpy dev

config/application-prod.yml

@@ -45,3 +45,5 @@ logging:
       clean-history-on-start: true
       #保留天数
       max-history: 3
+  pattern:
+    console: "%clr(%d{${LOG_DATEFORMAT_PATTERN:yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:%wEx}"

src/main/java/com/sonic/agent/automation/AppiumServer.java

@@ -21,8 +21,7 @@ public class AppiumServer {
     public static void start() {
         service = AppiumDriverLocalService.buildService(new AppiumServiceBuilder().usingAnyFreePort()
                 .withArgument(GeneralServerFlag.LOG_LEVEL, "error")
-                .withArgument(GeneralServerFlag.ALLOW_INSECURE, "chromedriver_autodownload")
-                .withArgument(GeneralServerFlag.SESSION_OVERRIDE));
+                .withArgument(GeneralServerFlag.ALLOW_INSECURE, "chromedriver_autodownload"));
         service.start();
     }
 

src/main/java/com/sonic/agent/cv/FindResult.java → src/main/java/com/sonic/agent/automation/FindResult.java

@@ -1,57 +1,57 @@
-package com.sonic.agent.cv;
-
-public class FindResult {
-    private int x;
-    private int y;
-    private String url;
-    private int time;
-
-    public FindResult() {
-        x = 0;
-        y = 0;
-        url = "";
-        time = 0;
-    }
-
-    public int getX() {
-        return x;
-    }
-
-    public void setX(int x) {
-        this.x = x;
-    }
-
-    public int getY() {
-        return y;
-    }
-
-    public void setY(int y) {
-        this.y = y;
-    }
-
-    public String getUrl() {
-        return url;
-    }
-
-    public void setUrl(String url) {
-        this.url = url;
-    }
-
-    public int getTime() {
-        return time;
-    }
-
-    public void setTime(int time) {
-        this.time = time;
-    }
-
-    @Override
-    public String toString() {
-        return "FindResult{" +
-                "x=" + x +
-                ", y=" + y +
-                ", url='" + url + '\'' +
-                ", time=" + time +
-                '}';
-    }
-}
+package com.sonic.agent.automation;
+
+public class FindResult {
+    private int x;
+    private int y;
+    private String url;
+    private int time;
+
+    public FindResult() {
+        x = 0;
+        y = 0;
+        url = "";
+        time = 0;
+    }
+
+    public int getX() {
+        return x;
+    }
+
+    public void setX(int x) {
+        this.x = x;
+    }
+
+    public int getY() {
+        return y;
+    }
+
+    public void setY(int y) {
+        this.y = y;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public void setUrl(String url) {
+        this.url = url;
+    }
+
+    public int getTime() {
+        return time;
+    }
+
+    public void setTime(int time) {
+        this.time = time;
+    }
+
+    @Override
+    public String toString() {
+        return "FindResult{" +
+                "x=" + x +
+                ", y=" + y +
+                ", url='" + url + '\'' +
+                ", time=" + time +
+                '}';
+    }
+}

src/main/java/com/sonic/agent/automation/IOSStepHandler.java

@@ -4,14 +4,14 @@
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.sonic.agent.bridge.ios.TIDeviceTool;
-import com.sonic.agent.cv.*;
 import com.sonic.agent.interfaces.ErrorType;
 import com.sonic.agent.interfaces.ResultDetailStatus;
 import com.sonic.agent.interfaces.StepType;
 import com.sonic.agent.maps.IOSProcessMap;
 import com.sonic.agent.maps.IOSSizeMap;
 import com.sonic.agent.tools.DownImageTool;
 import com.sonic.agent.tools.LogTool;
+import com.sonic.agent.tools.SpringTool;
 import com.sonic.agent.tools.UploadTools;
 import io.appium.java_client.MobileBy;
 import io.appium.java_client.MultiTouchAction;
@@ -38,8 +38,14 @@
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.remote.DesiredCapabilities;
 import org.openqa.selenium.support.ui.ExpectedConditions;
+import org.springframework.core.env.Environment;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.http.ResponseEntity;
 import org.springframework.util.Base64Utils;
 import org.springframework.util.FileCopyUtils;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+import org.springframework.web.client.RestTemplate;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -53,8 +59,17 @@
 
 import static org.testng.Assert.*;
 
+/**
+ * @author ZhouYiXun
+ * @des iOS自动化处理类
+ * @date 2021/8/16 20:10
+ */
 public class IOSStepHandler {
     public LogTool log = new LogTool();
+    private RestTemplate restTemplate = SpringTool.getBean(RestTemplate.class);
+    private Environment environment = SpringTool.getBean(Environment.class);
+    private String baseUrl = "http://" + environment.getProperty("sonic.server.host")
+            + ":" + environment.getProperty("sonic.server.folder-port") + "/api/folder";
     private IOSDriver iosDriver;
     private JSONObject globalParams = new JSONObject();
     private String testPackage = "";
@@ -94,7 +109,9 @@ public void startIOSDriver(String udId, int wdaPort) throws InterruptedException
         try {
             iosDriver = new IOSDriver(AppiumServer.service.getUrl(), desiredCapabilities);
             iosDriver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
-            iosDriver.setSetting(Setting.MJPEG_SERVER_FRAMERATE, 30);
+            iosDriver.setSetting(Setting.MJPEG_SERVER_FRAMERATE, 50);
+            iosDriver.setSetting(Setting.MJPEG_SCALING_FACTOR, 50);
+            iosDriver.setSetting(Setting.MJPEG_SERVER_SCREENSHOT_QUALITY, 10);
             log.sendStepLog(StepType.PASS, "连接设备驱动成功", "");
         } catch (Exception e) {
             log.sendStepLog(StepType.ERROR, "连接设备驱动失败！", "");
@@ -567,71 +584,55 @@ public void clickByImg(HandleDes handleDes, String des, String pathValue) throws
                 return;
             }
         }
-        FindResult findResult = null;
+        File localCap = getScreenToLocal();
+        FindResult findResult;
+        FileSystemResource resource1 = new FileSystemResource(file);
+        FileSystemResource resource2 = new FileSystemResource(localCap);
+        MultiValueMap<String, Object> param = new LinkedMultiValueMap<>();
+        param.add("file1", resource1);
+        param.add("file2", resource2);
+        param.add("type", "finder");
         try {
-            SIFTFinder siftFinder = new SIFTFinder();
-            findResult = siftFinder.getSIFTFindResult(file, getScreenToLocal());
-        } catch (Exception e) {
-            log.sendStepLog(StepType.WARN, "SIFT图像算法出错，切换算法中...",
-                    "");
-        }
-        if (findResult != null) {
-            log.sendStepLog(StepType.INFO, "图片定位到坐标：(" + findResult.getX() + "," + findResult.getY() + ")  耗时：" + findResult.getTime() + " ms",
-                    findResult.getUrl());
-        } else {
-            log.sendStepLog(StepType.INFO, "SIFT算法无法定位图片，切换AKAZE算法中...",
-                    "");
-            try {
-                AKAZEFinder akazeFinder = new AKAZEFinder();
-                findResult = akazeFinder.getAKAZEFindResult(file, getScreenToLocal());
-            } catch (Exception e) {
-                log.sendStepLog(StepType.WARN, "AKAZE图像算法出错，切换模版匹配算法中...",
-                        "");
-            }
-            if (findResult != null) {
-                log.sendStepLog(StepType.INFO, "图片定位到坐标：(" + findResult.getX() + "," + findResult.getY() + ")  耗时：" + findResult.getTime() + " ms",
-                        findResult.getUrl());
-            } else {
-                log.sendStepLog(StepType.INFO, "AKAZE算法无法定位图片，切换模版匹配算法中...",
-                        "");
-                try {
-                    TemMatcher temMatcher = new TemMatcher();
-                    findResult = temMatcher.getTemMatchResult(file, getScreenToLocal());
-                } catch (Exception e) {
-                    log.sendStepLog(StepType.WARN, "模版匹配算法出错",
-                            "");
-                }
+            ResponseEntity<JSONObject> responseEntity =
+                    restTemplate.postForEntity(baseUrl + "/upload/cv", param, JSONObject.class);
+            if (responseEntity.getBody().getInteger("code") == 2000) {
+                findResult = responseEntity.getBody().getJSONObject("data").toJavaObject(FindResult.class);
                 if (findResult != null) {
-                    log.sendStepLog(StepType.INFO, "图片定位到坐标：(" + findResult.getX() + "," + findResult.getY() + ")  耗时：" + findResult.getTime() + " ms",
-                            findResult.getUrl());
-                } else {
-                    handleDes.setE(new Exception("图片定位失败！"));
+                    try {
+                        log.sendStepLog(StepType.INFO, "图片定位到坐标：(" + findResult.getX() + "," + findResult.getY() + ")  耗时：" + findResult.getTime() + " ms",
+                                findResult.getUrl());
+                        TouchAction ta = new TouchAction(iosDriver);
+                        ta.tap(PointOption.point(findResult.getX(), findResult.getY())).perform();
+                    } catch (Exception e) {
+                        log.sendStepLog(StepType.ERROR, "点击" + des + "失败！", "");
+                        handleDes.setE(e);
+                    }
                 }
+            } else if (responseEntity.getBody().getInteger("code") == 4003) {
+                handleDes.setE(new Exception("图像匹配失败！"));
+            } else {
+                handleDes.setE(new Exception("点击失败！cv服务出错！"));
             }
-        }
-        if (findResult != null) {
-            try {
-                TouchAction ta = new TouchAction(iosDriver);
-                ta.tap(PointOption.point(findResult.getX(), findResult.getY())).perform();
-            } catch (Exception e) {
-                log.sendStepLog(StepType.ERROR, "点击" + des + "失败！", "");
-                handleDes.setE(e);
-            }
+        } catch (Exception e) {
+            handleDes.setE(new Exception("点击失败！cv服务访问出错！"));
+        } finally {
+            file.delete();
+            localCap.delete();
         }
     }
 
 
     public void readText(HandleDes handleDes, String language, String text) throws Exception {
-        TextReader textReader = new TextReader();
-        String result = textReader.getTessResult(getScreenToLocal(), language);
-        log.sendStepLog(StepType.INFO, "",
-                "图像文字识别结果：<br>" + result);
-        String filter = result.replaceAll(" ", "");
+//        TextReader textReader = new TextReader();
+//        String result = textReader.getTessResult(getScreenToLocal(), language);
+//        log.sendStepLog(StepType.INFO, "",
+//                "图像文字识别结果：<br>" + result);
+//        String filter = result.replaceAll(" ", "");
         handleDes.setStepDes("图像文字识别");
-        handleDes.setDetail("期望包含文本：" + text);
-        if (!filter.contains(text)) {
-            handleDes.setE(new Exception("图像文字识别不通过！"));
-        }
+        handleDes.setDetail("（该功能暂时关闭）期望包含文本：" + text);
+//        if (!filter.contains(text)) {
+//            handleDes.setE(new Exception("图像文字识别不通过！"));
+//        }
     }
 
     public File getScreenToLocal() {
@@ -667,13 +668,33 @@ public void checkImage(HandleDes handleDes, String des, String pathValue, double
         if (pathValue.startsWith("http")) {
             file = DownImageTool.download(pathValue);
         }
-        double score = SimilarityChecker.getSimilarMSSIMScore(file, getScreenToLocal(), true);
-        handleDes.setStepDes("检测" + des + "图片相似度");
-        handleDes.setDetail("相似度为" + score * 100 + "%");
-        if (score == 0) {
-            handleDes.setE(new Exception("图片相似度检测不通过！比对图片分辨率不一致！"));
-        } else if (score < (matchThreshold / 100)) {
-            handleDes.setE(new Exception("图片相似度检测不通过！expect " + matchThreshold + " but " + score * 100));
+        File localCap = getScreenToLocal();
+        FileSystemResource resource1 = new FileSystemResource(file);
+        FileSystemResource resource2 = new FileSystemResource(localCap);
+        MultiValueMap<String, Object> param = new LinkedMultiValueMap<>();
+        param.add("file1", resource1);
+        param.add("file2", resource2);
+        param.add("type", "checker");
+        try {
+            ResponseEntity<JSONObject> responseEntity =
+                    restTemplate.postForEntity(baseUrl + "/upload/cv", param, JSONObject.class);
+            if (responseEntity.getBody().getInteger("code") == 2000) {
+                double score = responseEntity.getBody().getDouble("data");
+                handleDes.setStepDes("检测" + des + "图片相似度");
+                handleDes.setDetail("相似度为" + score * 100 + "%");
+                if (score == 0) {
+                    handleDes.setE(new Exception("图片相似度检测不通过！比对图片分辨率不一致！"));
+                } else if (score < (matchThreshold / 100)) {
+                    handleDes.setE(new Exception("图片相似度检测不通过！expect " + matchThreshold + " but " + score * 100));
+                }
+            } else {
+                handleDes.setE(new Exception("图片相似度检测出错！cv服务出错！"));
+            }
+        } catch (Exception e) {
+            handleDes.setE(new Exception("图片相似度检测出错！cv服务访问出错！"));
+        } finally {
+            file.delete();
+            localCap.delete();
         }
     }
 

src/main/java/com/sonic/agent/bridge/android/AndroidDeviceBridgeTool.java

@@ -6,6 +6,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.ApplicationListener;
 import org.springframework.context.annotation.DependsOn;
@@ -28,6 +29,7 @@
 public class AndroidDeviceBridgeTool implements ApplicationListener<ContextRefreshedEvent> {
     private static final Logger logger = LoggerFactory.getLogger(AndroidDeviceBridgeTool.class);
     public static AndroidDebugBridge androidDebugBridge = null;
+    private AndroidTemperThread androidTemperThread = null;
 
     @Autowired
     private AndroidDeviceStatusListener androidDeviceStatusListener;
@@ -36,7 +38,6 @@ public class AndroidDeviceBridgeTool implements ApplicationListener<ContextRefre
     @Override
     public void onApplicationEvent(@NonNull ContextRefreshedEvent event) {
         logger.info("开启安卓相关功能");
-        init();
     }
 
     /**
@@ -67,7 +68,7 @@ public void init() {
         String systemADBPath = getADBPathFromSystemEnv();
         //添加设备上下线监听
         androidDebugBridge.addDeviceChangeListener(androidDeviceStatusListener);
-        AndroidDebugBridge.init(false);
+        AndroidDebugBridge.initIfNeeded(false);
         //开始创建ADB
         androidDebugBridge = AndroidDebugBridge.createBridge(systemADBPath, true);
         if (androidDebugBridge != null) {
@@ -86,7 +87,10 @@ public void init() {
                 break;
             }
         }
-        new AndroidTemperThread().start();
+        if (androidTemperThread == null || !androidTemperThread.isAlive()) {
+            androidTemperThread = new AndroidTemperThread();
+            androidTemperThread.start();
+        }
     }
 
     /**