Update doctest code for more consistent runs

Signed-off-by: Simeon Widdis <[email protected]>

doctest/build.gradle

@@ -21,8 +21,16 @@ def path = project(':').projectDir
 def plugin_path = project(':doctest').projectDir
 
 task cloneSqlCli(type: Exec) {
-    // clone the sql-cli repo locally
-    commandLine 'git', 'clone', 'https://github.com/opensearch-project/sql-cli.git'
+    def repoDir = new File("${project.projectDir}/sql-cli")
+
+    if (repoDir.exists()) {
+        // Repository already exists, fetch and checkout latest
+        commandLine 'git', '-C', repoDir.absolutePath, 'fetch', 'origin', 'main'
+        commandLine 'git', '-C', repoDir.absolutePath, 'checkout', 'origin/main'
+    } else {
+        // Repository doesn't exist, clone it
+        commandLine 'git', 'clone', 'https://github.com/opensearch-project/sql-cli.git', repoDir.absolutePath
+    }
 }
 
 task bootstrap(type: Exec, dependsOn: ['cloneSqlCli']) {

doctest/test_docs.py

@@ -48,10 +48,34 @@ def process(self, statement):
         click.echo(output)
 
 
+"""
+For _explain requests, there are several additional request fields that will inconsistently
+appear/change depending on underlying cluster state. This method normalizes these responses in-place
+to make _explain doctests more consistent.
+
+If the passed response is not an _explain response, the input is left unmodified.
+"""
+def normalize_explain_response(data):
+    if "root" in data:
+        data = data["root"]
+
+    if (request := data.get("description", {}).get("request", None)) and request.startswith("OpenSearchQueryRequest("):
+        for filter_field in ["needClean", "pitId", "cursorKeepAlive", "searchAfter", "searchResponse"]:
+            request = re.sub(f", {filter_field}=\\w+", "", request)
+        data["description"]["request"] = request
+
+    for child in data.get("children", []):
+        normalize_explain_response(child)
+
+    return data
+
+
 def pretty_print(s):
     try:
-        d = json.loads(s)
-        print(json.dumps(d, indent=2))
+        data = json.loads(s)
+        normalize_explain_response(data)
+
+        print(json.dumps(data, indent=2))
     except json.decoder.JSONDecodeError:
         print(s)