Added Go based automation to update treesitter source. Also bumped to v0.20.8

_automation/treesitter_updater/README.md

@@ -0,0 +1,23 @@
+# Tree Sitter Updater
+
+This Go program automates the process of downloading, extracting, and processing the specified version of the [Tree Sitter library](https://github.com/tree-sitter/tree-sitter). It's designed to simplify the updating of the upstream Tree Sitter library for use in this project.
+
+
+## Usage
+
+If you want to change the version of the treesitter library that is retrieved, update the `sitterVersion` variable in `main.go`.
+
+Note, you must run this script from within the `_automation/treesitter_updater` directory because it makes an assumption that the final destination for the .C and .H files is 2 directories up from it's current directory.
+
+```bash
+cd _automation/treesitter_updater
+go run main.go
+```
+
+The success / failure will be printed to stdout, as well as a list of (1) new files and (2) replaced files.
+
+## Constants
+
+- `sitterVersion`: Specifies the version of Tree Sitter to download.
+- `sitterURL`: The URL to the Tree Sitter source for the specified version.
+

_automation/treesitter_updater/main.go

@@ -0,0 +1,245 @@
+package main
+
+import (
+	// Import necessary packages
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// Constants for the Tree Sitter version and download URL
+const sitterVersion = "0.20.8"
+const sitterURL = "https://github.com/tree-sitter/tree-sitter/archive/refs/tags/v" + sitterVersion + ".tar.gz"
+
+func main() {
+	// Get the current working directory
+	currentDir, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("Error getting current directory: %v", err)
+	}
+
+	// Construct the directory path for the downloaded Tree Sitter files
+	treeSitterDir := "tree-sitter-" + sitterVersion
+	parentPath := filepath.Join(currentDir, "tmpts", treeSitterDir)
+
+	// Download and extract the Tree Sitter source code
+	if err := downloadAndExtractSitter(sitterURL, sitterVersion); err != nil {
+		log.Fatalf("Error: %v", err)
+	}
+
+	// Copy necessary files to tmpts directory
+	copyFiles(filepath.Join(parentPath, "lib", "include", "tree_sitter"), filepath.Join(currentDir, "tmpts"), "*.h")
+	copyFiles(filepath.Join(parentPath, "lib", "src"), filepath.Join(currentDir, "tmpts"), "*.c")
+	copyFiles(filepath.Join(parentPath, "lib", "src"), filepath.Join(currentDir, "tmpts"), "*.h")
+	copyFiles(filepath.Join(parentPath, "lib", "src", "unicode"), filepath.Join(currentDir, "tmpts"), "*.h")
+
+	// Remove the original extracted directory
+	err = os.RemoveAll(parentPath)
+	if err != nil {
+		log.Fatalf("Error removing extracted treesitter directory: %v", err)
+	}
+
+	// Modify include paths in the copied files
+	if err := modifyIncludePaths(filepath.Join(currentDir, "tmpts")); err != nil {
+		log.Fatalf("Error modifying include paths: %v", err)
+	}
+
+	// Clean up unnecessary files
+	cleanup(filepath.Join(currentDir, "tmpts"))
+
+	// Copy and report files from tmpts to two levels up in the directory structure
+	err = copyAndReportFiles(filepath.Join(currentDir, "tmpts"), filepath.Join(currentDir, "..", ".."))
+	if err != nil {
+		log.Fatalf("Error copying and reporting files: %v", err)
+	}
+
+	err = os.RemoveAll(filepath.Join(currentDir, "tmpts"))
+	if err != nil {
+		log.Fatalf("Error removing tmpts directory: %v", err)
+	}
+
+	fmt.Printf("\n\nDone!\n")
+}
+
+// Function to copy and report files from source to destination directory
+func copyAndReportFiles(srcDir, dstDir string) error {
+	// Walk through the source directory
+	return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		// Calculate relative file path and destination file path
+		relPath, err := filepath.Rel(srcDir, path)
+		if err != nil {
+			return err
+		}
+		dstFilePath := filepath.Join(dstDir, relPath)
+
+		// Check if file exists at destination and print appropriate message
+		if _, err := os.Stat(dstFilePath); err == nil {
+			fmt.Printf("%-39s %s\n", filepath.Base(dstFilePath), "[replaced]")
+		} else if os.IsNotExist(err) {
+			fmt.Printf("%-39s %s\n", filepath.Base(dstFilePath), "[new file]")
+		}
+
+		// Copy the file to destination
+		return copyFile(path, dstFilePath)
+	})
+}
+
+// Function to copy files matching a pattern from source to destination directory
+func copyFiles(srcDir, dstDir, pattern string) {
+	files, err := ioutil.ReadDir(srcDir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Iterate through files and copy if they match the pattern
+	for _, file := range files {
+		if matched, _ := filepath.Match(pattern, file.Name()); matched {
+			srcFilePath := filepath.Join(srcDir, file.Name())
+			dstFilePath := filepath.Join(dstDir, file.Name())
+			copyFile(srcFilePath, dstFilePath)
+		}
+	}
+}
+
+// Function to copy a single file from source to destination
+func copyFile(src, dst string) error {
+	// Read the file from source
+	input, err := ioutil.ReadFile(src)
+	if err != nil {
+		return err
+	}
+
+	// Write the file to destination
+	err = ioutil.WriteFile(dst, input, 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Function to modify include paths in .c and .h files
+func modifyIncludePaths(path string) error {
+	// Walk through the directory and modify files
+	return filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Skip directories and non .c/.h files
+		if info.IsDir() || (filepath.Ext(filePath) != ".c" && filepath.Ext(filePath) != ".h") {
+			return nil
+		}
+
+		// Read the file content
+		content, err := os.ReadFile(filePath)
+		if err != nil {
+			return err
+		}
+
+		// Modify the content and write back
+		modifiedContent := strings.ReplaceAll(string(content), `"tree_sitter/`, `"`)
+		modifiedContent = strings.ReplaceAll(modifiedContent, `"unicode/`, `"`)
+		return os.WriteFile(filePath, []byte(modifiedContent), info.Mode())
+	})
+}
+
+// Function to download and extract Tree Sitter from the given URL
+func downloadAndExtractSitter(url, version string) error {
+	// Send HTTP request to download the file
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	// Prepare gzip reader
+	gzr, err := gzip.NewReader(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer gzr.Close()
+
+	// Prepare tar reader and extract files
+	tr := tar.NewReader(gzr)
+	for {
+		header, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		// Process files within specified directories
+		if !strings.HasPrefix(header.Name, "tree-sitter-"+version+"/lib/src") && !strings.HasPrefix(header.Name, "tree-sitter-"+version+"/lib/include") {
+			continue
+		}
+
+		relPath := strings.TrimPrefix(header.Name, version+"/")
+		target := filepath.Join("tmpts", relPath)
+
+		// Create directories and files as needed
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, 0755); err != nil {
+				return err
+			}
+		case tar.TypeReg:
+			outFile, err := os.Create(target)
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(outFile, tr); err != nil {
+				outFile.Close()
+				return err
+			}
+			outFile.Close()
+		}
+	}
+
+	return nil
+}
+
+// Function to clean up the specified directory
+func cleanup(path string) {
+	// Walk through the directory and remove unnecessary files
+	err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		if filepath.Ext(path) != ".h" && filepath.Ext(path) != ".c" || filepath.Base(path) == "lib.c" {
+			return os.Remove(path)
+		}
+		return nil
+	})
+
+	if err != nil {
+		// Handle the error
+	}
+}
+
+// Function to run a command and pipe its output
+func runCmd(name string, args ...string) error {
+	cmd := exec.Command(name, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}

api.h

@@ -381,6 +381,13 @@ TSNode ts_tree_root_node_with_offset(
  */
 const TSLanguage *ts_tree_language(const TSTree *);
 
+/**
+ * Get the array of included ranges that was used to parse the syntax tree.
+ *
+ * The returned pointer must be freed by the caller.
+ */
+TSRange *ts_tree_included_ranges(const TSTree *, uint32_t *length);
+
 /**
  * Edit the syntax tree to keep it in sync with source code that has been
  * edited.
@@ -413,7 +420,7 @@ TSRange *ts_tree_get_changed_ranges(
 /**
  * Write a DOT graph describing the syntax tree to the given file.
  */
-void ts_tree_print_dot_graph(const TSTree *, FILE *);
+void ts_tree_print_dot_graph(const TSTree *, int file_descriptor);
 
 /******************/
 /* Section - Node */
@@ -743,15 +750,26 @@ const TSQueryPredicateStep *ts_query_predicates_for_pattern(
   uint32_t *length
 );
 
-bool ts_query_is_pattern_rooted(
-  const TSQuery *self,
-  uint32_t pattern_index
-);
+/*
+ * Check if the given pattern in the query has a single root node.
+ */
+bool ts_query_is_pattern_rooted(const TSQuery *self, uint32_t pattern_index);
 
-bool ts_query_is_pattern_guaranteed_at_step(
-  const TSQuery *self,
-  uint32_t byte_offset
-);
+/*
+ * Check if the given pattern in the query is 'non local'.
+ *
+ * A non-local pattern has multiple root nodes and can match within a
+ * repeating sequence of nodes, as specified by the grammar. Non-local
+ * patterns disable certain optimizations that would otherwise be possible
+ * when executing a query on a specific range of a syntax tree.
+ */
+bool ts_query_is_pattern_non_local(const TSQuery *self, uint32_t pattern_index);
+
+/*
+ * Check if a given pattern is guaranteed to match once a given step is reached.
+ * The step is specified by its byte offset in the query's source code.
+ */
+bool ts_query_is_pattern_guaranteed_at_step(const TSQuery *self, uint32_t byte_offset);
 
 /**
  * Get the name and length of one of the query's captures, or one of the

array.h

@@ -170,10 +170,10 @@ static inline void array__swap(VoidArray *self, VoidArray *other) {
   *self = swap;
 }
 
-static inline void array__grow(VoidArray *self, size_t count, size_t element_size) {
-  size_t new_size = self->size + count;
+static inline void array__grow(VoidArray *self, uint32_t count, size_t element_size) {
+  uint32_t new_size = self->size + count;
   if (new_size > self->capacity) {
-    size_t new_capacity = self->capacity * 2;
+    uint32_t new_capacity = self->capacity * 2;
     if (new_capacity < 8) new_capacity = 8;
     if (new_capacity < new_size) new_capacity = new_size;
     array__reserve(self, element_size, new_capacity);

clock.h

@@ -1,6 +1,7 @@
 #ifndef TREE_SITTER_CLOCK_H_
 #define TREE_SITTER_CLOCK_H_
 
+#include <stdbool.h>
 #include <stdint.h>
 
 typedef uint64_t TSDuration;
@@ -82,6 +83,10 @@ static inline TSClock clock_after(TSClock base, TSDuration duration) {
   TSClock result = base;
   result.tv_sec += duration / 1000000;
   result.tv_nsec += (duration % 1000000) * 1000;
+  if (result.tv_nsec >= 1000000000) {
+    result.tv_nsec -= 1000000000;
+    ++(result.tv_sec);
+  }
   return result;
 }