Merge pull request #80 from elm-tooling/fix-scanner-getting-stuck

Cargo.toml

@@ -20,7 +20,7 @@ include = [
 ]
 
 # Keep in sync with package.json
-version = "5.0.0"
+version = "5.0.1"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elm-tooling/tree-sitter-elm",
-  "version": "5.0.0",
+  "version": "5.0.1",
   "description": "Tree sitter definitions for elm",
   "main": "index.js",
   "publishConfig": {
@@ -32,7 +32,9 @@
     "parse-basic": "tree-sitter parse ./examples/Basic.elm",
     "parse-test": "tree-sitter parse --debug ./examples/test.elm",
     "test": "tree-sitter test && script/parse-examples",
+    "test-skip-download": "tree-sitter test && script/parse-examples -s",
     "test-full": "tree-sitter test && script/parse-examples-full",
+    "test-full-skip-download": "tree-sitter test && script/parse-examples-full -s",
     "test-only": "tree-sitter test",
     "test-highlighting": "tree-sitter highlight test/highlight/basic.elm",
     "test-tags": "tree-sitter tags test/highlight/basic.elm",

script/parse-examples

@@ -13,26 +13,42 @@ function checkout_at() {
   popd
 }
 
-checkout_at "examples/elm-spa-example" "rtfeldman/elm-spa-example" "c8c3201ec0488f17c1245e1fd2293ba5bc0748d5"
-checkout_at "examples/elm-browser" "elm/browser" "1d28cd625b3ce07be6dfad51660bea6de2c905f2"
-checkout_at "examples/elm-bytes" "elm/bytes" "2bce2aeda4ef18c3dcccd84084647d22a7af36a6"
-checkout_at "examples/elm-core" "elm/core" "22eefd207e7a63daab215ae497f683ff2319c2ca"
-checkout_at "examples/elm-file" "elm/file" "e4ca3864c93a5e766e24ed6916174753567b2f59"
-checkout_at "examples/elm-html" "elm/html" "94c079007f8a7ed282d5b53f4a49101dd0b6cf99"
-checkout_at "examples/elm-http" "elm/http" "81b6fdc67d8e5fb25644fd79e6b0edbe2e14e474"
-checkout_at "examples/elm-json" "elm/json" "af344039e8c014b06ed0f73ac3ffd22c60d30876"
-checkout_at "examples/elm-parser" "elm/parser" "7506b07eaa93a93d13b508b948c016105b0953c8"
-checkout_at "examples/elm-project-metadata-utils" "elm/project-metadata-utils" "b075be15046e151e36a79950412c4cb703605aeb"
-checkout_at "examples/elm-random" "elm/random" "ecf97bb43f0d5cd75243428f69f45323957bda25"
-checkout_at "examples/elm-regex" "elm/regex" "8810c41fb17ddf89165665489be213f44070bc4a"
-checkout_at "examples/elm-svg" "elm/svg" "08bd432990862bab5b840654dd437fbb2e6176e7"
-checkout_at "examples/elm-time" "elm/time" "dc3b75b7366e59b99962706f7bf064d3634a4bba"
-checkout_at "examples/elm-url" "elm/url" "8602f4b48653ca93b6991fe2c6a764523cd1d462"
-checkout_at "examples/elm-virtual-dom" "elm/virtual-dom" "5a5bcf48720bc7d53461b3cd42a9f19f119c5503"
-checkout_at "examples/elm-ui" "mdgriffith/elm-ui" "7aa0e015499d4975a4dda02c2629ee21ec18eac3"
-checkout_at "examples/elm-markup" "mdgriffith/elm-markup" "e4b88aa04c7e9bafd02b5436c311cf07e255dab1"
-checkout_at "examples/elm-visualization" "gampleman/elm-visualization" "68631a3dbf840d30bc642e605a49ccd8040504f6"
+# Define list of arguments expected in the input
+optstring="s"
 
+while getopts ${optstring} arg; do
+  case ${arg} in
+    s)
+      SKIP_DOWNLOAD='true'
+      echo "Skip download"
+      ;;
+    ?)
+      echo "Invalid option: -${OPTARG}."
+      ;;
+  esac
+done
+
+if [[ $SKIP_DOWNLOAD != 'true' ]]; then
+  checkout_at "examples/elm-spa-example" "rtfeldman/elm-spa-example" "c8c3201ec0488f17c1245e1fd2293ba5bc0748d5"
+  checkout_at "examples/elm-browser" "elm/browser" "1d28cd625b3ce07be6dfad51660bea6de2c905f2"
+  checkout_at "examples/elm-bytes" "elm/bytes" "2bce2aeda4ef18c3dcccd84084647d22a7af36a6"
+  checkout_at "examples/elm-core" "elm/core" "22eefd207e7a63daab215ae497f683ff2319c2ca"
+  checkout_at "examples/elm-file" "elm/file" "e4ca3864c93a5e766e24ed6916174753567b2f59"
+  checkout_at "examples/elm-html" "elm/html" "94c079007f8a7ed282d5b53f4a49101dd0b6cf99"
+  checkout_at "examples/elm-http" "elm/http" "81b6fdc67d8e5fb25644fd79e6b0edbe2e14e474"
+  checkout_at "examples/elm-json" "elm/json" "af344039e8c014b06ed0f73ac3ffd22c60d30876"
+  checkout_at "examples/elm-parser" "elm/parser" "7506b07eaa93a93d13b508b948c016105b0953c8"
+  checkout_at "examples/elm-project-metadata-utils" "elm/project-metadata-utils" "b075be15046e151e36a79950412c4cb703605aeb"
+  checkout_at "examples/elm-random" "elm/random" "ecf97bb43f0d5cd75243428f69f45323957bda25"
+  checkout_at "examples/elm-regex" "elm/regex" "8810c41fb17ddf89165665489be213f44070bc4a"
+  checkout_at "examples/elm-svg" "elm/svg" "08bd432990862bab5b840654dd437fbb2e6176e7"
+  checkout_at "examples/elm-time" "elm/time" "dc3b75b7366e59b99962706f7bf064d3634a4bba"
+  checkout_at "examples/elm-url" "elm/url" "8602f4b48653ca93b6991fe2c6a764523cd1d462"
+  checkout_at "examples/elm-virtual-dom" "elm/virtual-dom" "5a5bcf48720bc7d53461b3cd42a9f19f119c5503"
+  checkout_at "examples/elm-ui" "mdgriffith/elm-ui" "7aa0e015499d4975a4dda02c2629ee21ec18eac3"
+  checkout_at "examples/elm-markup" "mdgriffith/elm-markup" "e4b88aa04c7e9bafd02b5436c311cf07e255dab1"
+  checkout_at "examples/elm-visualization" "gampleman/elm-visualization" "68631a3dbf840d30bc642e605a49ccd8040504f6"
+fi
 
 skipped_files=()
 examples_to_parse=()

script/parse-examples-full

@@ -13,24 +13,41 @@ function checkout() {
   popd
 }
 
-echo "Getting libs"
+# Define list of arguments expected in the input
+optstring="s"
+
+while getopts ${optstring} arg; do
+  case ${arg} in
+    s)
+      SKIP_DOWNLOAD='true'
+      echo "Skip download"
+      ;;
+    ?)
+      echo "Invalid option: -${OPTARG}."
+      ;;
+  esac
+done
+
+echo "Finding libs"
 libs_to_parse=$(grep -Po '"name":.*?[^\\]",' ./script/search.json | perl -pe 's/"name": "//; s/^"//; s/",$//')
 libs_not_to_parse=$(grep -Po '".+"' ./script/error-packages.json | perl -pe 's/^"//; s/"$//')
 
 for lib in $libs_to_parse; do
-    if [[ ! $libs_not_to_parse == *$lib* ]]; then
+    if [[ $libs_not_to_parse != *$lib* && $SKIP_DOWNLOAD != 'true' ]]; then
       echo $lib
       checkout "examples-full/$lib" "$lib"
     fi
 done
 
-echo "Getting applications"
+echo "Finding applications"
 applications_to_parse=$(grep -Po '".+"' ./script/applications.json | perl -pe 's/^"//; s/"$//')
 
-for project in $applications_to_parse; do
-      echo $project
-      checkout "examples-full/$project" "$project"
-done
+if [[ $SKIP_DOWNLOAD != 'true' ]]; then
+  for project in $applications_to_parse; do
+        echo $project
+        checkout "examples-full/$project" "$project"
+  done
+fi
 
 
 start=`date +%s.%N`

src/scanner.cc

@@ -252,10 +252,9 @@ namespace
             }
 
             // Open section if the grammar lets us but only push to indent stack if we go further down in the stack
-            if (valid_symbols[VIRTUAL_OPEN_SECTION])
+            if (valid_symbols[VIRTUAL_OPEN_SECTION] && !lexer->eof(lexer))
             {
-                indent_length_stack.push_back(lexer->get_column(lexer));
-
+                indent_length_stack.push_back(lexer->get_column(lexer)); // This needs a `!lexer->eof(lexer)` check or it will get stuck
                 lexer->result_symbol = VIRTUAL_OPEN_SECTION;
                 return true;
             }