Reason V4 [Stack 3/n] [Parse Hashtags for polymorphic variants]

docs/RELEASING.md

@@ -16,14 +16,32 @@ and `rtop.json` respectively in the repo root, you would run that script after
 committing/bumping some versions:
 
 
+**IMPORTANT: Update The Version Numbers In Packages:**
+1. Make sure the version number in `esy.json` and `reason.json` is the new
+   version number for the release.
+2. Make sure the file
+   [../../src/reason-version/reason_version.ml](../../src/reason-version/reason_version.ml)
+   also has that same version number that `refmt` has:
+
 ```sh
 git checkout -b MYRELEASE origin/master
 git rebase origin/master
-vim -O esy.json reason.json
-# Then edit the version number accordingly on BOTH files. With that same VERSION do:
-version=3.5.0 make pre_release
+vim -O esy.json reason.json src/reason-version/reason_version.ml
+
+# Edit version field in jsons, and make sure reason_version has the new version
+# let package_version = {
+#   major = 3;
+#   minor = 7;
+#   patch = 0;
+# }
+
 git commit -m "Bump version"
 git push origin HEAD:PullRequestForVersion # Commit these version bumps
+
+```
+
+**Perform The Release:**
+```sh
 node ./scripts/esy-prepublish.js ./reason.json ./rtop.json
 
 # Then publish. For example:

docs/TEMPLATE_LITERALS.md

@@ -0,0 +1,146 @@
+
+Contributors: Lexing and Parsing String Templates:
+===================================================
+Supporting string templates requires coordination between the lexer, parser and
+printer. The lexer (as always) creates a token stream, but when it encounters a
+backtick, it begins a special parsing mode that collects the (mostly) raw text,
+until either hitting a closing backtick, or a `${`. If it encounters the `${`
+(called an "interpolation region"), it will temporarily resume the "regular"
+lexing approach, instead of collecting the raw text - until it hits a balanced
+`}`, upon which it will enter the "raw text" mode again until it hits the
+closing backtick.
+
+- Parsing of raw text regions and regular tokenizing: Handled by
+  `reason_declarative_lexer.ml`.
+- Token balancing: Handled by `reason_lexer.ml`.
+
+The output of lexing becomes tokens streamed into the parser, and the parser
+`reason_parser.mly` turns those tokens into AST expressions.
+
+## Lexing:
+
+String templates are opened by:
+- A backtick.
+- Followed by any whitespace character (newline, or space/tab).
+
+- Any whitespace character (newline, or space/tab).
+- Followed by a backtick
+
+```reason
+let x = ` hi this is my string template `
+let x = `
+The newline counts as a whitespace character both for opening and closing.
+`
+
+```
+
+Within the string template literal, there may be regions of non-string
+"interpolation" where expressions are lexed/parsed.
+
+```reason
+let x = ` hi this is my ${expressionHere() ++ "!"} template `
+```
+
+Template strings are lexed into tokens, some of those tokens contain a string
+"payload" with portions of the string content.
+The opening backtick, closing backtick, and `${` characters do not become a
+token that is fed to the parser, and are not included in the text payload of
+any token. The Right Brace `}` closing an interpolation region `${` _does_
+become a token that is fed to the parser. There are three tokens that are
+produced when lexing string templates.
+
+- `STRING_TEMPLATE_TERMINATED(string)`: A string region that is terminated with
+  closing backtick. It may be the entire string template contents if there are
+  no interpolation regions `${}`, or it may be the final string segment after
+  an interpolation region `${}`, as long as it is the closing of the entire
+  template.
+- `STRING_TEMPLATE_SEGMENT_LBRACE(string)`: A string region occuring _before_
+  an interpolation region `${`. The `string` payload of this token is the
+  contents up until (but not including) the next `${`.
+- `RBRACE`: A `}` character that terminates an interpolation region that
+  started with `${`.
+
+Simple example:
+
+     STRING_TEMPLATE_TERMINATED
+     |                         |
+   ` lorem ipsum lorem ipsum bla `
+    ^                          ^
+    |                          |
+    |                The closing backtick also doesn't show up in the token
+    |                stream, but the last white space is part of the lexed
+    |                STRING_TEMPLATE_TERMINATED token
+    |                (it is used to compute indentation, but is stripped from
+    |                the string constant, or re-inserted in refmting if not present)
+    |
+    The backtick doesn't show up anywhere in the token stream.  The first
+    single white space after backtick is also not part of the lexed tokens.
+
+Multiline example:
+
+    All of this leading line whitespace remains parts of the tokens' payloads
+    but it is is normalized and stripped when the parser converts the tokens
+    into string expressions.
+    |
+    |   This newline not part of any token
+    |   |
+    |   v
+    |  `
+    +->   lorem ipsum lorem
+          ipsum bla
+       `
+      ^
+      |
+      All of this white space on final line is part of the token as well.
+
+
+For interpolation, the token `STRING_TEMPLATE_SEGMENT_LBRACE` represents the
+string contents (minus any single/first white space after backtick), up to the
+`${`. As with non-interpolated string templates, the opening and closing
+backtick does not show up in the token stream, the first white space character
+after opening backtick is not included in the lexed string contents, the final
+white space character before closing backtick *is* part of the lexed string
+token (to compute indentation), but that final white space character, along
+with leading line whitespace is stripped from the string expression when the
+parsing stage converts from lexed tokens to AST string expressions.
+
+   ` lorem ipsum lorem ipsum bla${expression}lorem ipsum lorem ip lorem`
+     |                         |            ||                        |
+     STRING_TEMPLATE_TERMINATED             |STRING_TEMPLATE_TERMINATED
+                                      RBRACE
+## Parsing:
+
+The string template tokens are turned into normal AST expressions.
+`STRING_TEMPLATE_SEGMENT_LBRACE` and `STRING_TEMPLATE_TERMINATED` lexed tokens
+contains all of the string contents, plus leading line whitespace for each
+line, including the final whitespace before the closing backtick. These are
+normalized in the parser by stripping that leading whitespace including two
+additional spaces for nice indentation, before turning them into some
+combination of string contants with a special attribute on the AST, or string
+concats with a special attribute on the concat AST node.
+
+```reason
+
+// This:
+let x = `
+  Hello there
+`;
+// Becomes:
+let x = [@reason.template] "Hello there";
+
+// This:
+let x = `
+  ${expr} Hello there
+`;
+// Becomes:
+let x = [@reason.template] (expr ++ [@reason.template] "Hello there");
+
+```
+
+User Documentation:
+===================
+> This section is the user documentation for string template literals, which
+> will be published to the [official Reason Syntax
+> documentation](https://reasonml.github.io/) when 
+
+TODO

esy.json

@@ -84,6 +84,6 @@
   },
   "scripts": {
     "test": "esy x make test-once-installed",
-    "doc": "esy dune build @doc"
+    "doc": "esy build dune build @doc"
   }
 }

formatTest/typeCheckedTests/expected_output/arityConversion.re

@@ -1,3 +1,5 @@
+[@reason.version 3.7];
+
 Some((1, 2, 3));
 
 type bcd =

formatTest/typeCheckedTests/expected_output/attributes.4.04.0.re

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 /* Pexp_letexception with attributes */
 let () = {
   [@attribute]

formatTest/typeCheckedTests/expected_output/attributes.re

@@ -9,6 +9,7 @@
  */;
 
 /**Floating comment text should be removed*/;
+[@reason.version 3.7];
 
 /**
  * Core language features:

formatTest/typeCheckedTests/expected_output/attributes.rei

@@ -1,6 +1,8 @@
 /* Copyright (c) 2015-present, Facebook, Inc. All rights reserved. */
 
 /**Floating comment text should be removed*/;
+[@reason.version 3.7];
+
 let test: int;
 
 /**

formatTest/typeCheckedTests/expected_output/autoUpgradeAngleBrackets.re

@@ -0,0 +1,14 @@
+/**
+ * Even if you have an explicit v3.6 marker.
+ * This whole file wil be auto-upaded to 3.8 becase something uses
+ * angle brackets.
+ */;
+[@reason.version 3.8];
+let watchThisIsOldStyle: list<int> = [1, 2];
+
+let watchThisIsOldStylePoly = #hello;
+
+/**
+ * This will cause the whole file to be promoted.
+ */
+let x: list<int> = [1, 3];

formatTest/typeCheckedTests/expected_output/autoUpgradeAngleBracketsNoVersionAttr.re

@@ -0,0 +1,13 @@
+[@reason.version 3.8];
+/**
+ * Test auto-promotion based on feature inference even if no version
+ * tag. By default you're using the old 3.7.
+ */
+let watchThisIsOldStyle: list<int> = [1, 2];
+
+let watchThisIsOldStylePoly = #hello;
+
+/**
+ * This will cause the whole file to be promoted.
+ */
+let x: list<int> = [1, 3];

formatTest/typeCheckedTests/expected_output/autoUpgradeDoNotAutoUpgrade.re

@@ -0,0 +1,5 @@
+[@reason.version 3.7];
+/**
+ * This should just print a 3.7 version attr at the top.
+ */
+let watchThisIsOldStyle: list(int) = [1, 2];

formatTest/typeCheckedTests/expected_output/basics.re

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 /* Copyright (c) 2015-present, Facebook, Inc. All rights reserved. */
 
 let l =

formatTest/typeCheckedTests/expected_output/basics_no_semi.re

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 /* Copyright (c) 2015-present, Facebook, Inc. All rights reserved. */
 
 let l =

formatTest/typeCheckedTests/expected_output/bigarraySyntax.re

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 /* https://github.com/facebook/reason/issues/2038 */
 let my_big_array1 =
   Bigarray.Array1.create(

formatTest/typeCheckedTests/expected_output/comments.re

@@ -1,6 +1,9 @@
 /* **** comment */
 /*** comment */
 /** docstring */;
+
+[@reason.version 3.7];
+
 /* comment */
 /** docstring */;
 /*** comment */

formatTest/typeCheckedTests/expected_output/comments.rei

@@ -1,8 +1,11 @@
 /* **** comment */
 /*** comment */
-/*** docstring */
+/** docstring */;
+
+[@reason.version 3.7];
+
 /* comment */
-/*** docstring */
+/** docstring */;
 /*** comment */
 /**** comment */
 /***** comment */

formatTest/typeCheckedTests/expected_output/comments.rei.4.07.0

@@ -1,6 +1,9 @@
 /* **** comment */
 /*** comment */
 /** docstring */;
+
+[@reason.version 3.7];
+
 /* comment */
 /** docstring */;
 /*** comment */

formatTest/typeCheckedTests/expected_output/comments.rei.4.07.1

@@ -1,6 +1,9 @@
 /* **** comment */
 /*** comment */
 /** docstring */;
+
+[@reason.version 3.7];
+
 /* comment */
 /** docstring */;
 /*** comment */

formatTest/typeCheckedTests/expected_output/comments.rei.4.08.0

@@ -1,6 +1,9 @@
 /* **** comment */
 /*** comment */
 /** docstring */;
+
+[@reason.version 3.7];
+
 /* comment */
 /** docstring */;
 /*** comment */

formatTest/typeCheckedTests/expected_output/comments.rei.4.09.0

@@ -1,6 +1,9 @@
 /* **** comment */
 /*** comment */
 /** docstring */;
+
+[@reason.version 3.7];
+
 /* comment */
 /** docstring */;
 /*** comment */

formatTest/typeCheckedTests/expected_output/features406.4.06.0.re

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 module EM = {
   /** Exception */
 

formatTest/typeCheckedTests/expected_output/features408.4.08.0.re

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 open {
        type t = string;
      };

formatTest/typeCheckedTests/expected_output/features408.4.08.0.rei

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 module X: {type t;};
 
 module M := X;

formatTest/typeCheckedTests/expected_output/imperative.re

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 /* Copyright (c) 2015-present, Facebook, Inc. All rights reserved. */
 
 /*

formatTest/typeCheckedTests/expected_output/jsx.re

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 type component = {displayName: string};
 
 module Bar = {

formatTest/typeCheckedTests/expected_output/knownMlIssues.re

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 /* [x] fixed */
 type t2 = (int, int); /* attributed to entire type not binding */
 

formatTest/typeCheckedTests/expected_output/knownReIssues.re

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 /**
    Issue 940: https://github.com/facebook/reason/issues/940
    The parens in the exception match case with an alias,

formatTest/typeCheckedTests/expected_output/lazy.re

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 let myComputation =
   lazy({
     let tmp = 10;

formatTest/typeCheckedTests/expected_output/letop.re

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 let (let.opt) = (x, f) =>
   switch (x) {
   | None => None

formatTest/typeCheckedTests/expected_output/mlSyntax.re

@@ -1,8 +1,9 @@
 /* Copyright (c) 2015-present, Facebook, Inc. All rights reserved. */
 
-/**
+/***
  * Testing pattern matching using ml syntax to exercise nesting of cases.
- */;
+ */
+[@reason.version 3.7];
 
 type xyz =
   | X

formatTest/typeCheckedTests/expected_output/mlVariants.re

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 /* Copyright (c) 2015-present, Facebook, Inc. All rights reserved. */
 
 type polyVariantsInMl = [

formatTest/typeCheckedTests/expected_output/mlVariants.re.4.06.1

@@ -1,3 +1,4 @@
+[@reason.version 3.7];
 /* Copyright (c) 2015-present, Facebook, Inc. All rights reserved. */
 
 type polyVariantsInMl = [