Add escaped characters to the AST (#36)

To represent characters escaped by a backslash, a new AST entry is
added. This improves their handling considerably compared to the
previous state, where the escaping was in some cases ignored and in
other cases turned into a literal backslash.

The change has not yet been pushed "all the way through": `expanded_words` and `intermediate_fields` still use backslashes for escaping. That fix is a longer-term issue (#37).

src/semantics.lem

@@ -463,6 +463,7 @@ let rec expand_control s0 split q k =
           Right (step, s1, [], [K (Quote (f ++ f1) w1)])
        | Left err -> Left err
        end
+   | Escape c -> Right (ESEscape "", s0, [DQuo [C c]], [])
   end
 
 and expand_words s0 split q sm (f,w) =

src/shim.ml

@@ -183,9 +183,8 @@ and parse_arg ?tilde_ok:(tilde_ok=false) ~assign:(assign:bool)
   | [],`CTLAri::_ -> failwith "End of string before CTLENDARI"
   | [],`CTLQuo::_ -> failwith "End of string before CTLQUOTEMARK"
   (* CTLESC *)
-  | '\129'::_ as s,_ -> 
-     let (str,s') = parse_string [] (in_quote stack) s in
-     arg_char assign (S (implode str)) s' bqlist stack
+  | '\129'::c::s',_ ->
+     arg_char assign (K (Escape c)) s' bqlist stack
   (* CTLVAR *)
   | '\130'::t::s,_ ->
      let var_name,s = Dash.split_at (fun c -> c = '=') s in
@@ -253,7 +252,7 @@ and parse_arg ?tilde_ok:(tilde_ok=false) ~assign:(assign:bool)
        end
   (* ordinary character *)
   | _::_,_ -> 
-     let (str,s') = parse_string [] (in_quote stack) s in
+     let (str,s') = parse_string [] s in
      arg_char assign (S (implode str)) s' bqlist stack
 
 and parse_tilde assign acc = 
@@ -275,29 +274,18 @@ and parse_tilde assign acc =
   (* ordinary char *)
   | c::s' -> parse_tilde assign (c::acc) s'  
 
-and parse_string acc quoted = function
+and parse_string acc = function
   | [] -> List.rev acc, []
+  | '\129'::_ as s -> List.rev acc, s
   | '\130'::_ as s -> List.rev acc, s
   | '\131'::_ as s -> List.rev acc, s
   | '\132'::_ as s -> List.rev acc, s
   | '\134'::_ as s -> List.rev acc, s
   | '\135'::_ as s -> List.rev acc, s
   | '\136'::_ as s -> List.rev acc, s
   | '~'   ::_ as s -> List.rev acc, s
-  | '\129'::'\\'::'\129'::c::s when quoted ->
-     let c' = ['\\'; c] in
-     parse_string (List.rev c' @ acc) quoted s
-  | '\129'::c::s  -> 
-     let c' =
-       match c with
-       | '*' when not quoted -> ['\\'; c]
-       | '?' when not quoted -> ['\\'; c]
-       | '[' when not quoted -> ['\\'; c]
-       | _   -> [c]
-     in
-     parse_string (List.rev c' @ acc) quoted s
-  | c::s -> parse_string (c::acc) quoted s
-
+  | c::s -> parse_string (c::acc) s
+
 and arg_char assign c s bqlist stack =
   let tilde_ok = 
     match c with
@@ -686,6 +674,7 @@ and json_of_control = function
                                             ("s", String s)]
   | Arith (f,w) ->  obj_fw "Arith" f w
   | Quote (f,w) -> obj_fw "Quote" f w
+  | Escape c -> Assoc [tag "Escape"; ("character", String (String.make 1 c))]
 and json_of_format = function
   | Normal -> obj "Normal"
   | Length -> obj "Length"
@@ -758,6 +747,7 @@ and json_of_expansion_step = function
   | ESSplit s -> Assoc [tag "ESSplit"; ("msg", String s)]
   | ESPath s -> Assoc [tag "ESPath"; ("msg", String s)]
   | ESQuote s -> Assoc [tag "ESQuote"; ("msg", String s)]
+  | ESEscape s -> Assoc [tag "ESEscape"; ("msg", String s)]
   | ESStep s -> Assoc [tag "ESStep"; ("msg", String s)]
   | ESNested (outer, inner) -> Assoc [tag "ESNested"; ("inner", json_of_expansion_step inner); ("outer", json_of_expansion_step outer)]
   | ESEval (exp_step, eval_step) ->  Assoc [tag "ESEval"; ("inner", json_of_evaluation_step eval_step); ("outer", json_of_expansion_step exp_step)]

src/smoosh_prelude.lem

@@ -904,6 +904,7 @@ and control =
   | LBacktickWait of stmt (* original stmt *) * pid * string (* result of command *)
   | Arith of expanded_words (* runtime technicality *) * words
   | Quote of expanded_words (* runtime technicality *) * words
+  | Escape of char
 
 and entry =
     S of string
@@ -1037,6 +1038,7 @@ and expansion_step =
   | ESSplit of string
   | ESPath of string
   | ESQuote of string
+  | ESEscape of string
   | ESStep of string
   | ESNested of expansion_step * expansion_step
   | ESEval of expansion_step * evaluation_step
@@ -1703,6 +1705,7 @@ let rec ran_command_substitution estep =
   | ESSplit _s -> false
   | ESPath _s -> false
   | ESQuote _s -> false
+  | ESEscape _s -> false
   | ESStep _s -> false
   | ESNested estep_outer estep_inner ->
      ran_command_substitution estep_outer || 
@@ -2034,6 +2037,7 @@ and string_of_control code =
   | LBacktickWait c _pid _s -> "$( " ^ string_of_stmt c ^ " )"
   | Arith ew w -> "$(( " ^ string_of_expanded_words ew ^ string_of_words w ^ " ))"
   | Quote ew w -> "\"" ^ string_of_expanded_words ew ^ string_of_words w ^ "\""
+  | Escape c -> (toString [#'\\'; c])
   end
 
 and string_of_format fmt = 
@@ -2083,6 +2087,7 @@ and string_of_expansion_step estep =
   | ESSplit s -> spaced "Split" s
   | ESPath s -> spaced "Path" s
   | ESQuote s -> spaced "Quote" s
+  | ESEscape s -> spaced "Escape" s
   | ESStep s -> spaced "Step" s
   | ESNested estep_outer estep_inner ->
      parens (string_of_expansion_step estep_outer) ^ " " ^

tests/shell/semantics.expansion.substring.out

@@ -0,0 +1 @@
+a

tests/shell/semantics.expansion.substring.test

@@ -0,0 +1,2 @@
+FOO="\\a"
+echo ${FOO#*\\}