Improved error messages (as pointed out in issue #181)

lark/exceptions.py

@@ -50,21 +50,22 @@ def match_examples(self, parse_fn, examples):
 
 class UnexpectedCharacters(LexError, UnexpectedInput):
     def __init__(self, seq, lex_pos, line, column, allowed=None, considered_tokens=None, state=None):
-        context = seq[lex_pos:lex_pos+10]
-        message = "No token defined for '%s' in %r at line %d col %d" % (seq[lex_pos], context, line, column)
-        if allowed:
-            message += '\n\nExpecting: %s\n' % allowed
-
-        super(UnexpectedCharacters, self).__init__(message)
+        message = "No terminal defined for '%s' at line %d col %d" % (seq[lex_pos], line, column)
 
         self.line = line
         self.column = column
-        self.context = context
         self.allowed = allowed
         self.considered_tokens = considered_tokens
         self.pos_in_stream = lex_pos
         self.state = state
 
+        message += '\n\n' + self.get_context(seq)
+        if allowed:
+            message += '\nExpecting: %s\n' % allowed
+
+        super(UnexpectedCharacters, self).__init__(message)
+
+
 
 class UnexpectedToken(ParseError, UnexpectedInput):
     def __init__(self, token, expected, considered_rules=None, state=None):

lark/load_grammar.py

@@ -311,7 +311,7 @@ def pattern(self, p):
             self.token_reverse[p] = tokendef
             self.tokens.append(tokendef)
 
-        return Terminal(Token('TERMINAL', token_name, -1), filter_out=isinstance(p, PatternStr))
+        return Terminal(token_name, filter_out=isinstance(p, PatternStr))
 
 
 def _rfind(s, choices):

lark/parsers/xearley.py

@@ -20,7 +20,7 @@
 
 from collections import defaultdict
 
-from ..exceptions import ParseError, UnexpectedInput
+from ..exceptions import ParseError, UnexpectedCharacters
 from ..lexer import Token
 from ..tree import Tree
 from .grammar_analysis import GrammarAnalyzer
@@ -116,7 +116,7 @@ def scan(i, column):
             del delayed_matches[i+1]    # No longer needed, so unburden memory
 
             if not next_set and not delayed_matches:
-                raise UnexpectedInput(stream, i, text_line, text_column, {item.expect for item in to_scan}, set(to_scan))
+                raise UnexpectedCharacters(stream, i, text_line, text_column, {item.expect for item in to_scan}, set(to_scan))
 
             return next_set