Expand ASI using the Parse Table.

crates/ast/ast.json

@@ -2,6 +2,9 @@
   "Void": {
     "_type": "enum"
   },
+  "Asi": {
+    "_type": "struct"
+  },
   "Argument": {
     "_type": "enum",
     "SpreadElement": "Box<Expression>",

crates/ast/src/source_location.rs

@@ -25,8 +25,10 @@ impl SourceLocation {
         self.start = start.start;
         self.end = end.end;
     }
+}
 
-    pub fn default() -> Self {
+impl Default for SourceLocation {
+    fn default() -> Self {
         Self { start: 0, end: 0 }
     }
 }

crates/generated_parser/src/ast_builder.rs

@@ -84,6 +84,11 @@ impl<'alloc> AstBuilder<'alloc> {
         list.append(elements);
     }
 
+    // Automatic Semicolon Insertion
+    pub fn asi(&self) -> arena::Box<'alloc, Asi> {
+        self.alloc_with(|| Asi::default())
+    }
+
     // IdentifierReference : Identifier
     pub fn identifier_reference(
         &self,

crates/generated_parser/src/traits/mod.rs

@@ -32,5 +32,5 @@ pub trait ParserTrait<'alloc, Value> {
     fn replay(&mut self, tv: TermValue<Value>);
     fn epsilon(&mut self, state: usize);
     fn top_state(&self) -> usize;
-    fn check_not_on_new_line(&mut self, peek: usize) -> Result<'alloc, bool>;
+    fn is_on_new_line(&self) -> Result<'alloc, bool>;
 }

crates/parser/src/parser.rs

@@ -96,19 +96,13 @@ impl<'alloc> ParserTrait<'alloc, StackValue<'alloc>> for Parser<'alloc> {
     fn top_state(&self) -> usize {
         self.state()
     }
-    fn check_not_on_new_line(&mut self, peek: usize) -> Result<'alloc, bool> {
+    fn is_on_new_line(&self) -> Result<'alloc, bool> {
         let sv = {
             let stack = self.node_stack.stack_slice();
-            &stack[stack.len() - peek].value
+            &stack[stack.len() - 1].value
         };
         if let StackValue::Token(ref token) = sv {
-            if !token.is_on_new_line {
-                return Ok(true);
-            }
-            self.rewind(peek - 1);
-            let tv = self.pop();
-            self.try_error_handling(tv)?;
-            return Ok(false);
+            return Ok(token.is_on_new_line);
         }
         Err(ParseError::NoLineTerminatorHereExpectedToken.into())
     }
@@ -194,51 +188,14 @@ impl<'alloc> Parser<'alloc> {
             }),
         });
         if let StackValue::Token(ref token) = t.value {
-            // Error tokens might them-self cause more errors to be reported.
-            // This happens due to the fact that the ErrorToken can be replayed,
-            // and while the ErrorToken might be in the lookahead rules, it
-            // might not be in the shifted terms coming after the reduced
-            // nonterminal.
-            if t.term == TerminalId::ErrorToken.into() {
-                return Err(Self::parse_error(token).into());
-            }
-
-            // Otherwise, check if the current rule accept an Automatic
-            // Semi-Colon insertion (ASI).
-            let state = self.state();
-            assert!(state < TABLES.shift_count);
-            let error_code = TABLES.error_codes[state];
-            if let Some(error_code) = error_code {
-                let err_token = (*token).clone();
-                Self::recover(token, error_code)?;
-                self.replay(t);
-                let err_token = self.handler.alloc(err_token);
-                self.replay(TermValue {
-                    term: TerminalId::ErrorToken.into(),
-                    value: StackValue::Token(err_token),
-                });
-                return Ok(false);
-            }
             // On error, don't attempt error handling again.
             return Err(Self::parse_error(token).into());
         }
         Err(ParseError::ParserCannotUnpackToken.into())
     }
 
-    pub(crate) fn recover(t: &Token, error_code: ErrorCode) -> Result<'alloc, ()> {
-        match error_code {
-            ErrorCode::Asi => {
-                if t.is_on_new_line
-                    || t.terminal_id == TerminalId::End
-                    || t.terminal_id == TerminalId::CloseBrace
-                {
-                    Ok(())
-                } else {
-                    Err(Self::parse_error(t).into())
-                }
-            }
-            ErrorCode::DoWhileAsi => Ok(()),
-        }
+    pub(crate) fn recover(t: &Token, _error_code: ErrorCode) -> Result<'alloc, ()> {
+        Err(Self::parse_error(t).into())
     }
 
     fn simulator<'a>(&'a self) -> Simulator<'alloc, 'a> {

crates/parser/src/simulator.rs

@@ -114,8 +114,8 @@ impl<'alloc, 'parser> ParserTrait<'alloc, ()> for Simulator<'alloc, 'parser> {
     fn top_state(&self) -> usize {
         self.state()
     }
-    fn check_not_on_new_line(&mut self, _peek: usize) -> Result<'alloc, bool> {
-        Ok(true)
+    fn is_on_new_line(&self) -> Result<'alloc, bool> {
+        Ok(false)
     }
 }
 

jsparagus/actions.py

@@ -96,7 +96,7 @@ def __init__(self) -> None:
     def is_inconsistent(self) -> bool:
         """Returns True if this action is inconsistent. An action can be
         inconsistent if the parameters it is given cannot be evaluated given
-        its current location in the parse table. Such as CheckNotOnNewLine.
+        its current location in the parse table. Such as CheckLineTerminator.
         """
         return False
 
@@ -108,6 +108,18 @@ def condition(self) -> Action:
         "Return the conditional action."
         raise TypeError("Action.condition not implemented")
 
+    def can_negate(self) -> bool:
+        "Whether the current condition (action) implemented the negate function."
+        assert self.is_condition()
+        return False
+
+    def negate(self, covered: typing.List[Action]) -> typing.List[Action]:
+        """Given a list of conditions, returns the condition which check the same
+        variable but all the values which are not covered by the rest of the
+        conditions, by adding it to the list which is returned."""
+        assert self.can_negate()
+        raise TypeError("Action.negate not implemented")
+
     def check_same_variable(self, other: Action) -> bool:
         "Return whether both conditionals are checking the same variable."
         assert self.is_condition()
@@ -214,6 +226,37 @@ def stable_str(self, states: typing.Any) -> str:
 ShiftedAction = typing.Union[Action, bool]
 
 
+class Shift(Action):
+    """Shift action is the implicit action performed when a terminal or nonterminal
+    is used on an edge. However this state is not supported as an epsilon edge,
+    but only serves to annotate delayed actions of states. """
+    __slots__ = ['term']
+
+    term: ShiftedTerm
+
+    def __init__(self, term: ShiftedTerm):
+        super().__init__()
+        self.term = term
+
+    def is_inconsistent(self) -> bool:
+        return True
+
+    def is_condition(self) -> bool:
+        return True
+
+    def condition(self) -> Shift:
+        return self
+
+    def update_stack(self) -> bool:
+        return True
+
+    def update_stack_with(self) -> StackDiff:
+        return StackDiff(0, None, -1)
+
+    def __str__(self) -> str:
+        return "Shift({})".format(str(self.term))
+
+
 class Replay(Action):
     """Replay a term which was previously saved by the Unwind function. Note that
     this does not Shift a term given as argument as the replay action should
@@ -279,21 +322,25 @@ def shifted_action(self, shifted_term: Element) -> Unwind:
 class Reduce(Action):
     """Prevent the fall-through to the epsilon transition and returns to the shift
     table execution to resume shifting or replaying terms."""
-    __slots__ = ['unwind']
+    __slots__ = ['unwind', 'lookahead']
 
     unwind: Unwind
 
-    def __init__(self, unwind: Unwind) -> None:
+    # List of lookahead tokens used to prevent aliasing of reduce states.
+    lookahead: typing.Tuple[Element, ...]
+
+    def __init__(self, unwind: Unwind, lookahead: typing.Tuple[Element, ...] = ()) -> None:
         nt_name = unwind.nt.name
         if isinstance(nt_name, InitNt):
             name = "Start_" + str(nt_name.goal.name)
         else:
             name = nt_name
         super().__init__()
         self.unwind = unwind
+        self.lookahead = lookahead
 
     def __str__(self) -> str:
-        return "Reduce({})".format(str(self.unwind))
+        return "Reduce({}, {})".format(str(self.unwind), str(self.lookahead))
 
     def follow_edge(self) -> bool:
         return False
@@ -306,11 +353,11 @@ def update_stack_with(self) -> StackDiff:
 
     def unshift_action(self, num: int) -> Reduce:
         unwind = self.unwind.unshift_action(num)
-        return Reduce(unwind)
+        return Reduce(unwind, lookahead=self.lookahead[:-num])
 
     def shifted_action(self, shifted_term: Element) -> Reduce:
         unwind = self.unwind.shifted_action(shifted_term)
-        return Reduce(unwind)
+        return Reduce(unwind, lookahead=(*self.lookahead, shifted_term))
 
 
 class Accept(Action):
@@ -374,18 +421,27 @@ def shifted_action(self, shifted_term: Element) -> ShiftedAction:
         return not self.accept
 
 
-class CheckNotOnNewLine(Action):
+class CheckLineTerminator(Action):
     """Check whether the terminal at the given stack offset is on a new line or
-    not. If not this would produce an Error, otherwise this rule would be
-    shifted."""
-    __slots__ = ['offset']
+    not. If the condition is true, then the edge is followed. """
+    __slots__ = ['offset', 'is_on_new_line']
 
+    # Offset of the token which is being checked.
+    #   - If this number is zero, then # this represent the next token.
+    #   - If this number is -1, this represents the last shifted token.
+    #   - If this number is -2, this represents the second to last shifted token.
     offset: int
 
-    def __init__(self, offset: int = 0) -> None:
+    # Check whether the token at the offset is (= True), or is not (= False) on
+    # a new line compared to the previous token.
+    is_on_new_line: bool
+
+    def __init__(self, offset: int = 0, is_on_new_line: bool = False) -> None:
         # assert offset >= -1 and "Smaller offsets are not supported on all backends."
         super().__init__()
+        assert offset >= -1
         self.offset = offset
+        self.is_on_new_line = is_on_new_line
 
     def is_inconsistent(self) -> bool:
         # We can only look at stacked terminals. Having an offset of 0 implies
@@ -397,22 +453,35 @@ def is_inconsistent(self) -> bool:
     def is_condition(self) -> bool:
         return True
 
-    def condition(self) -> CheckNotOnNewLine:
+    def condition(self) -> CheckLineTerminator:
         return self
 
+    def can_negate(self) -> bool:
+        "Unordered condition, which accept or not to reach the next state."
+        return True
+
+    def negate(self, covered: typing.List[Action]) -> typing.List[Action]:
+        assert len(covered) >= 1 and len(covered) <= 2
+        if len(covered) == 2:
+            return covered
+        assert covered[0] == self
+        negated = CheckLineTerminator(self.offset, not self.is_on_new_line)
+        return [self, negated]
+
     def check_same_variable(self, other: Action) -> bool:
-        return isinstance(other, CheckNotOnNewLine) and self.offset == other.offset
+        return isinstance(other, CheckLineTerminator) and self.offset == other.offset
 
     def check_different_values(self, other: Action) -> bool:
-        return False
+        assert isinstance(other, CheckLineTerminator)
+        return self.is_on_new_line != other.is_on_new_line
 
     def shifted_action(self, shifted_term: Element) -> ShiftedAction:
         if isinstance(shifted_term, Nt):
             return True
-        return CheckNotOnNewLine(self.offset - 1)
+        return CheckLineTerminator(self.offset - 1, self.is_on_new_line)
 
     def __str__(self) -> str:
-        return "CheckNotOnNewLine({})".format(self.offset)
+        return "CheckLineTerminator({}, {})".format(self.offset, self.is_on_new_line)
 
 
 class FilterStates(Action):
@@ -554,7 +623,7 @@ def __init__(
             args: typing.Tuple[OutputExpr, ...],
             trait: types.Type = types.Type("AstBuilder"),
             fallible: bool = False,
-            set_to: str = "val",
+            set_to: str = "value",
             offset: int = 0,
     ) -> None:
         super().__init__()

jsparagus/aps.py

@@ -217,6 +217,7 @@ def shift_next(self, pt: ParseTable) -> typing.Iterator[APS]:
         last_edge = sh[-1]
         state = pt.states[last_edge.src]
         state_match_shift_end = self.state == self.shift[-1].src
+        term: Term
         if self.replay == []:
             assert state_match_shift_end
             for term, to in state.shifted_edges():

jsparagus/emit/python.py

@@ -6,7 +6,7 @@
 import typing
 
 from ..grammar import ErrorSymbol, Nt, Some
-from ..actions import (Accept, Action, CheckNotOnNewLine, FilterFlag, FilterStates, FunCall,
+from ..actions import (Accept, Action, CheckLineTerminator, FilterFlag, FilterStates, FunCall,
                        Lookahead, OutputExpr, PopFlag, PushFlag, Reduce, Replay, Seq, Unwind)
 from ..runtime import ErrorToken, ErrorTokenClass
 from ..ordered import OrderedSet
@@ -73,7 +73,7 @@ def write_action(act: Action, indent: str = "") -> typing.Tuple[str, bool]:
             return indent, False
         if isinstance(act, Lookahead):
             raise ValueError("Unexpected Lookahead action")
-        if isinstance(act, CheckNotOnNewLine):
+        if isinstance(act, CheckLineTerminator):
             out.write("{}if not parser.check_not_on_new_line(lexer, {}):\n".format(indent, -act.offset))
             out.write("{}    return\n".format(indent))
             return indent, True