feat(library/init/lean/parser/command): improve how optional affects firstTokens field

library/init/lean/parser/command.lean

@@ -45,17 +45,7 @@ def declVal          := declValSimple <|> declValEqns
 def «def»            := parser! "def " >> declId >> optDeclSig >> declVal
 def «theorem»        := parser! "theorem " >> declId >> declSig >> declVal
 
-def declaration      := declModifiers >> («def» <|> «theorem»)
-
-/- TODO: remove this hack by improving how we compute `Parser.info.firstTokens` -/
-def declTokens := ["/--", "@[", "private", "protected", "noncomputable", "unsafe", "def", "theorem"]
-
-@[builtinCommandParser] def declEntry : Parser :=
-{ info := {
-    firstTokens := FirstTokens.tokens $ declTokens.map $ fun tk => { val := tk },
-    .. declaration.info
-  },
-  fn   := declaration.fn }
+@[builtinCommandParser] def declaration := declModifiers >> («def» <|> «theorem»)
 
 end Command
 

library/init/lean/parser/parser.lean

@@ -149,26 +149,37 @@ def ParserFn (k : ParserKind) := ParserArg k → BasicParserFn
 instance ParserFn.inhabited (k : ParserKind) : Inhabited (ParserFn k) := ⟨fun _ _ => id⟩
 
 inductive FirstTokens
-| epsilon : FirstTokens
-| unknown : FirstTokens
-| tokens  : List TokenConfig → FirstTokens
+| epsilon   : FirstTokens
+| unknown   : FirstTokens
+| tokens    : List TokenConfig → FirstTokens
+| optTokens : List TokenConfig → FirstTokens
 
 namespace FirstTokens
 
 def merge : FirstTokens → FirstTokens → FirstTokens
-| epsilon     tks         := tks
-| tks         epsilon     := tks
-| (tokens s₁) (tokens s₂) := tokens (s₁ ++ s₂)
-| _           _           := unknown
+| epsilon        tks            := tks
+| tks            epsilon        := tks
+| (tokens s₁)    (tokens s₂)    := tokens (s₁ ++ s₂)
+| (optTokens s₁) (optTokens s₂) := optTokens (s₁ ++ s₂)
+| (tokens s₁)    (optTokens s₂) := tokens (s₁ ++ s₂)
+| (optTokens s₁) (tokens s₂)    := tokens (s₁ ++ s₂)
+| _              _              := unknown
 
 def seq : FirstTokens → FirstTokens → FirstTokens
-| epsilon tks := tks
-| tks     _   := tks
+| epsilon        tks            := tks
+| (optTokens s₁) (optTokens s₂) := optTokens (s₁ ++ s₂)
+| (optTokens s₁) (tokens s₂)    := tokens (s₁ ++ s₂)
+| tks            _              := tks
+
+def toOptional : FirstTokens → FirstTokens
+| (tokens tks) := optTokens tks
+| tks          := tks
 
 def toStr : FirstTokens → String
-| epsilon      := "epsilon"
-| unknown      := "unknown"
-| (tokens tks) := toString tks
+| epsilon         := "epsilon"
+| unknown         := "unknown"
+| (tokens tks)    := toString tks
+| (optTokens tks) := "?" ++ toString tks
 
 instance : HasToString FirstTokens := ⟨toStr⟩
 
@@ -288,8 +299,12 @@ fun a c s =>
   let s      := if s.hasError && s.pos == iniPos then s.restore iniSz iniPos else s;
   s.mkNode nullKind iniSz
 
+@[noinline] def optionaInfo (p : ParserInfo) : ParserInfo :=
+{ updateTokens := p.updateTokens,
+  firstTokens  := p.firstTokens.toOptional }
+
 @[inline] def optional {k : ParserKind} (p : Parser k) : Parser k :=
-{ info := noFirstTokenInfo p.info,
+{ info := optionaInfo p.info,
   fn   := optionalFn p.fn }
 
 @[inline] def lookaheadFn {k : ParserKind} (p : ParserFn k) : ParserFn k :=

tests/playground/cmdparsertest1.lean

@@ -16,6 +16,7 @@ is.mfor $ fun input => do
 
 def main (xs : List String) : IO Unit :=
 do
+IO.println Command.declaration.info.firstTokens;
 test [
 "@[inline] def x := 2",
 "protected def length.{u} {α : Type u} : List α → Nat