feat: copy & store whole ref range in SourceInfo

src/Init/Data/String/Basic.lean

@@ -314,9 +314,6 @@ end String
 
 namespace Substring
 
-@[inline] def bsize : Substring → Nat
-  | ⟨_, b, e⟩ => e - b
-
 @[inline] def isEmpty (ss : Substring) : Bool :=
   ss.bsize == 0
 

src/Init/Meta.lean

@@ -148,12 +148,6 @@ partial def getTailInfo : Syntax → Option SourceInfo
   | node _ args   => args.findSomeRev? getTailInfo
   | _             => none
 
-partial def getTailPos : Syntax → Option String.Pos
-  | atom { pos := some pos, .. }  val    => some (pos + val.bsize)
-  | ident { pos := some pos, .. } val .. => some (pos + val.toString.bsize)
-  | node _ args                          => args.findSomeRev? getTailPos
-  | _                                    => none
-
 @[specialize] private partial def updateLast {α} [Inhabited α] (a : Array α) (f : α → Option α) (i : Nat) : Option (Array α) :=
   if i == 0 then
     none
@@ -180,8 +174,8 @@ def setTailInfo (stx : Syntax) (info : SourceInfo) : Syntax :=
 
 def unsetTrailing (stx : Syntax) : Syntax :=
   match stx.getTailInfo with
-  | none      => stx
-  | some info => stx.setTailInfo { info with trailing := none }
+  | SourceInfo.original lead pos trail => stx.setTailInfo (SourceInfo.original lead pos "".toSubstring)
+  | _                                  => stx
 
 @[specialize] private partial def updateFirst {α} [Inhabited α] (a : Array α) (f : α → Option α) (i : Nat) : Option (Array α) :=
   if h : i < a.size then
@@ -211,34 +205,10 @@ def setInfo (info : SourceInfo) : Syntax → Syntax
   | ident _ rawVal val pre => ident info rawVal val pre
   | stx                    => stx
 
-/--
-  Copy head and tail position information from `source` to `s`.
-  `leading` and `trailing` information is not preserved. -/
-def copyRangePos (s : Syntax) (source : Syntax) : Syntax :=
-  match source.getPos with
-  | none     => s
-  | some pos =>
-    let s := s.setHeadInfo { pos := pos }
-    match source.getTailInfo with
-    | some { pos := some pos, .. } =>
-      let s := s.setTailInfo { pos := pos }
-      /- The trailing token at `s` may be different from `source`.
-         So, we retrieve the tail positions and adjust `pos` to make sure the `s.getTailPos == source.getTailPos`. -/
-      match source.getTailPos, s.getTailPos with
-      | some pos₁, some pos₂ =>
-        if pos₁ < pos₂ then
-          s.setTailInfo { pos := some ((pos : Nat) - (pos₂ - pos₁) : Nat) }
-        else if pos₁ > pos₂ then
-          s.setTailInfo { pos := some ((pos : Nat) + (pos₁ - pos₂) : Nat) }
-        else
-          s
-      | _, _ => s
-    | _ => s
-
 /-- Return the first atom/identifier that has position information -/
 partial def getHead? : Syntax → Option Syntax
-  | stx@(atom { pos := some _, .. } ..)  => some stx
-  | stx@(ident { pos := some _, .. } ..) => some stx
+  | stx@(atom info ..)  => info.getPos.map fun _ => stx
+  | stx@(ident info ..) => info.getPos.map fun _ => stx
   | node _ args      => args.findSome? getHead?
   | _                => none
 
@@ -251,7 +221,7 @@ end Syntax
   | some ref => withRef ref x
 
 def mkAtom (val : String) : Syntax :=
-  Syntax.atom {} val
+  Syntax.atom SourceInfo.none val
 
 @[inline] def mkNode (k : SyntaxNodeKind) (args : Array Syntax) : Syntax :=
   Syntax.node k args
@@ -277,8 +247,7 @@ partial def expandMacros : Syntax → MacroM Syntax
   Create an identifier copying the position from `src`.
   To refer to a specific constant, use `mkCIdentFrom` instead. -/
 def mkIdentFrom (src : Syntax) (val : Name) : Syntax :=
-  let pos := src.getPos
-  Syntax.ident { pos := pos } (toString val).toSubstring val []
+  Syntax.ident (SourceInfo.fromRef src) (toString val).toSubstring val []
 
 def mkIdentFromRef [Monad m] [MonadRef m] (val : Name) : m Syntax := do
   return mkIdentFrom (← getRef) val
@@ -288,10 +257,9 @@ def mkIdentFromRef [Monad m] [MonadRef m] (val : Name) : m Syntax := do
   This variant of `mkIdentFrom` makes sure that the identifier cannot accidentally
   be captured. -/
 def mkCIdentFrom (src : Syntax) (c : Name) : Syntax :=
-  let pos := src.getPos
   -- Remark: We use the reserved macro scope to make sure there are no accidental collision with our frontend
   let id   := addMacroScope `_internal c reservedMacroScope
-  Syntax.ident { pos := pos } (toString id).toSubstring id [(c, [])]
+  Syntax.ident (SourceInfo.fromRef src) (toString id).toSubstring id [(c, [])]
 
 def mkCIdentFromRef [Monad m] [MonadRef m] (c : Name) : m Syntax := do
   return mkCIdentFrom (← getRef) c
@@ -301,7 +269,7 @@ def mkCIdent (c : Name) : Syntax :=
 
 @[export lean_mk_syntax_ident]
 def mkIdent (val : Name) : Syntax :=
-  Syntax.ident {} (toString val).toSubstring val []
+  Syntax.ident SourceInfo.none (toString val).toSubstring val []
 
 @[inline] def mkNullNode (args : Array Syntax := #[]) : Syntax :=
   Syntax.node nullKind args
@@ -348,17 +316,17 @@ def mkApp (fn : Syntax) : (args : Array Syntax) → Syntax
 def mkCApp (fn : Name) (args : Array Syntax) : Syntax :=
   mkApp (mkCIdent fn) args
 
-def mkLit (kind : SyntaxNodeKind) (val : String) (info : SourceInfo := {}) : Syntax :=
+def mkLit (kind : SyntaxNodeKind) (val : String) (info := SourceInfo.none) : Syntax :=
   let atom : Syntax := Syntax.atom info val
   Syntax.node kind #[atom]
 
-def mkStrLit (val : String) (info : SourceInfo := {}) : Syntax :=
+def mkStrLit (val : String) (info := SourceInfo.none) : Syntax :=
   mkLit strLitKind (String.quote val) info
 
-def mkNumLit (val : String) (info : SourceInfo := {}) : Syntax :=
+def mkNumLit (val : String) (info := SourceInfo.none) : Syntax :=
   mkLit numLitKind val info
 
-def mkScientificLit (val : String) (info : SourceInfo := {}) : Syntax :=
+def mkScientificLit (val : String) (info := SourceInfo.none) : Syntax :=
   mkLit scientificLitKind val info
 
 /- Recall that we don't have special Syntax constructors for storing numeric and string atoms.

src/Init/Prelude.lean

@@ -1015,6 +1015,9 @@ structure Substring where
   startPos : String.Pos
   stopPos : String.Pos
 
+@[inline] def Substring.bsize : Substring → Nat
+  | ⟨_, b, e⟩ => e.sub b
+
 def String.csize (c : Char) : Nat :=
   c.utf8Size.toNat
 
@@ -1582,19 +1585,31 @@ end Name
 
 /- Syntax -/
 
-/--
-  Source information of syntax atoms. All information is generally set for unquoted syntax and unset for syntax in
-  syntax quotations, but syntax transformations might want to invalidate only one side to make the pretty printer
-  reformat it. In the special case of the delaborator, we also use purely synthetic position information without
-  whitespace information. -/
-structure SourceInfo where
-  /- Will be inferred after parsing by `Syntax.updateLeading`. During parsing,
-     it is not at all clear what the preceding token was, especially with backtracking. -/
-  leading  : Option Substring  := none
-  pos      : Option String.Pos := none
-  trailing : Option Substring  := none
+/-- Source information of tokens. -/
+inductive SourceInfo where
+  /-
+    Token from original input with whitespace and position information.
+    `leading` will be inferred after parsing by `Syntax.updateLeading`. During parsing,
+    it is not at all clear what the preceding token was, especially with backtracking. -/
+  | original (leading : Substring) (pos : String.Pos) (trailing : Substring)
+  /-
+    Synthesized token (e.g. from a quotation) annotated with a span from the original source.
+    In the delaborator, we "misuse" this constructor to store synthetic positions identifying
+    subterms. -/
+  | synthetic (pos : String.Pos) (endPos : String.Pos)
+  /- Synthesized token without position information. -/
+  | protected none
 
-instance : Inhabited SourceInfo := ⟨{}⟩
+instance : Inhabited SourceInfo := ⟨SourceInfo.none⟩
+
+namespace SourceInfo
+
+def getPos : SourceInfo → Option String.Pos
+  | original (pos := pos) ..  => some pos
+  | synthetic (pos := pos) .. => some pos
+  | SourceInfo.none           => none
+
+end SourceInfo
 
 abbrev SyntaxNodeKind := Name
 
@@ -1672,23 +1687,42 @@ def setArg (stx : Syntax) (i : Nat) (arg : Syntax) : Syntax :=
   | stx         => stx
 
 /-- Retrieve the left-most leaf's info in the Syntax tree. -/
-partial def getHeadInfo : Syntax → Option SourceInfo
+partial def getHeadInfo? : Syntax → Option SourceInfo
   | atom info _      => some info
   | ident info _ _ _ => some info
   | node _ args      =>
     let rec loop (i : Nat) : Option SourceInfo :=
       match decide (Less i args.size) with
-      | true => match getHeadInfo (args.get! i) with
+      | true => match getHeadInfo? (args.get! i) with
          | some info => some info
          | none      => loop (hAdd i 1)
       | false => none
     loop 0
   | _                => none
 
+/-- Retrieve the left-most leaf's info in the Syntax tree, or `none` if there is no token. -/
+partial def getHeadInfo (stx : Syntax) : SourceInfo :=
+  match stx.getHeadInfo? with
+  | some info => info
+  | none      => SourceInfo.none
+
 def getPos (stx : Syntax) : Option String.Pos :=
-  match stx.getHeadInfo with
-  | some info => info.pos
-  | _         => none
+  stx.getHeadInfo.getPos
+
+partial def getTailPos : Syntax → Option String.Pos
+  | atom (SourceInfo.original (pos := pos) ..) val     => some (pos.add val.bsize)
+  | atom (SourceInfo.synthetic (endPos := pos) ..) _   => some pos
+  | ident (SourceInfo.original (pos := pos) ..) val .. => some (pos.add val.bsize)
+  | ident (SourceInfo.synthetic (endPos := pos) ..) .. => some pos
+  | node _ args                                        =>
+    let rec loop (i : Nat) : Option String.Pos :=
+      match decide (Less i args.size) with
+      | true => match getTailPos (args.get! ((args.size.sub i).sub 1)) with
+         | some info => some info
+         | none      => loop (hAdd i 1)
+      | false => none
+    loop 0
+  | _                                                  => none
 
 /--
   An array of syntax elements interspersed with separators. Can be coerced to/from `Array Syntax` to automatically
@@ -1698,10 +1732,13 @@ structure SepArray (sep : String) where
 
 end Syntax
 
+def SourceInfo.fromRef (ref : Syntax) : SourceInfo :=
+  match ref.getPos, ref.getTailPos with
+  | some pos, some tailPos => SourceInfo.synthetic pos tailPos
+  | _,        _            => SourceInfo.none
+
 def mkAtomFrom (src : Syntax) (val : String) : Syntax :=
-  match src.getHeadInfo with
-  | some info => Syntax.atom info val
-  | none      => Syntax.atom {} val
+  Syntax.atom src.getHeadInfo val
 
 /- Parser descriptions -/
 
@@ -1784,7 +1821,7 @@ class MonadQuotation (m : Type → Type) extends MonadRef m where
 export MonadQuotation (getCurrMacroScope getMainModule withFreshMacroScope)
 
 def MonadRef.mkInfoFromRefPos [Monad m] [MonadRef m] : m SourceInfo := do
-  return { pos := (← getRef).getPos }
+  SourceInfo.fromRef (← getRef)
 
 instance {m n : Type → Type} [MonadFunctor m n] [MonadLift m n] [MonadQuotation m] : MonadQuotation n where
   getCurrMacroScope   := liftM (m := m) getCurrMacroScope

src/Lean/Elab/Binders.lean

@@ -54,7 +54,7 @@ partial def quoteAutoTactic : Syntax → TermElabM Syntax
           let quotedArg ← quoteAutoTactic arg
           quotedArgs ← `(Array.push $quotedArgs $quotedArg)
       `(Syntax.node $(quote k) $quotedArgs)
-  | Syntax.atom info val => `(Syntax.atom {} $(quote val))
+  | Syntax.atom info val => `(mkAtom $(quote val))
   | Syntax.missing       => unreachable!
 
 def declareTacticSyntax (tactic : Syntax) : TermElabM Name :=

src/Lean/Elab/Do.lean

@@ -848,7 +848,7 @@ def seqToTerm (action : Syntax) (k : Syntax) : M Syntax := withRef action <| wit
     let cond := action[1]
     `(assert! $cond; $k)
   else
-    let action := Syntax.copyRangePos (← `(($action : $((←read).m) PUnit))) action
+    let action ← withRef action `(($action : $((←read).m) PUnit))
     `(Bind.bind $action (fun (_ : PUnit) => $k))
 
 def declToTerm (decl : Syntax) (k : Syntax) : M Syntax := withRef decl <| withFreshMacroScope do
@@ -870,7 +870,7 @@ def declToTerm (decl : Syntax) (k : Syntax) : M Syntax := withRef decl <| withFr
       -- `doElem` must be a `doExpr action`. See `doLetArrowToCode`
       match isDoExpr? doElem with
       | some action =>
-        let action := Syntax.copyRangePos (← `(($action : $((← read).m) $type))) action
+        let action ← withRef action `(($action : $((← read).m) $type))
         `(Bind.bind $action (fun ($id:ident : $type) => $k))
       | none        => Macro.throwErrorAt decl "unexpected kind of 'do' declaration"
     else

src/Lean/Elab/Term.lean

@@ -1292,21 +1292,21 @@ def resolveName (n : Name) (preresolved : List (Name × List String)) (explicitL
   `(v.head, id, [f₁, f₂])` where `id` is an identifier for `v.head`, and `f₁` and `f₂` are identifiers for fields `"bla"` and `"boo"`. -/
 def resolveName' (ident : Syntax) (explicitLevels : List Level) : TermElabM (List (Expr × Syntax × List Syntax)) := do
   match ident with
-  | Syntax.ident { pos := pos?, .. } rawStr n preresolved =>
+  | Syntax.ident info rawStr n preresolved =>
     let r ← resolveName n preresolved explicitLevels
     r.mapM fun (c, fields) => do
       let (cSstr, fields) := fields.foldr (init := (rawStr, [])) fun field (restSstr, fs) =>
         let fieldSstr := restSstr.takeRightWhile (· ≠ '.')
         ({ restSstr with stopPos := restSstr.stopPos - (fieldSstr.bsize + 1) }, (field, fieldSstr) :: fs)
       let id := mkIdentFrom ident cSstr.toString
-      match pos? with
+      match info.getPos with
       | none =>
         return (c, id, fields.map fun (field, _) => mkIdentFrom ident (Name.mkSimple field))
       | some pos =>
         let mut pos := pos + cSstr.bsize + 1
         let mut newFields := #[]
         for (field, fieldSstr) in fields do
-          newFields := newFields.push <| Syntax.ident { pos := some pos } fieldSstr (Name.mkSimple field) []
+          newFields := newFields.push <| Syntax.ident (SourceInfo.original "".toSubstring pos "".toSubstring) fieldSstr (Name.mkSimple field) []
           pos := pos + fieldSstr.bsize + 1
         return (c, id, newFields.toList)
   | _ => throwError! "identifier expected"

src/Lean/Parser/Basic.lean

@@ -723,11 +723,11 @@ private def rawAux (startPos : Nat) (trailingWs : Bool) : ParserFn := fun c s =>
     let s        := whitespace c s
     let stopPos' := s.pos
     let trailing := { str := input, startPos := stopPos, stopPos := stopPos' : Substring }
-    let atom     := mkAtom { leading := leading, pos := startPos, trailing := trailing } val
+    let atom     := mkAtom (SourceInfo.original leading startPos trailing) val
     s.pushSyntax atom
   else
     let trailing := mkEmptySubstringAt input stopPos
-    let atom     := mkAtom { leading := leading, pos := startPos, trailing := trailing } val
+    let atom     := mkAtom (SourceInfo.original leading startPos trailing) val
     s.pushSyntax atom
 
 /-- Match an arbitrary Parser and return the consumed String in a `Syntax.atom`. -/
@@ -782,7 +782,7 @@ def mkNodeToken (n : SyntaxNodeKind) (startPos : Nat) : ParserFn := fun c s =>
   let s         := whitespace c s
   let wsStopPos := s.pos
   let trailing  := { str := input, startPos := stopPos, stopPos := wsStopPos : Substring }
-  let info      := { leading := leading, pos := startPos, trailing := trailing : SourceInfo }
+  let info      := SourceInfo.original leading startPos trailing
   s.pushSyntax (Syntax.mkLit n val info)
 
 def charLitFnAux (startPos : Nat) : ParserFn := fun c s =>
@@ -924,7 +924,7 @@ def mkTokenAndFixPos (startPos : Nat) (tk : Option Token) : ParserFn := fun c s
       let s         := whitespace c s
       let wsStopPos := s.pos
       let trailing  := { str := input, startPos := stopPos, stopPos := wsStopPos : Substring }
-      let atom      := mkAtom { leading := leading, pos := startPos, trailing := trailing } tk
+      let atom      := mkAtom (SourceInfo.original leading startPos trailing) tk
       s.pushSyntax atom
 
 def mkIdResult (startPos : Nat) (tk : Option Token) (val : Name) : ParserFn := fun c s =>
@@ -938,7 +938,7 @@ def mkIdResult (startPos : Nat) (tk : Option Token) (val : Name) : ParserFn := f
     let trailingStopPos := s.pos
     let leading         := mkEmptySubstringAt input startPos
     let trailing        := { str := input, startPos := stopPos, stopPos := trailingStopPos : Substring }
-    let info            := { leading := leading, trailing := trailing, pos := startPos : SourceInfo }
+    let info            := SourceInfo.original leading startPos trailing
     let atom            := mkIdent info rawVal val
     s.pushSyntax atom
 
@@ -1081,8 +1081,8 @@ def symbolInfo (sym : String) : ParserInfo := {
 
 def checkTailNoWs (prev : Syntax) : Bool :=
   match prev.getTailInfo with
-  | some { trailing := some trailing, .. } => trailing.stopPos == trailing.startPos
-  | _                                      => false
+  | some (SourceInfo.original _ _ trailing) => trailing.stopPos == trailing.startPos
+  | _                                       => false
 
 /-- Check if the following token is the symbol _or_ identifier `sym`. Useful for
     parsing local tokens that have not been added to the token table (but may have
@@ -1135,8 +1135,8 @@ partial def strAux (sym : String) (errorMsg : String) (j : Nat) :ParserFn :=
 
 def checkTailWs (prev : Syntax) : Bool :=
   match prev.getTailInfo with
-  | some { trailing := some trailing, .. } => trailing.stopPos > trailing.startPos
-  | _                                      => false
+  | some (SourceInfo.original _ _ trailing) => trailing.stopPos > trailing.startPos
+  | _                                       => false
 
 def checkWsBeforeFn (errorMsg : String) : ParserFn := fun c s =>
   let prev := s.stxStack.back

src/Lean/Parser/Module.lean

@@ -53,7 +53,7 @@ def parseHeader (inputCtx : InputContext) : IO (Syntax × ModuleParserState × M
     pure (stx, { pos := s.pos }, {})
 
 private def mkEOI (pos : String.Pos) : Syntax :=
-  let atom := mkAtom { pos := pos, trailing := "".toSubstring, leading := "".toSubstring } ""
+  let atom := mkAtom (SourceInfo.original "".toSubstring pos "".toSubstring) ""
   Syntax.node `Lean.Parser.Module.eoi #[atom]
 
 def isEOI (s : Syntax) : Bool :=

src/Lean/Parser/Transform.lean

@@ -1,50 +0,0 @@
-/-
-Copyright (c) 2019 Microsoft Corporation. All rights reserved.
-Released under Apache 2.0 license as described in the file LICENSE.
-Authors: Leonardo de Moura, Sebastian Ullrich
--/
-import Lean.Parser.Basic
-
-namespace Lean
-namespace Syntax
-
-def manyToSepBy (stx : Syntax) (sepTk : String) : Syntax := do
-  match stx with
-  | node k args =>
-    if args.size == 0 then
-      stx
-    else
-      let mut argsNew := #[args[0]]
-      for i in [1:args.size] do
-        let arg  := args[i]
-        let prev := argsNew.back
-        match prev.getTailInfo with
-        | some info =>
-          let prevArg := prev.setTailInfo { trailing := none }
-          argsNew := argsNew.set! (argsNew.size - 1) prev
-          argsNew := argsNew.push (atom info sepTk)
-          argsNew := argsNew.push arg
-        | none =>
-          argsNew := argsNew.push (atom {} sepTk)
-          argsNew := argsNew.push arg
-      node k argsNew
-  | stx => stx
-
-def removeParen (stx : Syntax) : Syntax :=
-  stx.ifNodeKind `Lean.Parser.Term.paren
-    (fun stx =>
-      let body := stx.getArg 1
-      if body.getNumArgs != 2 then stx.val
-      else if (body.getArg 1).isNone then
-        let body := body.getArg 0
-        match stx.getArg 2, body.getTailInfo with
-        | atom { trailing := some outer } ")", some bodyInfo@{ trailing := some inner } =>
-          let bodyInfoTrail := inner.toString ++ "  "  -- add whithespaces for removed parentheses
-          let bodyInfoTrail := bodyInfoTrail ++ outer.toString -- add close paren trailing spaces
-          body.setTailInfo { bodyInfo with trailing := bodyInfoTrail.toSubstring }
-        | _, _ => stx.val
-      else stx.val)
-    (fun _ => stx)
-
-end Syntax
-end Lean

src/Lean/PrettyPrinter/Delaborator/Basic.lean

@@ -203,12 +203,12 @@ def withMDataExpr {α} (d : DelabM α) : DelabM α := do
   withReader ({ · with expr := e }) d
 
 partial def annotatePos (pos : Nat) : Syntax → Syntax
-  | stx@(Syntax.ident _ _ _ _)                   => stx.setInfo { pos := pos }
+  | stx@(Syntax.ident _ _ _ _)                   => stx.setInfo (SourceInfo.synthetic pos pos)
   -- app => annotate function
   | stx@(Syntax.node `Lean.Parser.Term.app args) => stx.modifyArg 0 (annotatePos pos)
   -- otherwise, annotate first direct child token if any
   | stx => match stx.getArgs.findIdx? Syntax.isAtom with
-    | some idx => stx.modifyArg idx (Syntax.setInfo { pos := pos })
+    | some idx => stx.modifyArg idx (Syntax.setInfo (SourceInfo.synthetic pos pos))
     | none     => stx
 
 def annotateCurPos (stx : Syntax) : Delab := do

src/Lean/PrettyPrinter/Formatter.lean

@@ -260,8 +260,8 @@ def pushTokenCore (tk : String) : FormatterM Unit := do
     push tk.trimRight
 
 def pushToken (info : SourceInfo) (tk : String) : FormatterM Unit := do
-  match info.trailing with
-  | some ss =>
+  match info with
+  | SourceInfo.original _ _ ss =>
     -- preserve non-whitespace content (i.e. comments)
     let ss' := ss.trim
     if !ss'.isEmpty then
@@ -271,7 +271,7 @@ def pushToken (info : SourceInfo) (tk : String) : FormatterM Unit := do
       else
         push s!"  {ss'}"
       modify fun st => { st with leadWord := "" }
-  | none    => pure ()
+  | _ => pure ()
 
   let st ← get
   -- If there is no space between `tk` and the next word, see if we would parse more than `tk` as a single token
@@ -291,8 +291,8 @@ def pushToken (info : SourceInfo) (tk : String) : FormatterM Unit := do
     pushTokenCore tk
     modify fun st => { st with leadWord := if tk.trimLeft == tk then tk else "" }
 
-  match info.leading with
-  | some ss =>
+  match info with
+  | SourceInfo.original ss _ _ =>
     -- preserve non-whitespace content (i.e. comments)
     let ss' := ss.trim
     if !ss'.isEmpty then
@@ -304,7 +304,7 @@ def pushToken (info : SourceInfo) (tk : String) : FormatterM Unit := do
       else
         push s!"{ss'} "
       modify fun st => { st with leadWord := "" }
-  | none    => pure ()
+  | _ => pure ()
 
 @[combinatorFormatter Lean.Parser.symbolNoAntiquot]
 def symbolNoAntiquot.formatter (sym : String) : Formatter := do

src/Lean/PrettyPrinter/Parenthesizer.lean

@@ -215,16 +215,19 @@ def maybeParenthesize (cat : Name) (canJuxtapose : Bool) (mkParen : Syntax → S
       -- the original node, we must first move to the right, except if we already were at the left-most child in the first
       -- place.
       if idx > 0 then goRight
-      let stx ← getCur
-      match stx.getHeadInfo, stx.getTailInfo with
-      | some hi, some ti =>
-        -- Move leading/trailing whitespace of `stx` outside of parentheses
-        let stx := (stx.setHeadInfo { hi with leading := "".toSubstring }).setTailInfo { ti with trailing := "".toSubstring }
-        let stx := mkParen stx
-        let stx := (stx.setHeadInfo { hi with trailing := "".toSubstring }).setTailInfo { ti with leading := "".toSubstring }
-        setCur stx
-      | _, _ => setCur (mkParen stx)
-      let stx ← getCur; trace! `PrettyPrinter.parenthesize m!"parenthesized: {stx.formatStx none}"
+      let mut stx ← getCur
+      -- Move leading/trailing whitespace of `stx` outside of parentheses
+      if let SourceInfo.original _ pos trail := stx.getHeadInfo then
+        stx := stx.setHeadInfo (SourceInfo.original "".toSubstring pos trail)
+      if let SourceInfo.original lead pos _ := stx.getTailInfo then
+        stx := stx.setTailInfo (SourceInfo.original lead pos "".toSubstring)
+      let mut stx' := mkParen stx
+      if let SourceInfo.original lead pos _ := stx.getHeadInfo then
+        stx' := stx'.setHeadInfo (SourceInfo.original lead pos "".toSubstring)
+      if let SourceInfo.original _ pos trail := stx.getTailInfo then
+        stx' := stx'.setTailInfo (SourceInfo.original "".toSubstring pos trail)
+      trace! `PrettyPrinter.parenthesize m!"parenthesized: {stx'.formatStx none}"
+      setCur stx'
       goLeft
       -- after parenthesization, there is no more trailing parser
       modify (fun st => { st with contPrec := Parser.maxPrec, contCat := cat, trailPrec := none })

src/Lean/Server/FileWorker.lean

@@ -431,7 +431,7 @@ section RequestHandling
         return #[]
 
   def rangeOfSyntax (text : FileMap) (stx : Syntax) : Range :=
-    ⟨text.utf8PosToLspPos <| stx.getHeadInfo.get!.pos.get!,
+    ⟨text.utf8PosToLspPos <| stx.getPos.get!,
      text.utf8PosToLspPos <| stx.getTailPos.get!⟩
 
   partial def handleDocumentSymbol (id : RequestID) (p : DocumentSymbolParams) :

src/Lean/Server/Snapshots.lean

@@ -106,7 +106,7 @@ def compileNextCmd (contents : String) (snap : Snapshot) : IO (Sum Snapshot Mess
   let pmctx := { env := cmdState.env, options := scope.opts, currNamespace := scope.currNamespace, openDecls := scope.openDecls }
   let (cmdStx, cmdParserState, msgLog) :=
     Parser.parseCommand inputCtx pmctx snap.mpState snap.msgLog
-  let cmdPos := cmdStx.getHeadInfo.get!.pos.get! -- TODO(WN): always `some`?
+  let cmdPos := cmdStx.getPos.get!
   if Parser.isEOI cmdStx || Parser.isExitCommand cmdStx then
     Sum.inr msgLog
   else

src/Lean/Syntax.lean

@@ -7,12 +7,10 @@ import Lean.Data.Name
 import Lean.Data.Format
 
 namespace Lean
-namespace SourceInfo
 
-def updateTrailing (info : SourceInfo) (trailing : Option Substring) : SourceInfo :=
-  { info with trailing := trailing }
-
-end SourceInfo
+def SourceInfo.updateTrailing (trailing : Substring) : SourceInfo → SourceInfo
+  | SourceInfo.original leading pos _ => SourceInfo.original leading pos trailing
+  | info                              => info
 
 /- Syntax AST -/
 
@@ -119,8 +117,8 @@ def getIdAt (stx : Syntax) (i : Nat) : Name :=
   Id.run $ stx.rewriteBottomUpM fn
 
 private def updateInfo : SourceInfo → String.Pos → String.Pos → SourceInfo
-  | {leading := some lead, pos := some pos, trailing := some trail}, leadStart, trailStop =>
-    {leading := some { lead with startPos := leadStart }, pos := some pos, trailing := some { trail with stopPos := trailStop }}
+  | SourceInfo.original lead pos trail, leadStart, trailStop =>
+    SourceInfo.original { lead with startPos := leadStart } pos { trail with stopPos := trailStop }
   | info, _, _ => info
 
 private def chooseNiceTrailStop (trail : Substring) : String.Pos :=
@@ -130,12 +128,12 @@ trail.startPos + trail.posOf '\n'
    or the beginning of the String. -/
 @[inline]
 private def updateLeadingAux : Syntax → StateM String.Pos (Option Syntax)
-  | atom info@{trailing := some trail, ..} val => do
+  | atom info@(SourceInfo.original lead pos trail) val => do
     let trailStop := chooseNiceTrailStop trail
     let newInfo := updateInfo info (← get) trailStop
     set trailStop
     pure $ some (atom newInfo val)
-  | ident info@{trailing := some trail, ..} rawVal val pre   => do
+  | ident info@(SourceInfo.original lead pos trail) rawVal val pre => do
     let trailStop := chooseNiceTrailStop trail
     let newInfo := updateInfo info (← get) trailStop
     set trailStop
@@ -160,7 +158,7 @@ private def updateLeadingAux : Syntax → StateM String.Pos (Option Syntax)
 def updateLeading : Syntax → Syntax :=
   fun stx => (replaceM updateLeadingAux stx).run' 0
 
-partial def updateTrailing (trailing : Option Substring) : Syntax → Syntax
+partial def updateTrailing (trailing : Substring) : Syntax → Syntax
   | Syntax.atom info val               => Syntax.atom (info.updateTrailing trailing) val
   | Syntax.ident info rawVal val pre   => Syntax.ident (info.updateTrailing trailing) rawVal val pre
   | n@(Syntax.node k args)             =>
@@ -173,17 +171,19 @@ partial def updateTrailing (trailing : Option Substring) : Syntax → Syntax
   | s => s
 
 partial def getTailWithPos : Syntax → Option Syntax
-  | stx@(atom { pos := some _, .. } _)   => some stx
-  | stx@(ident { pos := some _, .. } ..) => some stx
-  | node _ args                          => args.findSomeRev? getTailWithPos
-  | _                                    => none
+  | stx@(atom info _)   => info.getPos.map fun _ => stx
+  | stx@(ident info ..) => info.getPos.map fun _ => stx
+  | node _ args         => args.findSomeRev? getTailWithPos
+  | _                   => none
 
 private def reprintLeaf (info : SourceInfo) (val : String) : String :=
+  match info with
+  | SourceInfo.original lead _ trail => s!"{lead}{val}{trail}"
   -- no source info => add gracious amounts of whitespace to definitely separate tokens
   -- Note that the proper pretty printer does not use this function.
   -- The parser as well always produces source info, so round-tripping is still
   -- guaranteed.
-  (Substring.toString <$> info.leading).getD " " ++ val ++ (Substring.toString <$> info.trailing).getD " "
+  | _                                => s!" {val} "
 
 partial def reprint : Syntax → Option String
   | atom info val           => reprintLeaf info val
@@ -203,13 +203,10 @@ partial def reprint : Syntax → Option String
 open Std.Format
 
 private def formatInfo (showInfo : Bool) (info : SourceInfo) (f : Format) : Format :=
-  if showInfo then
-    (match info.leading with | some ss => repr ss.toString ++ ":" | _ => "") ++
-    f ++
-    (match info.pos with | some pos => ":" ++ toString info.pos | _ => "") ++
-    (match info.trailing with | some ss => ":" ++ repr ss.toString | _ => "")
-  else
-    f
+  match showInfo, info with
+  | true, SourceInfo.original lead pos trail => f!"{lead}:{f}:{pos}:{trail}"
+  | true, SourceInfo.synthetic pos endPos    => f!"{pos}:{f}:{endPos}"
+  | _,    _                                  => f
 
 partial def formatStxAux (maxDepth : Option Nat) (showInfo : Bool) : Nat → Syntax → Format
   | _,     atom info val     => formatInfo showInfo info $ format (repr val)
@@ -323,9 +320,6 @@ namespace SyntaxNode
 
 end SyntaxNode
 
-def mkSimpleAtom (val : String) : Syntax :=
-  Syntax.atom {} val
-
 def mkListNode (args : Array Syntax) : Syntax :=
   Syntax.node nullKind args
 

stage0/src/library/compiler/ir_interpreter.cpp

@@ -904,7 +904,7 @@ private:
     }
 public:
     explicit interpreter(environment const & env, options const & opts) : m_env(env), m_opts(opts) {
-        m_prefer_native = opts.get_bool(*g_interpreter_prefer_native, LEAN_DEFAULT_INTERPRETER_PREFER_NATIVE);
+        m_prefer_native = true; //opts.get_bool(*g_interpreter_prefer_native, LEAN_DEFAULT_INTERPRETER_PREFER_NATIVE);
     }
 
     ~interpreter() {

tests/lean/276.lean.expected.out

@@ -1,2 +1,2 @@
 fun {α} => PEmpty.rec.{v, u_1} fun x => α : {α : Sort v} → PEmpty.{u_1} → α
-276.lean:13:4-13:8: error: code generator does not support recursor 'PEmpty.rec' yet, consider using 'match ... with' and/or structural recursion
+276.lean:13:4-13:15: error: code generator does not support recursor 'PEmpty.rec' yet, consider using 'match ... with' and/or structural recursion

tests/lean/StxQuot.lean

@@ -90,6 +90,6 @@ open Parser.Term
 #check (match · with | `([1, $ys,*, 2, $zs,*, 3]) => _)
 
 #eval run do
-  match Syntax.setHeadInfo (← `(fun x =>%$(Syntax.atom {pos := some 2} "") x)) { pos := some 1 } with
-  | `(fun%$i1 $x =>%$i2 $y) => pure #[i1.getHeadInfo >>= SourceInfo.pos, i2.getHeadInfo >>= SourceInfo.pos]
+  match Syntax.setHeadInfo (← `(fun x =>%$(Syntax.atom (SourceInfo.synthetic 2 2) "") x)) (SourceInfo.synthetic 1 1) with
+  | `(fun%$i1 $x =>%$i2 $y) => pure #[i1.getPos, i2.getPos]
   | _ => unreachable!

tests/lean/doNotation1.lean.expected.out

@@ -9,11 +9,11 @@ doNotation1.lean:25:7-25:11: error: invalid reassignment, value has type
   Bool
 but is expected to have type
   Nat
-doNotation1.lean:24:0-24:10: error: type mismatch, 'for' has type
+doNotation1.lean:24:0-25:11: error: type mismatch, 'for' has type
   PUnit
 but is expected to have type
   List Bool
-doNotation1.lean:28:0-28:10: error: type mismatch, 'for' has type
+doNotation1.lean:28:0-29:14: error: type mismatch, 'for' has type
   PUnit
 but is expected to have type
   List Nat
@@ -28,7 +28,7 @@ has type
   EIO IO.Error (IO.Ref Bool)
 but is expected to have type
   EIO IO.Error Unit
-doNotation1.lean:58:2-58:3: error: type mismatch, result value has type
+doNotation1.lean:58:2-58:20: error: type mismatch, result value has type
   Unit
 but is expected to have type
   Bool

tests/lean/macroStack.lean.expected.out

@@ -7,7 +7,7 @@ while expanding
 while expanding
   if h : (x > 0) then 1 else 0
 macroStack.lean:11:9-11:15: error: invalid use of `(<- ...)`, must be nested inside a 'do' expression
-macroStack.lean:17:0-17:1: error: failed to synthesize instance
+macroStack.lean:17:0-17:6: error: failed to synthesize instance
   HAdd Nat String ?m
 with resulting expansion
   HAdd.hAdd✝ (x + x✝) x✝¹

tests/lean/patvar.lean.expected.out

@@ -1,6 +1,6 @@
-patvar.lean:3:0-3:5: error: missing cases:
+patvar.lean:3:0-3:22: error: missing cases:
 (List.cons _ _)
-patvar.lean:10:0-10:5: error: missing cases:
+patvar.lean:10:0-10:16: error: missing cases:
 (List.cons _ _)
 patvar.lean:14:0-14:21: error: invalid pattern variable, must be atomic
 patvar.lean:17:0-17:20: error: invalid pattern variable, must be atomic

tests/lean/run/CommandExtOverlap.lean

@@ -4,7 +4,7 @@ open Lean
 
 macro_rules [mycheck]
 | `(#check $es,*) =>
-  let cmds := es.getElems.map $ fun e => Syntax.node `Lean.Parser.Command.check #[Syntax.atom {} "#check", e]
+  let cmds := es.getElems.map $ fun e => Syntax.node `Lean.Parser.Command.check #[mkAtom "#check", e]
   pure $ mkNullNode cmds
 
 #check true

tests/lean/run/Reparen.lean

@@ -6,7 +6,9 @@ open Std.Format open Std
 
 def unparenAux (parens body : Syntax) : Syntax :=
 match parens.getHeadInfo, body.getHeadInfo, body.getTailInfo, parens.getTailInfo with
-| some bi', some bi, some ti, some ti' => (body.setHeadInfo { bi with leading := bi'.leading }).setTailInfo { ti with trailing := ti'.trailing }
+| SourceInfo.original lead _ _, SourceInfo.original _ pos trail,
+  SourceInfo.original endLead endPos _, SourceInfo.original _ _ endTrail =>
+    body.setHeadInfo (SourceInfo.original lead pos trail) |>.setTailInfo (SourceInfo.original endLead endPos endTrail)
 | _, _, _, _ => body
 
 partial def unparen : Syntax → Syntax

tests/lean/run/autoparam.lean

@@ -1,5 +1,4 @@
 
-
 def f (x y : Nat) (h : x = y := by assumption) : Nat :=
 x + x
 

tests/lean/run/tacticExtOverlap.lean

@@ -3,7 +3,7 @@ open Lean
 syntax (name := myintro) "intros" sepBy(ident, ",") : tactic
 
 macro_rules [myintro]
-| `(tactic| intros $x,*) => pure $ Syntax.node `Lean.Parser.Tactic.intros #[Syntax.atom {} "intros", mkNullNode x]
+| `(tactic| intros $x,*) => pure $ Syntax.node `Lean.Parser.Tactic.intros #[mkAtom "intros", mkNullNode x]
 
 theorem tst1 {p q : Prop} : p → q → p :=
 by {

tests/lean/sanitychecks.lean.expected.out

@@ -6,7 +6,7 @@ structural recursion cannot be used
 well founded recursion has not been implemented yet
 sanitychecks.lean:4:8-5:9: error: 'partial' theorems are not allowed, 'partial' is a code generation directive
 sanitychecks.lean:7:7-8:9: error: 'unsafe' theorems are not allowed
-sanitychecks.lean:10:0-10:9: error: failed to synthesize instance
+sanitychecks.lean:10:0-10:24: error: failed to synthesize instance
   Inhabited False
 sanitychecks.lean:18:0-18:40: error: failed to compile partial definition 'unsound3', failed to show that type is inhabited
 sanitychecks.lean:20:12-20:20: error: (kernel) invalid declaration, it uses unsafe declaration 'unsafeCast'

tests/lean/scopedInstanceOutsideNamespace.lean.expected.out

@@ -1,4 +1,4 @@
 scopedInstanceOutsideNamespace.lean:1:0-2:38: error: scoped attributes must be used inside namespaces
 "true"
-scopedInstanceOutsideNamespace.lean:6:0-6:3: error: scoped attributes must be used inside namespaces
+scopedInstanceOutsideNamespace.lean:6:0-6:30: error: scoped attributes must be used inside namespaces
 scopedInstanceOutsideNamespace.lean:8:0-10:38: error: scoped attributes must be used inside namespaces

tests/lean/scopedMacros.lean.expected.out

@@ -2,5 +2,5 @@
 scopedMacros.lean:11:7-11:11: error: unknown identifier 'foo!'
 10 + 1 : Nat
 scopedMacros.lean:19:0-19:37: error: scoped attributes must be used inside namespaces
-scopedMacros.lean:19:0-19:7: error: invalid syntax node kind 'termBla!_'
+scopedMacros.lean:19:0-19:50: error: invalid syntax node kind 'termBla!_'
 scopedMacros.lean:29:7-29:11: error: unknown identifier 'bla!'