From a24d7d593a4ae9b9d8784fd380efaef241f8aac7 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Fri, 28 May 2021 17:13:47 -0400 Subject: [PATCH 001/696] Initial Commit --- .gitignore | 1 + Leanpkg.lean | 2 ++ leanpkg.toml | 4 ++++ 3 files changed, 7 insertions(+) create mode 100644 .gitignore create mode 100644 Leanpkg.lean create mode 100644 leanpkg.toml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..796b96d1c4 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/build diff --git a/Leanpkg.lean b/Leanpkg.lean new file mode 100644 index 0000000000..ec2c947e80 --- /dev/null +++ b/Leanpkg.lean @@ -0,0 +1,2 @@ +def main : IO Unit := + IO.println "Hello, world!" diff --git a/leanpkg.toml b/leanpkg.toml new file mode 100644 index 0000000000..376b9c0d0e --- /dev/null +++ b/leanpkg.toml @@ -0,0 +1,4 @@ +[package] +name = "Leanpkg" +version = "0.1" +lean_version = "leanprover/lean4:nightly-2021-05-26" From 404d32c7304320ddaacf8b67e95721d14ba05d8d Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 29 May 2021 08:54:48 -0400 Subject: [PATCH 002/696] Port over Leanpkg and get an example building --- Leanpkg.lean | 2 - Leanpkg2.lean | 11 ++++ Leanpkg2/Build.lean | 37 ++++++++++++ Leanpkg2/BuildCore.lean | 103 ++++++++++++++++++++++++++++++++++ Leanpkg2/Cli.lean | 91 ++++++++++++++++++++++++++++++ Leanpkg2/Configure.lean | 45 +++++++++++++++ Leanpkg2/Git.lean | 38 +++++++++++++ Leanpkg2/Init.lean | 40 +++++++++++++ Leanpkg2/LeanVersion.lean | 27 +++++++++ Leanpkg2/Make.lean | 53 +++++++++++++++++ Leanpkg2/Manifest.lean | 95 +++++++++++++++++++++++++++++++ Leanpkg2/Proc.lean | 18 ++++++ Leanpkg2/Resolve.lean | 83 +++++++++++++++++++++++++++ Leanpkg2/Toml.lean | 72 ++++++++++++++++++++++++ examples/hello/.gitignore | 1 + examples/hello/Package.lean | 20 +++++++ examples/hello/package.sh | 2 + examples/hello/src/Hello.lean | 2 + leanpkg.toml | 3 +- 19 files changed, 739 insertions(+), 4 deletions(-) delete mode 100644 Leanpkg.lean create mode 100644 Leanpkg2.lean create mode 100644 Leanpkg2/Build.lean create mode 100644 Leanpkg2/BuildCore.lean create mode 100644 Leanpkg2/Cli.lean create mode 100644 Leanpkg2/Configure.lean create mode 100644 Leanpkg2/Git.lean create mode 100644 Leanpkg2/Init.lean create mode 100644 Leanpkg2/LeanVersion.lean create mode 100644 Leanpkg2/Make.lean create mode 100644 Leanpkg2/Manifest.lean create mode 100644 Leanpkg2/Proc.lean create mode 100644 Leanpkg2/Resolve.lean create mode 100644 Leanpkg2/Toml.lean create mode 100644 examples/hello/.gitignore create mode 100644 examples/hello/Package.lean create mode 100644 examples/hello/package.sh create mode 100644 examples/hello/src/Hello.lean diff --git a/Leanpkg.lean b/Leanpkg.lean deleted file mode 100644 index ec2c947e80..0000000000 --- a/Leanpkg.lean +++ /dev/null @@ -1,2 +0,0 @@ -def main : IO Unit := - IO.println "Hello, world!" diff --git a/Leanpkg2.lean b/Leanpkg2.lean new file mode 100644 index 0000000000..dfe5c12f3a --- /dev/null +++ b/Leanpkg2.lean @@ -0,0 +1,11 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone +-/ +import Leanpkg2.Cli + +def main (args : List String) : IO Unit := do + Lean.initSearchPath none -- HACK + let (cmd, outerArgs, innerArgs) ← Leanpkg2.splitCmdlineArgs args + Leanpkg2.cli cmd outerArgs innerArgs diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean new file mode 100644 index 0000000000..07936f0c4e --- /dev/null +++ b/Leanpkg2/Build.lean @@ -0,0 +1,37 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone +-/ +import Leanpkg2.Resolve +import Leanpkg2.BuildCore +import Leanpkg2.Configure +import Leanpkg2.Make + +open System + +namespace Leanpkg2 + +def buildImports (manifest : Manifest) (imports : List String) (leanArgs : List String) : IO Unit := do + let cfg ← configure manifest + let imports := imports.map (·.toName) + let root ← getRootPart <| manifest.effectivePath + let localImports := imports.filter (·.getRoot == root) + if localImports != [] then + let buildCfg : Build.Config := { pkg := root, leanArgs, leanPath := cfg.leanPath, moreDeps := cfg.moreDeps } + if ← FilePath.pathExists "Makefile" then + let oleans := localImports.map fun i => Lean.modToFilePath "build" i "olean" |>.toString + execMake manifest oleans buildCfg + else + Build.buildModules buildCfg localImports + IO.println cfg.leanPath + IO.println cfg.leanSrcPath + +def build (manifest : Manifest) (makeArgs leanArgs : List String := []) : IO Unit := do + let cfg ← configure manifest + let root ← getRootPart <| manifest.effectivePath + let buildCfg : Build.Config := { pkg := root, leanArgs, leanPath := cfg.leanPath, moreDeps := cfg.moreDeps } + --if makeArgs != [] || (← FilePath.pathExists "Makefile") then + execMake manifest makeArgs buildCfg + --else + -- Build.buildModules buildCfg [root] diff --git a/Leanpkg2/BuildCore.lean b/Leanpkg2/BuildCore.lean new file mode 100644 index 0000000000..2ea6b0dddb --- /dev/null +++ b/Leanpkg2/BuildCore.lean @@ -0,0 +1,103 @@ +/- +Copyright (c) 2021 Sebastian Ullrich. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Sebastian Ullrich +-/ +import Lean.Data.Name +import Lean.Elab.Import +import Leanpkg2.Proc + +open Lean +open System + +namespace Leanpkg2.Build + +def buildPath : FilePath := "build" +def tempBuildPath := buildPath / "temp" + +structure Config where + pkg : Name + leanArgs : List String + leanPath : String + -- things like `leanpkg.toml` and olean roots of dependencies that should also trigger rebuilds + moreDeps : List FilePath + +structure Context extends Config where + parents : List Name := [] + moreDepsMTime : IO.FS.SystemTime + +deriving instance Inhabited for IO.FS.SystemTime +deriving instance Inhabited for Task + +structure Result where + maxMTime : IO.FS.SystemTime + task : Task (Except IO.Error Unit) + deriving Inhabited + +structure State where + modTasks : NameMap Result := ∅ + +abbrev BuildM := ReaderT Context <| StateT State IO + +partial def buildModule (mod : Name) : BuildM Result := do + let ctx ← read + if ctx.parents.contains mod then + -- cyclic import + let cycle := ctx.parents.dropWhile (· != mod) ++ [mod] + let cycle := cycle.reverse.map (s!" {·}") + throw <| IO.userError s!"import cycle detected:\n{"\n".intercalate cycle}" + + if let some r := (← get).modTasks.find? mod then + -- already visited + return r + + let leanFile := modToFilePath "." mod "lean" + let leanMData ← leanFile.metadata + + -- recursively build dependencies and calculate transitive `maxMTime` + let (imports, _, _) ← Elab.parseImports (← IO.FS.readFile leanFile) leanFile.toString + let localImports := imports.filter (·.module.getRoot == ctx.pkg) + let deps ← localImports.mapM (buildModule ·.module) + let depMTimes ← deps.mapM (·.maxMTime) + let maxMTime := List.maximum? (leanMData.modified :: ctx.moreDepsMTime :: depMTimes) |>.get! + + -- check whether we have an up-to-date .olean + let oleanFile := modToFilePath buildPath mod "olean" + try + if (← oleanFile.metadata).modified >= maxMTime then + let r := { maxMTime, task := Task.pure (Except.ok ()) } + modify fun st => { st with modTasks := st.modTasks.insert mod r } + return r + catch + | IO.Error.noFileOrDirectory .. => pure () + | e => throw e + + let task ← IO.mapTasks (tasks := deps.map (·.task)) fun rs => do + if let some e := rs.findSome? (fun | Except.error e => some e | Except.ok _ => none) then + -- propagate failure + throw e + try + let cFile := modToFilePath tempBuildPath mod "c" + IO.createDirAll oleanFile.parent.get! + IO.createDirAll cFile.parent.get! + execCmd { + cmd := (← IO.appDir) / "lean" |>.withExtension FilePath.exeExtension |>.toString + args := ctx.leanArgs.toArray ++ #["-o", oleanFile.toString, "-c", cFile.toString, leanFile.toString] + env := #[("LEAN_PATH", ctx.leanPath)] + } + catch e => + -- print errors early + IO.eprintln e + throw e + + let r := { maxMTime, task := task } + modify fun st => { st with modTasks := st.modTasks.insert mod r } + return r + +def buildModules (cfg : Config) (mods : List Name) : IO Unit := do + let moreDepsMTime := (← cfg.moreDeps.mapM (·.metadata)).map (·.modified) |>.maximum? |>.getD ⟨0, 0⟩ + let rs ← mods.mapM buildModule |>.run { toConfig := cfg, moreDepsMTime } |>.run' {} + for r in rs do + if let Except.error _ ← IO.wait r.task then + -- actual error has already been printed above + throw <| IO.userError "Build failed." diff --git a/Leanpkg2/Cli.lean b/Leanpkg2/Cli.lean new file mode 100644 index 0000000000..9ddd491486 --- /dev/null +++ b/Leanpkg2/Cli.lean @@ -0,0 +1,91 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone +-/ +import Leanpkg2.Init +import Leanpkg2.Configure +import Leanpkg2.Make +import Leanpkg2.Build +import Leanpkg2.Manifest + +namespace Leanpkg2 + +def usage := +"Lean package manager, version " ++ uiLeanVersionString ++ " +Usage: leanpkg + +init create a Lean package in the current directory +configure download and build dependencies +build [] configure and build *.olean files + +See `leanpkg help ` for more information on a specific command." + +def helpConfigure := +"Download dependencies + +Usage: + leanpkg configure + +This command sets up the `build/deps` directory. + +For each (transitive) git dependency, the specified commit is checked out +into a sub-directory of `build/deps`. If there are dependencies on multiple +versions of the same package, the version materialized is undefined. No copy +is made of local dependencies." + +def helpBuild := +"Download dependencies and build *.olean files + +Usage: + leanpkg build [] [-- ] + +This command invokes `leanpkg configure` followed by `leanmake LEAN_OPTS=`. +If defined, the `package.timeout` configuration value is passed to Lean via its `-T` parameter. +If no are given, only .olean files will be produced in `build/`. If `lib` or `bin` +is passed instead, the extracted C code is compiled with `c++` and a static library in `build/lib` +or an executable in `build/bin`, respectively, is created. `leanpkg build bin` requires a declaration +of name `main` in the root namespace, which must return `IO Unit` or `IO UInt32` (the exit code) and +may accept the program's command line arguments as a `List String` parameter. + +NOTE: building and linking dependent libraries currently has to be done manually, e.g. +``` +$ (cd a; leanpkg build lib) +$ (cd b; leanpkg build bin LINK_OPTS=../a/build/lib/libA.a) +```" + +def helpInit := +"Create a new Lean package in the current directory + +Usage: + leanpkg init + +This command creates a new Lean package with the given name in the current +directory." + +def cli : (cmd : String) → (leanpkgArgs leanArgs : List String) → IO Unit +| "init", [name], [] => init name +| "configure", [], [] => discard do configure (← readManifest) +| "print-paths", leanpkgArgs, leanArgs => do buildImports (← readManifest) leanpkgArgs leanArgs +| "build", makeArgs, leanArgs => do build (← readManifest) makeArgs leanArgs +| "help", ["init"], [] => IO.println helpInit +| "help", ["configure"], [] => IO.println helpConfigure +| "help", ["build"], [] => IO.println helpBuild +| "help", _, [] => IO.println usage +| _, _, _ => throw <| IO.userError usage + +private def splitCmdlineArgsCore : List String → List String × List String +| [] => ([], []) +| (arg::args) => + if arg == "--" then + ([], args) + else + let (outerArgs, innerArgs) := splitCmdlineArgsCore args + (arg::outerArgs, innerArgs) + +def splitCmdlineArgs : List String → IO (String × List String × List String) +| [] => throw <| IO.userError usage +| [cmd] => return (cmd, [], []) +| (cmd::rest) => + let (outerArgs, innerArgs) := splitCmdlineArgsCore rest + return (cmd, outerArgs, innerArgs) diff --git a/Leanpkg2/Configure.lean b/Leanpkg2/Configure.lean new file mode 100644 index 0000000000..397b5cc5be --- /dev/null +++ b/Leanpkg2/Configure.lean @@ -0,0 +1,45 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone +-/ +import Leanpkg2.BuildCore +import Leanpkg2.Resolve + +open System + +namespace Leanpkg2 + +def getRootPart (pkg : FilePath := ".") : IO Lean.Name := do + let entries ← pkg.readDir + match entries.filter (FilePath.extension ·.fileName == "lean") with + | #[rootFile] => FilePath.withExtension rootFile.fileName "" |>.toString + | #[] => throw <| IO.userError s!"no '.lean' file found in {← IO.realPath "."}" + | _ => throw <| IO.userError s!"{← IO.realPath "."} must contain a unique '.lean' file as the package root" + +structure Configuration := + leanPath : String + leanSrcPath : String + moreDeps : List FilePath + +def configure (manifest : Manifest) : IO Configuration := do + IO.eprintln $ + "configuring " ++ manifest.name ++ " " ++ manifest.version + let paths ← solveDeps manifest + let mut moreDeps := [] + for (pkgpath, srcpath) in paths do + unless pkgpath == "." do + -- build recursively + -- TODO: share build of common dependencies + execCmd { + cmd := (← IO.appPath).toString + cwd := pkgpath + args := #["build"] + } + let pkgRoot := srcpath / Build.buildPath / (← getRootPart srcpath).toString + moreDeps := pkgRoot.withExtension "olean" :: moreDeps + return { + leanPath := SearchPath.toString <| paths.map (·.2 / Build.buildPath) + leanSrcPath := SearchPath.toString <| paths.map (·.2) + moreDeps + } diff --git a/Leanpkg2/Git.lean b/Leanpkg2/Git.lean new file mode 100644 index 0000000000..fb8ca06de2 --- /dev/null +++ b/Leanpkg2/Git.lean @@ -0,0 +1,38 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich +-/ +import Leanpkg2.LeanVersion + +open System + +namespace Leanpkg2 + +def upstreamGitBranch := + "master" + +def gitdefaultRevision : Option String → String + | none => upstreamGitBranch + | some branch => branch + +def gitParseRevision (gitRepo : FilePath) (rev : String) : IO String := do + let rev ← IO.Process.run {cmd := "git", args := #["rev-parse", "-q", "--verify", rev], cwd := gitRepo} + rev.trim -- remove newline at end + +def gitHeadRevision (gitRepo : FilePath) : IO String := + gitParseRevision gitRepo "HEAD" + +def gitParseOriginRevision (gitRepo : FilePath) (rev : String) : IO String := + (gitParseRevision gitRepo $ "origin/" ++ rev) <|> gitParseRevision gitRepo rev + <|> throw (IO.userError s!"cannot find revision {rev} in repository {gitRepo}") + +def gitLatestOriginRevision (gitRepo : FilePath) (branch : Option String) : IO String := do + discard <| IO.Process.run {cmd := "git", args := #["fetch"], cwd := gitRepo} + gitParseOriginRevision gitRepo (gitdefaultRevision branch) + +def gitRevisionExists (gitRepo : FilePath) (rev : String) : IO Bool := do + try + discard <| gitParseRevision gitRepo (rev ++ "^{commit}") + true + catch _ => false diff --git a/Leanpkg2/Init.lean b/Leanpkg2/Init.lean new file mode 100644 index 0000000000..41c772a120 --- /dev/null +++ b/Leanpkg2/Init.lean @@ -0,0 +1,40 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone +-/ +import Leanpkg2.Git +import Leanpkg2.Proc +import Leanpkg2.Manifest + +namespace Leanpkg2 + +def initGitignoreContents := +"/build +" + +def mainFileContents := +"def main : IO Unit := + IO.println \"Hello, world!\" +" + +def leanpkgFileContents (pkgName : String) := +s!"[package] +name = \"{pkgName}\" +version = \"0.1\" +lean_version = \"{leanVersionString}\" +" + +def initPkg (pkgName : String) (fromNew : Bool) : IO Unit := do + IO.FS.writeFile leanpkgTomlFn (leanpkgFileContents pkgName) + IO.FS.writeFile ⟨s!"{pkgName.capitalize}.lean"⟩ mainFileContents + let h ← IO.FS.Handle.mk ⟨".gitignore"⟩ IO.FS.Mode.append (bin := false) + h.putStr initGitignoreContents + unless ← System.FilePath.isDir ⟨".git"⟩ do + (do + execCmd {cmd := "git", args := #["init", "-q"]} + unless upstreamGitBranch = "master" do + execCmd {cmd := "git", args := #["checkout", "-B", upstreamGitBranch]} + ) <|> IO.eprintln "WARNING: failed to initialize git repository" + +def init (pkgName : String) := initPkg pkgName false diff --git a/Leanpkg2/LeanVersion.lean b/Leanpkg2/LeanVersion.lean new file mode 100644 index 0000000000..eb8984fef4 --- /dev/null +++ b/Leanpkg2/LeanVersion.lean @@ -0,0 +1,27 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich +-/ +namespace Leanpkg2 + +def leanVersionStringCore := + s!"{Lean.version.major}.{Lean.version.minor}.{Lean.version.patch}" + +def origin := "leanprover/lean4" + +def leanVersionString := + if Lean.version.isRelease then + s!"{origin}:{leanVersionStringCore}" + else if Lean.version.specialDesc ≠ "" then + s!"{origin}:{Lean.version.specialDesc}" + else + s!"{origin}:master" + +def uiLeanVersionString := +if Lean.version.isRelease then + leanVersionStringCore +else if Lean.version.specialDesc ≠ "" then + Lean.version.specialDesc +else + s!"master ({leanVersionStringCore})" diff --git a/Leanpkg2/Make.lean b/Leanpkg2/Make.lean new file mode 100644 index 0000000000..aceda30ccb --- /dev/null +++ b/Leanpkg2/Make.lean @@ -0,0 +1,53 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone +-/ +import Leanpkg2.Manifest +import Leanpkg2.BuildCore + +open System + +namespace Leanpkg2 + +def lockFileName : System.FilePath := ⟨".leanpkg-lock"⟩ + +partial def withLockFile (x : IO α) : IO α := do + acquire + try + x + finally + IO.removeFile lockFileName + where + acquire (firstTime := true) := + try + -- TODO: lock file should ideally contain PID + if !System.Platform.isWindows then + discard <| IO.Prim.Handle.mk lockFileName "wx" + else + -- `x` mode doesn't seem to work on Windows even though it's listed at + -- https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/fopen-wfopen?view=msvc-160 + -- ...? Let's use the slightly racy approach then. + if ← lockFileName.pathExists then + throw <| IO.Error.alreadyExists 0 "" + discard <| IO.Prim.Handle.mk lockFileName "w" + catch + | IO.Error.alreadyExists _ _ => do + if firstTime then + IO.eprintln s!"Waiting for prior leanpkg invocation to finish... (remove '{lockFileName}' if stuck)" + IO.sleep (ms := 300) + acquire (firstTime := false) + | e => throw e + +def execMake (manifest : Manifest) (makeArgs : List String) (cfg : Build.Config) : IO Unit := withLockFile do + let timeoutArgs := + match manifest.timeout with + | some t => ["-T", toString t] + | none => [] + let leanArgs := timeoutArgs ++ cfg.leanArgs + let mut spawnArgs := { + cmd := "sh" + cwd := manifest.effectivePath + args := #["-c", s!"\"{← IO.appDir}/leanmake\" PKG={cfg.pkg} LEAN_OPTS=\"{" ".intercalate leanArgs}\" LEAN_PATH=\"{cfg.leanPath}\" {" ".intercalate makeArgs} MORE_DEPS+=\"{" ".intercalate (cfg.moreDeps.map toString)}\" >&2"] + } + execCmd spawnArgs diff --git a/Leanpkg2/Manifest.lean b/Leanpkg2/Manifest.lean new file mode 100644 index 0000000000..3efa16cd6e --- /dev/null +++ b/Leanpkg2/Manifest.lean @@ -0,0 +1,95 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich +-/ +import Leanpkg2.Toml +import Leanpkg2.LeanVersion + +open System + +namespace Leanpkg2 + +inductive Source where + | path (dir : System.FilePath) : Source + | git (url rev : String) (branch : Option String) : Source + +namespace Source + +def fromToml (v : Toml.Value) : Option Source := + (do let Toml.Value.str dir ← v.lookup "path" | none + path ⟨dir⟩) <|> + (do let Toml.Value.str url ← v.lookup "git" | none + let Toml.Value.str rev ← v.lookup "rev" | none + match v.lookup "branch" with + | none => git url rev none + | some (Toml.Value.str branch) => git url rev (some branch) + | _ => none) + +def toToml : Source → Toml.Value + | path dir => Toml.Value.table [("path", Toml.Value.str dir.toString)] + | git url rev none => + Toml.Value.table [("git", Toml.Value.str url), ("rev", Toml.Value.str rev)] + | git url rev (some branch) => + Toml.Value.table [("git", Toml.Value.str url), ("branch", Toml.Value.str branch), ("rev", Toml.Value.str rev)] + +end Source + +structure Dependency where + name : String + src : Source + +structure Manifest where + name : String + version : String + leanVersion : String := leanVersionString + timeout : Option Nat := none + path : Option FilePath := none + dependencies : List Dependency := [] + +namespace Manifest + +def effectivePath (m : Manifest) : FilePath := + m.path.getD ⟨"."⟩ + +def fromToml (t : Toml.Value) : Option Manifest := OptionM.run do + let pkg ← t.lookup "package" + let Toml.Value.str n ← pkg.lookup "name" | none + let Toml.Value.str ver ← pkg.lookup "version" | none + let leanVer ← match pkg.lookup "lean_version" with + | some (Toml.Value.str leanVer) => some leanVer + | none => some leanVersionString + | _ => none + let tm ← match pkg.lookup "timeout" with + | some (Toml.Value.nat timeout) => some (some timeout) + | none => some none + | _ => none + let path ← match pkg.lookup "path" with + | some (Toml.Value.str path) => some (some ⟨path⟩) + | none => some none + | _ => none + let Toml.Value.table deps ← t.lookup "dependencies" <|> some (Toml.Value.table []) | none + let deps ← deps.mapM fun ⟨n, src⟩ => do Dependency.mk n (← Source.fromToml src) + return { name := n, version := ver, leanVersion := leanVer, + path := path, dependencies := deps, timeout := tm } + +def fromFile (fn : System.FilePath) : IO Manifest := do + let cnts ← IO.FS.readFile fn + let toml ← Toml.parse cnts + let some manifest ← pure (fromToml toml) + | throw <| IO.userError s!"cannot read manifest from {fn}" + manifest + +end Manifest + +def leanpkgTomlFn : System.FilePath := ⟨"leanpkg.toml"⟩ + +def readManifest : IO Manifest := do + let m ← Manifest.fromFile leanpkgTomlFn + if m.leanVersion ≠ leanVersionString then + IO.eprintln $ "\nWARNING: Lean version mismatch: installed version is " ++ leanVersionString + ++ ", but package requires " ++ m.leanVersion ++ "\n" + return m + +def writeManifest (manifest : Lean.Syntax) (fn : FilePath) : IO Unit := do + IO.FS.writeFile fn manifest.reprint.get! diff --git a/Leanpkg2/Proc.lean b/Leanpkg2/Proc.lean new file mode 100644 index 0000000000..1d3a125ab3 --- /dev/null +++ b/Leanpkg2/Proc.lean @@ -0,0 +1,18 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich +-/ +namespace Leanpkg2 + +def execCmd (args : IO.Process.SpawnArgs) : IO Unit := do + let envstr := String.join <| args.env.toList.map fun (k, v) => s!"{k}={v.getD ""} " + let cmdstr := " ".intercalate (args.cmd :: args.args.toList) + IO.eprintln <| "> " ++ envstr ++ + match args.cwd with + | some cwd => s!"{cmdstr} # in directory {cwd}" + | none => cmdstr + let child ← IO.Process.spawn args + let exitCode ← child.wait + if exitCode != 0 then + throw <| IO.userError <| s!"external command exited with status {exitCode}" diff --git a/Leanpkg2/Resolve.lean b/Leanpkg2/Resolve.lean new file mode 100644 index 0000000000..a127adb605 --- /dev/null +++ b/Leanpkg2/Resolve.lean @@ -0,0 +1,83 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich +-/ +import Leanpkg2.Manifest +import Leanpkg2.Proc +import Leanpkg2.Git + +open System + +namespace Leanpkg2 + +def Assignment := List (String × FilePath) + +namespace Assignment +def empty : Assignment := [] + +def contains (a : Assignment) (s : String) : Bool := + (a.lookup s).isSome + +def insert (a : Assignment) (k : String) (v : FilePath) : Assignment := + if a.contains k then a else (k, v) :: a + +def fold {α} (i : α) (f : α → String → FilePath → α) : Assignment → α := + List.foldl (fun a ⟨k, v⟩ => f a k v) i + +end Assignment + +abbrev Solver := StateT Assignment IO + +def notYetAssigned (d : String) : Solver Bool := do + ¬ (← get).contains d + +def resolvedPath (d : String) : Solver FilePath := do + let some path ← pure ((← get).lookup d) | unreachable! + path + +def materialize (relpath : FilePath) (dep : Dependency) : Solver Unit := + match dep.src with + | Source.path dir => do + let depdir := dir / relpath + IO.eprintln s!"{dep.name}: using local path {depdir}" + modify (·.insert dep.name depdir) + | Source.git url rev branch => do + let depdir := FilePath.mk "build" / "deps" / dep.name + if ← depdir.isDir then + IO.eprint s!"{dep.name}: trying to update {depdir} to revision {rev}" + IO.eprintln (match branch with | none => "" | some branch => "@" ++ branch) + let hash ← gitParseOriginRevision depdir rev + let revEx ← gitRevisionExists depdir hash + unless revEx do + execCmd {cmd := "git", args := #["fetch"], cwd := depdir} + else + IO.eprintln s!"{dep.name}: cloning {url} to {depdir}" + execCmd {cmd := "git", args := #["clone", url, depdir.toString]} + let hash ← gitParseOriginRevision depdir rev + execCmd {cmd := "git", args := #["checkout", "--detach", hash], cwd := depdir} + modify (·.insert dep.name depdir) + +def solveDepsCore (relPath : FilePath) (d : Manifest) : (maxDepth : Nat) → Solver Unit + | 0 => throw <| IO.userError "maximum dependency resolution depth reached" + | maxDepth + 1 => do + let deps ← d.dependencies.filterM (notYetAssigned ·.name) + deps.forM (materialize relPath) + for dep in deps do + let p ← resolvedPath dep.name + let d' ← Manifest.fromFile $ p / "leanpkg.toml" + unless d'.name = dep.name do + throw <| IO.userError s!"{d.name} (in {relPath}) depends on {d'.name}, but resolved dependency has name {dep.name} (in {p})" + solveDepsCore p d' maxDepth + +def constructPath (depname : String) (dirname : FilePath) : IO (FilePath × FilePath) := do + let path ← Manifest.effectivePath (← Manifest.fromFile <| dirname / leanpkgTomlFn) + (dirname, dirname / path) + +def solveDeps (d : Manifest) : IO (List (FilePath × FilePath)) := do + let (_, assg) ← (solveDepsCore ⟨"."⟩ d 1024).run <| Assignment.empty.insert d.name ⟨"."⟩ + assg.reverse.mapM fun ⟨depname, dirname⟩ => + if dirname == "." then + (dirname, dirname / d.effectivePath) + else + constructPath depname dirname diff --git a/Leanpkg2/Toml.lean b/Leanpkg2/Toml.lean new file mode 100644 index 0000000000..7337ef887b --- /dev/null +++ b/Leanpkg2/Toml.lean @@ -0,0 +1,72 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich +-/ +import Lean.Parser + +namespace Toml + +inductive Value : Type where + | str : String → Value + | nat : Nat → Value + | bool : Bool → Value + | table : List (String × Value) → Value + deriving Inhabited + +def Value.lookup : Value → String → Option Value + | Value.table cs, k => cs.lookup k + | _, _ => none + +-- TODO: custom whitespace and other inaccuracies +declare_syntax_cat val +syntax "True" : val +syntax "False" : val +syntax str : val +syntax num : val +syntax bareKey := ident -- TODO +syntax key := bareKey <|> str +declare_syntax_cat keyCat @[keyCatParser] def key' := key -- HACK: for the antiquotation +syntax keyVal := key " = " val +syntax table := "[" key "]" keyVal* +syntax inlineTable := "{" keyVal,* "}" +syntax inlineTable : val +syntax file := table* +declare_syntax_cat fileCat @[fileCatParser] def file' := file -- HACK: for the antiquotation + +open Lean + +partial def ofSyntax : Syntax → Value + | `(val|True) => Value.bool true + | `(val|False) => Value.bool false + | `(val|$s:strLit) => Value.str <| s.isStrLit?.get! + | `(val|$n:numLit) => Value.nat <| n.isNatLit?.get! + | `(val|{$[$keys:key = $values],*}) => toTable keys (values.map ofSyntax) + | `(fileCat|$[[$keys] $kvss*]*) => toTable keys <| kvss.map fun kvs => ofSyntax <| Lean.Unhygienic.run `(val|{$kvs,*}) + | stx => unreachable! + where + toKey : Syntax → String + | `(keyCat|$key:ident) => key.getId.toString + | `(keyCat|$key:strLit) => key.isStrLit?.get! + | _ => unreachable! + toTable (keys : Array Syntax) (vals : Array Value) : Value := + Value.table <| Array.toList <| keys.zipWith vals fun k v => (toKey k, v) + +open Lean.Parser + +def parse (input : String) : IO Value := do + -- HACKHACKHACK + let env ← importModules [{ module := `Leanpkg2.Toml }] {} + let fileParser ← compileParserDescr (parserExtension.getState env).categories file { env := env, opts := {} } + let c := mkParserContext (mkInputContext input "") { env := env, options := {} } + let s := mkParserState input + let s := whitespace c s + let s := fileParser.fn c s + if s.hasError then + throw <| IO.userError (s.toErrorMsg c) + else if input.atEnd s.pos then + ofSyntax s.stxStack.back + else + throw <| IO.userError ((s.mkError "end of input").toErrorMsg c) + +end Toml diff --git a/examples/hello/.gitignore b/examples/hello/.gitignore new file mode 100644 index 0000000000..2ea799ec8c --- /dev/null +++ b/examples/hello/.gitignore @@ -0,0 +1 @@ +/src/build diff --git a/examples/hello/Package.lean b/examples/hello/Package.lean new file mode 100644 index 0000000000..4ec8c7eca7 --- /dev/null +++ b/examples/hello/Package.lean @@ -0,0 +1,20 @@ +import Leanpkg2.Build +import Leanpkg2.Configure +import Leanpkg2.Manifest + +open Leanpkg2 + +def manifest : Manifest := { + name := "hello", + version := "1.0", + path := "src" +} + +def configure : IO Unit := + discard <| Leanpkg2.configure manifest + +def build : IO Unit := + Leanpkg2.build manifest ["bin"] + +def main : IO Unit := + build diff --git a/examples/hello/package.sh b/examples/hello/package.sh new file mode 100644 index 0000000000..8b9f18965e --- /dev/null +++ b/examples/hello/package.sh @@ -0,0 +1,2 @@ +export LEAN_PATH=../../build +lean --run Package.lean diff --git a/examples/hello/src/Hello.lean b/examples/hello/src/Hello.lean new file mode 100644 index 0000000000..690d791c80 --- /dev/null +++ b/examples/hello/src/Hello.lean @@ -0,0 +1,2 @@ +def main : IO Unit := + IO.println "Hello from Leanpkg2!" diff --git a/leanpkg.toml b/leanpkg.toml index 376b9c0d0e..30c2cced3c 100644 --- a/leanpkg.toml +++ b/leanpkg.toml @@ -1,4 +1,3 @@ [package] -name = "Leanpkg" +name = "leanpkg2" version = "0.1" -lean_version = "leanprover/lean4:nightly-2021-05-26" From 93732bc7db7101b6b064b1d632c5703fd6d1d1cf Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 29 May 2021 09:02:26 -0400 Subject: [PATCH 003/696] Add LICENSE --- LICENSE | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..4c3f8ab0fb --- /dev/null +++ b/LICENSE @@ -0,0 +1,70 @@ +Apache License 2.0 (Apache) +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + +1. You must give any other recipients of the Work or Derivative Works a copy of this License; and + +2. You must cause any modified files to carry prominent notices stating that You changed the files; and + +3. You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + +4. If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. From 84ae7466a42ff7044e8df1dbe8d6fd9a2ee9feff Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 29 May 2021 09:07:42 -0400 Subject: [PATCH 004/696] Update Authors --- Leanpkg2/Manifest.lean | 2 +- Leanpkg2/Resolve.lean | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Leanpkg2/Manifest.lean b/Leanpkg2/Manifest.lean index 3efa16cd6e..d052ff79b3 100644 --- a/Leanpkg2/Manifest.lean +++ b/Leanpkg2/Manifest.lean @@ -1,7 +1,7 @@ /- Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. -Authors: Gabriel Ebner, Sebastian Ullrich +Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Leanpkg2.Toml import Leanpkg2.LeanVersion diff --git a/Leanpkg2/Resolve.lean b/Leanpkg2/Resolve.lean index a127adb605..79a6d21cee 100644 --- a/Leanpkg2/Resolve.lean +++ b/Leanpkg2/Resolve.lean @@ -1,7 +1,7 @@ /- Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. -Authors: Gabriel Ebner, Sebastian Ullrich +Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Leanpkg2.Manifest import Leanpkg2.Proc From 82b368e838ebad6abc3f1a7f15cf46ad381daf50 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 29 May 2021 09:07:48 -0400 Subject: [PATCH 005/696] Add README --- README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000000..775a56c0b6 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Leanpkg2 + +An overhaul of Leanpkg for Lean 4 where the package configuration and build scripts are written in Lean. Currently just a prototype. A more detailed README will come when the project has matured. From ea9382643e3b00a7127c3dd9a0efd8728672b6ed Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 29 May 2021 11:27:31 -0400 Subject: [PATCH 006/696] Builds with `buildModules` now work as well --- Leanpkg2/Build.lean | 10 +++++----- Leanpkg2/BuildCore.lean | 13 ++++++++----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean index 07936f0c4e..31c04627fe 100644 --- a/Leanpkg2/Build.lean +++ b/Leanpkg2/Build.lean @@ -23,7 +23,7 @@ def buildImports (manifest : Manifest) (imports : List String) (leanArgs : List let oleans := localImports.map fun i => Lean.modToFilePath "build" i "olean" |>.toString execMake manifest oleans buildCfg else - Build.buildModules buildCfg localImports + Build.buildModules manifest buildCfg localImports IO.println cfg.leanPath IO.println cfg.leanSrcPath @@ -31,7 +31,7 @@ def build (manifest : Manifest) (makeArgs leanArgs : List String := []) : IO Uni let cfg ← configure manifest let root ← getRootPart <| manifest.effectivePath let buildCfg : Build.Config := { pkg := root, leanArgs, leanPath := cfg.leanPath, moreDeps := cfg.moreDeps } - --if makeArgs != [] || (← FilePath.pathExists "Makefile") then - execMake manifest makeArgs buildCfg - --else - -- Build.buildModules buildCfg [root] + if makeArgs != [] || (← FilePath.pathExists "Makefile") then + execMake manifest makeArgs buildCfg + else + Build.buildModules manifest buildCfg [root] diff --git a/Leanpkg2/BuildCore.lean b/Leanpkg2/BuildCore.lean index 2ea6b0dddb..622d27773d 100644 --- a/Leanpkg2/BuildCore.lean +++ b/Leanpkg2/BuildCore.lean @@ -5,6 +5,7 @@ Authors: Sebastian Ullrich -/ import Lean.Data.Name import Lean.Elab.Import +import Leanpkg2.Manifest import Leanpkg2.Proc open Lean @@ -25,6 +26,7 @@ structure Config where structure Context extends Config where parents : List Name := [] moreDepsMTime : IO.FS.SystemTime + manifest : Manifest deriving instance Inhabited for IO.FS.SystemTime deriving instance Inhabited for Task @@ -51,7 +53,8 @@ partial def buildModule (mod : Name) : BuildM Result := do -- already visited return r - let leanFile := modToFilePath "." mod "lean" + let srcPath := ctx.manifest.effectivePath + let leanFile := modToFilePath srcPath mod "lean" let leanMData ← leanFile.metadata -- recursively build dependencies and calculate transitive `maxMTime` @@ -62,7 +65,7 @@ partial def buildModule (mod : Name) : BuildM Result := do let maxMTime := List.maximum? (leanMData.modified :: ctx.moreDepsMTime :: depMTimes) |>.get! -- check whether we have an up-to-date .olean - let oleanFile := modToFilePath buildPath mod "olean" + let oleanFile := modToFilePath (srcPath / buildPath) mod "olean" try if (← oleanFile.metadata).modified >= maxMTime then let r := { maxMTime, task := Task.pure (Except.ok ()) } @@ -77,7 +80,7 @@ partial def buildModule (mod : Name) : BuildM Result := do -- propagate failure throw e try - let cFile := modToFilePath tempBuildPath mod "c" + let cFile := modToFilePath (srcPath / tempBuildPath) mod "c" IO.createDirAll oleanFile.parent.get! IO.createDirAll cFile.parent.get! execCmd { @@ -94,9 +97,9 @@ partial def buildModule (mod : Name) : BuildM Result := do modify fun st => { st with modTasks := st.modTasks.insert mod r } return r -def buildModules (cfg : Config) (mods : List Name) : IO Unit := do +def buildModules (manifest : Manifest) (cfg : Config) (mods : List Name) : IO Unit := do let moreDepsMTime := (← cfg.moreDeps.mapM (·.metadata)).map (·.modified) |>.maximum? |>.getD ⟨0, 0⟩ - let rs ← mods.mapM buildModule |>.run { toConfig := cfg, moreDepsMTime } |>.run' {} + let rs ← mods.mapM buildModule |>.run { toConfig := cfg, moreDepsMTime, manifest } |>.run' {} for r in rs do if let Except.error _ ← IO.wait r.task then -- actual error has already been printed above From 44b9ad2a30942c714fc63fc0aad16e497bf3bd38 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 29 May 2021 12:38:29 -0400 Subject: [PATCH 007/696] Assorted code cleanup and reogranization --- Leanpkg2/Build.lean | 108 +++++++++++++++++++++++++++++++++---- Leanpkg2/BuildConfig.lean | 24 +++++++++ Leanpkg2/BuildCore.lean | 106 ------------------------------------ Leanpkg2/Cli.lean | 4 +- Leanpkg2/Configure.lean | 12 ++--- Leanpkg2/Init.lean | 4 +- Leanpkg2/Make.lean | 6 +-- Leanpkg2/Manifest.lean | 64 +--------------------- Leanpkg2/Resolve.lean | 8 +-- Leanpkg2/TomlManifest.lean | 77 ++++++++++++++++++++++++++ 10 files changed, 218 insertions(+), 195 deletions(-) create mode 100644 Leanpkg2/BuildConfig.lean delete mode 100644 Leanpkg2/BuildCore.lean create mode 100644 Leanpkg2/TomlManifest.lean diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean index 31c04627fe..7cc2946c4d 100644 --- a/Leanpkg2/Build.lean +++ b/Leanpkg2/Build.lean @@ -3,35 +3,125 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ +import Lean.Data.Name +import Lean.Elab.Import import Leanpkg2.Resolve -import Leanpkg2.BuildCore +import Leanpkg2.Manifest +import Leanpkg2.BuildConfig import Leanpkg2.Configure import Leanpkg2.Make +import Leanpkg2.Proc -open System +open Lean System namespace Leanpkg2 -def buildImports (manifest : Manifest) (imports : List String) (leanArgs : List String) : IO Unit := do - let cfg ← configure manifest +structure BuildContext extends BuildConfig where + parents : List Name := [] + moreDepsMTime : IO.FS.SystemTime + manifest : Manifest + +deriving instance Inhabited for IO.FS.SystemTime +deriving instance Inhabited for Task + +structure Result where + maxMTime : IO.FS.SystemTime + task : Task (Except IO.Error Unit) + deriving Inhabited + +structure State where + modTasks : NameMap Result := ∅ + +abbrev BuildM := ReaderT BuildContext <| StateT State IO + +partial def buildModule (mod : Name) : BuildM Result := do + let ctx ← read + if ctx.parents.contains mod then + -- cyclic import + let cycle := ctx.parents.dropWhile (· != mod) ++ [mod] + let cycle := cycle.reverse.map (s!" {·}") + throw <| IO.userError s!"import cycle detected:\n{"\n".intercalate cycle}" + + if let some r := (← get).modTasks.find? mod then + -- already visited + return r + + let srcPath := ctx.manifest.effectivePath + let leanFile := modToFilePath srcPath mod "lean" + let leanMData ← leanFile.metadata + + -- recursively build dependencies and calculate transitive `maxMTime` + let (imports, _, _) ← Elab.parseImports (← IO.FS.readFile leanFile) leanFile.toString + let localImports := imports.filter (·.module.getRoot == ctx.pkg) + let deps ← localImports.mapM (buildModule ·.module) + let depMTimes ← deps.mapM (·.maxMTime) + let maxMTime := List.maximum? (leanMData.modified :: ctx.moreDepsMTime :: depMTimes) |>.get! + + -- check whether we have an up-to-date .olean + let oleanFile := modToFilePath (srcPath / buildPath) mod "olean" + try + if (← oleanFile.metadata).modified >= maxMTime then + let r := { maxMTime, task := Task.pure (Except.ok ()) } + modify fun st => { st with modTasks := st.modTasks.insert mod r } + return r + catch + | IO.Error.noFileOrDirectory .. => pure () + | e => throw e + + let task ← IO.mapTasks (tasks := deps.map (·.task)) fun rs => do + if let some e := rs.findSome? (fun | Except.error e => some e | Except.ok _ => none) then + -- propagate failure + throw e + try + let cFile := modToFilePath (srcPath / tempBuildPath) mod "c" + IO.createDirAll oleanFile.parent.get! + IO.createDirAll cFile.parent.get! + execCmd { + cmd := (← IO.appDir) / "lean" |>.withExtension FilePath.exeExtension |>.toString + args := ctx.leanArgs.toArray ++ #["-o", oleanFile.toString, "-c", cFile.toString, leanFile.toString] + env := #[("LEAN_PATH", ctx.leanPath)] + } + catch e => + -- print errors early + IO.eprintln e + throw e + + let r := { maxMTime, task := task } + modify fun st => { st with modTasks := st.modTasks.insert mod r } + return r + +def buildModules (manifest : Manifest) (cfg : BuildConfig) (mods : List Name) : IO Unit := do + let moreDepsMTime := (← cfg.moreDeps.mapM (·.metadata)).map (·.modified) |>.maximum? |>.getD ⟨0, 0⟩ + let rs ← mods.mapM buildModule |>.run { toBuildConfig := cfg, moreDepsMTime, manifest } |>.run' {} + for r in rs do + if let Except.error _ ← IO.wait r.task then + -- actual error has already been printed above + throw <| IO.userError "Build failed." + +def buildImports (manifest : Manifest) (cfg : Configuration) (imports leanArgs : List String := []) : IO Unit := do let imports := imports.map (·.toName) let root ← getRootPart <| manifest.effectivePath let localImports := imports.filter (·.getRoot == root) if localImports != [] then - let buildCfg : Build.Config := { pkg := root, leanArgs, leanPath := cfg.leanPath, moreDeps := cfg.moreDeps } + let buildCfg : BuildConfig := { pkg := root, leanArgs, leanPath := cfg.leanPath, moreDeps := cfg.moreDeps } if ← FilePath.pathExists "Makefile" then - let oleans := localImports.map fun i => Lean.modToFilePath "build" i "olean" |>.toString + let oleans := localImports.map fun i => + Lean.modToFilePath (manifest.effectivePath / buildPath) i "olean" |>.toString execMake manifest oleans buildCfg else - Build.buildModules manifest buildCfg localImports + buildModules manifest buildCfg localImports + +def printPaths (manifest : Manifest) (imports leanArgs : List String := []) : IO Unit := do + let cfg ← configure manifest + buildImports manifest cfg imports leanArgs IO.println cfg.leanPath IO.println cfg.leanSrcPath def build (manifest : Manifest) (makeArgs leanArgs : List String := []) : IO Unit := do let cfg ← configure manifest let root ← getRootPart <| manifest.effectivePath - let buildCfg : Build.Config := { pkg := root, leanArgs, leanPath := cfg.leanPath, moreDeps := cfg.moreDeps } + let buildCfg : BuildConfig := { pkg := root, leanArgs, leanPath := cfg.leanPath, moreDeps := cfg.moreDeps } if makeArgs != [] || (← FilePath.pathExists "Makefile") then execMake manifest makeArgs buildCfg else - Build.buildModules manifest buildCfg [root] + buildModules manifest buildCfg [root] diff --git a/Leanpkg2/BuildConfig.lean b/Leanpkg2/BuildConfig.lean new file mode 100644 index 0000000000..390de4d48a --- /dev/null +++ b/Leanpkg2/BuildConfig.lean @@ -0,0 +1,24 @@ +/- +Copyright (c) 2021 Sebastian Ullrich. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Sebastian Ullrich +-/ +import Lean.Data.Name +import Lean.Elab.Import +import Leanpkg2.Manifest +import Leanpkg2.Proc + +open Lean +open System + +namespace Leanpkg2 + +def buildPath : FilePath := "build" +def tempBuildPath := buildPath / "temp" + +structure BuildConfig where + pkg : Name + leanArgs : List String + leanPath : String + -- things like `leanpkg.toml` and olean roots of dependencies that should also trigger rebuilds + moreDeps : List FilePath diff --git a/Leanpkg2/BuildCore.lean b/Leanpkg2/BuildCore.lean deleted file mode 100644 index 622d27773d..0000000000 --- a/Leanpkg2/BuildCore.lean +++ /dev/null @@ -1,106 +0,0 @@ -/- -Copyright (c) 2021 Sebastian Ullrich. All rights reserved. -Released under Apache 2.0 license as described in the file LICENSE. -Authors: Sebastian Ullrich --/ -import Lean.Data.Name -import Lean.Elab.Import -import Leanpkg2.Manifest -import Leanpkg2.Proc - -open Lean -open System - -namespace Leanpkg2.Build - -def buildPath : FilePath := "build" -def tempBuildPath := buildPath / "temp" - -structure Config where - pkg : Name - leanArgs : List String - leanPath : String - -- things like `leanpkg.toml` and olean roots of dependencies that should also trigger rebuilds - moreDeps : List FilePath - -structure Context extends Config where - parents : List Name := [] - moreDepsMTime : IO.FS.SystemTime - manifest : Manifest - -deriving instance Inhabited for IO.FS.SystemTime -deriving instance Inhabited for Task - -structure Result where - maxMTime : IO.FS.SystemTime - task : Task (Except IO.Error Unit) - deriving Inhabited - -structure State where - modTasks : NameMap Result := ∅ - -abbrev BuildM := ReaderT Context <| StateT State IO - -partial def buildModule (mod : Name) : BuildM Result := do - let ctx ← read - if ctx.parents.contains mod then - -- cyclic import - let cycle := ctx.parents.dropWhile (· != mod) ++ [mod] - let cycle := cycle.reverse.map (s!" {·}") - throw <| IO.userError s!"import cycle detected:\n{"\n".intercalate cycle}" - - if let some r := (← get).modTasks.find? mod then - -- already visited - return r - - let srcPath := ctx.manifest.effectivePath - let leanFile := modToFilePath srcPath mod "lean" - let leanMData ← leanFile.metadata - - -- recursively build dependencies and calculate transitive `maxMTime` - let (imports, _, _) ← Elab.parseImports (← IO.FS.readFile leanFile) leanFile.toString - let localImports := imports.filter (·.module.getRoot == ctx.pkg) - let deps ← localImports.mapM (buildModule ·.module) - let depMTimes ← deps.mapM (·.maxMTime) - let maxMTime := List.maximum? (leanMData.modified :: ctx.moreDepsMTime :: depMTimes) |>.get! - - -- check whether we have an up-to-date .olean - let oleanFile := modToFilePath (srcPath / buildPath) mod "olean" - try - if (← oleanFile.metadata).modified >= maxMTime then - let r := { maxMTime, task := Task.pure (Except.ok ()) } - modify fun st => { st with modTasks := st.modTasks.insert mod r } - return r - catch - | IO.Error.noFileOrDirectory .. => pure () - | e => throw e - - let task ← IO.mapTasks (tasks := deps.map (·.task)) fun rs => do - if let some e := rs.findSome? (fun | Except.error e => some e | Except.ok _ => none) then - -- propagate failure - throw e - try - let cFile := modToFilePath (srcPath / tempBuildPath) mod "c" - IO.createDirAll oleanFile.parent.get! - IO.createDirAll cFile.parent.get! - execCmd { - cmd := (← IO.appDir) / "lean" |>.withExtension FilePath.exeExtension |>.toString - args := ctx.leanArgs.toArray ++ #["-o", oleanFile.toString, "-c", cFile.toString, leanFile.toString] - env := #[("LEAN_PATH", ctx.leanPath)] - } - catch e => - -- print errors early - IO.eprintln e - throw e - - let r := { maxMTime, task := task } - modify fun st => { st with modTasks := st.modTasks.insert mod r } - return r - -def buildModules (manifest : Manifest) (cfg : Config) (mods : List Name) : IO Unit := do - let moreDepsMTime := (← cfg.moreDeps.mapM (·.metadata)).map (·.modified) |>.maximum? |>.getD ⟨0, 0⟩ - let rs ← mods.mapM buildModule |>.run { toConfig := cfg, moreDepsMTime, manifest } |>.run' {} - for r in rs do - if let Except.error _ ← IO.wait r.task then - -- actual error has already been printed above - throw <| IO.userError "Build failed." diff --git a/Leanpkg2/Cli.lean b/Leanpkg2/Cli.lean index 9ddd491486..6f6014aa25 100644 --- a/Leanpkg2/Cli.lean +++ b/Leanpkg2/Cli.lean @@ -7,7 +7,7 @@ import Leanpkg2.Init import Leanpkg2.Configure import Leanpkg2.Make import Leanpkg2.Build -import Leanpkg2.Manifest +import Leanpkg2.TomlManifest namespace Leanpkg2 @@ -66,7 +66,7 @@ directory." def cli : (cmd : String) → (leanpkgArgs leanArgs : List String) → IO Unit | "init", [name], [] => init name | "configure", [], [] => discard do configure (← readManifest) -| "print-paths", leanpkgArgs, leanArgs => do buildImports (← readManifest) leanpkgArgs leanArgs +| "print-paths", imports, leanArgs => do printPaths (← readManifest) imports leanArgs | "build", makeArgs, leanArgs => do build (← readManifest) makeArgs leanArgs | "help", ["init"], [] => IO.println helpInit | "help", ["configure"], [] => IO.println helpConfigure diff --git a/Leanpkg2/Configure.lean b/Leanpkg2/Configure.lean index 397b5cc5be..d8e611e551 100644 --- a/Leanpkg2/Configure.lean +++ b/Leanpkg2/Configure.lean @@ -3,7 +3,7 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ -import Leanpkg2.BuildCore +import Leanpkg2.BuildConfig import Leanpkg2.Resolve open System @@ -27,19 +27,19 @@ def configure (manifest : Manifest) : IO Configuration := do "configuring " ++ manifest.name ++ " " ++ manifest.version let paths ← solveDeps manifest let mut moreDeps := [] - for (pkgpath, srcpath) in paths do - unless pkgpath == "." do + for (pkgPath, srcPath) in paths do + unless pkgPath == "." do -- build recursively -- TODO: share build of common dependencies execCmd { cmd := (← IO.appPath).toString - cwd := pkgpath + cwd := pkgPath args := #["build"] } - let pkgRoot := srcpath / Build.buildPath / (← getRootPart srcpath).toString + let pkgRoot := srcPath / buildPath / (← getRootPart srcPath).toString moreDeps := pkgRoot.withExtension "olean" :: moreDeps return { - leanPath := SearchPath.toString <| paths.map (·.2 / Build.buildPath) + leanPath := SearchPath.toString <| paths.map (·.2 / buildPath) leanSrcPath := SearchPath.toString <| paths.map (·.2) moreDeps } diff --git a/Leanpkg2/Init.lean b/Leanpkg2/Init.lean index 41c772a120..bf26c46b1e 100644 --- a/Leanpkg2/Init.lean +++ b/Leanpkg2/Init.lean @@ -5,7 +5,7 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Leanpkg2.Git import Leanpkg2.Proc -import Leanpkg2.Manifest +import Leanpkg2.TomlManifest namespace Leanpkg2 @@ -26,7 +26,7 @@ lean_version = \"{leanVersionString}\" " def initPkg (pkgName : String) (fromNew : Bool) : IO Unit := do - IO.FS.writeFile leanpkgTomlFn (leanpkgFileContents pkgName) + IO.FS.writeFile leanpkgToml (leanpkgFileContents pkgName) IO.FS.writeFile ⟨s!"{pkgName.capitalize}.lean"⟩ mainFileContents let h ← IO.FS.Handle.mk ⟨".gitignore"⟩ IO.FS.Mode.append (bin := false) h.putStr initGitignoreContents diff --git a/Leanpkg2/Make.lean b/Leanpkg2/Make.lean index aceda30ccb..a83aba726e 100644 --- a/Leanpkg2/Make.lean +++ b/Leanpkg2/Make.lean @@ -4,13 +4,13 @@ Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Leanpkg2.Manifest -import Leanpkg2.BuildCore +import Leanpkg2.BuildConfig open System namespace Leanpkg2 -def lockFileName : System.FilePath := ⟨".leanpkg-lock"⟩ +def lockFileName : System.FilePath := ".leanpkg-lock" partial def withLockFile (x : IO α) : IO α := do acquire @@ -39,7 +39,7 @@ partial def withLockFile (x : IO α) : IO α := do acquire (firstTime := false) | e => throw e -def execMake (manifest : Manifest) (makeArgs : List String) (cfg : Build.Config) : IO Unit := withLockFile do +def execMake (manifest : Manifest) (makeArgs : List String) (cfg : BuildConfig) : IO Unit := withLockFile do let timeoutArgs := match manifest.timeout with | some t => ["-T", toString t] diff --git a/Leanpkg2/Manifest.lean b/Leanpkg2/Manifest.lean index d052ff79b3..6353d3dff5 100644 --- a/Leanpkg2/Manifest.lean +++ b/Leanpkg2/Manifest.lean @@ -3,7 +3,6 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ -import Leanpkg2.Toml import Leanpkg2.LeanVersion open System @@ -11,30 +10,9 @@ open System namespace Leanpkg2 inductive Source where - | path (dir : System.FilePath) : Source + | path (dir : FilePath) : Source | git (url rev : String) (branch : Option String) : Source -namespace Source - -def fromToml (v : Toml.Value) : Option Source := - (do let Toml.Value.str dir ← v.lookup "path" | none - path ⟨dir⟩) <|> - (do let Toml.Value.str url ← v.lookup "git" | none - let Toml.Value.str rev ← v.lookup "rev" | none - match v.lookup "branch" with - | none => git url rev none - | some (Toml.Value.str branch) => git url rev (some branch) - | _ => none) - -def toToml : Source → Toml.Value - | path dir => Toml.Value.table [("path", Toml.Value.str dir.toString)] - | git url rev none => - Toml.Value.table [("git", Toml.Value.str url), ("rev", Toml.Value.str rev)] - | git url rev (some branch) => - Toml.Value.table [("git", Toml.Value.str url), ("branch", Toml.Value.str branch), ("rev", Toml.Value.str rev)] - -end Source - structure Dependency where name : String src : Source @@ -52,44 +30,4 @@ namespace Manifest def effectivePath (m : Manifest) : FilePath := m.path.getD ⟨"."⟩ -def fromToml (t : Toml.Value) : Option Manifest := OptionM.run do - let pkg ← t.lookup "package" - let Toml.Value.str n ← pkg.lookup "name" | none - let Toml.Value.str ver ← pkg.lookup "version" | none - let leanVer ← match pkg.lookup "lean_version" with - | some (Toml.Value.str leanVer) => some leanVer - | none => some leanVersionString - | _ => none - let tm ← match pkg.lookup "timeout" with - | some (Toml.Value.nat timeout) => some (some timeout) - | none => some none - | _ => none - let path ← match pkg.lookup "path" with - | some (Toml.Value.str path) => some (some ⟨path⟩) - | none => some none - | _ => none - let Toml.Value.table deps ← t.lookup "dependencies" <|> some (Toml.Value.table []) | none - let deps ← deps.mapM fun ⟨n, src⟩ => do Dependency.mk n (← Source.fromToml src) - return { name := n, version := ver, leanVersion := leanVer, - path := path, dependencies := deps, timeout := tm } - -def fromFile (fn : System.FilePath) : IO Manifest := do - let cnts ← IO.FS.readFile fn - let toml ← Toml.parse cnts - let some manifest ← pure (fromToml toml) - | throw <| IO.userError s!"cannot read manifest from {fn}" - manifest - end Manifest - -def leanpkgTomlFn : System.FilePath := ⟨"leanpkg.toml"⟩ - -def readManifest : IO Manifest := do - let m ← Manifest.fromFile leanpkgTomlFn - if m.leanVersion ≠ leanVersionString then - IO.eprintln $ "\nWARNING: Lean version mismatch: installed version is " ++ leanVersionString - ++ ", but package requires " ++ m.leanVersion ++ "\n" - return m - -def writeManifest (manifest : Lean.Syntax) (fn : FilePath) : IO Unit := do - IO.FS.writeFile fn manifest.reprint.get! diff --git a/Leanpkg2/Resolve.lean b/Leanpkg2/Resolve.lean index 79a6d21cee..5af57d81a4 100644 --- a/Leanpkg2/Resolve.lean +++ b/Leanpkg2/Resolve.lean @@ -3,9 +3,9 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ -import Leanpkg2.Manifest -import Leanpkg2.Proc import Leanpkg2.Git +import Leanpkg2.Proc +import Leanpkg2.TomlManifest open System @@ -65,13 +65,13 @@ def solveDepsCore (relPath : FilePath) (d : Manifest) : (maxDepth : Nat) → Sol deps.forM (materialize relPath) for dep in deps do let p ← resolvedPath dep.name - let d' ← Manifest.fromFile $ p / "leanpkg.toml" + let d' ← Manifest.fromTomlFile <| p / "leanpkg.toml" unless d'.name = dep.name do throw <| IO.userError s!"{d.name} (in {relPath}) depends on {d'.name}, but resolved dependency has name {dep.name} (in {p})" solveDepsCore p d' maxDepth def constructPath (depname : String) (dirname : FilePath) : IO (FilePath × FilePath) := do - let path ← Manifest.effectivePath (← Manifest.fromFile <| dirname / leanpkgTomlFn) + let path ← Manifest.effectivePath (← Manifest.fromTomlFile <| dirname / leanpkgToml) (dirname, dirname / path) def solveDeps (d : Manifest) : IO (List (FilePath × FilePath)) := do diff --git a/Leanpkg2/TomlManifest.lean b/Leanpkg2/TomlManifest.lean new file mode 100644 index 0000000000..43a1965fbe --- /dev/null +++ b/Leanpkg2/TomlManifest.lean @@ -0,0 +1,77 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone +-/ +import Leanpkg2.Toml +import Leanpkg2.LeanVersion +import Leanpkg2.Manifest + +open System + +namespace Leanpkg2 + +namespace Source + +def fromToml (v : Toml.Value) : Option Source := + (do let Toml.Value.str dir ← v.lookup "path" | none + path ⟨dir⟩) <|> + (do let Toml.Value.str url ← v.lookup "git" | none + let Toml.Value.str rev ← v.lookup "rev" | none + match v.lookup "branch" with + | none => git url rev none + | some (Toml.Value.str branch) => git url rev (some branch) + | _ => none) + +def toToml : Source → Toml.Value + | path dir => Toml.Value.table [("path", Toml.Value.str dir.toString)] + | git url rev none => + Toml.Value.table [("git", Toml.Value.str url), ("rev", Toml.Value.str rev)] + | git url rev (some branch) => + Toml.Value.table [("git", Toml.Value.str url), ("branch", Toml.Value.str branch), ("rev", Toml.Value.str rev)] + +end Source + +namespace Manifest + +def fromToml (t : Toml.Value) : Option Manifest := OptionM.run do + let pkg ← t.lookup "package" + let Toml.Value.str n ← pkg.lookup "name" | none + let Toml.Value.str ver ← pkg.lookup "version" | none + let leanVer ← match pkg.lookup "lean_version" with + | some (Toml.Value.str leanVer) => some leanVer + | none => some leanVersionString + | _ => none + let tm ← match pkg.lookup "timeout" with + | some (Toml.Value.nat timeout) => some (some timeout) + | none => some none + | _ => none + let path ← match pkg.lookup "path" with + | some (Toml.Value.str path) => some (some ⟨path⟩) + | none => some none + | _ => none + let Toml.Value.table deps ← t.lookup "dependencies" <|> some (Toml.Value.table []) | none + let deps ← deps.mapM fun ⟨n, src⟩ => do Dependency.mk n (← Source.fromToml src) + return { name := n, version := ver, leanVersion := leanVer, + path := path, dependencies := deps, timeout := tm } + +def fromTomlFile (fn : System.FilePath) : IO Manifest := do + let cnts ← IO.FS.readFile fn + let toml ← Toml.parse cnts + let some manifest ← pure (fromToml toml) + | throw <| IO.userError s!"cannot read manifest from {fn}" + manifest + +end Manifest + +def leanpkgToml : System.FilePath := "leanpkg.toml" + +def readManifest : IO Manifest := do + let m ← Manifest.fromTomlFile leanpkgToml + if m.leanVersion ≠ leanVersionString then + IO.eprintln $ "\nWARNING: Lean version mismatch: installed version is " ++ leanVersionString + ++ ", but package requires " ++ m.leanVersion ++ "\n" + return m + +def writeManifest (manifest : Lean.Syntax) (fn : FilePath) : IO Unit := do + IO.FS.writeFile fn manifest.reprint.get! From ef0135ac9e57bccaa14370509d9b2238d504436c Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 29 May 2021 21:29:33 -0400 Subject: [PATCH 008/696] Add working example expackage with a (local) dependency --- Leanpkg2/Build.lean | 8 ++++---- Leanpkg2/BuildConfig.lean | 10 ++++++++++ Leanpkg2/Configure.lean | 29 ++++++++++------------------- Leanpkg2/Manifest.lean | 18 +++++++++++++++++- Leanpkg2/Resolve.lean | 16 ++++++---------- Leanpkg2/TomlManifest.lean | 16 +++++++++------- examples/hello/.gitignore | 2 +- examples/hello/{src => }/Hello.lean | 0 examples/hello/Package.lean | 1 - examples/helloDeps/a/.gitignore | 1 + examples/helloDeps/a/A.lean | 1 + examples/helloDeps/a/Package.lean | 16 ++++++++++++++++ examples/helloDeps/a/leanpkg.toml | 3 +++ examples/helloDeps/b/.gitignore | 1 + examples/helloDeps/b/B.lean | 6 ++++++ examples/helloDeps/b/B/Bar.lean | 3 +++ examples/helloDeps/b/B/Baz.lean | 3 +++ examples/helloDeps/b/B/Foo.lean | 2 ++ examples/helloDeps/b/Package.lean | 19 +++++++++++++++++++ examples/helloDeps/package.sh | 3 +++ 20 files changed, 115 insertions(+), 43 deletions(-) rename examples/hello/{src => }/Hello.lean (100%) create mode 100644 examples/helloDeps/a/.gitignore create mode 100644 examples/helloDeps/a/A.lean create mode 100644 examples/helloDeps/a/Package.lean create mode 100644 examples/helloDeps/a/leanpkg.toml create mode 100644 examples/helloDeps/b/.gitignore create mode 100644 examples/helloDeps/b/B.lean create mode 100644 examples/helloDeps/b/B/Bar.lean create mode 100644 examples/helloDeps/b/B/Baz.lean create mode 100644 examples/helloDeps/b/B/Foo.lean create mode 100644 examples/helloDeps/b/Package.lean create mode 100644 examples/helloDeps/package.sh diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean index 7cc2946c4d..69bf566942 100644 --- a/Leanpkg2/Build.lean +++ b/Leanpkg2/Build.lean @@ -99,8 +99,8 @@ def buildModules (manifest : Manifest) (cfg : BuildConfig) (mods : List Name) : throw <| IO.userError "Build failed." def buildImports (manifest : Manifest) (cfg : Configuration) (imports leanArgs : List String := []) : IO Unit := do + let root : Name := manifest.module let imports := imports.map (·.toName) - let root ← getRootPart <| manifest.effectivePath let localImports := imports.filter (·.getRoot == root) if localImports != [] then let buildCfg : BuildConfig := { pkg := root, leanArgs, leanPath := cfg.leanPath, moreDeps := cfg.moreDeps } @@ -119,9 +119,9 @@ def printPaths (manifest : Manifest) (imports leanArgs : List String := []) : IO def build (manifest : Manifest) (makeArgs leanArgs : List String := []) : IO Unit := do let cfg ← configure manifest - let root ← getRootPart <| manifest.effectivePath - let buildCfg : BuildConfig := { pkg := root, leanArgs, leanPath := cfg.leanPath, moreDeps := cfg.moreDeps } + IO.eprintln $ "building " ++ manifest.name ++ " " ++ manifest.version + let buildCfg : BuildConfig := { pkg := manifest.module, leanArgs, leanPath := cfg.leanPath, moreDeps := cfg.moreDeps } if makeArgs != [] || (← FilePath.pathExists "Makefile") then execMake manifest makeArgs buildCfg else - buildModules manifest buildCfg [root] + buildModules manifest buildCfg [manifest.module] diff --git a/Leanpkg2/BuildConfig.lean b/Leanpkg2/BuildConfig.lean index 390de4d48a..94e89cb355 100644 --- a/Leanpkg2/BuildConfig.lean +++ b/Leanpkg2/BuildConfig.lean @@ -16,6 +16,16 @@ namespace Leanpkg2 def buildPath : FilePath := "build" def tempBuildPath := buildPath / "temp" +namespace Package + +def buildDir (self : Package) : FilePath := + self.dir / Leanpkg2.buildPath + +def buildRoot (self : Package) : FilePath := + self.buildDir / self.manifest.module + +end Package + structure BuildConfig where pkg : Name leanArgs : List String diff --git a/Leanpkg2/Configure.lean b/Leanpkg2/Configure.lean index d8e611e551..7c324fa4d3 100644 --- a/Leanpkg2/Configure.lean +++ b/Leanpkg2/Configure.lean @@ -10,36 +10,27 @@ open System namespace Leanpkg2 -def getRootPart (pkg : FilePath := ".") : IO Lean.Name := do - let entries ← pkg.readDir - match entries.filter (FilePath.extension ·.fileName == "lean") with - | #[rootFile] => FilePath.withExtension rootFile.fileName "" |>.toString - | #[] => throw <| IO.userError s!"no '.lean' file found in {← IO.realPath "."}" - | _ => throw <| IO.userError s!"{← IO.realPath "."} must contain a unique '.lean' file as the package root" - structure Configuration := leanPath : String leanSrcPath : String moreDeps : List FilePath def configure (manifest : Manifest) : IO Configuration := do - IO.eprintln $ - "configuring " ++ manifest.name ++ " " ++ manifest.version - let paths ← solveDeps manifest + IO.eprintln $ "configuring " ++ manifest.name ++ " " ++ manifest.version + let pkgs ← solveDeps manifest let mut moreDeps := [] - for (pkgPath, srcPath) in paths do - unless pkgPath == "." do + for pkg in pkgs do + unless pkg.dir.toString == "." do -- build recursively -- TODO: share build of common dependencies execCmd { - cmd := (← IO.appPath).toString - cwd := pkgPath - args := #["build"] + cwd := pkg.dir + cmd := (← IO.appDir) / "lean" |>.toString + args := #["--run", "Package.lean"] } - let pkgRoot := srcPath / buildPath / (← getRootPart srcPath).toString - moreDeps := pkgRoot.withExtension "olean" :: moreDeps + moreDeps := pkg.buildRoot.withExtension "olean" :: moreDeps return { - leanPath := SearchPath.toString <| paths.map (·.2 / buildPath) - leanSrcPath := SearchPath.toString <| paths.map (·.2) + leanPath := SearchPath.toString <| pkgs.map (·.buildDir) + leanSrcPath := SearchPath.toString <| pkgs.map (·.sourceDir) moreDeps } diff --git a/Leanpkg2/Manifest.lean b/Leanpkg2/Manifest.lean index 6353d3dff5..850203fc96 100644 --- a/Leanpkg2/Manifest.lean +++ b/Leanpkg2/Manifest.lean @@ -3,9 +3,10 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ +import Lean.Data.Name import Leanpkg2.LeanVersion -open System +open Lean System namespace Leanpkg2 @@ -23,6 +24,7 @@ structure Manifest where leanVersion : String := leanVersionString timeout : Option Nat := none path : Option FilePath := none + module : String := name.capitalize dependencies : List Dependency := [] namespace Manifest @@ -31,3 +33,17 @@ def effectivePath (m : Manifest) : FilePath := m.path.getD ⟨"."⟩ end Manifest + +structure Package where + dir : FilePath + manifest : Manifest + +namespace Package + +def sourceDir (self : Package) : FilePath := + self.dir / self.manifest.effectivePath + +def sourceRoot (self : Package) : FilePath := + self.sourceDir / self.manifest.module + +end Package diff --git a/Leanpkg2/Resolve.lean b/Leanpkg2/Resolve.lean index 5af57d81a4..b906ab06d5 100644 --- a/Leanpkg2/Resolve.lean +++ b/Leanpkg2/Resolve.lean @@ -65,19 +65,15 @@ def solveDepsCore (relPath : FilePath) (d : Manifest) : (maxDepth : Nat) → Sol deps.forM (materialize relPath) for dep in deps do let p ← resolvedPath dep.name - let d' ← Manifest.fromTomlFile <| p / "leanpkg.toml" + let d' ← Manifest.fromTomlFile <| p / leanpkgToml unless d'.name = dep.name do throw <| IO.userError s!"{d.name} (in {relPath}) depends on {d'.name}, but resolved dependency has name {dep.name} (in {p})" solveDepsCore p d' maxDepth -def constructPath (depname : String) (dirname : FilePath) : IO (FilePath × FilePath) := do - let path ← Manifest.effectivePath (← Manifest.fromTomlFile <| dirname / leanpkgToml) - (dirname, dirname / path) - -def solveDeps (d : Manifest) : IO (List (FilePath × FilePath)) := do - let (_, assg) ← (solveDepsCore ⟨"."⟩ d 1024).run <| Assignment.empty.insert d.name ⟨"."⟩ - assg.reverse.mapM fun ⟨depname, dirname⟩ => +def solveDeps (m : Manifest) : IO (List Package) := do + let (_, assg) ← (solveDepsCore ⟨"."⟩ m 1024).run <| Assignment.empty.insert m.name ⟨"."⟩ + assg.reverse.mapM fun ⟨depname, dirname⟩ => do if dirname == "." then - (dirname, dirname / d.effectivePath) + return ⟨ dirname, m ⟩ else - constructPath depname dirname + return ⟨ dirname, ← Manifest.fromTomlFile <| dirname / leanpkgToml ⟩ diff --git a/Leanpkg2/TomlManifest.lean b/Leanpkg2/TomlManifest.lean index 43a1965fbe..4e04111987 100644 --- a/Leanpkg2/TomlManifest.lean +++ b/Leanpkg2/TomlManifest.lean @@ -36,13 +36,16 @@ namespace Manifest def fromToml (t : Toml.Value) : Option Manifest := OptionM.run do let pkg ← t.lookup "package" - let Toml.Value.str n ← pkg.lookup "name" | none - let Toml.Value.str ver ← pkg.lookup "version" | none - let leanVer ← match pkg.lookup "lean_version" with + let Toml.Value.str name ← pkg.lookup "name" | none + let Toml.Value.str version ← pkg.lookup "version" | none + let module := match pkg.lookup "module" with + | some (Toml.Value.str mod) => mod + | _ => name.capitalize + let leanVersion ← match pkg.lookup "lean_version" with | some (Toml.Value.str leanVer) => some leanVer | none => some leanVersionString | _ => none - let tm ← match pkg.lookup "timeout" with + let timeout ← match pkg.lookup "timeout" with | some (Toml.Value.nat timeout) => some (some timeout) | none => some none | _ => none @@ -51,9 +54,8 @@ def fromToml (t : Toml.Value) : Option Manifest := OptionM.run do | none => some none | _ => none let Toml.Value.table deps ← t.lookup "dependencies" <|> some (Toml.Value.table []) | none - let deps ← deps.mapM fun ⟨n, src⟩ => do Dependency.mk n (← Source.fromToml src) - return { name := n, version := ver, leanVersion := leanVer, - path := path, dependencies := deps, timeout := tm } + let dependencies ← deps.mapM fun ⟨n, src⟩ => do Dependency.mk n (← Source.fromToml src) + return { name, module, version, leanVersion, path, dependencies, timeout } def fromTomlFile (fn : System.FilePath) : IO Manifest := do let cnts ← IO.FS.readFile fn diff --git a/examples/hello/.gitignore b/examples/hello/.gitignore index 2ea799ec8c..796b96d1c4 100644 --- a/examples/hello/.gitignore +++ b/examples/hello/.gitignore @@ -1 +1 @@ -/src/build +/build diff --git a/examples/hello/src/Hello.lean b/examples/hello/Hello.lean similarity index 100% rename from examples/hello/src/Hello.lean rename to examples/hello/Hello.lean diff --git a/examples/hello/Package.lean b/examples/hello/Package.lean index 4ec8c7eca7..0329c6feda 100644 --- a/examples/hello/Package.lean +++ b/examples/hello/Package.lean @@ -7,7 +7,6 @@ open Leanpkg2 def manifest : Manifest := { name := "hello", version := "1.0", - path := "src" } def configure : IO Unit := diff --git a/examples/helloDeps/a/.gitignore b/examples/helloDeps/a/.gitignore new file mode 100644 index 0000000000..796b96d1c4 --- /dev/null +++ b/examples/helloDeps/a/.gitignore @@ -0,0 +1 @@ +/build diff --git a/examples/helloDeps/a/A.lean b/examples/helloDeps/a/A.lean new file mode 100644 index 0000000000..7ee5f37600 --- /dev/null +++ b/examples/helloDeps/a/A.lean @@ -0,0 +1 @@ +def name := "world" diff --git a/examples/helloDeps/a/Package.lean b/examples/helloDeps/a/Package.lean new file mode 100644 index 0000000000..0db27b9bd5 --- /dev/null +++ b/examples/helloDeps/a/Package.lean @@ -0,0 +1,16 @@ +import Leanpkg2.Build +import Leanpkg2.Configure +import Leanpkg2.Manifest + +open Leanpkg2 + +def manifest : Manifest := { + name := "a", + version := "1.0", +} + +def build : IO Unit := + Leanpkg2.build manifest ["lib"] + +def main : IO Unit := + build diff --git a/examples/helloDeps/a/leanpkg.toml b/examples/helloDeps/a/leanpkg.toml new file mode 100644 index 0000000000..123e4c53fb --- /dev/null +++ b/examples/helloDeps/a/leanpkg.toml @@ -0,0 +1,3 @@ +[package] +name = "a" +version = "0.1" diff --git a/examples/helloDeps/b/.gitignore b/examples/helloDeps/b/.gitignore new file mode 100644 index 0000000000..796b96d1c4 --- /dev/null +++ b/examples/helloDeps/b/.gitignore @@ -0,0 +1 @@ +/build diff --git a/examples/helloDeps/b/B.lean b/examples/helloDeps/b/B.lean new file mode 100644 index 0000000000..fc5594b7a6 --- /dev/null +++ b/examples/helloDeps/b/B.lean @@ -0,0 +1,6 @@ +import A +import B.Bar +import B.Baz + +def main : IO Unit := + IO.println s!"Hello, {foo} {name}!" diff --git a/examples/helloDeps/b/B/Bar.lean b/examples/helloDeps/b/B/Bar.lean new file mode 100644 index 0000000000..7ca89339f6 --- /dev/null +++ b/examples/helloDeps/b/B/Bar.lean @@ -0,0 +1,3 @@ +import B.Foo + +def bar := "bar" diff --git a/examples/helloDeps/b/B/Baz.lean b/examples/helloDeps/b/B/Baz.lean new file mode 100644 index 0000000000..4823066822 --- /dev/null +++ b/examples/helloDeps/b/B/Baz.lean @@ -0,0 +1,3 @@ +import B.Foo + +def baz := "baz" diff --git a/examples/helloDeps/b/B/Foo.lean b/examples/helloDeps/b/B/Foo.lean new file mode 100644 index 0000000000..060b281a84 --- /dev/null +++ b/examples/helloDeps/b/B/Foo.lean @@ -0,0 +1,2 @@ +def foo := "foo" +--#eval IO.sleep 3000 diff --git a/examples/helloDeps/b/Package.lean b/examples/helloDeps/b/Package.lean new file mode 100644 index 0000000000..f71e4a0d2e --- /dev/null +++ b/examples/helloDeps/b/Package.lean @@ -0,0 +1,19 @@ +import Leanpkg2.Build +import Leanpkg2.Configure +import Leanpkg2.Manifest + +open Leanpkg2 System + +def manifest : Manifest := { + name := "b", + version := "1.0", + dependencies := [ + { name := "a", src := Source.path (FilePath.mk ".." / "a") } + ] +} + +def build : IO Unit := do + Leanpkg2.build manifest ["bin", "LINK_OPTS=../a/build/lib/libA.a"] + +def main : IO Unit := + build diff --git a/examples/helloDeps/package.sh b/examples/helloDeps/package.sh new file mode 100644 index 0000000000..0af6f92968 --- /dev/null +++ b/examples/helloDeps/package.sh @@ -0,0 +1,3 @@ +cd b +export LEAN_PATH=../../../build +lean --run Package.lean From 543656c24b6c5e0b3ec9b855705f6eb3aa49d3ad Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 29 May 2021 22:57:24 -0400 Subject: [PATCH 009/696] Merge Configure into Build --- Leanpkg2.lean | 13 +++++++--- Leanpkg2/Build.lean | 43 ++++++++++++++++++++----------- Leanpkg2/BuildConfig.lean | 12 ++++++++- Leanpkg2/Cli.lean | 4 +-- Leanpkg2/Configure.lean | 36 -------------------------- Leanpkg2/Make.lean | 2 +- examples/hello/Package.lean | 3 +-- examples/helloDeps/a/Package.lean | 1 - examples/helloDeps/b/Package.lean | 1 - 9 files changed, 51 insertions(+), 64 deletions(-) delete mode 100644 Leanpkg2/Configure.lean diff --git a/Leanpkg2.lean b/Leanpkg2.lean index dfe5c12f3a..1c650d078c 100644 --- a/Leanpkg2.lean +++ b/Leanpkg2.lean @@ -5,7 +5,12 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Leanpkg2.Cli -def main (args : List String) : IO Unit := do - Lean.initSearchPath none -- HACK - let (cmd, outerArgs, innerArgs) ← Leanpkg2.splitCmdlineArgs args - Leanpkg2.cli cmd outerArgs innerArgs +def main (args : List String) : IO UInt32 := do + try + Lean.initSearchPath none -- HACK + let (cmd, outerArgs, innerArgs) ← Leanpkg2.splitCmdlineArgs args + Leanpkg2.cli cmd outerArgs innerArgs + pure 0 + catch e => + IO.eprintln e -- avoid "uncaught exception: ..." + pure 1 diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean index 69bf566942..42203bfe77 100644 --- a/Leanpkg2/Build.lean +++ b/Leanpkg2/Build.lean @@ -8,7 +8,6 @@ import Lean.Elab.Import import Leanpkg2.Resolve import Leanpkg2.Manifest import Leanpkg2.BuildConfig -import Leanpkg2.Configure import Leanpkg2.Make import Leanpkg2.Proc @@ -52,7 +51,7 @@ partial def buildModule (mod : Name) : BuildM Result := do -- recursively build dependencies and calculate transitive `maxMTime` let (imports, _, _) ← Elab.parseImports (← IO.FS.readFile leanFile) leanFile.toString - let localImports := imports.filter (·.module.getRoot == ctx.pkg) + let localImports := imports.filter (·.module.getRoot == ctx.module) let deps ← localImports.mapM (buildModule ·.module) let depMTimes ← deps.mapM (·.maxMTime) let maxMTime := List.maximum? (leanMData.modified :: ctx.moreDepsMTime :: depMTimes) |>.get! @@ -98,30 +97,44 @@ def buildModules (manifest : Manifest) (cfg : BuildConfig) (mods : List Name) : -- actual error has already been printed above throw <| IO.userError "Build failed." -def buildImports (manifest : Manifest) (cfg : Configuration) (imports leanArgs : List String := []) : IO Unit := do - let root : Name := manifest.module +def buildImports (manifest : Manifest) (cfg : BuildConfig) (imports leanArgs : List String := []) : IO Unit := do let imports := imports.map (·.toName) - let localImports := imports.filter (·.getRoot == root) + let localImports := imports.filter (·.getRoot == cfg.module) if localImports != [] then - let buildCfg : BuildConfig := { pkg := root, leanArgs, leanPath := cfg.leanPath, moreDeps := cfg.moreDeps } if ← FilePath.pathExists "Makefile" then let oleans := localImports.map fun i => Lean.modToFilePath (manifest.effectivePath / buildPath) i "olean" |>.toString - execMake manifest oleans buildCfg + execMake manifest oleans cfg else - buildModules manifest buildCfg localImports + buildModules manifest cfg localImports + +def buildDeps (manifest : Manifest) : IO (List Package) := do + let pkgs ← solveDeps manifest + for pkg in pkgs do + unless pkg.dir.toString == "." do + -- build recursively + -- TODO: share build of common dependencies + execCmd { + cwd := pkg.dir + cmd := (← IO.appDir) / "lean" |>.toString + args := #["--run", "Package.lean"] + } + return pkgs + +def configure (manifest : Manifest) : IO Unit := do + discard <| buildDeps manifest def printPaths (manifest : Manifest) (imports leanArgs : List String := []) : IO Unit := do - let cfg ← configure manifest + let pkgs ← buildDeps manifest + let cfg := BuildConfig.fromPackages manifest.module leanArgs pkgs buildImports manifest cfg imports leanArgs IO.println cfg.leanPath - IO.println cfg.leanSrcPath + IO.println <| SearchPath.toString <| pkgs.map (·.sourceDir) def build (manifest : Manifest) (makeArgs leanArgs : List String := []) : IO Unit := do - let cfg ← configure manifest - IO.eprintln $ "building " ++ manifest.name ++ " " ++ manifest.version - let buildCfg : BuildConfig := { pkg := manifest.module, leanArgs, leanPath := cfg.leanPath, moreDeps := cfg.moreDeps } + let pkgs ← buildDeps manifest + let cfg := BuildConfig.fromPackages manifest.module leanArgs pkgs if makeArgs != [] || (← FilePath.pathExists "Makefile") then - execMake manifest makeArgs buildCfg + execMake manifest makeArgs cfg else - buildModules manifest buildCfg [manifest.module] + buildModules manifest cfg [manifest.module] diff --git a/Leanpkg2/BuildConfig.lean b/Leanpkg2/BuildConfig.lean index 94e89cb355..6e123cb813 100644 --- a/Leanpkg2/BuildConfig.lean +++ b/Leanpkg2/BuildConfig.lean @@ -27,8 +27,18 @@ def buildRoot (self : Package) : FilePath := end Package structure BuildConfig where - pkg : Name + module : Name leanArgs : List String leanPath : String -- things like `leanpkg.toml` and olean roots of dependencies that should also trigger rebuilds moreDeps : List FilePath + +namespace BuildConfig + +def fromPackages (module : Name) (leanArgs : List String) (pkgs : List Package) : BuildConfig := { + module, leanArgs, + leanPath := SearchPath.toString <| pkgs.map (·.buildDir) + moreDeps := pkgs.filter (·.dir.toString != ".") |>.map (·.buildRoot.withExtension "olean") +} + +end BuildConfig diff --git a/Leanpkg2/Cli.lean b/Leanpkg2/Cli.lean index 6f6014aa25..e94674e429 100644 --- a/Leanpkg2/Cli.lean +++ b/Leanpkg2/Cli.lean @@ -4,8 +4,6 @@ Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Leanpkg2.Init -import Leanpkg2.Configure -import Leanpkg2.Make import Leanpkg2.Build import Leanpkg2.TomlManifest @@ -65,7 +63,7 @@ directory." def cli : (cmd : String) → (leanpkgArgs leanArgs : List String) → IO Unit | "init", [name], [] => init name -| "configure", [], [] => discard do configure (← readManifest) +| "configure", [], [] => do configure (← readManifest) | "print-paths", imports, leanArgs => do printPaths (← readManifest) imports leanArgs | "build", makeArgs, leanArgs => do build (← readManifest) makeArgs leanArgs | "help", ["init"], [] => IO.println helpInit diff --git a/Leanpkg2/Configure.lean b/Leanpkg2/Configure.lean deleted file mode 100644 index 7c324fa4d3..0000000000 --- a/Leanpkg2/Configure.lean +++ /dev/null @@ -1,36 +0,0 @@ -/- -Copyright (c) 2017 Microsoft Corporation. All rights reserved. -Released under Apache 2.0 license as described in the file LICENSE. -Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone --/ -import Leanpkg2.BuildConfig -import Leanpkg2.Resolve - -open System - -namespace Leanpkg2 - -structure Configuration := - leanPath : String - leanSrcPath : String - moreDeps : List FilePath - -def configure (manifest : Manifest) : IO Configuration := do - IO.eprintln $ "configuring " ++ manifest.name ++ " " ++ manifest.version - let pkgs ← solveDeps manifest - let mut moreDeps := [] - for pkg in pkgs do - unless pkg.dir.toString == "." do - -- build recursively - -- TODO: share build of common dependencies - execCmd { - cwd := pkg.dir - cmd := (← IO.appDir) / "lean" |>.toString - args := #["--run", "Package.lean"] - } - moreDeps := pkg.buildRoot.withExtension "olean" :: moreDeps - return { - leanPath := SearchPath.toString <| pkgs.map (·.buildDir) - leanSrcPath := SearchPath.toString <| pkgs.map (·.sourceDir) - moreDeps - } diff --git a/Leanpkg2/Make.lean b/Leanpkg2/Make.lean index a83aba726e..37954aeb57 100644 --- a/Leanpkg2/Make.lean +++ b/Leanpkg2/Make.lean @@ -48,6 +48,6 @@ def execMake (manifest : Manifest) (makeArgs : List String) (cfg : BuildConfig) let mut spawnArgs := { cmd := "sh" cwd := manifest.effectivePath - args := #["-c", s!"\"{← IO.appDir}/leanmake\" PKG={cfg.pkg} LEAN_OPTS=\"{" ".intercalate leanArgs}\" LEAN_PATH=\"{cfg.leanPath}\" {" ".intercalate makeArgs} MORE_DEPS+=\"{" ".intercalate (cfg.moreDeps.map toString)}\" >&2"] + args := #["-c", s!"\"{← IO.appDir}/leanmake\" PKG={cfg.module} LEAN_OPTS=\"{" ".intercalate leanArgs}\" LEAN_PATH=\"{cfg.leanPath}\" {" ".intercalate makeArgs} MORE_DEPS+=\"{" ".intercalate (cfg.moreDeps.map toString)}\" >&2"] } execCmd spawnArgs diff --git a/examples/hello/Package.lean b/examples/hello/Package.lean index 0329c6feda..294b023114 100644 --- a/examples/hello/Package.lean +++ b/examples/hello/Package.lean @@ -1,5 +1,4 @@ import Leanpkg2.Build -import Leanpkg2.Configure import Leanpkg2.Manifest open Leanpkg2 @@ -10,7 +9,7 @@ def manifest : Manifest := { } def configure : IO Unit := - discard <| Leanpkg2.configure manifest + Leanpkg2.configure manifest def build : IO Unit := Leanpkg2.build manifest ["bin"] diff --git a/examples/helloDeps/a/Package.lean b/examples/helloDeps/a/Package.lean index 0db27b9bd5..c1c41dd810 100644 --- a/examples/helloDeps/a/Package.lean +++ b/examples/helloDeps/a/Package.lean @@ -1,5 +1,4 @@ import Leanpkg2.Build -import Leanpkg2.Configure import Leanpkg2.Manifest open Leanpkg2 diff --git a/examples/helloDeps/b/Package.lean b/examples/helloDeps/b/Package.lean index f71e4a0d2e..dc7902dd08 100644 --- a/examples/helloDeps/b/Package.lean +++ b/examples/helloDeps/b/Package.lean @@ -1,5 +1,4 @@ import Leanpkg2.Build -import Leanpkg2.Configure import Leanpkg2.Manifest open Leanpkg2 System From 436d3213de64156e67ee2092fdabf9fd1521336a Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sun, 30 May 2021 17:30:33 -0400 Subject: [PATCH 010/696] Now built off Lean master --- Leanpkg2/Build.lean | 3 --- 1 file changed, 3 deletions(-) diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean index 42203bfe77..854f58964d 100644 --- a/Leanpkg2/Build.lean +++ b/Leanpkg2/Build.lean @@ -20,9 +20,6 @@ structure BuildContext extends BuildConfig where moreDepsMTime : IO.FS.SystemTime manifest : Manifest -deriving instance Inhabited for IO.FS.SystemTime -deriving instance Inhabited for Task - structure Result where maxMTime : IO.FS.SystemTime task : Task (Except IO.Error Unit) From a462864d789356217d4d637f014e82d25f647680 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Wed, 2 Jun 2021 15:21:07 -0400 Subject: [PATCH 011/696] Add cleaner script for 'helloDeps' example --- examples/helloDeps/clean.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 examples/helloDeps/clean.sh diff --git a/examples/helloDeps/clean.sh b/examples/helloDeps/clean.sh new file mode 100644 index 0000000000..143901d23e --- /dev/null +++ b/examples/helloDeps/clean.sh @@ -0,0 +1,2 @@ +rm -r a/build +rm -r b/build From d5d8ff588abffc7e6ef771e8864aaaeaf89cc910 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Wed, 2 Jun 2021 15:21:36 -0400 Subject: [PATCH 012/696] Solver now returns `Package` --- Leanpkg2/Manifest.lean | 8 +++++++ Leanpkg2/Resolve.lean | 52 +++++++++++++++++++++--------------------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/Leanpkg2/Manifest.lean b/Leanpkg2/Manifest.lean index 850203fc96..f7195eca4c 100644 --- a/Leanpkg2/Manifest.lean +++ b/Leanpkg2/Manifest.lean @@ -26,6 +26,7 @@ structure Manifest where path : Option FilePath := none module : String := name.capitalize dependencies : List Dependency := [] + deriving Inhabited namespace Manifest @@ -37,9 +38,16 @@ end Manifest structure Package where dir : FilePath manifest : Manifest + deriving Inhabited namespace Package +def name (self : Package) : String := + self.manifest.name + +def dependencies (self : Package) : List Dependency := + self.manifest.dependencies + def sourceDir (self : Package) : FilePath := self.dir / self.manifest.effectivePath diff --git a/Leanpkg2/Resolve.lean b/Leanpkg2/Resolve.lean index b906ab06d5..6ed62180b7 100644 --- a/Leanpkg2/Resolve.lean +++ b/Leanpkg2/Resolve.lean @@ -6,12 +6,15 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone import Leanpkg2.Git import Leanpkg2.Proc import Leanpkg2.TomlManifest +import Leanpkg2.BuildConfig open System namespace Leanpkg2 -def Assignment := List (String × FilePath) +def depsPath := buildPath / "deps" + +def Assignment := List (String × Package) namespace Assignment def empty : Assignment := [] @@ -19,10 +22,10 @@ def empty : Assignment := [] def contains (a : Assignment) (s : String) : Bool := (a.lookup s).isSome -def insert (a : Assignment) (k : String) (v : FilePath) : Assignment := +def insert (a : Assignment) (k : String) (v : Package) : Assignment := if a.contains k then a else (k, v) :: a -def fold {α} (i : α) (f : α → String → FilePath → α) : Assignment → α := +def fold {α} (i : α) (f : α → String → Package → α) : Assignment → α := List.foldl (fun a ⟨k, v⟩ => f a k v) i end Assignment @@ -32,18 +35,19 @@ abbrev Solver := StateT Assignment IO def notYetAssigned (d : String) : Solver Bool := do ¬ (← get).contains d -def resolvedPath (d : String) : Solver FilePath := do - let some path ← pure ((← get).lookup d) | unreachable! - path +def resolvedPackage (d : String) : Solver Package := do + let some pkg ← pure ((← get).lookup d) | unreachable! + pkg -def materialize (relpath : FilePath) (dep : Dependency) : Solver Unit := +def materialize (relPath : FilePath) (dep : Dependency) : Solver Unit := match dep.src with | Source.path dir => do - let depdir := dir / relpath + let depdir := dir / relPath IO.eprintln s!"{dep.name}: using local path {depdir}" - modify (·.insert dep.name depdir) + let m ← Manifest.fromTomlFile <| depdir / leanpkgToml + modify (·.insert dep.name ⟨depdir, m⟩) | Source.git url rev branch => do - let depdir := FilePath.mk "build" / "deps" / dep.name + let depdir := depsPath / dep.name if ← depdir.isDir then IO.eprint s!"{dep.name}: trying to update {depdir} to revision {rev}" IO.eprintln (match branch with | none => "" | some branch => "@" ++ branch) @@ -56,24 +60,20 @@ def materialize (relpath : FilePath) (dep : Dependency) : Solver Unit := execCmd {cmd := "git", args := #["clone", url, depdir.toString]} let hash ← gitParseOriginRevision depdir rev execCmd {cmd := "git", args := #["checkout", "--detach", hash], cwd := depdir} - modify (·.insert dep.name depdir) + let m ← Manifest.fromTomlFile <| depdir / leanpkgToml + modify (·.insert dep.name ⟨depdir, m⟩) -def solveDepsCore (relPath : FilePath) (d : Manifest) : (maxDepth : Nat) → Solver Unit +def solveDepsCore (pkg : String) (relPath : FilePath) (deps : List Dependency) : (maxDepth : Nat) → Solver Unit | 0 => throw <| IO.userError "maximum dependency resolution depth reached" | maxDepth + 1 => do - let deps ← d.dependencies.filterM (notYetAssigned ·.name) - deps.forM (materialize relPath) - for dep in deps do - let p ← resolvedPath dep.name - let d' ← Manifest.fromTomlFile <| p / leanpkgToml - unless d'.name = dep.name do - throw <| IO.userError s!"{d.name} (in {relPath}) depends on {d'.name}, but resolved dependency has name {dep.name} (in {p})" - solveDepsCore p d' maxDepth + let newDeps ← deps.filterM (notYetAssigned ·.name) + newDeps.forM (materialize relPath) + for dep in newDeps do + let depPkg ← resolvedPackage dep.name + unless depPkg.name = dep.name do + throw <| IO.userError s!"{pkg} (in {relPath}) depends on {depPkg.name}, but resolved dependency has name {dep.name} (in {depPkg.dir})" + solveDepsCore depPkg.name depPkg.dir depPkg.dependencies maxDepth def solveDeps (m : Manifest) : IO (List Package) := do - let (_, assg) ← (solveDepsCore ⟨"."⟩ m 1024).run <| Assignment.empty.insert m.name ⟨"."⟩ - assg.reverse.mapM fun ⟨depname, dirname⟩ => do - if dirname == "." then - return ⟨ dirname, m ⟩ - else - return ⟨ dirname, ← Manifest.fromTomlFile <| dirname / leanpkgToml ⟩ + let (_, assg) ← (solveDepsCore m.name ⟨"."⟩ m.dependencies 1024).run <| Assignment.empty.insert m.name ⟨".", m⟩ + assg.reverse.mapM (·.2) From f2041789b7383a260948bdd8085c19c6bd0ee796 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Wed, 2 Jun 2021 16:02:22 -0400 Subject: [PATCH 013/696] Renamed `Manifest.lean` to `Pakcage.lean` --- Leanpkg2/Build.lean | 2 +- Leanpkg2/BuildConfig.lean | 17 +---------------- Leanpkg2/Make.lean | 6 +++--- Leanpkg2/{Manifest.lean => Package.lean} | 10 ++++++++++ Leanpkg2/Resolve.lean | 2 -- Leanpkg2/TomlManifest.lean | 2 +- examples/hello/Package.lean | 1 - examples/helloDeps/a/Package.lean | 1 - examples/helloDeps/b/Package.lean | 1 - 9 files changed, 16 insertions(+), 26 deletions(-) rename Leanpkg2/{Manifest.lean => Package.lean} (82%) diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean index 854f58964d..c60606ad63 100644 --- a/Leanpkg2/Build.lean +++ b/Leanpkg2/Build.lean @@ -6,7 +6,7 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone import Lean.Data.Name import Lean.Elab.Import import Leanpkg2.Resolve -import Leanpkg2.Manifest +import Leanpkg2.Package import Leanpkg2.BuildConfig import Leanpkg2.Make import Leanpkg2.Proc diff --git a/Leanpkg2/BuildConfig.lean b/Leanpkg2/BuildConfig.lean index 6e123cb813..de71b456fc 100644 --- a/Leanpkg2/BuildConfig.lean +++ b/Leanpkg2/BuildConfig.lean @@ -5,7 +5,7 @@ Authors: Sebastian Ullrich -/ import Lean.Data.Name import Lean.Elab.Import -import Leanpkg2.Manifest +import Leanpkg2.Package import Leanpkg2.Proc open Lean @@ -13,19 +13,6 @@ open System namespace Leanpkg2 -def buildPath : FilePath := "build" -def tempBuildPath := buildPath / "temp" - -namespace Package - -def buildDir (self : Package) : FilePath := - self.dir / Leanpkg2.buildPath - -def buildRoot (self : Package) : FilePath := - self.buildDir / self.manifest.module - -end Package - structure BuildConfig where module : Name leanArgs : List String @@ -40,5 +27,3 @@ def fromPackages (module : Name) (leanArgs : List String) (pkgs : List Package) leanPath := SearchPath.toString <| pkgs.map (·.buildDir) moreDeps := pkgs.filter (·.dir.toString != ".") |>.map (·.buildRoot.withExtension "olean") } - -end BuildConfig diff --git a/Leanpkg2/Make.lean b/Leanpkg2/Make.lean index 37954aeb57..26ab4bfb30 100644 --- a/Leanpkg2/Make.lean +++ b/Leanpkg2/Make.lean @@ -3,14 +3,14 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ -import Leanpkg2.Manifest +import Leanpkg2.Package import Leanpkg2.BuildConfig open System namespace Leanpkg2 -def lockFileName : System.FilePath := ".leanpkg-lock" +def lockFileName : FilePath := ".leanpkg-lock" partial def withLockFile (x : IO α) : IO α := do acquire @@ -22,7 +22,7 @@ partial def withLockFile (x : IO α) : IO α := do acquire (firstTime := true) := try -- TODO: lock file should ideally contain PID - if !System.Platform.isWindows then + if !Platform.isWindows then discard <| IO.Prim.Handle.mk lockFileName "wx" else -- `x` mode doesn't seem to work on Windows even though it's listed at diff --git a/Leanpkg2/Manifest.lean b/Leanpkg2/Package.lean similarity index 82% rename from Leanpkg2/Manifest.lean rename to Leanpkg2/Package.lean index f7195eca4c..cce75d61b5 100644 --- a/Leanpkg2/Manifest.lean +++ b/Leanpkg2/Package.lean @@ -10,6 +10,10 @@ open Lean System namespace Leanpkg2 +def buildPath : FilePath := "build" +def tempBuildPath := buildPath / "temp" +def depsPath := buildPath / "deps" + inductive Source where | path (dir : FilePath) : Source | git (url rev : String) (branch : Option String) : Source @@ -54,4 +58,10 @@ def sourceDir (self : Package) : FilePath := def sourceRoot (self : Package) : FilePath := self.sourceDir / self.manifest.module +def buildDir (self : Package) : FilePath := + self.dir / Leanpkg2.buildPath + +def buildRoot (self : Package) : FilePath := + self.buildDir / self.manifest.module + end Package diff --git a/Leanpkg2/Resolve.lean b/Leanpkg2/Resolve.lean index 6ed62180b7..f4152b5d0f 100644 --- a/Leanpkg2/Resolve.lean +++ b/Leanpkg2/Resolve.lean @@ -12,8 +12,6 @@ open System namespace Leanpkg2 -def depsPath := buildPath / "deps" - def Assignment := List (String × Package) namespace Assignment diff --git a/Leanpkg2/TomlManifest.lean b/Leanpkg2/TomlManifest.lean index 4e04111987..99dce01964 100644 --- a/Leanpkg2/TomlManifest.lean +++ b/Leanpkg2/TomlManifest.lean @@ -5,7 +5,7 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Leanpkg2.Toml import Leanpkg2.LeanVersion -import Leanpkg2.Manifest +import Leanpkg2.Package open System diff --git a/examples/hello/Package.lean b/examples/hello/Package.lean index 294b023114..5de8e84f0b 100644 --- a/examples/hello/Package.lean +++ b/examples/hello/Package.lean @@ -1,5 +1,4 @@ import Leanpkg2.Build -import Leanpkg2.Manifest open Leanpkg2 diff --git a/examples/helloDeps/a/Package.lean b/examples/helloDeps/a/Package.lean index c1c41dd810..11d316b818 100644 --- a/examples/helloDeps/a/Package.lean +++ b/examples/helloDeps/a/Package.lean @@ -1,5 +1,4 @@ import Leanpkg2.Build -import Leanpkg2.Manifest open Leanpkg2 diff --git a/examples/helloDeps/b/Package.lean b/examples/helloDeps/b/Package.lean index dc7902dd08..b5e24b0fc5 100644 --- a/examples/helloDeps/b/Package.lean +++ b/examples/helloDeps/b/Package.lean @@ -1,5 +1,4 @@ import Leanpkg2.Build -import Leanpkg2.Manifest open Leanpkg2 System From 9544f3dad81a08f6ea003d5b1cdffd0799d878ab Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Wed, 2 Jun 2021 18:19:31 -0400 Subject: [PATCH 014/696] Refactor materlize / git code --- Leanpkg2/Git.lean | 50 ++++++++++++++++++++++----------- Leanpkg2/Init.lean | 10 ++++--- Leanpkg2/Resolve.lean | 65 ++++++++++++++++++++++++------------------- 3 files changed, 76 insertions(+), 49 deletions(-) diff --git a/Leanpkg2/Git.lean b/Leanpkg2/Git.lean index fb8ca06de2..59eb4500ba 100644 --- a/Leanpkg2/Git.lean +++ b/Leanpkg2/Git.lean @@ -1,38 +1,54 @@ /- Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. -Authors: Gabriel Ebner, Sebastian Ullrich +Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ +import Leanpkg2.Proc import Leanpkg2.LeanVersion open System -namespace Leanpkg2 +namespace Leanpkg2.Git -def upstreamGitBranch := +def upstreamBranch := "master" -def gitdefaultRevision : Option String → String - | none => upstreamGitBranch +def defaultRevision : Option String → String + | none => upstreamBranch | some branch => branch -def gitParseRevision (gitRepo : FilePath) (rev : String) : IO String := do - let rev ← IO.Process.run {cmd := "git", args := #["rev-parse", "-q", "--verify", rev], cwd := gitRepo} +def clone (url : String) (dir : FilePath) := + execCmd {cmd := "git", args := #["clone", url, dir.toString]} + +def quietInit (repo : Option FilePath := none) := + execCmd {cmd := "git", args := #["init", "-q"]} + +def fetch (repo : Option FilePath := none) := + execCmd {cmd := "git", args := #["fetch"], cwd := repo} + +def checkoutBranch (branch : String) (repo : Option FilePath := none) := + execCmd {cmd := "git", args := #["checkout", "-B", branch]} + +def checkoutDetach (hash : String) (repo : Option FilePath := none) := + execCmd {cmd := "git", args := #["checkout", "--detach", hash], cwd := repo} + +def parseRevision (rev : String) (repo : Option FilePath := none) : IO String := do + let rev ← IO.Process.run {cmd := "git", args := #["rev-parse", "-q", "--verify", rev], cwd := repo} rev.trim -- remove newline at end -def gitHeadRevision (gitRepo : FilePath) : IO String := - gitParseRevision gitRepo "HEAD" +def headRevision (repo : Option FilePath := none) : IO String := + parseRevision "HEAD" repo -def gitParseOriginRevision (gitRepo : FilePath) (rev : String) : IO String := - (gitParseRevision gitRepo $ "origin/" ++ rev) <|> gitParseRevision gitRepo rev - <|> throw (IO.userError s!"cannot find revision {rev} in repository {gitRepo}") +def parseOriginRevision (rev : String) (repo : Option FilePath := none) : IO String := + (parseRevision ("origin/" ++ rev) repo) <|> parseRevision rev repo + <|> throw (IO.userError s!"cannot find revision {rev} in repository {repo}") -def gitLatestOriginRevision (gitRepo : FilePath) (branch : Option String) : IO String := do - discard <| IO.Process.run {cmd := "git", args := #["fetch"], cwd := gitRepo} - gitParseOriginRevision gitRepo (gitdefaultRevision branch) +def latestOriginRevision (branch : Option String) (repo : Option FilePath := none) : IO String := do + discard <| IO.Process.run {cmd := "git", args := #["fetch"], cwd := repo} + parseOriginRevision (defaultRevision branch) repo -def gitRevisionExists (gitRepo : FilePath) (rev : String) : IO Bool := do +def revisionExists (rev : String) (repo : Option FilePath := none) : IO Bool := do try - discard <| gitParseRevision gitRepo (rev ++ "^{commit}") + discard <| parseRevision (rev ++ "^{commit}") repo true catch _ => false diff --git a/Leanpkg2/Init.lean b/Leanpkg2/Init.lean index bf26c46b1e..97026ba820 100644 --- a/Leanpkg2/Init.lean +++ b/Leanpkg2/Init.lean @@ -25,6 +25,7 @@ version = \"0.1\" lean_version = \"{leanVersionString}\" " +open Git in def initPkg (pkgName : String) (fromNew : Bool) : IO Unit := do IO.FS.writeFile leanpkgToml (leanpkgFileContents pkgName) IO.FS.writeFile ⟨s!"{pkgName.capitalize}.lean"⟩ mainFileContents @@ -32,9 +33,10 @@ def initPkg (pkgName : String) (fromNew : Bool) : IO Unit := do h.putStr initGitignoreContents unless ← System.FilePath.isDir ⟨".git"⟩ do (do - execCmd {cmd := "git", args := #["init", "-q"]} - unless upstreamGitBranch = "master" do - execCmd {cmd := "git", args := #["checkout", "-B", upstreamGitBranch]} - ) <|> IO.eprintln "WARNING: failed to initialize git repository" + quietInit + unless upstreamBranch = "master" do + checkoutBranch upstreamBranch + ) <|> + IO.eprintln "WARNING: failed to initialize git repository" def init (pkgName : String) := initPkg pkgName false diff --git a/Leanpkg2/Resolve.lean b/Leanpkg2/Resolve.lean index f4152b5d0f..780644cfb4 100644 --- a/Leanpkg2/Resolve.lean +++ b/Leanpkg2/Resolve.lean @@ -12,6 +12,33 @@ open System namespace Leanpkg2 +open Git in +def materializeGit + (name : String) (dir : FilePath) (url rev : String) (branch : Option String) +: IO Unit := do + if ← dir.isDir then + IO.eprint s!"{name}: trying to update {dir} to revision {rev}" + IO.eprintln (match branch with | none => "" | some branch => "@" ++ branch) + let hash ← parseOriginRevision rev dir + unless ← revisionExists hash dir do fetch dir + checkoutDetach hash dir + else + IO.eprintln s!"{name}: cloning {url} to {dir}" + clone url dir + let hash ← parseOriginRevision rev dir + checkoutDetach hash dir + +def materialize (relPath : FilePath) (dep : Dependency) : IO FilePath := + match dep.src with + | Source.path dir => do + let depdir := dir / relPath + IO.eprintln s!"{dep.name}: using local path {depdir}" + depdir + | Source.git url rev branch => do + let depdir := depsPath / dep.name + materializeGit dep.name depdir url rev branch + depdir + def Assignment := List (String × Package) namespace Assignment @@ -37,41 +64,23 @@ def resolvedPackage (d : String) : Solver Package := do let some pkg ← pure ((← get).lookup d) | unreachable! pkg -def materialize (relPath : FilePath) (dep : Dependency) : Solver Unit := - match dep.src with - | Source.path dir => do - let depdir := dir / relPath - IO.eprintln s!"{dep.name}: using local path {depdir}" - let m ← Manifest.fromTomlFile <| depdir / leanpkgToml - modify (·.insert dep.name ⟨depdir, m⟩) - | Source.git url rev branch => do - let depdir := depsPath / dep.name - if ← depdir.isDir then - IO.eprint s!"{dep.name}: trying to update {depdir} to revision {rev}" - IO.eprintln (match branch with | none => "" | some branch => "@" ++ branch) - let hash ← gitParseOriginRevision depdir rev - let revEx ← gitRevisionExists depdir hash - unless revEx do - execCmd {cmd := "git", args := #["fetch"], cwd := depdir} - else - IO.eprintln s!"{dep.name}: cloning {url} to {depdir}" - execCmd {cmd := "git", args := #["clone", url, depdir.toString]} - let hash ← gitParseOriginRevision depdir rev - execCmd {cmd := "git", args := #["checkout", "--detach", hash], cwd := depdir} - let m ← Manifest.fromTomlFile <| depdir / leanpkgToml - modify (·.insert dep.name ⟨depdir, m⟩) - -def solveDepsCore (pkg : String) (relPath : FilePath) (deps : List Dependency) : (maxDepth : Nat) → Solver Unit +def solveDepsCore + (pkgName : String) (relPath : FilePath) (deps : List Dependency) +: (maxDepth : Nat) → Solver Unit | 0 => throw <| IO.userError "maximum dependency resolution depth reached" | maxDepth + 1 => do let newDeps ← deps.filterM (notYetAssigned ·.name) - newDeps.forM (materialize relPath) + for dep in newDeps do + let dir ← materialize relPath dep + let m ← Manifest.fromTomlFile <| dir / leanpkgToml + modify (·.insert dep.name ⟨dir, m⟩) for dep in newDeps do let depPkg ← resolvedPackage dep.name unless depPkg.name = dep.name do - throw <| IO.userError s!"{pkg} (in {relPath}) depends on {depPkg.name}, but resolved dependency has name {dep.name} (in {depPkg.dir})" + throw <| IO.userError s!"{pkgName} (in {relPath}) depends on {depPkg.name}, but resolved dependency has name {dep.name} (in {depPkg.dir})" solveDepsCore depPkg.name depPkg.dir depPkg.dependencies maxDepth def solveDeps (m : Manifest) : IO (List Package) := do - let (_, assg) ← (solveDepsCore m.name ⟨"."⟩ m.dependencies 1024).run <| Assignment.empty.insert m.name ⟨".", m⟩ + let solver := solveDepsCore m.name ⟨"."⟩ m.dependencies 1024 + let (_, assg) ← solver.run (Assignment.empty.insert m.name ⟨".", m⟩) assg.reverse.mapM (·.2) From 158838bf63143573bcc314aedefff520a27d8fe0 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Wed, 2 Jun 2021 18:25:30 -0400 Subject: [PATCH 015/696] Fix local depdir calculation --- Leanpkg2/Resolve.lean | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Leanpkg2/Resolve.lean b/Leanpkg2/Resolve.lean index 780644cfb4..e141405b63 100644 --- a/Leanpkg2/Resolve.lean +++ b/Leanpkg2/Resolve.lean @@ -31,7 +31,7 @@ def materializeGit def materialize (relPath : FilePath) (dep : Dependency) : IO FilePath := match dep.src with | Source.path dir => do - let depdir := dir / relPath + let depdir := relPath / dir IO.eprintln s!"{dep.name}: using local path {depdir}" depdir | Source.git url rev branch => do From bf15f7156813bc6fee26a70b0d4969d8b94cac21 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Wed, 2 Jun 2021 18:55:08 -0400 Subject: [PATCH 016/696] Removed `Manifest.path` --- Leanpkg2/Build.lean | 12 +++++------- Leanpkg2/Make.lean | 1 - Leanpkg2/Package.lean | 10 +--------- Leanpkg2/TomlManifest.lean | 6 +----- 4 files changed, 7 insertions(+), 22 deletions(-) diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean index c60606ad63..0a1bdaa5cc 100644 --- a/Leanpkg2/Build.lean +++ b/Leanpkg2/Build.lean @@ -18,7 +18,6 @@ namespace Leanpkg2 structure BuildContext extends BuildConfig where parents : List Name := [] moreDepsMTime : IO.FS.SystemTime - manifest : Manifest structure Result where maxMTime : IO.FS.SystemTime @@ -42,8 +41,7 @@ partial def buildModule (mod : Name) : BuildM Result := do -- already visited return r - let srcPath := ctx.manifest.effectivePath - let leanFile := modToFilePath srcPath mod "lean" + let leanFile := modToFilePath "." mod "lean" let leanMData ← leanFile.metadata -- recursively build dependencies and calculate transitive `maxMTime` @@ -54,7 +52,7 @@ partial def buildModule (mod : Name) : BuildM Result := do let maxMTime := List.maximum? (leanMData.modified :: ctx.moreDepsMTime :: depMTimes) |>.get! -- check whether we have an up-to-date .olean - let oleanFile := modToFilePath (srcPath / buildPath) mod "olean" + let oleanFile := modToFilePath buildPath mod "olean" try if (← oleanFile.metadata).modified >= maxMTime then let r := { maxMTime, task := Task.pure (Except.ok ()) } @@ -69,7 +67,7 @@ partial def buildModule (mod : Name) : BuildM Result := do -- propagate failure throw e try - let cFile := modToFilePath (srcPath / tempBuildPath) mod "c" + let cFile := modToFilePath tempBuildPath mod "c" IO.createDirAll oleanFile.parent.get! IO.createDirAll cFile.parent.get! execCmd { @@ -88,7 +86,7 @@ partial def buildModule (mod : Name) : BuildM Result := do def buildModules (manifest : Manifest) (cfg : BuildConfig) (mods : List Name) : IO Unit := do let moreDepsMTime := (← cfg.moreDeps.mapM (·.metadata)).map (·.modified) |>.maximum? |>.getD ⟨0, 0⟩ - let rs ← mods.mapM buildModule |>.run { toBuildConfig := cfg, moreDepsMTime, manifest } |>.run' {} + let rs ← mods.mapM buildModule |>.run { toBuildConfig := cfg, moreDepsMTime } |>.run' {} for r in rs do if let Except.error _ ← IO.wait r.task then -- actual error has already been printed above @@ -100,7 +98,7 @@ def buildImports (manifest : Manifest) (cfg : BuildConfig) (imports leanArgs : L if localImports != [] then if ← FilePath.pathExists "Makefile" then let oleans := localImports.map fun i => - Lean.modToFilePath (manifest.effectivePath / buildPath) i "olean" |>.toString + Lean.modToFilePath buildPath i "olean" |>.toString execMake manifest oleans cfg else buildModules manifest cfg localImports diff --git a/Leanpkg2/Make.lean b/Leanpkg2/Make.lean index 26ab4bfb30..8a61c24d4e 100644 --- a/Leanpkg2/Make.lean +++ b/Leanpkg2/Make.lean @@ -47,7 +47,6 @@ def execMake (manifest : Manifest) (makeArgs : List String) (cfg : BuildConfig) let leanArgs := timeoutArgs ++ cfg.leanArgs let mut spawnArgs := { cmd := "sh" - cwd := manifest.effectivePath args := #["-c", s!"\"{← IO.appDir}/leanmake\" PKG={cfg.module} LEAN_OPTS=\"{" ".intercalate leanArgs}\" LEAN_PATH=\"{cfg.leanPath}\" {" ".intercalate makeArgs} MORE_DEPS+=\"{" ".intercalate (cfg.moreDeps.map toString)}\" >&2"] } execCmd spawnArgs diff --git a/Leanpkg2/Package.lean b/Leanpkg2/Package.lean index cce75d61b5..e5ffc41bc5 100644 --- a/Leanpkg2/Package.lean +++ b/Leanpkg2/Package.lean @@ -27,18 +27,10 @@ structure Manifest where version : String leanVersion : String := leanVersionString timeout : Option Nat := none - path : Option FilePath := none module : String := name.capitalize dependencies : List Dependency := [] deriving Inhabited -namespace Manifest - -def effectivePath (m : Manifest) : FilePath := - m.path.getD ⟨"."⟩ - -end Manifest - structure Package where dir : FilePath manifest : Manifest @@ -53,7 +45,7 @@ def dependencies (self : Package) : List Dependency := self.manifest.dependencies def sourceDir (self : Package) : FilePath := - self.dir / self.manifest.effectivePath + self.dir def sourceRoot (self : Package) : FilePath := self.sourceDir / self.manifest.module diff --git a/Leanpkg2/TomlManifest.lean b/Leanpkg2/TomlManifest.lean index 99dce01964..c7d1a58041 100644 --- a/Leanpkg2/TomlManifest.lean +++ b/Leanpkg2/TomlManifest.lean @@ -49,13 +49,9 @@ def fromToml (t : Toml.Value) : Option Manifest := OptionM.run do | some (Toml.Value.nat timeout) => some (some timeout) | none => some none | _ => none - let path ← match pkg.lookup "path" with - | some (Toml.Value.str path) => some (some ⟨path⟩) - | none => some none - | _ => none let Toml.Value.table deps ← t.lookup "dependencies" <|> some (Toml.Value.table []) | none let dependencies ← deps.mapM fun ⟨n, src⟩ => do Dependency.mk n (← Source.fromToml src) - return { name, module, version, leanVersion, path, dependencies, timeout } + return { name, module, version, leanVersion, dependencies, timeout } def fromTomlFile (fn : System.FilePath) : IO Manifest := do let cnts ← IO.FS.readFile fn From 3ca6b0bf516c18a4fd42bb4834762f71bcacce63 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Thu, 3 Jun 2021 15:17:46 -0400 Subject: [PATCH 017/696] Minor code style cleanup --- Leanpkg2/BuildConfig.lean | 12 +++++++----- Leanpkg2/Resolve.lean | 3 ++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Leanpkg2/BuildConfig.lean b/Leanpkg2/BuildConfig.lean index de71b456fc..009ec6b06f 100644 --- a/Leanpkg2/BuildConfig.lean +++ b/Leanpkg2/BuildConfig.lean @@ -1,15 +1,14 @@ /- Copyright (c) 2021 Sebastian Ullrich. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. -Authors: Sebastian Ullrich +Authors: Sebastian Ullrich, Mac Malone -/ import Lean.Data.Name import Lean.Elab.Import import Leanpkg2.Package import Leanpkg2.Proc -open Lean -open System +open Lean System namespace Leanpkg2 @@ -22,8 +21,11 @@ structure BuildConfig where namespace BuildConfig -def fromPackages (module : Name) (leanArgs : List String) (pkgs : List Package) : BuildConfig := { +def fromPackages +(module : Name) (leanArgs : List String) (pkgs : List Package) +: BuildConfig := { module, leanArgs, leanPath := SearchPath.toString <| pkgs.map (·.buildDir) - moreDeps := pkgs.filter (·.dir.toString != ".") |>.map (·.buildRoot.withExtension "olean") + moreDeps := pkgs.filter (·.dir.toString != ".") |>.map + (·.buildRoot.withExtension "olean") } diff --git a/Leanpkg2/Resolve.lean b/Leanpkg2/Resolve.lean index e141405b63..cd20d1d4aa 100644 --- a/Leanpkg2/Resolve.lean +++ b/Leanpkg2/Resolve.lean @@ -42,6 +42,7 @@ def materialize (relPath : FilePath) (dep : Dependency) : IO FilePath := def Assignment := List (String × Package) namespace Assignment + def empty : Assignment := [] def contains (a : Assignment) (s : String) : Bool := @@ -65,7 +66,7 @@ def resolvedPackage (d : String) : Solver Package := do pkg def solveDepsCore - (pkgName : String) (relPath : FilePath) (deps : List Dependency) +(pkgName : String) (relPath : FilePath) (deps : List Dependency) : (maxDepth : Nat) → Solver Unit | 0 => throw <| IO.userError "maximum dependency resolution depth reached" | maxDepth + 1 => do From 07e804ad168a3b16346fbeeb862ba0f72368b787 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Thu, 3 Jun 2021 16:58:55 -0400 Subject: [PATCH 018/696] Cleanup TOML manifest code --- Leanpkg2/TomlManifest.lean | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/Leanpkg2/TomlManifest.lean b/Leanpkg2/TomlManifest.lean index c7d1a58041..b37d893c25 100644 --- a/Leanpkg2/TomlManifest.lean +++ b/Leanpkg2/TomlManifest.lean @@ -13,7 +13,7 @@ namespace Leanpkg2 namespace Source -def fromToml (v : Toml.Value) : Option Source := +def fromToml? (v : Toml.Value) : Option Source := (do let Toml.Value.str dir ← v.lookup "path" | none path ⟨dir⟩) <|> (do let Toml.Value.str url ← v.lookup "git" | none @@ -34,13 +34,14 @@ end Source namespace Manifest -def fromToml (t : Toml.Value) : Option Manifest := OptionM.run do +def fromToml? (t : Toml.Value) : Option Manifest := OptionM.run do let pkg ← t.lookup "package" let Toml.Value.str name ← pkg.lookup "name" | none let Toml.Value.str version ← pkg.lookup "version" | none - let module := match pkg.lookup "module" with + let module ← match pkg.lookup "module" with | some (Toml.Value.str mod) => mod - | _ => name.capitalize + | none => some name.capitalize + | _ => none let leanVersion ← match pkg.lookup "lean_version" with | some (Toml.Value.str leanVer) => some leanVer | none => some leanVersionString @@ -50,19 +51,23 @@ def fromToml (t : Toml.Value) : Option Manifest := OptionM.run do | none => some none | _ => none let Toml.Value.table deps ← t.lookup "dependencies" <|> some (Toml.Value.table []) | none - let dependencies ← deps.mapM fun ⟨n, src⟩ => do Dependency.mk n (← Source.fromToml src) + let dependencies ← deps.mapM fun ⟨n, src⟩ => do Dependency.mk n (← Source.fromToml? src) return { name, module, version, leanVersion, dependencies, timeout } -def fromTomlFile (fn : System.FilePath) : IO Manifest := do - let cnts ← IO.FS.readFile fn - let toml ← Toml.parse cnts - let some manifest ← pure (fromToml toml) - | throw <| IO.userError s!"cannot read manifest from {fn}" +def fromTomlString? (str : String) : IO (Option Manifest) := do + fromToml? (← Toml.parse str) + +def fromTomlFile? (path : FilePath) : IO (Option Manifest) := do + fromTomlString? (← IO.FS.readFile path) + +def fromTomlFile (path : FilePath) : IO Manifest := do + let some manifest ← fromTomlFile? path + | throw <| IO.userError s!"manifest (at {path}) is ill-formed" manifest end Manifest -def leanpkgToml : System.FilePath := "leanpkg.toml" +def leanpkgToml : FilePath := "leanpkg.toml" def readManifest : IO Manifest := do let m ← Manifest.fromTomlFile leanpkgToml @@ -70,6 +75,3 @@ def readManifest : IO Manifest := do IO.eprintln $ "\nWARNING: Lean version mismatch: installed version is " ++ leanVersionString ++ ", but package requires " ++ m.leanVersion ++ "\n" return m - -def writeManifest (manifest : Lean.Syntax) (fn : FilePath) : IO Unit := do - IO.FS.writeFile fn manifest.reprint.get! From 3cc0c3e3705e09d3511ecc2316433945cc91a630 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sun, 6 Jun 2021 16:09:26 -0400 Subject: [PATCH 019/696] `Package.lean` => `package.lean` --- examples/hello/package.lean | 17 +++++++++++++++++ examples/helloDeps/a/package.lean | 14 ++++++++++++++ examples/helloDeps/b/package.lean | 17 +++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 examples/hello/package.lean create mode 100644 examples/helloDeps/a/package.lean create mode 100644 examples/helloDeps/b/package.lean diff --git a/examples/hello/package.lean b/examples/hello/package.lean new file mode 100644 index 0000000000..5de8e84f0b --- /dev/null +++ b/examples/hello/package.lean @@ -0,0 +1,17 @@ +import Leanpkg2.Build + +open Leanpkg2 + +def manifest : Manifest := { + name := "hello", + version := "1.0", +} + +def configure : IO Unit := + Leanpkg2.configure manifest + +def build : IO Unit := + Leanpkg2.build manifest ["bin"] + +def main : IO Unit := + build diff --git a/examples/helloDeps/a/package.lean b/examples/helloDeps/a/package.lean new file mode 100644 index 0000000000..11d316b818 --- /dev/null +++ b/examples/helloDeps/a/package.lean @@ -0,0 +1,14 @@ +import Leanpkg2.Build + +open Leanpkg2 + +def manifest : Manifest := { + name := "a", + version := "1.0", +} + +def build : IO Unit := + Leanpkg2.build manifest ["lib"] + +def main : IO Unit := + build diff --git a/examples/helloDeps/b/package.lean b/examples/helloDeps/b/package.lean new file mode 100644 index 0000000000..b5e24b0fc5 --- /dev/null +++ b/examples/helloDeps/b/package.lean @@ -0,0 +1,17 @@ +import Leanpkg2.Build + +open Leanpkg2 System + +def manifest : Manifest := { + name := "b", + version := "1.0", + dependencies := [ + { name := "a", src := Source.path (FilePath.mk ".." / "a") } + ] +} + +def build : IO Unit := do + Leanpkg2.build manifest ["bin", "LINK_OPTS=../a/build/lib/libA.a"] + +def main : IO Unit := + build From 6b999dcb21ebd5e10db4772d51be032f3d072d63 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sun, 6 Jun 2021 19:27:18 -0400 Subject: [PATCH 020/696] Refactored away the old notion of a manifest --- Leanpkg2/Build.lean | 63 +++++++++++-------- Leanpkg2/BuildConfig.lean | 31 --------- Leanpkg2/Cli.lean | 15 +++-- Leanpkg2/Init.lean | 2 +- Leanpkg2/Make.lean | 30 +++++---- Leanpkg2/Package.lean | 35 +++++++---- Leanpkg2/Resolve.lean | 40 ++++++------ .../{TomlManifest.lean => TomlConfig.lean} | 29 ++++----- examples/hello/Package.lean | 6 +- examples/hello/package.lean | 6 +- examples/helloDeps/a/Package.lean | 4 +- examples/helloDeps/a/package.lean | 4 +- examples/helloDeps/b/Package.lean | 4 +- examples/helloDeps/b/package.lean | 4 +- 14 files changed, 138 insertions(+), 135 deletions(-) delete mode 100644 Leanpkg2/BuildConfig.lean rename Leanpkg2/{TomlManifest.lean => TomlConfig.lean} (74%) diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean index 0a1bdaa5cc..c6a4be4d76 100644 --- a/Leanpkg2/Build.lean +++ b/Leanpkg2/Build.lean @@ -7,7 +7,6 @@ import Lean.Data.Name import Lean.Elab.Import import Leanpkg2.Resolve import Leanpkg2.Package -import Leanpkg2.BuildConfig import Leanpkg2.Make import Leanpkg2.Proc @@ -15,6 +14,22 @@ open Lean System namespace Leanpkg2 +structure BuildConfig where + module : Name + leanArgs : List String + leanPath : String + -- things like `leanpkg.toml` and olean roots of dependencies that should also trigger rebuilds + moreDeps : List FilePath + +def mkBuildConfig +(pkg : Package) (deps : List Package) (leanArgs : List String) +: BuildConfig := { + leanArgs, + module := pkg.module + leanPath := SearchPath.toString <| deps.map (·.buildDir) + moreDeps := deps.filter (·.dir != pkg.dir) |>.map (·.oleanRoot) +} + structure BuildContext extends BuildConfig where parents : List Name := [] moreDepsMTime : IO.FS.SystemTime @@ -84,7 +99,7 @@ partial def buildModule (mod : Name) : BuildM Result := do modify fun st => { st with modTasks := st.modTasks.insert mod r } return r -def buildModules (manifest : Manifest) (cfg : BuildConfig) (mods : List Name) : IO Unit := do +def buildModules (cfg : BuildConfig) (mods : List Name) : IO Unit := do let moreDepsMTime := (← cfg.moreDeps.mapM (·.metadata)).map (·.modified) |>.maximum? |>.getD ⟨0, 0⟩ let rs ← mods.mapM buildModule |>.run { toBuildConfig := cfg, moreDepsMTime } |>.run' {} for r in rs do @@ -92,44 +107,42 @@ def buildModules (manifest : Manifest) (cfg : BuildConfig) (mods : List Name) : -- actual error has already been printed above throw <| IO.userError "Build failed." -def buildImports (manifest : Manifest) (cfg : BuildConfig) (imports leanArgs : List String := []) : IO Unit := do +def buildImports (pkg : Package) (deps : List Package) (imports leanArgs : List String := []) : IO Unit := do let imports := imports.map (·.toName) - let localImports := imports.filter (·.getRoot == cfg.module) + let localImports := imports.filter (·.getRoot == pkg.module) if localImports != [] then if ← FilePath.pathExists "Makefile" then let oleans := localImports.map fun i => Lean.modToFilePath buildPath i "olean" |>.toString - execMake manifest oleans cfg + execMake pkg deps oleans leanArgs else - buildModules manifest cfg localImports + buildModules (mkBuildConfig pkg deps leanArgs) localImports -def buildDeps (manifest : Manifest) : IO (List Package) := do - let pkgs ← solveDeps manifest - for pkg in pkgs do - unless pkg.dir.toString == "." do +def buildDeps (pkg : Package) : IO (List Package) := do + let deps ← solveDeps pkg + for dep in deps do + unless dep.dir == pkg.dir do -- build recursively -- TODO: share build of common dependencies execCmd { - cwd := pkg.dir + cwd := dep.dir cmd := (← IO.appDir) / "lean" |>.toString args := #["--run", "Package.lean"] } - return pkgs + return deps -def configure (manifest : Manifest) : IO Unit := do - discard <| buildDeps manifest +def configure (pkg : Package) : IO Unit := do + discard <| buildDeps pkg -def printPaths (manifest : Manifest) (imports leanArgs : List String := []) : IO Unit := do - let pkgs ← buildDeps manifest - let cfg := BuildConfig.fromPackages manifest.module leanArgs pkgs - buildImports manifest cfg imports leanArgs - IO.println cfg.leanPath - IO.println <| SearchPath.toString <| pkgs.map (·.sourceDir) +def printPaths (pkg : Package) (imports leanArgs : List String := []) : IO Unit := do + let deps ← buildDeps pkg + buildImports pkg deps imports leanArgs + IO.println <| SearchPath.toString <| deps.map (·.buildDir) + IO.println <| SearchPath.toString <| deps.map (·.sourceDir) -def build (manifest : Manifest) (makeArgs leanArgs : List String := []) : IO Unit := do - let pkgs ← buildDeps manifest - let cfg := BuildConfig.fromPackages manifest.module leanArgs pkgs +def build (pkg : Package) (makeArgs leanArgs : List String := []) : IO Unit := do + let deps ← buildDeps pkg if makeArgs != [] || (← FilePath.pathExists "Makefile") then - execMake manifest makeArgs cfg + execMake pkg deps makeArgs leanArgs else - buildModules manifest cfg [manifest.module] + buildModules (mkBuildConfig pkg deps leanArgs) [pkg.module] diff --git a/Leanpkg2/BuildConfig.lean b/Leanpkg2/BuildConfig.lean deleted file mode 100644 index 009ec6b06f..0000000000 --- a/Leanpkg2/BuildConfig.lean +++ /dev/null @@ -1,31 +0,0 @@ -/- -Copyright (c) 2021 Sebastian Ullrich. All rights reserved. -Released under Apache 2.0 license as described in the file LICENSE. -Authors: Sebastian Ullrich, Mac Malone --/ -import Lean.Data.Name -import Lean.Elab.Import -import Leanpkg2.Package -import Leanpkg2.Proc - -open Lean System - -namespace Leanpkg2 - -structure BuildConfig where - module : Name - leanArgs : List String - leanPath : String - -- things like `leanpkg.toml` and olean roots of dependencies that should also trigger rebuilds - moreDeps : List FilePath - -namespace BuildConfig - -def fromPackages -(module : Name) (leanArgs : List String) (pkgs : List Package) -: BuildConfig := { - module, leanArgs, - leanPath := SearchPath.toString <| pkgs.map (·.buildDir) - moreDeps := pkgs.filter (·.dir.toString != ".") |>.map - (·.buildRoot.withExtension "olean") -} diff --git a/Leanpkg2/Cli.lean b/Leanpkg2/Cli.lean index e94674e429..9ab76db53e 100644 --- a/Leanpkg2/Cli.lean +++ b/Leanpkg2/Cli.lean @@ -5,7 +5,7 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Leanpkg2.Init import Leanpkg2.Build -import Leanpkg2.TomlManifest +import Leanpkg2.TomlConfig namespace Leanpkg2 @@ -61,11 +61,18 @@ Usage: This command creates a new Lean package with the given name in the current directory." +def getRootPkg : IO Package := do + let cfg ← PackageConfig.fromTomlFile leanpkgToml + if cfg.leanVersion ≠ leanVersionString then + IO.eprintln $ "\nWARNING: Lean version mismatch: installed version is " ++ + leanVersionString ++ ", but package requires " ++ cfg.leanVersion ++ "\n" + return ⟨".", cfg⟩ + def cli : (cmd : String) → (leanpkgArgs leanArgs : List String) → IO Unit | "init", [name], [] => init name -| "configure", [], [] => do configure (← readManifest) -| "print-paths", imports, leanArgs => do printPaths (← readManifest) imports leanArgs -| "build", makeArgs, leanArgs => do build (← readManifest) makeArgs leanArgs +| "configure", [], [] => do configure (← getRootPkg) +| "print-paths", imports, leanArgs => do printPaths (← getRootPkg) imports leanArgs +| "build", makeArgs, leanArgs => do build (← getRootPkg) makeArgs leanArgs | "help", ["init"], [] => IO.println helpInit | "help", ["configure"], [] => IO.println helpConfigure | "help", ["build"], [] => IO.println helpBuild diff --git a/Leanpkg2/Init.lean b/Leanpkg2/Init.lean index 97026ba820..b08495b872 100644 --- a/Leanpkg2/Init.lean +++ b/Leanpkg2/Init.lean @@ -5,7 +5,7 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Leanpkg2.Git import Leanpkg2.Proc -import Leanpkg2.TomlManifest +import Leanpkg2.TomlConfig namespace Leanpkg2 diff --git a/Leanpkg2/Make.lean b/Leanpkg2/Make.lean index 8a61c24d4e..55f164fc4e 100644 --- a/Leanpkg2/Make.lean +++ b/Leanpkg2/Make.lean @@ -3,50 +3,58 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ +import Leanpkg2.Proc import Leanpkg2.Package -import Leanpkg2.BuildConfig open System namespace Leanpkg2 -def lockFileName : FilePath := ".leanpkg-lock" +def lockfile : FilePath := ".leanpkg-lock" partial def withLockFile (x : IO α) : IO α := do acquire try x finally - IO.removeFile lockFileName + IO.removeFile lockfile where acquire (firstTime := true) := try -- TODO: lock file should ideally contain PID if !Platform.isWindows then - discard <| IO.Prim.Handle.mk lockFileName "wx" + discard <| IO.Prim.Handle.mk lockfile "wx" else -- `x` mode doesn't seem to work on Windows even though it's listed at -- https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/fopen-wfopen?view=msvc-160 -- ...? Let's use the slightly racy approach then. - if ← lockFileName.pathExists then + if ← lockfile.pathExists then throw <| IO.Error.alreadyExists 0 "" - discard <| IO.Prim.Handle.mk lockFileName "w" + discard <| IO.Prim.Handle.mk lockfile "w" catch | IO.Error.alreadyExists _ _ => do if firstTime then - IO.eprintln s!"Waiting for prior leanpkg invocation to finish... (remove '{lockFileName}' if stuck)" + IO.eprintln s!"Waiting for prior leanpkg invocation to finish... (remove '{lockfile}' if stuck)" IO.sleep (ms := 300) acquire (firstTime := false) | e => throw e -def execMake (manifest : Manifest) (makeArgs : List String) (cfg : BuildConfig) : IO Unit := withLockFile do +def execMake +(pkg : Package) (deps : List Package) (makeArgs leanArgs : List String := []) +: IO Unit := withLockFile do let timeoutArgs := - match manifest.timeout with + match pkg.timeout with | some t => ["-T", toString t] | none => [] - let leanArgs := timeoutArgs ++ cfg.leanArgs + let leanmake := (← IO.appDir) / "leanmake" + let leanOptsStr := " ".intercalate <| timeoutArgs ++ leanArgs + let leanPathStr := SearchPath.toString <| deps.map (·.buildDir) + let makeArgsStr := " ".intercalate makeArgs + let moreDepsStr := " ".intercalate $ + deps.filter (·.dir.toString != pkg.dir) |>.map (·.oleanRoot.toString) let mut spawnArgs := { cmd := "sh" - args := #["-c", s!"\"{← IO.appDir}/leanmake\" PKG={cfg.module} LEAN_OPTS=\"{" ".intercalate leanArgs}\" LEAN_PATH=\"{cfg.leanPath}\" {" ".intercalate makeArgs} MORE_DEPS+=\"{" ".intercalate (cfg.moreDeps.map toString)}\" >&2"] + cwd := pkg.dir + args := #["-c", s!"\"{leanmake}\" PKG={pkg.module} LEAN_OPTS=\"{leanOptsStr}\" LEAN_PATH=\"{leanPathStr}\" {makeArgsStr} MORE_DEPS+=\"{moreDepsStr}\" >&2"] } execCmd spawnArgs diff --git a/Leanpkg2/Package.lean b/Leanpkg2/Package.lean index e5ffc41bc5..27c4111cd1 100644 --- a/Leanpkg2/Package.lean +++ b/Leanpkg2/Package.lean @@ -22,38 +22,47 @@ structure Dependency where name : String src : Source -structure Manifest where +structure PackageConfig where name : String version : String leanVersion : String := leanVersionString timeout : Option Nat := none - module : String := name.capitalize + module : Name := name.capitalize dependencies : List Dependency := [] deriving Inhabited structure Package where dir : FilePath - manifest : Manifest + config : PackageConfig deriving Inhabited namespace Package -def name (self : Package) : String := - self.manifest.name +def name (self : Package) := + self.config.name -def dependencies (self : Package) : List Dependency := - self.manifest.dependencies +def module (self : Package) := + self.config.module -def sourceDir (self : Package) : FilePath := +def dependencies (self : Package) := + self.config.dependencies + +def timeout (self : Package) := + self.config.timeout + +def sourceDir (self : Package) := self.dir -def sourceRoot (self : Package) : FilePath := - self.sourceDir / self.manifest.module +def sourceRoot (self : Package) := + self.sourceDir / self.config.module.toString -def buildDir (self : Package) : FilePath := +def buildDir (self : Package) := self.dir / Leanpkg2.buildPath -def buildRoot (self : Package) : FilePath := - self.buildDir / self.manifest.module +def buildRoot (self : Package) := + self.buildDir / self.config.module.toString + +def oleanRoot (self : Package) := + self.buildRoot.withExtension "olean" end Package diff --git a/Leanpkg2/Resolve.lean b/Leanpkg2/Resolve.lean index cd20d1d4aa..732def2140 100644 --- a/Leanpkg2/Resolve.lean +++ b/Leanpkg2/Resolve.lean @@ -4,9 +4,7 @@ Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Leanpkg2.Git -import Leanpkg2.Proc -import Leanpkg2.TomlManifest -import Leanpkg2.BuildConfig +import Leanpkg2.TomlConfig open System @@ -28,14 +26,14 @@ def materializeGit let hash ← parseOriginRevision rev dir checkoutDetach hash dir -def materialize (relPath : FilePath) (dep : Dependency) : IO FilePath := +def materialize (pkgDir : FilePath) (dep : Dependency) : IO FilePath := match dep.src with | Source.path dir => do - let depdir := relPath / dir + let depdir := pkgDir / dir IO.eprintln s!"{dep.name}: using local path {depdir}" depdir | Source.git url rev branch => do - let depdir := depsPath / dep.name + let depdir := pkgDir / depsPath / dep.name materializeGit dep.name depdir url rev branch depdir @@ -65,23 +63,29 @@ def resolvedPackage (d : String) : Solver Package := do let some pkg ← pure ((← get).lookup d) | unreachable! pkg -def solveDepsCore -(pkgName : String) (relPath : FilePath) (deps : List Dependency) -: (maxDepth : Nat) → Solver Unit +def solveDepsCore (pkg : Package) : (maxDepth : Nat) → Solver Unit | 0 => throw <| IO.userError "maximum dependency resolution depth reached" | maxDepth + 1 => do - let newDeps ← deps.filterM (notYetAssigned ·.name) + let newDeps ← pkg.dependencies.filterM (notYetAssigned ·.name) for dep in newDeps do - let dir ← materialize relPath dep - let m ← Manifest.fromTomlFile <| dir / leanpkgToml - modify (·.insert dep.name ⟨dir, m⟩) + let dir ← materialize pkg.dir dep + let cfg ← PackageConfig.fromTomlFile <| dir / leanpkgToml + modify (·.insert dep.name ⟨dir, cfg⟩) for dep in newDeps do let depPkg ← resolvedPackage dep.name unless depPkg.name = dep.name do - throw <| IO.userError s!"{pkgName} (in {relPath}) depends on {depPkg.name}, but resolved dependency has name {dep.name} (in {depPkg.dir})" - solveDepsCore depPkg.name depPkg.dir depPkg.dependencies maxDepth + throw <| IO.userError <| + s!"{pkg.name} (in {pkg.dir}) depends on {dep.name}, " ++ + s!"but resolved dependency has name {depPkg.name} (in {depPkg.dir})" + solveDepsCore depPkg maxDepth -def solveDeps (m : Manifest) : IO (List Package) := do - let solver := solveDepsCore m.name ⟨"."⟩ m.dependencies 1024 - let (_, assg) ← solver.run (Assignment.empty.insert m.name ⟨".", m⟩) +/-- + Resolves the dependency tree for the given package, + downloading and/or updating missing dependencies as necessary. + + Note that resulting list of dependencies *will* include the given package. +-/ +def solveDeps (pkg : Package) : IO (List Package) := do + let solver := solveDepsCore pkg 1024 + let (_, assg) ← solver.run (Assignment.empty.insert pkg.name ⟨pkg.dir, pkg.config⟩) assg.reverse.mapM (·.2) diff --git a/Leanpkg2/TomlManifest.lean b/Leanpkg2/TomlConfig.lean similarity index 74% rename from Leanpkg2/TomlManifest.lean rename to Leanpkg2/TomlConfig.lean index b37d893c25..fc82ff24e5 100644 --- a/Leanpkg2/TomlManifest.lean +++ b/Leanpkg2/TomlConfig.lean @@ -11,6 +11,8 @@ open System namespace Leanpkg2 +def leanpkgToml : FilePath := "leanpkg.toml" + namespace Source def fromToml? (v : Toml.Value) : Option Source := @@ -32,9 +34,9 @@ def toToml : Source → Toml.Value end Source -namespace Manifest +namespace PackageConfig -def fromToml? (t : Toml.Value) : Option Manifest := OptionM.run do +def fromToml? (t : Toml.Value) : Option PackageConfig := OptionM.run do let pkg ← t.lookup "package" let Toml.Value.str name ← pkg.lookup "name" | none let Toml.Value.str version ← pkg.lookup "version" | none @@ -54,24 +56,15 @@ def fromToml? (t : Toml.Value) : Option Manifest := OptionM.run do let dependencies ← deps.mapM fun ⟨n, src⟩ => do Dependency.mk n (← Source.fromToml? src) return { name, module, version, leanVersion, dependencies, timeout } -def fromTomlString? (str : String) : IO (Option Manifest) := do +def fromTomlString? (str : String) : IO (Option PackageConfig) := do fromToml? (← Toml.parse str) -def fromTomlFile? (path : FilePath) : IO (Option Manifest) := do +def fromTomlFile? (path : FilePath) : IO (Option PackageConfig) := do fromTomlString? (← IO.FS.readFile path) -def fromTomlFile (path : FilePath) : IO Manifest := do - let some manifest ← fromTomlFile? path - | throw <| IO.userError s!"manifest (at {path}) is ill-formed" - manifest +def fromTomlFile (path : FilePath) : IO PackageConfig := do + let some config ← fromTomlFile? path + | throw <| IO.userError s!"configuration (at {path}) is ill-formed" + config -end Manifest - -def leanpkgToml : FilePath := "leanpkg.toml" - -def readManifest : IO Manifest := do - let m ← Manifest.fromTomlFile leanpkgToml - if m.leanVersion ≠ leanVersionString then - IO.eprintln $ "\nWARNING: Lean version mismatch: installed version is " ++ leanVersionString - ++ ", but package requires " ++ m.leanVersion ++ "\n" - return m +end PackageConfig diff --git a/examples/hello/Package.lean b/examples/hello/Package.lean index 5de8e84f0b..382b9f89cc 100644 --- a/examples/hello/Package.lean +++ b/examples/hello/Package.lean @@ -2,16 +2,16 @@ import Leanpkg2.Build open Leanpkg2 -def manifest : Manifest := { +def package : PackageConfig := { name := "hello", version := "1.0", } def configure : IO Unit := - Leanpkg2.configure manifest + Leanpkg2.configure ⟨".", package⟩ def build : IO Unit := - Leanpkg2.build manifest ["bin"] + Leanpkg2.build ⟨".", package⟩ ["bin"] def main : IO Unit := build diff --git a/examples/hello/package.lean b/examples/hello/package.lean index 5de8e84f0b..382b9f89cc 100644 --- a/examples/hello/package.lean +++ b/examples/hello/package.lean @@ -2,16 +2,16 @@ import Leanpkg2.Build open Leanpkg2 -def manifest : Manifest := { +def package : PackageConfig := { name := "hello", version := "1.0", } def configure : IO Unit := - Leanpkg2.configure manifest + Leanpkg2.configure ⟨".", package⟩ def build : IO Unit := - Leanpkg2.build manifest ["bin"] + Leanpkg2.build ⟨".", package⟩ ["bin"] def main : IO Unit := build diff --git a/examples/helloDeps/a/Package.lean b/examples/helloDeps/a/Package.lean index 11d316b818..bc4a057c30 100644 --- a/examples/helloDeps/a/Package.lean +++ b/examples/helloDeps/a/Package.lean @@ -2,13 +2,13 @@ import Leanpkg2.Build open Leanpkg2 -def manifest : Manifest := { +def package : PackageConfig := { name := "a", version := "1.0", } def build : IO Unit := - Leanpkg2.build manifest ["lib"] + Leanpkg2.build ⟨".", package⟩ ["lib"] def main : IO Unit := build diff --git a/examples/helloDeps/a/package.lean b/examples/helloDeps/a/package.lean index 11d316b818..bc4a057c30 100644 --- a/examples/helloDeps/a/package.lean +++ b/examples/helloDeps/a/package.lean @@ -2,13 +2,13 @@ import Leanpkg2.Build open Leanpkg2 -def manifest : Manifest := { +def package : PackageConfig := { name := "a", version := "1.0", } def build : IO Unit := - Leanpkg2.build manifest ["lib"] + Leanpkg2.build ⟨".", package⟩ ["lib"] def main : IO Unit := build diff --git a/examples/helloDeps/b/Package.lean b/examples/helloDeps/b/Package.lean index b5e24b0fc5..8bc0d71a06 100644 --- a/examples/helloDeps/b/Package.lean +++ b/examples/helloDeps/b/Package.lean @@ -2,7 +2,7 @@ import Leanpkg2.Build open Leanpkg2 System -def manifest : Manifest := { +def package : PackageConfig := { name := "b", version := "1.0", dependencies := [ @@ -11,7 +11,7 @@ def manifest : Manifest := { } def build : IO Unit := do - Leanpkg2.build manifest ["bin", "LINK_OPTS=../a/build/lib/libA.a"] + Leanpkg2.build ⟨".", package⟩ ["bin", "LINK_OPTS=../a/build/lib/libA.a"] def main : IO Unit := build diff --git a/examples/helloDeps/b/package.lean b/examples/helloDeps/b/package.lean index b5e24b0fc5..8bc0d71a06 100644 --- a/examples/helloDeps/b/package.lean +++ b/examples/helloDeps/b/package.lean @@ -2,7 +2,7 @@ import Leanpkg2.Build open Leanpkg2 System -def manifest : Manifest := { +def package : PackageConfig := { name := "b", version := "1.0", dependencies := [ @@ -11,7 +11,7 @@ def manifest : Manifest := { } def build : IO Unit := do - Leanpkg2.build manifest ["bin", "LINK_OPTS=../a/build/lib/libA.a"] + Leanpkg2.build ⟨".", package⟩ ["bin", "LINK_OPTS=../a/build/lib/libA.a"] def main : IO Unit := build From 8efd56d131ae317e8205d662df8ee35a9aea4364 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sun, 6 Jun 2021 19:33:12 -0400 Subject: [PATCH 021/696] Properly lowercase `Package.lean` configurations --- examples/hello/Package.lean | 17 ----------------- examples/helloDeps/a/Package.lean | 14 -------------- examples/helloDeps/b/Package.lean | 17 ----------------- 3 files changed, 48 deletions(-) delete mode 100644 examples/hello/Package.lean delete mode 100644 examples/helloDeps/a/Package.lean delete mode 100644 examples/helloDeps/b/Package.lean diff --git a/examples/hello/Package.lean b/examples/hello/Package.lean deleted file mode 100644 index 382b9f89cc..0000000000 --- a/examples/hello/Package.lean +++ /dev/null @@ -1,17 +0,0 @@ -import Leanpkg2.Build - -open Leanpkg2 - -def package : PackageConfig := { - name := "hello", - version := "1.0", -} - -def configure : IO Unit := - Leanpkg2.configure ⟨".", package⟩ - -def build : IO Unit := - Leanpkg2.build ⟨".", package⟩ ["bin"] - -def main : IO Unit := - build diff --git a/examples/helloDeps/a/Package.lean b/examples/helloDeps/a/Package.lean deleted file mode 100644 index bc4a057c30..0000000000 --- a/examples/helloDeps/a/Package.lean +++ /dev/null @@ -1,14 +0,0 @@ -import Leanpkg2.Build - -open Leanpkg2 - -def package : PackageConfig := { - name := "a", - version := "1.0", -} - -def build : IO Unit := - Leanpkg2.build ⟨".", package⟩ ["lib"] - -def main : IO Unit := - build diff --git a/examples/helloDeps/b/Package.lean b/examples/helloDeps/b/Package.lean deleted file mode 100644 index 8bc0d71a06..0000000000 --- a/examples/helloDeps/b/Package.lean +++ /dev/null @@ -1,17 +0,0 @@ -import Leanpkg2.Build - -open Leanpkg2 System - -def package : PackageConfig := { - name := "b", - version := "1.0", - dependencies := [ - { name := "a", src := Source.path (FilePath.mk ".." / "a") } - ] -} - -def build : IO Unit := do - Leanpkg2.build ⟨".", package⟩ ["bin", "LINK_OPTS=../a/build/lib/libA.a"] - -def main : IO Unit := - build From d066872549385c74458f7eecb8fbf1a2572ba913 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sun, 6 Jun 2021 21:40:11 -0400 Subject: [PATCH 022/696] CLI now uses configuration from `package.lean`' --- Leanpkg2/Build.lean | 19 ++++++++++--------- Leanpkg2/Cli.lean | 4 ++-- Leanpkg2/LeanConfig.lean | 28 ++++++++++++++++++++++++++++ Leanpkg2/Resolve.lean | 4 ++-- examples/hello/package.lean | 15 ++------------- examples/hello/package.sh | 2 +- examples/helloDeps/a/package.lean | 12 ++---------- examples/helloDeps/b/package.lean | 6 ------ examples/helloDeps/package.sh | 2 +- 9 files changed, 48 insertions(+), 44 deletions(-) create mode 100644 Leanpkg2/LeanConfig.lean diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean index c6a4be4d76..447d5d6107 100644 --- a/Leanpkg2/Build.lean +++ b/Leanpkg2/Build.lean @@ -118,17 +118,21 @@ def buildImports (pkg : Package) (deps : List Package) (imports leanArgs : List else buildModules (mkBuildConfig pkg deps leanArgs) localImports + +def doBuild (pkg : Package) (deps : List Package) (makeArgs leanArgs : List String := []) : IO Unit := do + if makeArgs != [] || (← FilePath.pathExists "Makefile") then + execMake pkg deps makeArgs leanArgs + else + buildModules (mkBuildConfig pkg deps leanArgs) [pkg.module] + def buildDeps (pkg : Package) : IO (List Package) := do let deps ← solveDeps pkg for dep in deps do unless dep.dir == pkg.dir do -- build recursively -- TODO: share build of common dependencies - execCmd { - cwd := dep.dir - cmd := (← IO.appDir) / "lean" |>.toString - args := #["--run", "Package.lean"] - } + let depDeps ← solveDeps dep + doBuild dep depDeps ["lib"] return deps def configure (pkg : Package) : IO Unit := do @@ -142,7 +146,4 @@ def printPaths (pkg : Package) (imports leanArgs : List String := []) : IO Unit def build (pkg : Package) (makeArgs leanArgs : List String := []) : IO Unit := do let deps ← buildDeps pkg - if makeArgs != [] || (← FilePath.pathExists "Makefile") then - execMake pkg deps makeArgs leanArgs - else - buildModules (mkBuildConfig pkg deps leanArgs) [pkg.module] + doBuild pkg deps makeArgs leanArgs diff --git a/Leanpkg2/Cli.lean b/Leanpkg2/Cli.lean index 9ab76db53e..7d92337ea0 100644 --- a/Leanpkg2/Cli.lean +++ b/Leanpkg2/Cli.lean @@ -5,7 +5,7 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Leanpkg2.Init import Leanpkg2.Build -import Leanpkg2.TomlConfig +import Leanpkg2.LeanConfig namespace Leanpkg2 @@ -62,7 +62,7 @@ This command creates a new Lean package with the given name in the current directory." def getRootPkg : IO Package := do - let cfg ← PackageConfig.fromTomlFile leanpkgToml + let cfg ← PackageConfig.fromLeanFile leanConfigFile if cfg.leanVersion ≠ leanVersionString then IO.eprintln $ "\nWARNING: Lean version mismatch: installed version is " ++ leanVersionString ++ ", but package requires " ++ cfg.leanVersion ++ "\n" diff --git a/Leanpkg2/LeanConfig.lean b/Leanpkg2/LeanConfig.lean new file mode 100644 index 0000000000..7a1f4b17c7 --- /dev/null +++ b/Leanpkg2/LeanConfig.lean @@ -0,0 +1,28 @@ + +/- +Copyright (c) 2021 Mac Malone. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Mac Malone +-/ +import Lean +import Leanpkg2.Package + +open Lean Elab System + +namespace Leanpkg2 + +def leanConfigFile : FilePath := "package.lean" + +namespace PackageConfig + +unsafe def fromLeanFileUnsafe (path : FilePath) : IO PackageConfig := do + let input ← IO.FS.readFile path + let (env, ok) ← runFrontend input Options.empty path.toString `package + if ok then + IO.ofExcept <| Id.run <| ExceptT.run <| + env.evalConstCheck PackageConfig Options.empty ``PackageConfig `package + else + throw <| IO.userError <| s!"package configuration (at {path}) has errors" + +@[implementedBy fromLeanFileUnsafe] +constant fromLeanFile (path : FilePath) : IO PackageConfig diff --git a/Leanpkg2/Resolve.lean b/Leanpkg2/Resolve.lean index 732def2140..5e16ad2dca 100644 --- a/Leanpkg2/Resolve.lean +++ b/Leanpkg2/Resolve.lean @@ -4,7 +4,7 @@ Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Leanpkg2.Git -import Leanpkg2.TomlConfig +import Leanpkg2.LeanConfig open System @@ -69,7 +69,7 @@ def solveDepsCore (pkg : Package) : (maxDepth : Nat) → Solver Unit let newDeps ← pkg.dependencies.filterM (notYetAssigned ·.name) for dep in newDeps do let dir ← materialize pkg.dir dep - let cfg ← PackageConfig.fromTomlFile <| dir / leanpkgToml + let cfg ← PackageConfig.fromLeanFile <| dir / leanConfigFile modify (·.insert dep.name ⟨dir, cfg⟩) for dep in newDeps do let depPkg ← resolvedPackage dep.name diff --git a/examples/hello/package.lean b/examples/hello/package.lean index 382b9f89cc..9d65d2aca1 100644 --- a/examples/hello/package.lean +++ b/examples/hello/package.lean @@ -1,17 +1,6 @@ -import Leanpkg2.Build +import Leanpkg2.Package -open Leanpkg2 - -def package : PackageConfig := { +def package : Leanpkg2.PackageConfig := { name := "hello", version := "1.0", } - -def configure : IO Unit := - Leanpkg2.configure ⟨".", package⟩ - -def build : IO Unit := - Leanpkg2.build ⟨".", package⟩ ["bin"] - -def main : IO Unit := - build diff --git a/examples/hello/package.sh b/examples/hello/package.sh index 8b9f18965e..2c09854503 100644 --- a/examples/hello/package.sh +++ b/examples/hello/package.sh @@ -1,2 +1,2 @@ export LEAN_PATH=../../build -lean --run Package.lean +lean --run ../../Leanpkg2.lean build bin diff --git a/examples/helloDeps/a/package.lean b/examples/helloDeps/a/package.lean index bc4a057c30..7e697633cb 100644 --- a/examples/helloDeps/a/package.lean +++ b/examples/helloDeps/a/package.lean @@ -1,14 +1,6 @@ -import Leanpkg2.Build +import Leanpkg2.Package -open Leanpkg2 - -def package : PackageConfig := { +def package : Leanpkg2.PackageConfig := { name := "a", version := "1.0", } - -def build : IO Unit := - Leanpkg2.build ⟨".", package⟩ ["lib"] - -def main : IO Unit := - build diff --git a/examples/helloDeps/b/package.lean b/examples/helloDeps/b/package.lean index 8bc0d71a06..8e6bc14eb9 100644 --- a/examples/helloDeps/b/package.lean +++ b/examples/helloDeps/b/package.lean @@ -9,9 +9,3 @@ def package : PackageConfig := { { name := "a", src := Source.path (FilePath.mk ".." / "a") } ] } - -def build : IO Unit := do - Leanpkg2.build ⟨".", package⟩ ["bin", "LINK_OPTS=../a/build/lib/libA.a"] - -def main : IO Unit := - build diff --git a/examples/helloDeps/package.sh b/examples/helloDeps/package.sh index 0af6f92968..f8c6ddc536 100644 --- a/examples/helloDeps/package.sh +++ b/examples/helloDeps/package.sh @@ -1,3 +1,3 @@ cd b export LEAN_PATH=../../../build -lean --run Package.lean +lean --run ../../../Leanpkg2.lean build bin LINK_OPTS=../a/build/lib/libA.a From 99d458c646e00fa76868edda68534bda6781beb2 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sun, 6 Jun 2021 21:56:58 -0400 Subject: [PATCH 023/696] Update init to produce `package.lean` --- Leanpkg2/Cli.lean | 2 +- Leanpkg2/Init.lean | 21 +++++++++++---------- Leanpkg2/LeanConfig.lean | 2 +- Leanpkg2/Resolve.lean | 2 +- examples/hello/package.lean | 4 ++-- examples/helloDeps/a/package.lean | 4 ++-- examples/helloDeps/b/package.lean | 4 ++-- 7 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Leanpkg2/Cli.lean b/Leanpkg2/Cli.lean index 7d92337ea0..e153100ee4 100644 --- a/Leanpkg2/Cli.lean +++ b/Leanpkg2/Cli.lean @@ -62,7 +62,7 @@ This command creates a new Lean package with the given name in the current directory." def getRootPkg : IO Package := do - let cfg ← PackageConfig.fromLeanFile leanConfigFile + let cfg ← PackageConfig.fromLeanFile leanPkgFile if cfg.leanVersion ≠ leanVersionString then IO.eprintln $ "\nWARNING: Lean version mismatch: installed version is " ++ leanVersionString ++ ", but package requires " ++ cfg.leanVersion ++ "\n" diff --git a/Leanpkg2/Init.lean b/Leanpkg2/Init.lean index b08495b872..e7877486ff 100644 --- a/Leanpkg2/Init.lean +++ b/Leanpkg2/Init.lean @@ -5,7 +5,7 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Leanpkg2.Git import Leanpkg2.Proc -import Leanpkg2.TomlConfig +import Leanpkg2.LeanConfig namespace Leanpkg2 @@ -18,16 +18,19 @@ def mainFileContents := IO.println \"Hello, world!\" " -def leanpkgFileContents (pkgName : String) := -s!"[package] -name = \"{pkgName}\" -version = \"0.1\" -lean_version = \"{leanVersionString}\" +def leanPkgFileContents (pkgName : String) := +s!"import Leanpkg2.Package + +def package : Leanpkg2.PackageConfig := \{ + name := \"{pkgName}\" + version := \"0.1\" + leanVersion := \"{leanVersionString}\" +} " open Git in -def initPkg (pkgName : String) (fromNew : Bool) : IO Unit := do - IO.FS.writeFile leanpkgToml (leanpkgFileContents pkgName) +def init (pkgName : String) : IO Unit := do + IO.FS.writeFile leanPkgFile (leanPkgFileContents pkgName) IO.FS.writeFile ⟨s!"{pkgName.capitalize}.lean"⟩ mainFileContents let h ← IO.FS.Handle.mk ⟨".gitignore"⟩ IO.FS.Mode.append (bin := false) h.putStr initGitignoreContents @@ -38,5 +41,3 @@ def initPkg (pkgName : String) (fromNew : Bool) : IO Unit := do checkoutBranch upstreamBranch ) <|> IO.eprintln "WARNING: failed to initialize git repository" - -def init (pkgName : String) := initPkg pkgName false diff --git a/Leanpkg2/LeanConfig.lean b/Leanpkg2/LeanConfig.lean index 7a1f4b17c7..2430f1e6bf 100644 --- a/Leanpkg2/LeanConfig.lean +++ b/Leanpkg2/LeanConfig.lean @@ -11,7 +11,7 @@ open Lean Elab System namespace Leanpkg2 -def leanConfigFile : FilePath := "package.lean" +def leanPkgFile : FilePath := "package.lean" namespace PackageConfig diff --git a/Leanpkg2/Resolve.lean b/Leanpkg2/Resolve.lean index 5e16ad2dca..9dce9f2c15 100644 --- a/Leanpkg2/Resolve.lean +++ b/Leanpkg2/Resolve.lean @@ -69,7 +69,7 @@ def solveDepsCore (pkg : Package) : (maxDepth : Nat) → Solver Unit let newDeps ← pkg.dependencies.filterM (notYetAssigned ·.name) for dep in newDeps do let dir ← materialize pkg.dir dep - let cfg ← PackageConfig.fromLeanFile <| dir / leanConfigFile + let cfg ← PackageConfig.fromLeanFile <| dir / leanPkgFile modify (·.insert dep.name ⟨dir, cfg⟩) for dep in newDeps do let depPkg ← resolvedPackage dep.name diff --git a/examples/hello/package.lean b/examples/hello/package.lean index 9d65d2aca1..16c26d7ec4 100644 --- a/examples/hello/package.lean +++ b/examples/hello/package.lean @@ -1,6 +1,6 @@ import Leanpkg2.Package def package : Leanpkg2.PackageConfig := { - name := "hello", - version := "1.0", + name := "hello" + version := "1.0" } diff --git a/examples/helloDeps/a/package.lean b/examples/helloDeps/a/package.lean index 7e697633cb..db385e4029 100644 --- a/examples/helloDeps/a/package.lean +++ b/examples/helloDeps/a/package.lean @@ -1,6 +1,6 @@ import Leanpkg2.Package def package : Leanpkg2.PackageConfig := { - name := "a", - version := "1.0", + name := "a" + version := "1.0" } diff --git a/examples/helloDeps/b/package.lean b/examples/helloDeps/b/package.lean index 8e6bc14eb9..0c84296a53 100644 --- a/examples/helloDeps/b/package.lean +++ b/examples/helloDeps/b/package.lean @@ -3,8 +3,8 @@ import Leanpkg2.Build open Leanpkg2 System def package : PackageConfig := { - name := "b", - version := "1.0", + name := "b" + version := "1.0" dependencies := [ { name := "a", src := Source.path (FilePath.mk ".." / "a") } ] From 6317ab22e7a7be2c86565fd8bd4717a04a81f333 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sun, 6 Jun 2021 22:50:40 -0400 Subject: [PATCH 024/696] Only build dependency lib if bin is passed to `leanpkg buld` --- Leanpkg2/Build.lean | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean index 447d5d6107..02a52b65f1 100644 --- a/Leanpkg2/Build.lean +++ b/Leanpkg2/Build.lean @@ -118,24 +118,23 @@ def buildImports (pkg : Package) (deps : List Package) (imports leanArgs : List else buildModules (mkBuildConfig pkg deps leanArgs) localImports - -def doBuild (pkg : Package) (deps : List Package) (makeArgs leanArgs : List String := []) : IO Unit := do +def buildPkg (pkg : Package) (deps : List Package) (makeArgs leanArgs : List String := []) : IO Unit := do if makeArgs != [] || (← FilePath.pathExists "Makefile") then execMake pkg deps makeArgs leanArgs else buildModules (mkBuildConfig pkg deps leanArgs) [pkg.module] -def buildDeps (pkg : Package) : IO (List Package) := do +def buildDeps (pkg : Package) (makeArgs leanArgs : List String := []) : IO (List Package) := do let deps ← solveDeps pkg for dep in deps do unless dep.dir == pkg.dir do -- build recursively -- TODO: share build of common dependencies let depDeps ← solveDeps dep - doBuild dep depDeps ["lib"] + buildPkg dep depDeps makeArgs leanArgs return deps -def configure (pkg : Package) : IO Unit := do +def configure (pkg : Package) : IO Unit := discard <| buildDeps pkg def printPaths (pkg : Package) (imports leanArgs : List String := []) : IO Unit := do @@ -145,5 +144,5 @@ def printPaths (pkg : Package) (imports leanArgs : List String := []) : IO Unit IO.println <| SearchPath.toString <| deps.map (·.sourceDir) def build (pkg : Package) (makeArgs leanArgs : List String := []) : IO Unit := do - let deps ← buildDeps pkg - doBuild pkg deps makeArgs leanArgs + let deps ← buildDeps pkg (if makeArgs.contains "bin" then ["lib"] else []) + buildPkg pkg deps makeArgs leanArgs From 2ba39f56f0e8067ed7d4d26303e477fd4922942e Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sun, 6 Jun 2021 23:04:52 -0400 Subject: [PATCH 025/696] The solved dependency list no longer includes the root package --- Leanpkg2/Build.lean | 17 ++++++++--------- Leanpkg2/Make.lean | 5 ++--- Leanpkg2/Resolve.lean | 4 +--- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean index 02a52b65f1..acabad4092 100644 --- a/Leanpkg2/Build.lean +++ b/Leanpkg2/Build.lean @@ -26,8 +26,8 @@ def mkBuildConfig : BuildConfig := { leanArgs, module := pkg.module - leanPath := SearchPath.toString <| deps.map (·.buildDir) - moreDeps := deps.filter (·.dir != pkg.dir) |>.map (·.oleanRoot) + leanPath := SearchPath.toString <| pkg.buildDir :: deps.map (·.buildDir) + moreDeps := deps.map (·.oleanRoot) } structure BuildContext extends BuildConfig where @@ -127,11 +127,10 @@ def buildPkg (pkg : Package) (deps : List Package) (makeArgs leanArgs : List Str def buildDeps (pkg : Package) (makeArgs leanArgs : List String := []) : IO (List Package) := do let deps ← solveDeps pkg for dep in deps do - unless dep.dir == pkg.dir do - -- build recursively - -- TODO: share build of common dependencies - let depDeps ← solveDeps dep - buildPkg dep depDeps makeArgs leanArgs + -- build recursively + -- TODO: share build of common dependencies + let depDeps ← solveDeps dep + buildPkg dep depDeps makeArgs leanArgs return deps def configure (pkg : Package) : IO Unit := @@ -140,8 +139,8 @@ def configure (pkg : Package) : IO Unit := def printPaths (pkg : Package) (imports leanArgs : List String := []) : IO Unit := do let deps ← buildDeps pkg buildImports pkg deps imports leanArgs - IO.println <| SearchPath.toString <| deps.map (·.buildDir) - IO.println <| SearchPath.toString <| deps.map (·.sourceDir) + IO.println <| SearchPath.toString <| pkg.buildDir :: deps.map (·.buildDir) + IO.println <| SearchPath.toString <| pkg.sourceDir :: deps.map (·.sourceDir) def build (pkg : Package) (makeArgs leanArgs : List String := []) : IO Unit := do let deps ← buildDeps pkg (if makeArgs.contains "bin" then ["lib"] else []) diff --git a/Leanpkg2/Make.lean b/Leanpkg2/Make.lean index 55f164fc4e..5c23dc7f44 100644 --- a/Leanpkg2/Make.lean +++ b/Leanpkg2/Make.lean @@ -48,10 +48,9 @@ def execMake | none => [] let leanmake := (← IO.appDir) / "leanmake" let leanOptsStr := " ".intercalate <| timeoutArgs ++ leanArgs - let leanPathStr := SearchPath.toString <| deps.map (·.buildDir) + let leanPathStr := SearchPath.toString <| pkg.buildDir :: deps.map (·.buildDir) let makeArgsStr := " ".intercalate makeArgs - let moreDepsStr := " ".intercalate $ - deps.filter (·.dir.toString != pkg.dir) |>.map (·.oleanRoot.toString) + let moreDepsStr := " ".intercalate <| deps.map (·.oleanRoot.toString) let mut spawnArgs := { cmd := "sh" cwd := pkg.dir diff --git a/Leanpkg2/Resolve.lean b/Leanpkg2/Resolve.lean index 9dce9f2c15..ad809679ca 100644 --- a/Leanpkg2/Resolve.lean +++ b/Leanpkg2/Resolve.lean @@ -82,10 +82,8 @@ def solveDepsCore (pkg : Package) : (maxDepth : Nat) → Solver Unit /-- Resolves the dependency tree for the given package, downloading and/or updating missing dependencies as necessary. - - Note that resulting list of dependencies *will* include the given package. -/ def solveDeps (pkg : Package) : IO (List Package) := do let solver := solveDepsCore pkg 1024 let (_, assg) ← solver.run (Assignment.empty.insert pkg.name ⟨pkg.dir, pkg.config⟩) - assg.reverse.mapM (·.2) + assg.reverse.tail!.mapM (·.2) From 0f6b07e4344b46e4218bfc1714ec071a7451fd46 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sun, 6 Jun 2021 23:06:19 -0400 Subject: [PATCH 026/696] Remove unused `examples/helloDeps/a/leanpkg.toml` --- examples/helloDeps/a/leanpkg.toml | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 examples/helloDeps/a/leanpkg.toml diff --git a/examples/helloDeps/a/leanpkg.toml b/examples/helloDeps/a/leanpkg.toml deleted file mode 100644 index 123e4c53fb..0000000000 --- a/examples/helloDeps/a/leanpkg.toml +++ /dev/null @@ -1,3 +0,0 @@ -[package] -name = "a" -version = "0.1" From 76183aa6d18438b94bfd678011f927ee7942e3c7 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sun, 6 Jun 2021 23:30:32 -0400 Subject: [PATCH 027/696] Remove TOML code --- Leanpkg2/Build.lean | 3 +- Leanpkg2/Toml.lean | 72 ---------------------------------------- Leanpkg2/TomlConfig.lean | 70 -------------------------------------- 3 files changed, 2 insertions(+), 143 deletions(-) delete mode 100644 Leanpkg2/Toml.lean delete mode 100644 Leanpkg2/TomlConfig.lean diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean index acabad4092..51fe23fc65 100644 --- a/Leanpkg2/Build.lean +++ b/Leanpkg2/Build.lean @@ -18,7 +18,8 @@ structure BuildConfig where module : Name leanArgs : List String leanPath : String - -- things like `leanpkg.toml` and olean roots of dependencies that should also trigger rebuilds + -- things that should also trigger rebuilds + -- ex. olean roots of dependencies moreDeps : List FilePath def mkBuildConfig diff --git a/Leanpkg2/Toml.lean b/Leanpkg2/Toml.lean deleted file mode 100644 index 7337ef887b..0000000000 --- a/Leanpkg2/Toml.lean +++ /dev/null @@ -1,72 +0,0 @@ -/- -Copyright (c) 2017 Microsoft Corporation. All rights reserved. -Released under Apache 2.0 license as described in the file LICENSE. -Authors: Gabriel Ebner, Sebastian Ullrich --/ -import Lean.Parser - -namespace Toml - -inductive Value : Type where - | str : String → Value - | nat : Nat → Value - | bool : Bool → Value - | table : List (String × Value) → Value - deriving Inhabited - -def Value.lookup : Value → String → Option Value - | Value.table cs, k => cs.lookup k - | _, _ => none - --- TODO: custom whitespace and other inaccuracies -declare_syntax_cat val -syntax "True" : val -syntax "False" : val -syntax str : val -syntax num : val -syntax bareKey := ident -- TODO -syntax key := bareKey <|> str -declare_syntax_cat keyCat @[keyCatParser] def key' := key -- HACK: for the antiquotation -syntax keyVal := key " = " val -syntax table := "[" key "]" keyVal* -syntax inlineTable := "{" keyVal,* "}" -syntax inlineTable : val -syntax file := table* -declare_syntax_cat fileCat @[fileCatParser] def file' := file -- HACK: for the antiquotation - -open Lean - -partial def ofSyntax : Syntax → Value - | `(val|True) => Value.bool true - | `(val|False) => Value.bool false - | `(val|$s:strLit) => Value.str <| s.isStrLit?.get! - | `(val|$n:numLit) => Value.nat <| n.isNatLit?.get! - | `(val|{$[$keys:key = $values],*}) => toTable keys (values.map ofSyntax) - | `(fileCat|$[[$keys] $kvss*]*) => toTable keys <| kvss.map fun kvs => ofSyntax <| Lean.Unhygienic.run `(val|{$kvs,*}) - | stx => unreachable! - where - toKey : Syntax → String - | `(keyCat|$key:ident) => key.getId.toString - | `(keyCat|$key:strLit) => key.isStrLit?.get! - | _ => unreachable! - toTable (keys : Array Syntax) (vals : Array Value) : Value := - Value.table <| Array.toList <| keys.zipWith vals fun k v => (toKey k, v) - -open Lean.Parser - -def parse (input : String) : IO Value := do - -- HACKHACKHACK - let env ← importModules [{ module := `Leanpkg2.Toml }] {} - let fileParser ← compileParserDescr (parserExtension.getState env).categories file { env := env, opts := {} } - let c := mkParserContext (mkInputContext input "") { env := env, options := {} } - let s := mkParserState input - let s := whitespace c s - let s := fileParser.fn c s - if s.hasError then - throw <| IO.userError (s.toErrorMsg c) - else if input.atEnd s.pos then - ofSyntax s.stxStack.back - else - throw <| IO.userError ((s.mkError "end of input").toErrorMsg c) - -end Toml diff --git a/Leanpkg2/TomlConfig.lean b/Leanpkg2/TomlConfig.lean deleted file mode 100644 index fc82ff24e5..0000000000 --- a/Leanpkg2/TomlConfig.lean +++ /dev/null @@ -1,70 +0,0 @@ -/- -Copyright (c) 2017 Microsoft Corporation. All rights reserved. -Released under Apache 2.0 license as described in the file LICENSE. -Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone --/ -import Leanpkg2.Toml -import Leanpkg2.LeanVersion -import Leanpkg2.Package - -open System - -namespace Leanpkg2 - -def leanpkgToml : FilePath := "leanpkg.toml" - -namespace Source - -def fromToml? (v : Toml.Value) : Option Source := - (do let Toml.Value.str dir ← v.lookup "path" | none - path ⟨dir⟩) <|> - (do let Toml.Value.str url ← v.lookup "git" | none - let Toml.Value.str rev ← v.lookup "rev" | none - match v.lookup "branch" with - | none => git url rev none - | some (Toml.Value.str branch) => git url rev (some branch) - | _ => none) - -def toToml : Source → Toml.Value - | path dir => Toml.Value.table [("path", Toml.Value.str dir.toString)] - | git url rev none => - Toml.Value.table [("git", Toml.Value.str url), ("rev", Toml.Value.str rev)] - | git url rev (some branch) => - Toml.Value.table [("git", Toml.Value.str url), ("branch", Toml.Value.str branch), ("rev", Toml.Value.str rev)] - -end Source - -namespace PackageConfig - -def fromToml? (t : Toml.Value) : Option PackageConfig := OptionM.run do - let pkg ← t.lookup "package" - let Toml.Value.str name ← pkg.lookup "name" | none - let Toml.Value.str version ← pkg.lookup "version" | none - let module ← match pkg.lookup "module" with - | some (Toml.Value.str mod) => mod - | none => some name.capitalize - | _ => none - let leanVersion ← match pkg.lookup "lean_version" with - | some (Toml.Value.str leanVer) => some leanVer - | none => some leanVersionString - | _ => none - let timeout ← match pkg.lookup "timeout" with - | some (Toml.Value.nat timeout) => some (some timeout) - | none => some none - | _ => none - let Toml.Value.table deps ← t.lookup "dependencies" <|> some (Toml.Value.table []) | none - let dependencies ← deps.mapM fun ⟨n, src⟩ => do Dependency.mk n (← Source.fromToml? src) - return { name, module, version, leanVersion, dependencies, timeout } - -def fromTomlString? (str : String) : IO (Option PackageConfig) := do - fromToml? (← Toml.parse str) - -def fromTomlFile? (path : FilePath) : IO (Option PackageConfig) := do - fromTomlString? (← IO.FS.readFile path) - -def fromTomlFile (path : FilePath) : IO PackageConfig := do - let some config ← fromTomlFile? path - | throw <| IO.userError s!"configuration (at {path}) is ill-formed" - config - -end PackageConfig From 7770d4b421b51201517b9cdff3c765ca1b92791a Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Mon, 7 Jun 2021 02:38:23 -0400 Subject: [PATCH 028/696] Removed leftover hack for TOML --- Leanpkg2.lean | 1 - 1 file changed, 1 deletion(-) diff --git a/Leanpkg2.lean b/Leanpkg2.lean index 1c650d078c..d3f23cd45f 100644 --- a/Leanpkg2.lean +++ b/Leanpkg2.lean @@ -7,7 +7,6 @@ import Leanpkg2.Cli def main (args : List String) : IO UInt32 := do try - Lean.initSearchPath none -- HACK let (cmd, outerArgs, innerArgs) ← Leanpkg2.splitCmdlineArgs args Leanpkg2.cli cmd outerArgs innerArgs pure 0 From 12537427c2a5a0b2d9f85bc5e0b2bc54807b5372 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Mon, 7 Jun 2021 05:30:17 -0400 Subject: [PATCH 029/696] Fix some errors when running leanpkg2 in executable form --- Leanpkg2.lean | 9 +++++++++ Leanpkg2/Build.lean | 2 +- Leanpkg2/Make.lean | 3 +-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Leanpkg2.lean b/Leanpkg2.lean index d3f23cd45f..b732c85fff 100644 --- a/Leanpkg2.lean +++ b/Leanpkg2.lean @@ -7,6 +7,15 @@ import Leanpkg2.Cli def main (args : List String) : IO UInt32 := do try + /- + Initializes the search path the Leanpkg2 executable + uses when intepreting package configuration files. + + Also, in order to find the Lean stdlib (e.g., `Init`), + the executable needs to be either colocated with Lean or + have LEAN_PATH include the directory with its dynamic libraries. + -/ + Lean.initSearchPath none let (cmd, outerArgs, innerArgs) ← Leanpkg2.splitCmdlineArgs args Leanpkg2.cli cmd outerArgs innerArgs pure 0 diff --git a/Leanpkg2/Build.lean b/Leanpkg2/Build.lean index 51fe23fc65..ba8b533070 100644 --- a/Leanpkg2/Build.lean +++ b/Leanpkg2/Build.lean @@ -87,7 +87,7 @@ partial def buildModule (mod : Name) : BuildM Result := do IO.createDirAll oleanFile.parent.get! IO.createDirAll cFile.parent.get! execCmd { - cmd := (← IO.appDir) / "lean" |>.withExtension FilePath.exeExtension |>.toString + cmd := FilePath.withExtension "lean" FilePath.exeExtension |>.toString args := ctx.leanArgs.toArray ++ #["-o", oleanFile.toString, "-c", cFile.toString, leanFile.toString] env := #[("LEAN_PATH", ctx.leanPath)] } diff --git a/Leanpkg2/Make.lean b/Leanpkg2/Make.lean index 5c23dc7f44..46ead54e40 100644 --- a/Leanpkg2/Make.lean +++ b/Leanpkg2/Make.lean @@ -46,7 +46,6 @@ def execMake match pkg.timeout with | some t => ["-T", toString t] | none => [] - let leanmake := (← IO.appDir) / "leanmake" let leanOptsStr := " ".intercalate <| timeoutArgs ++ leanArgs let leanPathStr := SearchPath.toString <| pkg.buildDir :: deps.map (·.buildDir) let makeArgsStr := " ".intercalate makeArgs @@ -54,6 +53,6 @@ def execMake let mut spawnArgs := { cmd := "sh" cwd := pkg.dir - args := #["-c", s!"\"{leanmake}\" PKG={pkg.module} LEAN_OPTS=\"{leanOptsStr}\" LEAN_PATH=\"{leanPathStr}\" {makeArgsStr} MORE_DEPS+=\"{moreDepsStr}\" >&2"] + args := #["-c", s!"leanmake PKG={pkg.module} LEAN_OPTS=\"{leanOptsStr}\" LEAN_PATH=\"{leanPathStr}\" {makeArgsStr} MORE_DEPS+=\"{moreDepsStr}\" >&2"] } execCmd spawnArgs From 6fc398133d0b6064796cf6fd6ed5104888739f70 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Mon, 7 Jun 2021 05:42:42 -0400 Subject: [PATCH 030/696] Rename Leanpkg2 to Lake --- Leanpkg2.lean => Lake.lean | 8 ++++---- {Leanpkg2 => Lake}/Build.lean | 10 +++++----- {Leanpkg2 => Lake}/Cli.lean | 30 ++++++++++++++--------------- {Leanpkg2 => Lake}/Git.lean | 6 +++--- {Leanpkg2 => Lake}/Init.lean | 12 ++++++------ {Leanpkg2 => Lake}/LeanConfig.lean | 4 ++-- {Leanpkg2 => Lake}/LeanVersion.lean | 2 +- {Leanpkg2 => Lake}/Make.lean | 10 +++++----- {Leanpkg2 => Lake}/Package.lean | 6 +++--- {Leanpkg2 => Lake}/Proc.lean | 2 +- {Leanpkg2 => Lake}/Resolve.lean | 6 +++--- README.md | 2 +- examples/hello/Hello.lean | 2 +- examples/hello/package.lean | 4 ++-- examples/hello/package.sh | 2 +- examples/helloDeps/a/package.lean | 4 ++-- examples/helloDeps/b/package.lean | 4 ++-- examples/helloDeps/package.sh | 2 +- leanpkg.toml | 2 +- 19 files changed, 59 insertions(+), 59 deletions(-) rename Leanpkg2.lean => Lake.lean (76%) rename {Leanpkg2 => Lake}/Build.lean (98%) rename {Leanpkg2 => Lake}/Cli.lean (80%) rename {Leanpkg2 => Lake}/Git.lean (96%) rename {Leanpkg2 => Lake}/Init.lean (86%) rename {Leanpkg2 => Lake}/LeanConfig.lean (94%) rename {Leanpkg2 => Lake}/LeanVersion.lean (97%) rename {Leanpkg2 => Lake}/Make.lean (89%) rename {Leanpkg2 => Lake}/Package.lean (94%) rename {Leanpkg2 => Lake}/Proc.lean (97%) rename {Leanpkg2 => Lake}/Resolve.lean (97%) diff --git a/Leanpkg2.lean b/Lake.lean similarity index 76% rename from Leanpkg2.lean rename to Lake.lean index b732c85fff..abc445f59d 100644 --- a/Leanpkg2.lean +++ b/Lake.lean @@ -3,12 +3,12 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ -import Leanpkg2.Cli +import Lake.Cli def main (args : List String) : IO UInt32 := do try /- - Initializes the search path the Leanpkg2 executable + Initializes the search path the Lake executable uses when intepreting package configuration files. Also, in order to find the Lean stdlib (e.g., `Init`), @@ -16,8 +16,8 @@ def main (args : List String) : IO UInt32 := do have LEAN_PATH include the directory with its dynamic libraries. -/ Lean.initSearchPath none - let (cmd, outerArgs, innerArgs) ← Leanpkg2.splitCmdlineArgs args - Leanpkg2.cli cmd outerArgs innerArgs + let (cmd, outerArgs, innerArgs) ← Lake.splitCmdlineArgs args + Lake.cli cmd outerArgs innerArgs pure 0 catch e => IO.eprintln e -- avoid "uncaught exception: ..." diff --git a/Leanpkg2/Build.lean b/Lake/Build.lean similarity index 98% rename from Leanpkg2/Build.lean rename to Lake/Build.lean index ba8b533070..4bd02fa876 100644 --- a/Leanpkg2/Build.lean +++ b/Lake/Build.lean @@ -5,14 +5,14 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Lean.Data.Name import Lean.Elab.Import -import Leanpkg2.Resolve -import Leanpkg2.Package -import Leanpkg2.Make -import Leanpkg2.Proc +import Lake.Resolve +import Lake.Package +import Lake.Make +import Lake.Proc open Lean System -namespace Leanpkg2 +namespace Lake structure BuildConfig where module : Name diff --git a/Leanpkg2/Cli.lean b/Lake/Cli.lean similarity index 80% rename from Leanpkg2/Cli.lean rename to Lake/Cli.lean index e153100ee4..aa3f558679 100644 --- a/Leanpkg2/Cli.lean +++ b/Lake/Cli.lean @@ -3,27 +3,27 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ -import Leanpkg2.Init -import Leanpkg2.Build -import Leanpkg2.LeanConfig +import Lake.Init +import Lake.Build +import Lake.LeanConfig -namespace Leanpkg2 +namespace Lake def usage := -"Lean package manager, version " ++ uiLeanVersionString ++ " -Usage: leanpkg +"Lake, version " ++ uiLeanVersionString ++ " +Usage: lake init create a Lean package in the current directory configure download and build dependencies build [] configure and build *.olean files -See `leanpkg help ` for more information on a specific command." +See `lake help ` for more information on a specific command." def helpConfigure := "Download dependencies Usage: - leanpkg configure + lake configure This command sets up the `build/deps` directory. @@ -36,27 +36,27 @@ def helpBuild := "Download dependencies and build *.olean files Usage: - leanpkg build [] [-- ] + lake build [] [-- ] -This command invokes `leanpkg configure` followed by `leanmake LEAN_OPTS=`. +This command invokes `lake configure` followed by `leanmake LEAN_OPTS=`. If defined, the `package.timeout` configuration value is passed to Lean via its `-T` parameter. If no are given, only .olean files will be produced in `build/`. If `lib` or `bin` is passed instead, the extracted C code is compiled with `c++` and a static library in `build/lib` -or an executable in `build/bin`, respectively, is created. `leanpkg build bin` requires a declaration +or an executable in `build/bin`, respectively, is created. `lake build bin` requires a declaration of name `main` in the root namespace, which must return `IO Unit` or `IO UInt32` (the exit code) and may accept the program's command line arguments as a `List String` parameter. NOTE: building and linking dependent libraries currently has to be done manually, e.g. ``` -$ (cd a; leanpkg build lib) -$ (cd b; leanpkg build bin LINK_OPTS=../a/build/lib/libA.a) +$ (cd a; lake build lib) +$ (cd b; lake build bin LINK_OPTS=../a/build/lib/libA.a) ```" def helpInit := "Create a new Lean package in the current directory Usage: - leanpkg init + lake init This command creates a new Lean package with the given name in the current directory." @@ -68,7 +68,7 @@ def getRootPkg : IO Package := do leanVersionString ++ ", but package requires " ++ cfg.leanVersion ++ "\n" return ⟨".", cfg⟩ -def cli : (cmd : String) → (leanpkgArgs leanArgs : List String) → IO Unit +def cli : (cmd : String) → (lakeArgs leanArgs : List String) → IO Unit | "init", [name], [] => init name | "configure", [], [] => do configure (← getRootPkg) | "print-paths", imports, leanArgs => do printPaths (← getRootPkg) imports leanArgs diff --git a/Leanpkg2/Git.lean b/Lake/Git.lean similarity index 96% rename from Leanpkg2/Git.lean rename to Lake/Git.lean index 59eb4500ba..d6afd45501 100644 --- a/Leanpkg2/Git.lean +++ b/Lake/Git.lean @@ -3,12 +3,12 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ -import Leanpkg2.Proc -import Leanpkg2.LeanVersion +import Lake.Proc +import Lake.LeanVersion open System -namespace Leanpkg2.Git +namespace Lake.Git def upstreamBranch := "master" diff --git a/Leanpkg2/Init.lean b/Lake/Init.lean similarity index 86% rename from Leanpkg2/Init.lean rename to Lake/Init.lean index e7877486ff..3685453121 100644 --- a/Leanpkg2/Init.lean +++ b/Lake/Init.lean @@ -3,11 +3,11 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ -import Leanpkg2.Git -import Leanpkg2.Proc -import Leanpkg2.LeanConfig +import Lake.Git +import Lake.Proc +import Lake.LeanConfig -namespace Leanpkg2 +namespace Lake def initGitignoreContents := "/build @@ -19,9 +19,9 @@ def mainFileContents := " def leanPkgFileContents (pkgName : String) := -s!"import Leanpkg2.Package +s!"import Lake.Package -def package : Leanpkg2.PackageConfig := \{ +def package : Lake.PackageConfig := \{ name := \"{pkgName}\" version := \"0.1\" leanVersion := \"{leanVersionString}\" diff --git a/Leanpkg2/LeanConfig.lean b/Lake/LeanConfig.lean similarity index 94% rename from Leanpkg2/LeanConfig.lean rename to Lake/LeanConfig.lean index 2430f1e6bf..f0270f55ee 100644 --- a/Leanpkg2/LeanConfig.lean +++ b/Lake/LeanConfig.lean @@ -5,11 +5,11 @@ Released under Apache 2.0 license as described in the file LICENSE. Authors: Mac Malone -/ import Lean -import Leanpkg2.Package +import Lake.Package open Lean Elab System -namespace Leanpkg2 +namespace Lake def leanPkgFile : FilePath := "package.lean" diff --git a/Leanpkg2/LeanVersion.lean b/Lake/LeanVersion.lean similarity index 97% rename from Leanpkg2/LeanVersion.lean rename to Lake/LeanVersion.lean index eb8984fef4..b8dd8df778 100644 --- a/Leanpkg2/LeanVersion.lean +++ b/Lake/LeanVersion.lean @@ -3,7 +3,7 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich -/ -namespace Leanpkg2 +namespace Lake def leanVersionStringCore := s!"{Lean.version.major}.{Lean.version.minor}.{Lean.version.patch}" diff --git a/Leanpkg2/Make.lean b/Lake/Make.lean similarity index 89% rename from Leanpkg2/Make.lean rename to Lake/Make.lean index 46ead54e40..60ca57ce2a 100644 --- a/Leanpkg2/Make.lean +++ b/Lake/Make.lean @@ -3,14 +3,14 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ -import Leanpkg2.Proc -import Leanpkg2.Package +import Lake.Proc +import Lake.Package open System -namespace Leanpkg2 +namespace Lake -def lockfile : FilePath := ".leanpkg-lock" +def lockfile : FilePath := ".lake-lock" partial def withLockFile (x : IO α) : IO α := do acquire @@ -34,7 +34,7 @@ partial def withLockFile (x : IO α) : IO α := do catch | IO.Error.alreadyExists _ _ => do if firstTime then - IO.eprintln s!"Waiting for prior leanpkg invocation to finish... (remove '{lockfile}' if stuck)" + IO.eprintln s!"Waiting for prior lake invocation to finish... (remove '{lockfile}' if stuck)" IO.sleep (ms := 300) acquire (firstTime := false) | e => throw e diff --git a/Leanpkg2/Package.lean b/Lake/Package.lean similarity index 94% rename from Leanpkg2/Package.lean rename to Lake/Package.lean index 27c4111cd1..84da267379 100644 --- a/Leanpkg2/Package.lean +++ b/Lake/Package.lean @@ -4,11 +4,11 @@ Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Lean.Data.Name -import Leanpkg2.LeanVersion +import Lake.LeanVersion open Lean System -namespace Leanpkg2 +namespace Lake def buildPath : FilePath := "build" def tempBuildPath := buildPath / "temp" @@ -57,7 +57,7 @@ def sourceRoot (self : Package) := self.sourceDir / self.config.module.toString def buildDir (self : Package) := - self.dir / Leanpkg2.buildPath + self.dir / Lake.buildPath def buildRoot (self : Package) := self.buildDir / self.config.module.toString diff --git a/Leanpkg2/Proc.lean b/Lake/Proc.lean similarity index 97% rename from Leanpkg2/Proc.lean rename to Lake/Proc.lean index 1d3a125ab3..83d8db043c 100644 --- a/Leanpkg2/Proc.lean +++ b/Lake/Proc.lean @@ -3,7 +3,7 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich -/ -namespace Leanpkg2 +namespace Lake def execCmd (args : IO.Process.SpawnArgs) : IO Unit := do let envstr := String.join <| args.env.toList.map fun (k, v) => s!"{k}={v.getD ""} " diff --git a/Leanpkg2/Resolve.lean b/Lake/Resolve.lean similarity index 97% rename from Leanpkg2/Resolve.lean rename to Lake/Resolve.lean index ad809679ca..85a6d02dd9 100644 --- a/Leanpkg2/Resolve.lean +++ b/Lake/Resolve.lean @@ -3,12 +3,12 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ -import Leanpkg2.Git -import Leanpkg2.LeanConfig +import Lake.Git +import Lake.LeanConfig open System -namespace Leanpkg2 +namespace Lake open Git in def materializeGit diff --git a/README.md b/README.md index 775a56c0b6..ee5d036eb5 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ -# Leanpkg2 +# Lake An overhaul of Leanpkg for Lean 4 where the package configuration and build scripts are written in Lean. Currently just a prototype. A more detailed README will come when the project has matured. diff --git a/examples/hello/Hello.lean b/examples/hello/Hello.lean index 690d791c80..f7ef46da83 100644 --- a/examples/hello/Hello.lean +++ b/examples/hello/Hello.lean @@ -1,2 +1,2 @@ def main : IO Unit := - IO.println "Hello from Leanpkg2!" + IO.println "Hello from Lake!" diff --git a/examples/hello/package.lean b/examples/hello/package.lean index 16c26d7ec4..96bdde162c 100644 --- a/examples/hello/package.lean +++ b/examples/hello/package.lean @@ -1,6 +1,6 @@ -import Leanpkg2.Package +import Lake.Package -def package : Leanpkg2.PackageConfig := { +def package : Lake.PackageConfig := { name := "hello" version := "1.0" } diff --git a/examples/hello/package.sh b/examples/hello/package.sh index 2c09854503..7999e1fa9f 100644 --- a/examples/hello/package.sh +++ b/examples/hello/package.sh @@ -1,2 +1,2 @@ export LEAN_PATH=../../build -lean --run ../../Leanpkg2.lean build bin +lean --run ../../Lake.lean build bin diff --git a/examples/helloDeps/a/package.lean b/examples/helloDeps/a/package.lean index db385e4029..8db2e590f5 100644 --- a/examples/helloDeps/a/package.lean +++ b/examples/helloDeps/a/package.lean @@ -1,6 +1,6 @@ -import Leanpkg2.Package +import Lake.Package -def package : Leanpkg2.PackageConfig := { +def package : Lake.PackageConfig := { name := "a" version := "1.0" } diff --git a/examples/helloDeps/b/package.lean b/examples/helloDeps/b/package.lean index 0c84296a53..0a72162eb3 100644 --- a/examples/helloDeps/b/package.lean +++ b/examples/helloDeps/b/package.lean @@ -1,6 +1,6 @@ -import Leanpkg2.Build +import Lake.Build -open Leanpkg2 System +open Lake System def package : PackageConfig := { name := "b" diff --git a/examples/helloDeps/package.sh b/examples/helloDeps/package.sh index f8c6ddc536..7d91508111 100644 --- a/examples/helloDeps/package.sh +++ b/examples/helloDeps/package.sh @@ -1,3 +1,3 @@ cd b export LEAN_PATH=../../../build -lean --run ../../../Leanpkg2.lean build bin LINK_OPTS=../a/build/lib/libA.a +lean --run ../../../Lake.lean build bin LINK_OPTS=../a/build/lib/libA.a diff --git a/leanpkg.toml b/leanpkg.toml index 30c2cced3c..e982227eca 100644 --- a/leanpkg.toml +++ b/leanpkg.toml @@ -1,3 +1,3 @@ [package] -name = "leanpkg2" +name = "lake" version = "0.1" From c5c46798fbf8abce6130917eb42bcb9cbe82cd40 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Mon, 7 Jun 2021 05:46:08 -0400 Subject: [PATCH 031/696] Extend README --- README.md | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ee5d036eb5..ab2180d671 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,44 @@ # Lake -An overhaul of Leanpkg for Lean 4 where the package configuration and build scripts are written in Lean. Currently just a prototype. A more detailed README will come when the project has matured. +Lake (Lean Make) is a new build system and package manager for Lean 4. +With Lake, package configuration is written in Lean inside a dedicated `package.lean` file stored in the root of the package directory. Each `package.lean` includes a `package` definition (akin to `main`) which defines the package's configuration. + +## Creating and Building a Package + +We can set up a new package by running `lake init ` in a fresh directory. For example, we can create the package `hello` like so: + +``` +$ mkdir hello +$ cd hello +$ lake init hello +``` + +This will initialize a git repository in the directory with a basic `.gitignore` that ignores the build directory (i.e., `build`) where Lake outputs build files. + +It will also create the root Lean file for the package, which uses the capitalized version of the package's name (e.g., `Hello.lean` in this example). It contains the following dummy "Hello World" program: + +```lean +def main : IO Unit := + IO.println "Hello, world!" +``` + +Lake also creates a basic `package.lean` for the package: + +```lean +import Lake.Package + +def package : Lake.PackageConfig := { + name := "hello" + version := "0.1" + leanVersion := "" +} +``` + +We can use the command `lake build bin` to build the module (and its dependencies, if it has them) and a native executable. The latter of which will be written to `build/bin`. + +``` +$ lake build bin +... +$ ./build/bin/Hello +Hello, world! +``` From 8c39a6560947c8e09d457ad615d4fcd2fd68b32d Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Mon, 7 Jun 2021 05:55:13 -0400 Subject: [PATCH 032/696] Move `.lake-lock` into the build directory --- Lake/Make.lean | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lake/Make.lean b/Lake/Make.lean index 60ca57ce2a..75702baf24 100644 --- a/Lake/Make.lean +++ b/Lake/Make.lean @@ -10,7 +10,7 @@ open System namespace Lake -def lockfile : FilePath := ".lake-lock" +def lockfile : FilePath := buildPath / ".lake-lock" partial def withLockFile (x : IO α) : IO α := do acquire @@ -19,7 +19,8 @@ partial def withLockFile (x : IO α) : IO α := do finally IO.removeFile lockfile where - acquire (firstTime := true) := + acquire (firstTime := true) := do + IO.createDirAll lockfile.parent.get! try -- TODO: lock file should ideally contain PID if !Platform.isWindows then From af7e167deaa97002c1b0f08903ed75e91a92440a Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Mon, 7 Jun 2021 06:00:18 -0400 Subject: [PATCH 033/696] Bump to v1.0-pre --- leanpkg.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/leanpkg.toml b/leanpkg.toml index e982227eca..ba83586f70 100644 --- a/leanpkg.toml +++ b/leanpkg.toml @@ -1,3 +1,3 @@ [package] name = "lake" -version = "0.1" +version = "1.0-pre" From 96870779a27e256e10d585929fc6927e7b948c60 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Mon, 7 Jun 2021 17:03:51 -0400 Subject: [PATCH 034/696] Add Lake build/run instructions to README --- Lake.lean | 5 +++-- README.md | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Lake.lean b/Lake.lean index abc445f59d..942e31360b 100644 --- a/Lake.lean +++ b/Lake.lean @@ -12,8 +12,9 @@ def main (args : List String) : IO UInt32 := do uses when intepreting package configuration files. Also, in order to find the Lean stdlib (e.g., `Init`), - the executable needs to be either colocated with Lean or - have LEAN_PATH include the directory with its dynamic libraries. + the executable needs to either be colocated with Lean or + have `LEAN_PATH` include the directory with its `.olean` files + (e.g., `/lib/lean`). -/ Lean.initSearchPath none let (cmd, outerArgs, innerArgs) ← Lake.splitCmdlineArgs args diff --git a/README.md b/README.md index ab2180d671..2efe476b6d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,30 @@ Lake (Lean Make) is a new build system and package manager for Lean 4. With Lake, package configuration is written in Lean inside a dedicated `package.lean` file stored in the root of the package directory. Each `package.lean` includes a `package` definition (akin to `main`) which defines the package's configuration. +## Building and Running Lake + +In order to properly build Lake, you must provide `leanpkg build bin` with some additional linker options to have it create an executable that can correctly interpret the Lake package configuration files. + +On Unix: + +``` +$ leanpkg build bin LINK_OPTS=-rdynamic +``` + +On Windows (MSYS2): + +``` +$ leanpkg build bin LINK_OPTS=-Wl,--export-all +``` + +When running the built executable, you also must make sure that `LEAN_PATH` includes the build directory of Lake (for Lake's `.olean` files) and Lean's library directory for stdlib's (ex., `Init`'s) `.olean` files. + +For example: + +``` +$ LEAN_PATH="/lib/lean:/build" lake build bin +``` + ## Creating and Building a Package We can set up a new package by running `lake init ` in a fresh directory. For example, we can create the package `hello` like so: From 7791b49be9c720f0b1da8fb6311271904afb5353 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Mon, 7 Jun 2021 22:23:54 -0400 Subject: [PATCH 035/696] Add shell scripts for building Lake and its examples --- build-msys2.sh | 1 + build-unix.sh | 1 + examples/hello/clean.sh | 1 + examples/hello/package-msys2.sh | 4 ++++ examples/hello/package-unix.sh | 4 ++++ examples/helloDeps/package-msys2.sh | 5 +++++ examples/helloDeps/package-unix.sh | 5 +++++ 7 files changed, 21 insertions(+) create mode 100644 build-msys2.sh create mode 100644 build-unix.sh create mode 100644 examples/hello/clean.sh create mode 100644 examples/hello/package-msys2.sh create mode 100644 examples/hello/package-unix.sh create mode 100644 examples/helloDeps/package-msys2.sh create mode 100644 examples/helloDeps/package-unix.sh diff --git a/build-msys2.sh b/build-msys2.sh new file mode 100644 index 0000000000..99ef39c4bb --- /dev/null +++ b/build-msys2.sh @@ -0,0 +1 @@ +leanpkg build bin LINK_OPTS=-Wl,--export-all diff --git a/build-unix.sh b/build-unix.sh new file mode 100644 index 0000000000..b1f67e21cc --- /dev/null +++ b/build-unix.sh @@ -0,0 +1 @@ +leanpkg build bin LINK_OPTS=-rdynamic diff --git a/examples/hello/clean.sh b/examples/hello/clean.sh new file mode 100644 index 0000000000..07cbd15340 --- /dev/null +++ b/examples/hello/clean.sh @@ -0,0 +1 @@ +rm -r build diff --git a/examples/hello/package-msys2.sh b/examples/hello/package-msys2.sh new file mode 100644 index 0000000000..47ae625f46 --- /dev/null +++ b/examples/hello/package-msys2.sh @@ -0,0 +1,4 @@ +LAKE_HOME=../.. +LEAN_HOME=$(dirname $(dirname $(cygpath -u $(elan which lean)))) +export LEAN_PATH="${LEAN_HOME}/lib/lean:${LAKE_HOME}/build" +${LAKE_HOME}/build/bin/Lake.exe build bin diff --git a/examples/hello/package-unix.sh b/examples/hello/package-unix.sh new file mode 100644 index 0000000000..69aaf4a6ba --- /dev/null +++ b/examples/hello/package-unix.sh @@ -0,0 +1,4 @@ +LAKE_HOME=../.. +LEAN_HOME=$(dirname $(dirname $(elan which lean))) +export LEAN_PATH="${LEAN_HOME}/lib/lean:${LAKE_HOME}/build" +${LAKE_HOME}/build/bin/Lake build bin diff --git a/examples/helloDeps/package-msys2.sh b/examples/helloDeps/package-msys2.sh new file mode 100644 index 0000000000..358e765939 --- /dev/null +++ b/examples/helloDeps/package-msys2.sh @@ -0,0 +1,5 @@ +cd b +LAKE_HOME=../../.. +LEAN_HOME=$(dirname $(dirname $(cygpath -u $(elan which lean)))) +export LEAN_PATH="${LEAN_HOME}/lib/lean:${LAKE_HOME}/build" +${LAKE_HOME}/build/bin/Lake.exe build bin LINK_OPTS=../a/build/lib/libA.a diff --git a/examples/helloDeps/package-unix.sh b/examples/helloDeps/package-unix.sh new file mode 100644 index 0000000000..7cd3ef3fe8 --- /dev/null +++ b/examples/helloDeps/package-unix.sh @@ -0,0 +1,5 @@ +cd b +LAKE_HOME=../../.. +LEAN_HOME=$(dirname $(dirname $(elan which lean))) +export LEAN_PATH="${LEAN_HOME}/lib/lean:${LAKE_HOME}/build" +${LAKE_HOME}/build/bin/Lake build bin LINK_OPTS=../a/build/lib/libA.a From 6fad405294538926791a3c6b684ea6483b89bf39 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Tue, 8 Jun 2021 17:17:58 -0400 Subject: [PATCH 036/696] Intelligently initialize Lean search path --- Lake.lean | 12 ++----- Lake/SearchPath.lean | 54 +++++++++++++++++++++++++++++ README.md | 7 ++-- examples/hello/package-msys2.sh | 4 --- examples/hello/package-unix.sh | 4 --- examples/hello/package.sh | 3 +- examples/helloDeps/package-msys2.sh | 5 --- examples/helloDeps/package-unix.sh | 5 --- examples/helloDeps/package.sh | 3 +- 9 files changed, 63 insertions(+), 34 deletions(-) create mode 100644 Lake/SearchPath.lean delete mode 100644 examples/hello/package-msys2.sh delete mode 100644 examples/hello/package-unix.sh delete mode 100644 examples/helloDeps/package-msys2.sh delete mode 100644 examples/helloDeps/package-unix.sh diff --git a/Lake.lean b/Lake.lean index 942e31360b..160597a07f 100644 --- a/Lake.lean +++ b/Lake.lean @@ -4,19 +4,11 @@ Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Lake.Cli +import Lake.SearchPath def main (args : List String) : IO UInt32 := do try - /- - Initializes the search path the Lake executable - uses when intepreting package configuration files. - - Also, in order to find the Lean stdlib (e.g., `Init`), - the executable needs to either be colocated with Lean or - have `LEAN_PATH` include the directory with its `.olean` files - (e.g., `/lib/lean`). - -/ - Lean.initSearchPath none + Lake.setupLeanSearchPath let (cmd, outerArgs, innerArgs) ← Lake.splitCmdlineArgs args Lake.cli cmd outerArgs innerArgs pure 0 diff --git a/Lake/SearchPath.lean b/Lake/SearchPath.lean new file mode 100644 index 0000000000..33a735c2c2 --- /dev/null +++ b/Lake/SearchPath.lean @@ -0,0 +1,54 @@ +/- +Copyright (c) 2021 Mac Malone. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Mac Malone +-/ +import Lean.Util.Path + +open System + +namespace Lake + +def getLeanPath? : IO (Option FilePath) := do + let out ← IO.Process.output { + cmd := "sh", + args := #["-c", "echo \"#eval IO.appPath >>= IO.println\" | lean --stdin"] + } + if out.exitCode == 0 then + some <| FilePath.mk <| out.stdout.trim + else + none + +def getLeanHome? : IO (Option FilePath) := do + let leanPath? ← getLeanPath? + OptionM.run do (← (← leanPath?).parent).parent + +def getLakeHome? : IO (Option FilePath) := do + (← IO.appPath).parent.bind FilePath.parent + +/-- + Initializes the search path the Lake executable + uses when intepreting package configuration files. + + In order to use the Lean stdlib (e.g., `Init`), + the executable needs the search path to include the directory + with the stdlib's `.olean` files (e.g., from `/lib/lean`). + In order to use Lake's modules as well, the search path also + needs to include Lake's `.olean` files (e.g., from `build`). + + While this can be done by having the user augment `LEAN_PATH` with + the necessary directories, Lake also intelligently derives an initial + search path from the location of the `lean` executable and itself. + + It will assume that `lean` is located at `/bin/lean` + with its `.olean` files at `/lib/lean` and that `lake` + is at `/bin/lake` with its `.olean` files at ``. + If this is correct, the user will not need to augment `LEAN_PATH`. +-/ +def setupLeanSearchPath : IO Unit := do + let mut sp : SearchPath := [] + if let some lakeHome ← getLakeHome? then + sp := lakeHome :: sp + if let some leanHome ← getLeanHome? then + sp := leanHome / "lib" / "lean" :: sp + Lean.searchPathRef.set (← Lean.addSearchPathFromEnv sp) diff --git a/README.md b/README.md index 2efe476b6d..7e95ff6bb6 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,17 @@ On Windows (MSYS2): $ leanpkg build bin LINK_OPTS=-Wl,--export-all ``` -When running the built executable, you also must make sure that `LEAN_PATH` includes the build directory of Lake (for Lake's `.olean` files) and Lean's library directory for stdlib's (ex., `Init`'s) `.olean` files. +When running the built executable, you may need to esnure that `LEAN_PATH` includes the build directory of Lake (for Lake's `.olean` files) and Lean's library directory for stdlib's (ex., `Init`'s) `.olean` files. For example: ``` -$ LEAN_PATH="/lib/lean:/build" lake build bin +$ LEAN_PATH="/lib/lean:" lake build bin ``` +Lake will intelligently setup an initial search path based on the location +of its own executable and `lean`. It will assume that `lean` is located at `/bin/lean` with its `.olean` files at `/lib/lean` and that `lake` is at `/bin/lake` with its `.olean` files at ``. If this is correct, you will not need to augment `LEAN_PATH`. + ## Creating and Building a Package We can set up a new package by running `lake init ` in a fresh directory. For example, we can create the package `hello` like so: diff --git a/examples/hello/package-msys2.sh b/examples/hello/package-msys2.sh deleted file mode 100644 index 47ae625f46..0000000000 --- a/examples/hello/package-msys2.sh +++ /dev/null @@ -1,4 +0,0 @@ -LAKE_HOME=../.. -LEAN_HOME=$(dirname $(dirname $(cygpath -u $(elan which lean)))) -export LEAN_PATH="${LEAN_HOME}/lib/lean:${LAKE_HOME}/build" -${LAKE_HOME}/build/bin/Lake.exe build bin diff --git a/examples/hello/package-unix.sh b/examples/hello/package-unix.sh deleted file mode 100644 index 69aaf4a6ba..0000000000 --- a/examples/hello/package-unix.sh +++ /dev/null @@ -1,4 +0,0 @@ -LAKE_HOME=../.. -LEAN_HOME=$(dirname $(dirname $(elan which lean))) -export LEAN_PATH="${LEAN_HOME}/lib/lean:${LAKE_HOME}/build" -${LAKE_HOME}/build/bin/Lake build bin diff --git a/examples/hello/package.sh b/examples/hello/package.sh index 7999e1fa9f..dfeb2433bc 100644 --- a/examples/hello/package.sh +++ b/examples/hello/package.sh @@ -1,2 +1 @@ -export LEAN_PATH=../../build -lean --run ../../Lake.lean build bin +../../build/bin/Lake build bin diff --git a/examples/helloDeps/package-msys2.sh b/examples/helloDeps/package-msys2.sh deleted file mode 100644 index 358e765939..0000000000 --- a/examples/helloDeps/package-msys2.sh +++ /dev/null @@ -1,5 +0,0 @@ -cd b -LAKE_HOME=../../.. -LEAN_HOME=$(dirname $(dirname $(cygpath -u $(elan which lean)))) -export LEAN_PATH="${LEAN_HOME}/lib/lean:${LAKE_HOME}/build" -${LAKE_HOME}/build/bin/Lake.exe build bin LINK_OPTS=../a/build/lib/libA.a diff --git a/examples/helloDeps/package-unix.sh b/examples/helloDeps/package-unix.sh deleted file mode 100644 index 7cd3ef3fe8..0000000000 --- a/examples/helloDeps/package-unix.sh +++ /dev/null @@ -1,5 +0,0 @@ -cd b -LAKE_HOME=../../.. -LEAN_HOME=$(dirname $(dirname $(elan which lean))) -export LEAN_PATH="${LEAN_HOME}/lib/lean:${LAKE_HOME}/build" -${LAKE_HOME}/build/bin/Lake build bin LINK_OPTS=../a/build/lib/libA.a diff --git a/examples/helloDeps/package.sh b/examples/helloDeps/package.sh index 7d91508111..24dd6d7ecf 100644 --- a/examples/helloDeps/package.sh +++ b/examples/helloDeps/package.sh @@ -1,3 +1,2 @@ cd b -export LEAN_PATH=../../../build -lean --run ../../../Lake.lean build bin LINK_OPTS=../a/build/lib/libA.a +../../../build/bin/Lake build bin LINK_OPTS=../a/build/lib/libA.a From 68f0eb16bdec8f9e3fc448d8aa12502aeda3b373 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Tue, 8 Jun 2021 17:18:27 -0400 Subject: [PATCH 037/696] Minor LeanConfig code cleanup --- Lake/LeanConfig.lean | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Lake/LeanConfig.lean b/Lake/LeanConfig.lean index f0270f55ee..6fece98b92 100644 --- a/Lake/LeanConfig.lean +++ b/Lake/LeanConfig.lean @@ -1,13 +1,12 @@ - /- Copyright (c) 2021 Mac Malone. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Mac Malone -/ -import Lean +import Lean.Elab.Frontend import Lake.Package -open Lean Elab System +open Lean System namespace Lake @@ -17,7 +16,7 @@ namespace PackageConfig unsafe def fromLeanFileUnsafe (path : FilePath) : IO PackageConfig := do let input ← IO.FS.readFile path - let (env, ok) ← runFrontend input Options.empty path.toString `package + let (env, ok) ← Elab.runFrontend input Options.empty path.toString `package if ok then IO.ofExcept <| Id.run <| ExceptT.run <| env.evalConstCheck PackageConfig Options.empty ``PackageConfig `package From 9b1d958f9c98b786ca6763b41322f19ffb41768a Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Tue, 8 Jun 2021 17:40:02 -0400 Subject: [PATCH 038/696] Fixed a typo in the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e95ff6bb6..9397799104 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ On Windows (MSYS2): $ leanpkg build bin LINK_OPTS=-Wl,--export-all ``` -When running the built executable, you may need to esnure that `LEAN_PATH` includes the build directory of Lake (for Lake's `.olean` files) and Lean's library directory for stdlib's (ex., `Init`'s) `.olean` files. +When running the built executable, you may need to ensure that `LEAN_PATH` includes the build directory of Lake (for Lake's `.olean` files) and Lean's library directory for stdlib's (ex., `Init`'s) `.olean` files. For example: From 93f536816296d46fc58f5c30bedb3a5ba88e001a Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Fri, 11 Jun 2021 17:15:31 -0400 Subject: [PATCH 039/696] Rename build state/result to `BuildState`/`BuildResult` --- Lake/Build.lean | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index 4bd02fa876..a1c0ce8205 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -35,17 +35,17 @@ structure BuildContext extends BuildConfig where parents : List Name := [] moreDepsMTime : IO.FS.SystemTime -structure Result where +structure BuildResult where maxMTime : IO.FS.SystemTime task : Task (Except IO.Error Unit) deriving Inhabited -structure State where - modTasks : NameMap Result := ∅ +structure BuildState where + modTasks : NameMap BuildResult := ∅ -abbrev BuildM := ReaderT BuildContext <| StateT State IO +abbrev BuildM := ReaderT BuildContext <| StateT BuildState IO -partial def buildModule (mod : Name) : BuildM Result := do +partial def buildModule (mod : Name) : BuildM BuildResult := do let ctx ← read if ctx.parents.contains mod then -- cyclic import From ecfd65ffb7131978da6f66bb57d4c9c997f1d610 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 12 Jun 2021 20:00:07 -0400 Subject: [PATCH 040/696] Clean up `buildModule` code and integrate cycle fix --- Lake/Build.lean | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index a1c0ce8205..1c9221ae8c 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -47,23 +47,31 @@ abbrev BuildM := ReaderT BuildContext <| StateT BuildState IO partial def buildModule (mod : Name) : BuildM BuildResult := do let ctx ← read + + -- detect cyclic imports if ctx.parents.contains mod then - -- cyclic import - let cycle := ctx.parents.dropWhile (· != mod) ++ [mod] - let cycle := cycle.reverse.map (s!" {·}") + let cycle := mod :: (ctx.parents.partition (· != mod)).1 ++ [mod] + let cycle := cycle.map (s!" {·}") throw <| IO.userError s!"import cycle detected:\n{"\n".intercalate cycle}" - if let some r := (← get).modTasks.find? mod then - -- already visited - return r + -- skip if already visited + if let some res := (← get).modTasks.find? mod then + return res + -- deduce lean file let leanFile := modToFilePath "." mod "lean" - let leanMData ← leanFile.metadata - -- recursively build dependencies and calculate transitive `maxMTime` + -- parse imports let (imports, _, _) ← Elab.parseImports (← IO.FS.readFile leanFile) leanFile.toString let localImports := imports.filter (·.module.getRoot == ctx.module) - let deps ← localImports.mapM (buildModule ·.module) + + -- recursively build local dependencies + let deps ← localImports.mapM fun i => + withReader (fun ctx => { ctx with parents := mod :: ctx.parents }) <| + buildModule i.module + + -- calculate transitive `maxMTime` + let leanMData ← leanFile.metadata let depMTimes ← deps.mapM (·.maxMTime) let maxMTime := List.maximum? (leanMData.modified :: ctx.moreDepsMTime :: depMTimes) |>.get! @@ -71,34 +79,35 @@ partial def buildModule (mod : Name) : BuildM BuildResult := do let oleanFile := modToFilePath buildPath mod "olean" try if (← oleanFile.metadata).modified >= maxMTime then - let r := { maxMTime, task := Task.pure (Except.ok ()) } - modify fun st => { st with modTasks := st.modTasks.insert mod r } - return r + let res := { maxMTime, task := Task.pure (Except.ok ()) } + modify fun st => { st with modTasks := st.modTasks.insert mod res } + return res catch | IO.Error.noFileOrDirectory .. => pure () | e => throw e + -- (try to) compile the olean and c file let task ← IO.mapTasks (tasks := deps.map (·.task)) fun rs => do if let some e := rs.findSome? (fun | Except.error e => some e | Except.ok _ => none) then - -- propagate failure + -- propagate failure from dependencies throw e try let cFile := modToFilePath tempBuildPath mod "c" IO.createDirAll oleanFile.parent.get! IO.createDirAll cFile.parent.get! execCmd { - cmd := FilePath.withExtension "lean" FilePath.exeExtension |>.toString + cmd := "lean" args := ctx.leanArgs.toArray ++ #["-o", oleanFile.toString, "-c", cFile.toString, leanFile.toString] env := #[("LEAN_PATH", ctx.leanPath)] } catch e => - -- print errors early + -- print compile errors early IO.eprintln e throw e - let r := { maxMTime, task := task } - modify fun st => { st with modTasks := st.modTasks.insert mod r } - return r + let res := { maxMTime, task := task } + modify fun st => { st with modTasks := st.modTasks.insert mod res } + return res def buildModules (cfg : BuildConfig) (mods : List Name) : IO Unit := do let moreDepsMTime := (← cfg.moreDeps.mapM (·.metadata)).map (·.modified) |>.maximum? |>.getD ⟨0, 0⟩ From f512a9a9341228f0967915c24d54586ae1a5a0b2 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 12 Jun 2021 21:34:34 -0400 Subject: [PATCH 041/696] Auto link dep libs for bin (+ more pkg path helpers) --- Lake/Build.lean | 19 +++++++++++++++---- Lake/Make.lean | 2 +- Lake/Package.lean | 31 +++++++++++++++++++++++++------ examples/helloDeps/package.sh | 2 +- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index 1c9221ae8c..e82cfb2ab5 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -27,7 +27,7 @@ def mkBuildConfig : BuildConfig := { leanArgs, module := pkg.module - leanPath := SearchPath.toString <| pkg.buildDir :: deps.map (·.buildDir) + leanPath := SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) moreDeps := deps.map (·.oleanRoot) } @@ -149,9 +149,20 @@ def configure (pkg : Package) : IO Unit := def printPaths (pkg : Package) (imports leanArgs : List String := []) : IO Unit := do let deps ← buildDeps pkg buildImports pkg deps imports leanArgs - IO.println <| SearchPath.toString <| pkg.buildDir :: deps.map (·.buildDir) + IO.println <| SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) IO.println <| SearchPath.toString <| pkg.sourceDir :: deps.map (·.sourceDir) +private def relPathToUnixString (path : FilePath) : String := + if Platform.isWindows then + path.toString.map fun c => if c == '\\' then '/' else c + else + path.toString + def build (pkg : Package) (makeArgs leanArgs : List String := []) : IO Unit := do - let deps ← buildDeps pkg (if makeArgs.contains "bin" then ["lib"] else []) - buildPkg pkg deps makeArgs leanArgs + if makeArgs.contains "bin" then + let deps ← buildDeps pkg ["lib"] + let depLibs := SearchPath.toString <| deps.map (relPathToUnixString ·.staticLibPath) + buildPkg pkg deps (s!"LINK_OPTS=\"{depLibs}\"" :: makeArgs) leanArgs + else + let deps ← buildDeps pkg + buildPkg pkg deps makeArgs leanArgs diff --git a/Lake/Make.lean b/Lake/Make.lean index 75702baf24..1b9ca6d19d 100644 --- a/Lake/Make.lean +++ b/Lake/Make.lean @@ -48,7 +48,7 @@ def execMake | some t => ["-T", toString t] | none => [] let leanOptsStr := " ".intercalate <| timeoutArgs ++ leanArgs - let leanPathStr := SearchPath.toString <| pkg.buildDir :: deps.map (·.buildDir) + let leanPathStr := SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) let makeArgsStr := " ".intercalate makeArgs let moreDepsStr := " ".intercalate <| deps.map (·.oleanRoot.toString) let mut spawnArgs := { diff --git a/Lake/Package.lean b/Lake/Package.lean index 84da267379..ac1fe046e5 100644 --- a/Lake/Package.lean +++ b/Lake/Package.lean @@ -44,6 +44,9 @@ def name (self : Package) := def module (self : Package) := self.config.module +def moduleName (self : Package) := + self.config.module.toString + def dependencies (self : Package) := self.config.dependencies @@ -54,15 +57,31 @@ def sourceDir (self : Package) := self.dir def sourceRoot (self : Package) := - self.sourceDir / self.config.module.toString + self.sourceDir / self.moduleName def buildDir (self : Package) := self.dir / Lake.buildPath -def buildRoot (self : Package) := - self.buildDir / self.config.module.toString +def binDir (self : Package) := + self.buildDir / "bin" + +def binName (self : Package) := + self.moduleName + +def binPath (self : Package) := + self.binDir / FilePath.withExtension self.binName FilePath.exeExtension + +def libDir (self : Package) := + self.buildDir / "lib" + +def staticLibFile (self : Package) := + s!"lib{self.module}.a" + +def staticLibPath (self : Package) := + self.libDir / self.staticLibFile + +def oleanDir (self : Package) := + self.dir / Lake.buildPath def oleanRoot (self : Package) := - self.buildRoot.withExtension "olean" - -end Package + self.oleanDir / FilePath.withExtension self.moduleName "olean" diff --git a/examples/helloDeps/package.sh b/examples/helloDeps/package.sh index 24dd6d7ecf..9748d40d81 100644 --- a/examples/helloDeps/package.sh +++ b/examples/helloDeps/package.sh @@ -1,2 +1,2 @@ cd b -../../../build/bin/Lake build bin LINK_OPTS=../a/build/lib/libA.a +../../../build/bin/Lake build bin From c0d9917f7cc2a6dd2143aef7c94f9fd05234bca9 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 12 Jun 2021 22:01:17 -0400 Subject: [PATCH 042/696] Update to Lean nightly 06-13 --- Lake/Build.lean | 4 ++-- Lake/Make.lean | 8 ++++---- Lake/SearchPath.lean | 13 +++++-------- leanpkg.toml | 1 + 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index e82cfb2ab5..073d206100 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -93,8 +93,8 @@ partial def buildModule (mod : Name) : BuildM BuildResult := do throw e try let cFile := modToFilePath tempBuildPath mod "c" - IO.createDirAll oleanFile.parent.get! - IO.createDirAll cFile.parent.get! + IO.FS.createDirAll oleanFile.parent.get! + IO.FS.createDirAll cFile.parent.get! execCmd { cmd := "lean" args := ctx.leanArgs.toArray ++ #["-o", oleanFile.toString, "-c", cFile.toString, leanFile.toString] diff --git a/Lake/Make.lean b/Lake/Make.lean index 1b9ca6d19d..5a542fe091 100644 --- a/Lake/Make.lean +++ b/Lake/Make.lean @@ -17,21 +17,21 @@ partial def withLockFile (x : IO α) : IO α := do try x finally - IO.removeFile lockfile + IO.FS.removeFile lockfile where acquire (firstTime := true) := do - IO.createDirAll lockfile.parent.get! + IO.FS.createDirAll lockfile.parent.get! try -- TODO: lock file should ideally contain PID if !Platform.isWindows then - discard <| IO.Prim.Handle.mk lockfile "wx" + discard <| IO.FS.Handle.mkPrim lockfile "wx" else -- `x` mode doesn't seem to work on Windows even though it's listed at -- https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/fopen-wfopen?view=msvc-160 -- ...? Let's use the slightly racy approach then. if ← lockfile.pathExists then throw <| IO.Error.alreadyExists 0 "" - discard <| IO.Prim.Handle.mk lockfile "w" + discard <| IO.FS.Handle.mk lockfile IO.FS.Mode.write catch | IO.Error.alreadyExists _ _ => do if firstTime then diff --git a/Lake/SearchPath.lean b/Lake/SearchPath.lean index 33a735c2c2..ee073b099b 100644 --- a/Lake/SearchPath.lean +++ b/Lake/SearchPath.lean @@ -9,20 +9,16 @@ open System namespace Lake -def getLeanPath? : IO (Option FilePath) := do +def getLeanHome? : IO (Option FilePath) := do let out ← IO.Process.output { - cmd := "sh", - args := #["-c", "echo \"#eval IO.appPath >>= IO.println\" | lean --stdin"] + cmd := "lean", + args := #["--print-prefix"] } if out.exitCode == 0 then some <| FilePath.mk <| out.stdout.trim else none -def getLeanHome? : IO (Option FilePath) := do - let leanPath? ← getLeanPath? - OptionM.run do (← (← leanPath?).parent).parent - def getLakeHome? : IO (Option FilePath) := do (← IO.appPath).parent.bind FilePath.parent @@ -51,4 +47,5 @@ def setupLeanSearchPath : IO Unit := do sp := lakeHome :: sp if let some leanHome ← getLeanHome? then sp := leanHome / "lib" / "lean" :: sp - Lean.searchPathRef.set (← Lean.addSearchPathFromEnv sp) + sp ← Lean.addSearchPathFromEnv sp + Lean.searchPathRef.set sp diff --git a/leanpkg.toml b/leanpkg.toml index ba83586f70..129c5de5b3 100644 --- a/leanpkg.toml +++ b/leanpkg.toml @@ -1,3 +1,4 @@ [package] name = "lake" version = "1.0-pre" +lean_version = "leanprover/lean4:nightly-2021-06-13" From 9138d377816ba28a3f55f046fe322838c18bc478 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 12 Jun 2021 22:27:19 -0400 Subject: [PATCH 043/696] Fix dep lib separator --- Lake/Build.lean | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index 073d206100..bad5f9af18 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -161,7 +161,7 @@ private def relPathToUnixString (path : FilePath) : String := def build (pkg : Package) (makeArgs leanArgs : List String := []) : IO Unit := do if makeArgs.contains "bin" then let deps ← buildDeps pkg ["lib"] - let depLibs := SearchPath.toString <| deps.map (relPathToUnixString ·.staticLibPath) + let depLibs := " ".intercalate <| deps.map (relPathToUnixString ·.staticLibPath) buildPkg pkg deps (s!"LINK_OPTS=\"{depLibs}\"" :: makeArgs) leanArgs else let deps ← buildDeps pkg From 4532901112ce719ee83acd1c3dd15406381a5f9f Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 12 Jun 2021 22:28:59 -0400 Subject: [PATCH 044/696] Refactor `helloDeps` example to have two deps --- examples/helloDeps/a/A.lean | 2 +- examples/helloDeps/b/B.lean | 7 +------ examples/helloDeps/b/package.lean | 9 ++------- examples/helloDeps/clean.sh | 1 + examples/helloDeps/foo/.gitignore | 1 + examples/helloDeps/foo/Foo.lean | 7 +++++++ examples/helloDeps/{b/B => foo/Foo}/Bar.lean | 2 +- examples/helloDeps/{b/B => foo/Foo}/Baz.lean | 2 +- examples/helloDeps/{b/B => foo/Foo}/Foo.lean | 0 examples/helloDeps/foo/package.lean | 12 ++++++++++++ examples/helloDeps/package.sh | 2 +- examples/helloDeps/test.sh | 3 +++ 12 files changed, 31 insertions(+), 17 deletions(-) create mode 100644 examples/helloDeps/foo/.gitignore create mode 100644 examples/helloDeps/foo/Foo.lean rename examples/helloDeps/{b/B => foo/Foo}/Bar.lean (54%) rename examples/helloDeps/{b/B => foo/Foo}/Baz.lean (54%) rename examples/helloDeps/{b/B => foo/Foo}/Foo.lean (100%) create mode 100644 examples/helloDeps/foo/package.lean create mode 100644 examples/helloDeps/test.sh diff --git a/examples/helloDeps/a/A.lean b/examples/helloDeps/a/A.lean index 7ee5f37600..dffa6cdfd4 100644 --- a/examples/helloDeps/a/A.lean +++ b/examples/helloDeps/a/A.lean @@ -1 +1 @@ -def name := "world" +def a := "A" diff --git a/examples/helloDeps/b/B.lean b/examples/helloDeps/b/B.lean index fc5594b7a6..337123bfa8 100644 --- a/examples/helloDeps/b/B.lean +++ b/examples/helloDeps/b/B.lean @@ -1,6 +1 @@ -import A -import B.Bar -import B.Baz - -def main : IO Unit := - IO.println s!"Hello, {foo} {name}!" +def b := "B" diff --git a/examples/helloDeps/b/package.lean b/examples/helloDeps/b/package.lean index 0a72162eb3..ac19e6bd76 100644 --- a/examples/helloDeps/b/package.lean +++ b/examples/helloDeps/b/package.lean @@ -1,11 +1,6 @@ -import Lake.Build +import Lake.Package -open Lake System - -def package : PackageConfig := { +def package : Lake.PackageConfig := { name := "b" version := "1.0" - dependencies := [ - { name := "a", src := Source.path (FilePath.mk ".." / "a") } - ] } diff --git a/examples/helloDeps/clean.sh b/examples/helloDeps/clean.sh index 143901d23e..d6f800d026 100644 --- a/examples/helloDeps/clean.sh +++ b/examples/helloDeps/clean.sh @@ -1,2 +1,3 @@ rm -r a/build rm -r b/build +rm -r foo/build diff --git a/examples/helloDeps/foo/.gitignore b/examples/helloDeps/foo/.gitignore new file mode 100644 index 0000000000..796b96d1c4 --- /dev/null +++ b/examples/helloDeps/foo/.gitignore @@ -0,0 +1 @@ +/build diff --git a/examples/helloDeps/foo/Foo.lean b/examples/helloDeps/foo/Foo.lean new file mode 100644 index 0000000000..70f30e33be --- /dev/null +++ b/examples/helloDeps/foo/Foo.lean @@ -0,0 +1,7 @@ +import A +import B +import Foo.Bar +import Foo.Baz + +def main : IO Unit := + IO.println s!"Hello, {foo} {a} {b}!" diff --git a/examples/helloDeps/b/B/Bar.lean b/examples/helloDeps/foo/Foo/Bar.lean similarity index 54% rename from examples/helloDeps/b/B/Bar.lean rename to examples/helloDeps/foo/Foo/Bar.lean index 7ca89339f6..be6fe41c2e 100644 --- a/examples/helloDeps/b/B/Bar.lean +++ b/examples/helloDeps/foo/Foo/Bar.lean @@ -1,3 +1,3 @@ -import B.Foo +import Foo.Foo def bar := "bar" diff --git a/examples/helloDeps/b/B/Baz.lean b/examples/helloDeps/foo/Foo/Baz.lean similarity index 54% rename from examples/helloDeps/b/B/Baz.lean rename to examples/helloDeps/foo/Foo/Baz.lean index 4823066822..6e09870790 100644 --- a/examples/helloDeps/b/B/Baz.lean +++ b/examples/helloDeps/foo/Foo/Baz.lean @@ -1,3 +1,3 @@ -import B.Foo +import Foo.Foo def baz := "baz" diff --git a/examples/helloDeps/b/B/Foo.lean b/examples/helloDeps/foo/Foo/Foo.lean similarity index 100% rename from examples/helloDeps/b/B/Foo.lean rename to examples/helloDeps/foo/Foo/Foo.lean diff --git a/examples/helloDeps/foo/package.lean b/examples/helloDeps/foo/package.lean new file mode 100644 index 0000000000..58fee5cc9c --- /dev/null +++ b/examples/helloDeps/foo/package.lean @@ -0,0 +1,12 @@ +import Lake.Build + +open Lake System + +def package : PackageConfig := { + name := "foo" + version := "1.0" + dependencies := [ + { name := "a", src := Source.path (FilePath.mk ".." / "a") }, + { name := "b", src := Source.path (FilePath.mk ".." / "b") } + ] +} diff --git a/examples/helloDeps/package.sh b/examples/helloDeps/package.sh index 9748d40d81..285426058b 100644 --- a/examples/helloDeps/package.sh +++ b/examples/helloDeps/package.sh @@ -1,2 +1,2 @@ -cd b +cd foo ../../../build/bin/Lake build bin diff --git a/examples/helloDeps/test.sh b/examples/helloDeps/test.sh new file mode 100644 index 0000000000..7694066425 --- /dev/null +++ b/examples/helloDeps/test.sh @@ -0,0 +1,3 @@ +./clean.sh +./package.sh +./foo/build/bin/foo From 2aa3c1e0cbf124013da51365a33af81454109753 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Mon, 14 Jun 2021 00:29:09 -0400 Subject: [PATCH 045/696] Add test script to hello example --- examples/hello/test.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 examples/hello/test.sh diff --git a/examples/hello/test.sh b/examples/hello/test.sh new file mode 100644 index 0000000000..25f7f89409 --- /dev/null +++ b/examples/hello/test.sh @@ -0,0 +1,3 @@ +./clean.sh +./package.sh +./build/bin/Hello From b7b0217241807244e1a285629c992495d0af3874 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Mon, 14 Jun 2021 01:08:05 -0400 Subject: [PATCH 046/696] Bump Lean version --- leanpkg.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/leanpkg.toml b/leanpkg.toml index 129c5de5b3..ea897ddafe 100644 --- a/leanpkg.toml +++ b/leanpkg.toml @@ -1,4 +1,4 @@ [package] name = "lake" version = "1.0-pre" -lean_version = "leanprover/lean4:nightly-2021-06-13" +lean_version = "leanprover/lean4:nightly-2021-06-14" From f7a858c0decdd6d1b0506c1be45e9bee08f3fcdb Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Mon, 14 Jun 2021 01:08:35 -0400 Subject: [PATCH 047/696] Add search path heading to README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9397799104..a37598bc9a 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ On Windows (MSYS2): $ leanpkg build bin LINK_OPTS=-Wl,--export-all ``` +### Correcting Lake's Search Path + When running the built executable, you may need to ensure that `LEAN_PATH` includes the build directory of Lake (for Lake's `.olean` files) and Lean's library directory for stdlib's (ex., `Init`'s) `.olean` files. For example: From 9aa78a536129cb68414319438922b32a2b3df6e7 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Mon, 14 Jun 2021 01:36:19 -0400 Subject: [PATCH 048/696] More README edits --- README.md | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a37598bc9a..3972fba499 100644 --- a/README.md +++ b/README.md @@ -19,18 +19,13 @@ On Windows (MSYS2): $ leanpkg build bin LINK_OPTS=-Wl,--export-all ``` -### Correcting Lake's Search Path +Alternatively, you can build Lake by running the the pre-packaged `build-msys2.sh` and `build-unix.sh` shell scripts which include these commands. -When running the built executable, you may need to ensure that `LEAN_PATH` includes the build directory of Lake (for Lake's `.olean` files) and Lean's library directory for stdlib's (ex., `Init`'s) `.olean` files. +### Augmenting Lake's Search Path -For example: +The built executable also needs to know where to find the `.olean` files for the modules used in the package configuration file. Lake will intelligently setup an initial search path based on the location of its own executable and `lean`. It will assume that `lean` is located at `/bin/lean` with its `.olean` files (e.g., for `Init`) at `/lib/lean` and that `lake` is at `/bin/lake` with its `.olean` files at ``. -``` -$ LEAN_PATH="/lib/lean:" lake build bin -``` - -Lake will intelligently setup an initial search path based on the location -of its own executable and `lean`. It will assume that `lean` is located at `/bin/lean` with its `.olean` files at `/lib/lean` and that `lake` is at `/bin/lake` with its `.olean` files at ``. If this is correct, you will not need to augment `LEAN_PATH`. +You can augment this search path by including other directories of `.olean` files in the `LEAN_PATH` environment variable. Such directories will take precedence over the initial search path, so `LEAN_PATH` can also be used to correct Lake's search if the `.olean` files for Lean (or Lake itself) are in non-standard locations. ## Creating and Building a Package From 22dc542445d0323b76f685d7833e690066fdf481 Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Mon, 14 Jun 2021 01:42:16 -0400 Subject: [PATCH 049/696] Release 1.0 --- leanpkg.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/leanpkg.toml b/leanpkg.toml index ea897ddafe..755c26ba8e 100644 --- a/leanpkg.toml +++ b/leanpkg.toml @@ -1,4 +1,4 @@ [package] name = "lake" -version = "1.0-pre" +version = "1.0" lean_version = "leanprover/lean4:nightly-2021-06-14" From 9034b6b79b45cc5c5a9b5cd5812e035c930e4b0b Mon Sep 17 00:00:00 2001 From: tydeu Date: Thu, 8 Jul 2021 17:42:17 -0400 Subject: [PATCH 050/696] chore: bump to v2.0-pre --- leanpkg.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/leanpkg.toml b/leanpkg.toml index 755c26ba8e..7c54a930ab 100644 --- a/leanpkg.toml +++ b/leanpkg.toml @@ -1,4 +1,4 @@ [package] name = "lake" -version = "1.0" +version = "2.0-pre" lean_version = "leanprover/lean4:nightly-2021-06-14" From 981db940e82ab62637c2cea2e713481f4c95f39a Mon Sep 17 00:00:00 2001 From: tydeu Date: Thu, 8 Jul 2021 19:46:10 -0400 Subject: [PATCH 051/696] feat: build packages without make --- Lake/Build.lean | 299 ++++++++++++++++++++-------------- Lake/Cli.lean | 22 +-- Lake/Compile.lean | 51 ++++++ Lake/Make.lean | 59 ------- Lake/Package.lean | 62 +++++-- Lake/Resolve.lean | 1 - README.md | 4 +- build-msys2.sh | 2 +- build-unix.sh | 2 +- examples/hello/package.sh | 2 +- examples/helloDeps/package.sh | 3 +- 11 files changed, 298 insertions(+), 209 deletions(-) create mode 100644 Lake/Compile.lean delete mode 100644 Lake/Make.lean diff --git a/Lake/Build.lean b/Lake/Build.lean index bad5f9af18..0530e3f382 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -7,162 +7,217 @@ import Lean.Data.Name import Lean.Elab.Import import Lake.Resolve import Lake.Package -import Lake.Make -import Lake.Proc +import Lake.Compile -open Lean System +open System +open Lean hiding SearchPath namespace Lake -structure BuildConfig where - module : Name - leanArgs : List String - leanPath : String - -- things that should also trigger rebuilds - -- ex. olean roots of dependencies - moreDeps : List FilePath +-- # Basic Build Infos -def mkBuildConfig -(pkg : Package) (deps : List Package) (leanArgs : List String) -: BuildConfig := { - leanArgs, - module := pkg.module - leanPath := SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) - moreDeps := deps.map (·.oleanRoot) -} - -structure BuildContext extends BuildConfig where - parents : List Name := [] - moreDepsMTime : IO.FS.SystemTime - -structure BuildResult where - maxMTime : IO.FS.SystemTime - task : Task (Except IO.Error Unit) +structure BuildInfo (α : Type) where + artifact : α + maxMTime : IO.FS.SystemTime + task : Task (Except IO.Error PUnit) deriving Inhabited -structure BuildState where - modTasks : NameMap BuildResult := ∅ +abbrev ModuleBuildInfo := BuildInfo (FilePath × FilePath) -abbrev BuildM := ReaderT BuildContext <| StateT BuildState IO +namespace ModuleBuildInfo +def oleanFile (self : ModuleBuildInfo) := self.artifact.1 +def cFile (self : ModuleBuildInfo) := self.artifact.2 +end ModuleBuildInfo -partial def buildModule (mod : Name) : BuildM BuildResult := do - let ctx ← read - - -- detect cyclic imports - if ctx.parents.contains mod then - let cycle := mod :: (ctx.parents.partition (· != mod)).1 ++ [mod] - let cycle := cycle.map (s!" {·}") - throw <| IO.userError s!"import cycle detected:\n{"\n".intercalate cycle}" - - -- skip if already visited - if let some res := (← get).modTasks.find? mod then - return res - - -- deduce lean file - let leanFile := modToFilePath "." mod "lean" - - -- parse imports - let (imports, _, _) ← Elab.parseImports (← IO.FS.readFile leanFile) leanFile.toString - let localImports := imports.filter (·.module.getRoot == ctx.module) - - -- recursively build local dependencies - let deps ← localImports.mapM fun i => - withReader (fun ctx => { ctx with parents := mod :: ctx.parents }) <| - buildModule i.module - - -- calculate transitive `maxMTime` - let leanMData ← leanFile.metadata - let depMTimes ← deps.mapM (·.maxMTime) - let maxMTime := List.maximum? (leanMData.modified :: ctx.moreDepsMTime :: depMTimes) |>.get! - - -- check whether we have an up-to-date .olean - let oleanFile := modToFilePath buildPath mod "olean" +def buildOleanAndC (leanFile oleanFile cFile : FilePath) +(depInfos : List ModuleBuildInfo) (maxMTime : IO.FS.SystemTime) +(leanPath : String := "") (rootDir : FilePath := ".") (leanArgs : Array String := #[]) +: IO ModuleBuildInfo := do + let artifact := (oleanFile, cFile) + -- check whether we have an up-to-date .olean and .c try - if (← oleanFile.metadata).modified >= maxMTime then - let res := { maxMTime, task := Task.pure (Except.ok ()) } - modify fun st => { st with modTasks := st.modTasks.insert mod res } - return res + let cMTime := (← cFile.metadata).modified + let oleanMTime := (← oleanFile.metadata).modified + if cMTime >= maxMTime && oleanMTime >= maxMTime then + let task := Task.pure (Except.ok ()) + return { artifact, maxMTime, task } catch | IO.Error.noFileOrDirectory .. => pure () | e => throw e - -- (try to) compile the olean and c file - let task ← IO.mapTasks (tasks := deps.map (·.task)) fun rs => do - if let some e := rs.findSome? (fun | Except.error e => some e | Except.ok _ => none) then - -- propagate failure from dependencies - throw e + let task ← IO.mapTasks (tasks := depInfos.map (·.task)) fun rs => do + rs.forM IO.ofExcept -- propagate first failure from dependencies try - let cFile := modToFilePath tempBuildPath mod "c" - IO.FS.createDirAll oleanFile.parent.get! - IO.FS.createDirAll cFile.parent.get! - execCmd { - cmd := "lean" - args := ctx.leanArgs.toArray ++ #["-o", oleanFile.toString, "-c", cFile.toString, leanFile.toString] - env := #[("LEAN_PATH", ctx.leanPath)] - } + compileOleanAndC leanFile oleanFile cFile leanPath rootDir leanArgs catch e => -- print compile errors early IO.eprintln e throw e + return { artifact, maxMTime, task } - let res := { maxMTime, task := task } - modify fun st => { st with modTasks := st.modTasks.insert mod res } - return res +def buildO (oFile : FilePath) +(cInfo : BuildInfo FilePath) (leancArgs : Array String := #[]) +: IO (BuildInfo FilePath) := do + -- skip if we have an up-to-date .o + try + let cMTime := cInfo.maxMTime + if (← oFile.metadata).modified >= cMTime then + return {artifact := oFile, maxMTime := cMTime, task := Task.pure (Except.ok ()) } + catch + | IO.Error.noFileOrDirectory .. => pure () + | e => throw e + -- compile it otherwise + let task ← IO.mapTask (t := cInfo.task) fun x => do + IO.ofExcept x -- propagate failure from building .c + try + compileO oFile cInfo.artifact leancArgs + catch e => + -- print compile errors early + IO.eprintln e + throw e + return {artifact := oFile, maxMTime := cInfo.maxMTime, task } -def buildModules (cfg : BuildConfig) (mods : List Name) : IO Unit := do - let moreDepsMTime := (← cfg.moreDeps.mapM (·.metadata)).map (·.modified) |>.maximum? |>.getD ⟨0, 0⟩ - let rs ← mods.mapM buildModule |>.run { toBuildConfig := cfg, moreDepsMTime } |>.run' {} - for r in rs do - if let Except.error _ ← IO.wait r.task then - -- actual error has already been printed above - throw <| IO.userError "Build failed." +-- # Build Modules -def buildImports (pkg : Package) (deps : List Package) (imports leanArgs : List String := []) : IO Unit := do - let imports := imports.map (·.toName) - let localImports := imports.filter (·.getRoot == pkg.module) - if localImports != [] then - if ← FilePath.pathExists "Makefile" then - let oleans := localImports.map fun i => - Lean.modToFilePath buildPath i "olean" |>.toString - execMake pkg deps oleans leanArgs - else - buildModules (mkBuildConfig pkg deps leanArgs) localImports +structure BuildContext where + package : Package + leanPath : String + -- things that should also trigger rebuilds + -- ex. olean roots of dependencies + moreDeps : List FilePath + buildParents : List Name := [] + moreDepsMTime : IO.FS.SystemTime -def buildPkg (pkg : Package) (deps : List Package) (makeArgs leanArgs : List String := []) : IO Unit := do - if makeArgs != [] || (← FilePath.pathExists "Makefile") then - execMake pkg deps makeArgs leanArgs - else - buildModules (mkBuildConfig pkg deps leanArgs) [pkg.module] +structure BuildState where + buildInfos : NameMap ModuleBuildInfo := ∅ -def buildDeps (pkg : Package) (makeArgs leanArgs : List String := []) : IO (List Package) := do +abbrev BuildM := ReaderT BuildContext <| StateT BuildState IO + +partial def buildModule (mod : Name) : BuildM ModuleBuildInfo := do + let ctx ← read + let pkg := ctx.package + + -- detect cyclic imports + if ctx.buildParents.contains mod then + let cycle := mod :: (ctx.buildParents.partition (· != mod)).1 ++ [mod] + let cycle := cycle.map (s!" {·}") + throw <| IO.userError s!"import cycle detected:\n{"\n".intercalate cycle}" + + -- return previous result if already visited + if let some info := (← get).buildInfos.find? mod then + return info + + -- deduce lean file + let leanFile := ctx.package.modToSource mod + + -- parse imports + let (imports, _, _) ← Elab.parseImports (← IO.FS.readFile leanFile) leanFile.toString + let directLocalImports := imports.map (·.module) |>.filter (·.getRoot == pkg.module) + + -- recursively build local dependencies + let depInfos ← directLocalImports.mapM fun i => + withReader (fun ctx => { ctx with buildParents := mod :: ctx.buildParents }) <| + buildModule i + + -- calculate transitive `maxMTime` + let leanMData ← leanFile.metadata + let depMTimes ← depInfos.mapM (·.maxMTime) + let maxMTime := List.maximum? (leanMData.modified :: ctx.moreDepsMTime :: depMTimes) |>.get! + + -- do build + let cFile := pkg.modToC mod + let oleanFile := pkg.modToOlean mod + let info ← buildOleanAndC leanFile oleanFile cFile + depInfos maxMTime ctx.leanPath pkg.dir pkg.leanArgs + modify fun st => { st with buildInfos := st.buildInfos.insert mod info } + return info + +def mkBuildContext (pkg : Package) (deps : List Package) : IO BuildContext := do + let moreDeps := deps.map (·.oleanRoot) + let moreDepsMTime := (← moreDeps.mapM (·.metadata)).map (·.modified) |>.maximum? |>.getD ⟨0, 0⟩ + let leanPath := SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) + return { package := pkg, leanPath, moreDeps, moreDepsMTime } + +def buildPackageModulesCore +(pkg : Package) (deps : List Package) : IO (ModuleBuildInfo × BuildState) := do + let crx ← mkBuildContext pkg deps + buildModule pkg.module |>.run crx |>.run {} + +def buildPackageModuleDAG +(pkg : Package) (deps : List Package) : IO (NameMap ModuleBuildInfo) := do + (← buildPackageModulesCore pkg deps).2.buildInfos + +-- # Configure/Build Packages + +def buildPackageModules +(pkg : Package) (deps : List Package) : IO PUnit := do + let (info, _) ← buildPackageModulesCore pkg deps + if let Except.error _ ← IO.wait info.task then + -- actual error has already been printed above + throw <| IO.userError "Build failed." + +def buildDeps (pkg : Package) : IO (List Package) := do let deps ← solveDeps pkg for dep in deps do -- build recursively -- TODO: share build of common dependencies let depDeps ← solveDeps dep - buildPkg dep depDeps makeArgs leanArgs + buildPackageModules pkg deps return deps -def configure (pkg : Package) : IO Unit := +def configure (pkg : Package) : IO Unit := do discard <| buildDeps pkg -def printPaths (pkg : Package) (imports leanArgs : List String := []) : IO Unit := do +def build (pkg : Package) : IO Unit := do let deps ← buildDeps pkg - buildImports pkg deps imports leanArgs + buildPackageModules pkg deps + +-- # Build Package Lib/Bin + +def buildPackageOFiles (pkg : Package) (buildMap : NameMap ModuleBuildInfo) +: IO (List FilePath) := do + let oInfos ← buildMap.toList.mapM fun (mod, info) => + let oFile := pkg.modToO mod + buildO oFile {info with artifact := info.cFile} pkg.leancArgs + oInfos.mapM fun info => do + IO.ofExcept (← IO.wait info.task) + info.artifact + +def buildStaticLib (pkg : Package) : IO FilePath := do + let deps ← buildDeps pkg + let buildMap ← buildPackageModuleDAG pkg deps + let oFiles ← buildPackageOFiles pkg buildMap + compileLib pkg.staticLibFile oFiles.toArray + pkg.staticLibFile + +def buildBin (pkg : Package) : IO FilePath := do + let deps ← solveDeps pkg + let depLibs ← deps.mapM buildStaticLib + let buildMap ← buildPackageModuleDAG pkg deps + let oFiles ← buildPackageOFiles pkg buildMap + compileBin pkg.binFile (oFiles ++ depLibs).toArray pkg.linkArgs + pkg.binFile + +-- # Print Paths + +def buildModulesInPackage (pkg : Package) (deps : List Package) (mods : List Name) : IO Unit := do + let ctx ← mkBuildContext pkg deps + let rs ← mods.mapM buildModule |>.run ctx |>.run' {} + for r in rs do + if let Except.error _ ← IO.wait r.task then + -- actual error has already been printed above + throw <| IO.userError "Build failed." + +def buildImports +(pkg : Package) (deps : List Package) (imports : List String := []) +: IO Unit := do + let imports := imports.map (·.toName) + let localImports := imports.filter (·.getRoot == pkg.module) + buildModulesInPackage pkg deps localImports + +def printPaths (pkg : Package) (imports : List String := []) : IO Unit := do + let deps ← buildDeps pkg + buildImports pkg deps imports IO.println <| SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) IO.println <| SearchPath.toString <| pkg.sourceDir :: deps.map (·.sourceDir) -private def relPathToUnixString (path : FilePath) : String := - if Platform.isWindows then - path.toString.map fun c => if c == '\\' then '/' else c - else - path.toString - -def build (pkg : Package) (makeArgs leanArgs : List String := []) : IO Unit := do - if makeArgs.contains "bin" then - let deps ← buildDeps pkg ["lib"] - let depLibs := " ".intercalate <| deps.map (relPathToUnixString ·.staticLibPath) - buildPkg pkg deps (s!"LINK_OPTS=\"{depLibs}\"" :: makeArgs) leanArgs - else - let deps ← buildDeps pkg - buildPkg pkg deps makeArgs leanArgs diff --git a/Lake/Cli.lean b/Lake/Cli.lean index aa3f558679..3daddd06db 100644 --- a/Lake/Cli.lean +++ b/Lake/Cli.lean @@ -68,16 +68,18 @@ def getRootPkg : IO Package := do leanVersionString ++ ", but package requires " ++ cfg.leanVersion ++ "\n" return ⟨".", cfg⟩ -def cli : (cmd : String) → (lakeArgs leanArgs : List String) → IO Unit -| "init", [name], [] => init name -| "configure", [], [] => do configure (← getRootPkg) -| "print-paths", imports, leanArgs => do printPaths (← getRootPkg) imports leanArgs -| "build", makeArgs, leanArgs => do build (← getRootPkg) makeArgs leanArgs -| "help", ["init"], [] => IO.println helpInit -| "help", ["configure"], [] => IO.println helpConfigure -| "help", ["build"], [] => IO.println helpBuild -| "help", _, [] => IO.println usage -| _, _, _ => throw <| IO.userError usage +def cli : (cmd : String) → (lakeArgs pkgArgs : List String) → IO Unit +| "init", [name], [] => init name +| "configure", [], [] => do configure (← getRootPkg) +| "print-paths", imports, [] => do printPaths (← getRootPkg) imports +| "build", [], [] => do build (← getRootPkg) +| "build-bin", [], [] => do discard <| buildBin (← getRootPkg) +| "build-lib", [], [] => do discard <| buildStaticLib (← getRootPkg) +| "help", ["init"], [] => IO.println helpInit +| "help", ["configure"], [] => IO.println helpConfigure +| "help", ["build"], [] => IO.println helpBuild +| "help", _, [] => IO.println usage +| _, _, _ => throw <| IO.userError usage private def splitCmdlineArgsCore : List String → List String × List String | [] => ([], []) diff --git a/Lake/Compile.lean b/Lake/Compile.lean new file mode 100644 index 0000000000..5e7014e2ff --- /dev/null +++ b/Lake/Compile.lean @@ -0,0 +1,51 @@ +/- +Copyright (c) 2021 Mac Malone. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Mac Malone +-/ +import Lake.Proc + +namespace Lake +open System + +def compileOleanAndC +(leanFile oleanFile cFile : FilePath) +(leanPath : String := "") (rootDir : FilePath := ".") (leanArgs : Array String := #[]) +: IO Unit := do + if let some dir := cFile.parent then IO.FS.createDirAll dir + if let some dir := oleanFile.parent then IO.FS.createDirAll dir + execCmd { + cmd := "lean" + args := leanArgs ++ #[ + "-R", rootDir.toString, "-o", oleanFile.toString, "-c", + cFile.toString, leanFile.toString + ] + env := #[("LEAN_PATH", leanPath)] + } + +def compileO +(oFile cFile : FilePath) (leancArgs : Array String := #[]) +: IO Unit := do + if let some dir := oFile.parent then IO.FS.createDirAll dir + execCmd { + cmd := "leanc" + args := #["-c", "-o", oFile.toString, cFile.toString] ++ leancArgs + } + +def compileBin +(binFile : FilePath) (oFiles : Array FilePath) (linkArgs : Array String := #[]) +: IO Unit := do + if let some dir := binFile.parent then IO.FS.createDirAll dir + execCmd { + cmd := "leanc" + args := #["-o", binFile.toString] ++ oFiles.map toString ++ linkArgs + } + +def compileLib +(libFile : FilePath) (oFiles : Array FilePath) +: IO Unit := do + if let some dir := libFile.parent then IO.FS.createDirAll dir + execCmd { + cmd := "ar" + args := #["rcs", libFile.toString] ++ oFiles.map toString + } diff --git a/Lake/Make.lean b/Lake/Make.lean deleted file mode 100644 index 5a542fe091..0000000000 --- a/Lake/Make.lean +++ /dev/null @@ -1,59 +0,0 @@ -/- -Copyright (c) 2017 Microsoft Corporation. All rights reserved. -Released under Apache 2.0 license as described in the file LICENSE. -Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone --/ -import Lake.Proc -import Lake.Package - -open System - -namespace Lake - -def lockfile : FilePath := buildPath / ".lake-lock" - -partial def withLockFile (x : IO α) : IO α := do - acquire - try - x - finally - IO.FS.removeFile lockfile - where - acquire (firstTime := true) := do - IO.FS.createDirAll lockfile.parent.get! - try - -- TODO: lock file should ideally contain PID - if !Platform.isWindows then - discard <| IO.FS.Handle.mkPrim lockfile "wx" - else - -- `x` mode doesn't seem to work on Windows even though it's listed at - -- https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/fopen-wfopen?view=msvc-160 - -- ...? Let's use the slightly racy approach then. - if ← lockfile.pathExists then - throw <| IO.Error.alreadyExists 0 "" - discard <| IO.FS.Handle.mk lockfile IO.FS.Mode.write - catch - | IO.Error.alreadyExists _ _ => do - if firstTime then - IO.eprintln s!"Waiting for prior lake invocation to finish... (remove '{lockfile}' if stuck)" - IO.sleep (ms := 300) - acquire (firstTime := false) - | e => throw e - -def execMake -(pkg : Package) (deps : List Package) (makeArgs leanArgs : List String := []) -: IO Unit := withLockFile do - let timeoutArgs := - match pkg.timeout with - | some t => ["-T", toString t] - | none => [] - let leanOptsStr := " ".intercalate <| timeoutArgs ++ leanArgs - let leanPathStr := SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) - let makeArgsStr := " ".intercalate makeArgs - let moreDepsStr := " ".intercalate <| deps.map (·.oleanRoot.toString) - let mut spawnArgs := { - cmd := "sh" - cwd := pkg.dir - args := #["-c", s!"leanmake PKG={pkg.module} LEAN_OPTS=\"{leanOptsStr}\" LEAN_PATH=\"{leanPathStr}\" {makeArgsStr} MORE_DEPS+=\"{moreDepsStr}\" >&2"] - } - execCmd spawnArgs diff --git a/Lake/Package.lean b/Lake/Package.lean index ac1fe046e5..3c37090375 100644 --- a/Lake/Package.lean +++ b/Lake/Package.lean @@ -4,6 +4,7 @@ Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Lean.Data.Name +import Lean.Elab.Import import Lake.LeanVersion open Lean System @@ -26,6 +27,9 @@ structure PackageConfig where name : String version : String leanVersion : String := leanVersionString + leanArgs : Array String := #[] + leancArgs : Array String := #[] + linkArgs : Array String := #[] timeout : Option Nat := none module : Name := name.capitalize dependencies : List Dependency := [] @@ -50,6 +54,15 @@ def moduleName (self : Package) := def dependencies (self : Package) := self.config.dependencies +def leanArgs (self : Package) := + self.config.leanArgs + +def leancArgs (self : Package) := + self.config.leancArgs + +def linkArgs (self : Package) := + self.config.linkArgs + def timeout (self : Package) := self.config.timeout @@ -59,29 +72,56 @@ def sourceDir (self : Package) := def sourceRoot (self : Package) := self.sourceDir / self.moduleName +def modToSource (mod : Name) (self : Package) := + Lean.modToFilePath self.sourceDir mod "lean" + def buildDir (self : Package) := self.dir / Lake.buildPath +def oleanDir (self : Package) := + self.buildDir + +def oleanRoot (self : Package) := + self.oleanDir / FilePath.withExtension self.moduleName "olean" + +def modToOlean (mod : Name) (self : Package) := + Lean.modToFilePath self.oleanDir mod "olean" + +def tempBuildDir (self : Package) := + self.dir / tempBuildPath + +def cDir (self : Package) := + self.tempBuildDir + +def modToC (mod : Name) (self : Package) := + Lean.modToFilePath self.cDir mod "c" + +def oDir (self : Package) := + self.tempBuildDir + +def modToO (mod : Name) (self : Package) := + Lean.modToFilePath self.oDir mod "o" + def binDir (self : Package) := self.buildDir / "bin" def binName (self : Package) := self.moduleName -def binPath (self : Package) := - self.binDir / FilePath.withExtension self.binName FilePath.exeExtension +def binFileName (self : Package) : FilePath := + FilePath.withExtension self.binName FilePath.exeExtension + +def binFile (self : Package) := + self.binDir / self.binFileName def libDir (self : Package) := self.buildDir / "lib" -def staticLibFile (self : Package) := +def staticLibName (self : Package) := + self.moduleName + +def staticLibFileName (self : Package) := s!"lib{self.module}.a" -def staticLibPath (self : Package) := - self.libDir / self.staticLibFile - -def oleanDir (self : Package) := - self.dir / Lake.buildPath - -def oleanRoot (self : Package) := - self.oleanDir / FilePath.withExtension self.moduleName "olean" +def staticLibFile (self : Package) := + self.libDir / self.staticLibFileName diff --git a/Lake/Resolve.lean b/Lake/Resolve.lean index 85a6d02dd9..95211eb027 100644 --- a/Lake/Resolve.lean +++ b/Lake/Resolve.lean @@ -30,7 +30,6 @@ def materialize (pkgDir : FilePath) (dep : Dependency) : IO FilePath := match dep.src with | Source.path dir => do let depdir := pkgDir / dir - IO.eprintln s!"{dep.name}: using local path {depdir}" depdir | Source.git url rev branch => do let depdir := pkgDir / depsPath / dep.name diff --git a/README.md b/README.md index 3972fba499..15f70b5c6f 100644 --- a/README.md +++ b/README.md @@ -58,10 +58,10 @@ def package : Lake.PackageConfig := { } ``` -We can use the command `lake build bin` to build the module (and its dependencies, if it has them) and a native executable. The latter of which will be written to `build/bin`. +We can use the command `lake build-bin` to build the package (and its dependencies, if it has them) into a native executable. The result will be placed in to `build/bin`. ``` -$ lake build bin +$ lake build-bin ... $ ./build/bin/Hello Hello, world! diff --git a/build-msys2.sh b/build-msys2.sh index 99ef39c4bb..06c3e42b7a 100644 --- a/build-msys2.sh +++ b/build-msys2.sh @@ -1 +1 @@ -leanpkg build bin LINK_OPTS=-Wl,--export-all +leanpkg build bin LINK_OPTS=-Wl,--export-all "$@" diff --git a/build-unix.sh b/build-unix.sh index b1f67e21cc..1d82876db7 100644 --- a/build-unix.sh +++ b/build-unix.sh @@ -1 +1 @@ -leanpkg build bin LINK_OPTS=-rdynamic +leanpkg build bin LINK_OPTS=-rdynamic "$@" diff --git a/examples/hello/package.sh b/examples/hello/package.sh index dfeb2433bc..b6449c2dd0 100644 --- a/examples/hello/package.sh +++ b/examples/hello/package.sh @@ -1 +1 @@ -../../build/bin/Lake build bin +../../build/bin/Lake build-bin diff --git a/examples/helloDeps/package.sh b/examples/helloDeps/package.sh index 285426058b..4a895847e5 100644 --- a/examples/helloDeps/package.sh +++ b/examples/helloDeps/package.sh @@ -1,2 +1,3 @@ cd foo -../../../build/bin/Lake build bin +echo "in directory 'foo'" +../../../build/bin/Lake build-bin From a4622f61caee7cdd81d9ca1b06bad2ac503b7409 Mon Sep 17 00:00:00 2001 From: tydeu Date: Thu, 8 Jul 2021 20:58:25 -0400 Subject: [PATCH 052/696] doc: update help command text --- Lake/Cli.lean | 70 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/Lake/Cli.lean b/Lake/Cli.lean index 3daddd06db..799f0680e3 100644 --- a/Lake/Cli.lean +++ b/Lake/Cli.lean @@ -9,18 +9,27 @@ import Lake.LeanConfig namespace Lake +def version.major := 2 +def version.minor := 0 +def version.isPre := true +def versionString := s!"{version.major}.{version.minor}-pre" + def usage := -"Lake, version " ++ uiLeanVersionString ++ " -Usage: lake +"Lake, version " ++ versionString ++ " + +Usage: + lake init create a Lean package in the current directory configure download and build dependencies -build [] configure and build *.olean files +build configure and build *.olean files +build-lib configure and build a static library +build-bin configure and build a native binary executable See `lake help ` for more information on a specific command." def helpConfigure := -"Download dependencies +"Download and build dependencies Usage: lake configure @@ -33,24 +42,45 @@ versions of the same package, the version materialized is undefined. No copy is made of local dependencies." def helpBuild := -"Download dependencies and build *.olean files +"Configure this package and build *.olean files Usage: - lake build [] [-- ] + lake build -This command invokes `lake configure` followed by `leanmake LEAN_OPTS=`. -If defined, the `package.timeout` configuration value is passed to Lean via its `-T` parameter. -If no are given, only .olean files will be produced in `build/`. If `lib` or `bin` -is passed instead, the extracted C code is compiled with `c++` and a static library in `build/lib` -or an executable in `build/bin`, respectively, is created. `lake build bin` requires a declaration -of name `main` in the root namespace, which must return `IO Unit` or `IO UInt32` (the exit code) and -may accept the program's command line arguments as a `List String` parameter. +This command configures the package's dependencies and then builds the package +with `lean`. Additional arguments can be passed to `lean` by setting the +`leanArgs` field in the package's configuration." -NOTE: building and linking dependent libraries currently has to be done manually, e.g. -``` -$ (cd a; lake build lib) -$ (cd b; lake build bin LINK_OPTS=../a/build/lib/libA.a) -```" +def helpBuildLib := +"Configure this package and build a static library + +Usage: + lake build-lib + +This command configures this package's dependencies, builds the package itself, +compiles the extracted C code with `leanc`, and uses `ar` to produce a static +library in `build/lib`. + +Additional arguments can be passed to `lean` or `leanc` by setting the +`leanArgs` or `leancArgs` fields in the package's configuration." + +def helpBuildBin := +"Configure the package and build a native binary executable + +Usage: + lake build-bin + +This command configures this package's dependencies, builds the package itself, +compiles the extracted C code `leannc`, and then links the object files with +`leanc` to produce a native binary executable in `build/bin`. + +This requires a declaration of name `main` in the root namespace, which must +return `IO Unit` or `IO UInt32` (the exit code) and may accept the program's +command line arguments as a `List String` parameter. + +Additional arguments can be passed to `lean`, the `leanc` compiler, or the +`leanc` linker by setting the `leanArgs`, `leancArgs`, or `linkArgs` fields in +the package's configuration." def helpInit := "Create a new Lean package in the current directory @@ -73,11 +103,13 @@ def cli : (cmd : String) → (lakeArgs pkgArgs : List String) → IO Unit | "configure", [], [] => do configure (← getRootPkg) | "print-paths", imports, [] => do printPaths (← getRootPkg) imports | "build", [], [] => do build (← getRootPkg) -| "build-bin", [], [] => do discard <| buildBin (← getRootPkg) | "build-lib", [], [] => do discard <| buildStaticLib (← getRootPkg) +| "build-bin", [], [] => do discard <| buildBin (← getRootPkg) | "help", ["init"], [] => IO.println helpInit | "help", ["configure"], [] => IO.println helpConfigure | "help", ["build"], [] => IO.println helpBuild +| "help", ["build-lib"], [] => IO.println helpBuildLib +| "help", ["build-bin"], [] => IO.println helpBuildBin | "help", _, [] => IO.println usage | _, _, _ => throw <| IO.userError usage From f97f69b749b267ef2530b72f78e8cf3a2ff50031 Mon Sep 17 00:00:00 2001 From: tydeu Date: Fri, 9 Jul 2021 00:36:46 -0400 Subject: [PATCH 053/696] refactor: `BuildInfo` -> `BuildTarget` --- Lake/Build.lean | 174 ++++++++++++++++++++++-------------------- Lake/BuildTarget.lean | 52 +++++++++++++ 2 files changed, 142 insertions(+), 84 deletions(-) create mode 100644 Lake/BuildTarget.lean diff --git a/Lake/Build.lean b/Lake/Build.lean index 0530e3f382..0c9dcc08c3 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -5,6 +5,7 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Lean.Data.Name import Lean.Elab.Import +import Lake.BuildTarget import Lake.Resolve import Lake.Package import Lake.Compile @@ -14,86 +15,93 @@ open Lean hiding SearchPath namespace Lake --- # Basic Build Infos +-- # Build Targets -structure BuildInfo (α : Type) where - artifact : α - maxMTime : IO.FS.SystemTime - task : Task (Except IO.Error PUnit) +abbrev FileBuildTarget := BuildTarget FilePath + +structure LeanArtifact where + oleanFile : FilePath + cFile : FilePath deriving Inhabited -abbrev ModuleBuildInfo := BuildInfo (FilePath × FilePath) +abbrev LeanBuildTarget := BuildTarget LeanArtifact -namespace ModuleBuildInfo -def oleanFile (self : ModuleBuildInfo) := self.artifact.1 -def cFile (self : ModuleBuildInfo) := self.artifact.2 -end ModuleBuildInfo +namespace LeanBuildTarget -def buildOleanAndC (leanFile oleanFile cFile : FilePath) -(depInfos : List ModuleBuildInfo) (maxMTime : IO.FS.SystemTime) +def oleanFile (self : LeanBuildTarget) := self.artifact.oleanFile +def oleanTarget (self : LeanBuildTarget) : FileBuildTarget := + {self with artifact := self.oleanFile} + +def cFile (self : LeanBuildTarget) := self.artifact.cFile +def cTarget (self : LeanBuildTarget) : FileBuildTarget := + {self with artifact := self.cFile} + +end LeanBuildTarget + +def buildOleanAndC (leanFile oleanFile cFile : FilePath) +(depTargets : List LeanBuildTarget) (moreDepsMTime : IO.FS.SystemTime) (leanPath : String := "") (rootDir : FilePath := ".") (leanArgs : Array String := #[]) -: IO ModuleBuildInfo := do - let artifact := (oleanFile, cFile) - -- check whether we have an up-to-date .olean and .c +: IO LeanBuildTarget := do + let artifact := ⟨oleanFile, cFile⟩ + -- calculate transitive `maxMTime` + let leanMData ← leanFile.metadata + let depMTimes ← depTargets.mapM (·.maxMTime) + let maxMTime := List.maximum? (leanMData.modified :: moreDepsMTime :: depMTimes) |>.get! + -- construct a nop target if we have an up-to-date .olean and .c try let cMTime := (← cFile.metadata).modified let oleanMTime := (← oleanFile.metadata).modified if cMTime >= maxMTime && oleanMTime >= maxMTime then - let task := Task.pure (Except.ok ()) - return { artifact, maxMTime, task } + return BuildTarget.pure artifact maxMTime catch | IO.Error.noFileOrDirectory .. => pure () | e => throw e - -- (try to) compile the olean and c file - let task ← IO.mapTasks (tasks := depInfos.map (·.task)) fun rs => do - rs.forM IO.ofExcept -- propagate first failure from dependencies + -- construct a proper target otherwise + let buildTask ← BuildTask.afterTargets depTargets do try compileOleanAndC leanFile oleanFile cFile leanPath rootDir leanArgs catch e => -- print compile errors early IO.eprintln e throw e - return { artifact, maxMTime, task } + return { artifact, maxMTime, buildTask } def buildO (oFile : FilePath) -(cInfo : BuildInfo FilePath) (leancArgs : Array String := #[]) -: IO (BuildInfo FilePath) := do - -- skip if we have an up-to-date .o +(cTarget : FileBuildTarget) (leancArgs : Array String := #[]) +: IO FileBuildTarget := do + -- construct a nop target if we have an up-to-date .o + let cMTime := cTarget.maxMTime try - let cMTime := cInfo.maxMTime if (← oFile.metadata).modified >= cMTime then - return {artifact := oFile, maxMTime := cMTime, task := Task.pure (Except.ok ()) } + return BuildTarget.pure oFile cMTime catch | IO.Error.noFileOrDirectory .. => pure () | e => throw e - -- compile it otherwise - let task ← IO.mapTask (t := cInfo.task) fun x => do - IO.ofExcept x -- propagate failure from building .c + -- construct a proper target otherwise + let buildTask ← cTarget.afterBuild do try - compileO oFile cInfo.artifact leancArgs + compileO oFile cTarget.artifact leancArgs catch e => -- print compile errors early IO.eprintln e throw e - return {artifact := oFile, maxMTime := cInfo.maxMTime, task } + return {artifact := oFile, maxMTime := cMTime, buildTask} -- # Build Modules -structure BuildContext where +structure ModuleTargetContext where package : Package leanPath : String + buildParents : List Name := [] -- things that should also trigger rebuilds -- ex. olean roots of dependencies moreDeps : List FilePath - buildParents : List Name := [] moreDepsMTime : IO.FS.SystemTime -structure BuildState where - buildInfos : NameMap ModuleBuildInfo := ∅ +abbrev ModuleTargetM := + ReaderT ModuleTargetContext <| StateT (NameMap LeanBuildTarget) IO -abbrev BuildM := ReaderT BuildContext <| StateT BuildState IO - -partial def buildModule (mod : Name) : BuildM ModuleBuildInfo := do +partial def buildTargetsForModule (mod : Name) : ModuleTargetM LeanBuildTarget := do let ctx ← read let pkg := ctx.package @@ -104,58 +112,62 @@ partial def buildModule (mod : Name) : BuildM ModuleBuildInfo := do throw <| IO.userError s!"import cycle detected:\n{"\n".intercalate cycle}" -- return previous result if already visited - if let some info := (← get).buildInfos.find? mod then - return info + if let some target := (← get).find? mod then + return target -- deduce lean file - let leanFile := ctx.package.modToSource mod + let leanFile := pkg.modToSource mod -- parse imports let (imports, _, _) ← Elab.parseImports (← IO.FS.readFile leanFile) leanFile.toString let directLocalImports := imports.map (·.module) |>.filter (·.getRoot == pkg.module) -- recursively build local dependencies - let depInfos ← directLocalImports.mapM fun i => + let depTargets ← directLocalImports.mapM fun i => withReader (fun ctx => { ctx with buildParents := mod :: ctx.buildParents }) <| - buildModule i - - -- calculate transitive `maxMTime` - let leanMData ← leanFile.metadata - let depMTimes ← depInfos.mapM (·.maxMTime) - let maxMTime := List.maximum? (leanMData.modified :: ctx.moreDepsMTime :: depMTimes) |>.get! + buildTargetsForModule i -- do build let cFile := pkg.modToC mod let oleanFile := pkg.modToOlean mod - let info ← buildOleanAndC leanFile oleanFile cFile - depInfos maxMTime ctx.leanPath pkg.dir pkg.leanArgs - modify fun st => { st with buildInfos := st.buildInfos.insert mod info } - return info + let target ← buildOleanAndC leanFile oleanFile cFile + depTargets ctx.moreDepsMTime ctx.leanPath pkg.dir pkg.leanArgs + modify (·.insert mod target) + return target -def mkBuildContext (pkg : Package) (deps : List Package) : IO BuildContext := do +def mkModuleTargetContext +(pkg : Package) (deps : List Package) : IO ModuleTargetContext := do let moreDeps := deps.map (·.oleanRoot) let moreDepsMTime := (← moreDeps.mapM (·.metadata)).map (·.modified) |>.maximum? |>.getD ⟨0, 0⟩ let leanPath := SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) return { package := pkg, leanPath, moreDeps, moreDepsMTime } -def buildPackageModulesCore -(pkg : Package) (deps : List Package) : IO (ModuleBuildInfo × BuildState) := do - let crx ← mkBuildContext pkg deps - buildModule pkg.module |>.run crx |>.run {} - -def buildPackageModuleDAG -(pkg : Package) (deps : List Package) : IO (NameMap ModuleBuildInfo) := do - (← buildPackageModulesCore pkg deps).2.buildInfos - --- # Configure/Build Packages +def buildPackageModuleTargetMap +(pkg : Package) (deps : List Package) : IO (NameMap LeanBuildTarget) := do + let crx ← mkModuleTargetContext pkg deps + let (_, targetMap) ← buildTargetsForModule pkg.module |>.run crx |>.run {} + return targetMap def buildPackageModules (pkg : Package) (deps : List Package) : IO PUnit := do - let (info, _) ← buildPackageModulesCore pkg deps - if let Except.error _ ← IO.wait info.task then - -- actual error has already been printed above + let crx ← mkModuleTargetContext pkg deps + let target ← buildTargetsForModule pkg.module |>.run crx |>.run' {} + try target.materialize catch _ => + -- actual error has already been printed within buildTask throw <| IO.userError "Build failed." +def buildModulesInPackage +(pkg : Package) (deps : List Package) (mods : List Name) : IO Unit := do + let ctx ← mkModuleTargetContext pkg deps + let targets ← mods.mapM buildTargetsForModule |>.run ctx |>.run' {} + let tasks ← targets.mapM (·.buildTask) + for task in tasks do + try task.await catch e => + -- actual error has already been printed within buildTask + throw <| IO.userError "Build failed." + +-- # Configure/Build Packages + def buildDeps (pkg : Package) : IO (List Package) := do let deps ← solveDeps pkg for dep in deps do @@ -174,40 +186,34 @@ def build (pkg : Package) : IO Unit := do -- # Build Package Lib/Bin -def buildPackageOFiles (pkg : Package) (buildMap : NameMap ModuleBuildInfo) +def buildPackageOFiles +(pkg : Package) (targetMap : NameMap LeanBuildTarget) : IO (List FilePath) := do - let oInfos ← buildMap.toList.mapM fun (mod, info) => + let oTasks ← targetMap.toList.mapM fun (mod, target) => do let oFile := pkg.modToO mod - buildO oFile {info with artifact := info.cFile} pkg.leancArgs - oInfos.mapM fun info => do - IO.ofExcept (← IO.wait info.task) - info.artifact + let target ← buildO oFile target.cTarget pkg.leancArgs + (oFile, ← target.buildTask) + oTasks.mapM fun (oFile, task) => do + task.await + oFile def buildStaticLib (pkg : Package) : IO FilePath := do let deps ← buildDeps pkg - let buildMap ← buildPackageModuleDAG pkg deps - let oFiles ← buildPackageOFiles pkg buildMap + let targetMap ← buildPackageModuleTargetMap pkg deps + let oFiles ← buildPackageOFiles pkg targetMap compileLib pkg.staticLibFile oFiles.toArray pkg.staticLibFile def buildBin (pkg : Package) : IO FilePath := do let deps ← solveDeps pkg let depLibs ← deps.mapM buildStaticLib - let buildMap ← buildPackageModuleDAG pkg deps - let oFiles ← buildPackageOFiles pkg buildMap + let targetMap ← buildPackageModuleTargetMap pkg deps + let oFiles ← buildPackageOFiles pkg targetMap compileBin pkg.binFile (oFiles ++ depLibs).toArray pkg.linkArgs pkg.binFile -- # Print Paths -def buildModulesInPackage (pkg : Package) (deps : List Package) (mods : List Name) : IO Unit := do - let ctx ← mkBuildContext pkg deps - let rs ← mods.mapM buildModule |>.run ctx |>.run' {} - for r in rs do - if let Except.error _ ← IO.wait r.task then - -- actual error has already been printed above - throw <| IO.userError "Build failed." - def buildImports (pkg : Package) (deps : List Package) (imports : List String := []) : IO Unit := do diff --git a/Lake/BuildTarget.lean b/Lake/BuildTarget.lean new file mode 100644 index 0000000000..f5f2062ca4 --- /dev/null +++ b/Lake/BuildTarget.lean @@ -0,0 +1,52 @@ +/- +Copyright (c) 2021 Mac Malone. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Mac Malone +-/ + +namespace Lake + +def BuildTask := Task (Except IO.Error PUnit) + +namespace BuildTask + +def nop : BuildTask := + Task.pure (Except.ok ()) + +def await (self : BuildTask) : IO PUnit := do + IO.ofExcept (← IO.wait self) + +end BuildTask + +instance : Inhabited BuildTask := ⟨BuildTask.nop⟩ + +structure BuildTarget (α : Type) where + artifact : α + maxMTime : IO.FS.SystemTime + buildTask : BuildTask + +-- manually derive `Inhabited` instance because automatic deriving fails +instance [Inhabited α] : Inhabited (BuildTarget α) := + ⟨Inhabited.default, Inhabited.default, BuildTask.nop⟩ + +namespace BuildTarget + +def pure (artifact : α) (maxMTime : IO.FS.SystemTime := ⟨0, 0⟩) : BuildTarget α := + {artifact, maxMTime, buildTask := BuildTask.nop} + +def afterBuild (action : IO PUnit) (self : BuildTarget α) : IO BuildTask := + IO.mapTask (fun x => IO.ofExcept x *> action) self.buildTask + +def materialize (self : BuildTarget α) : IO PUnit := + self.buildTask.await + +end BuildTarget + +namespace BuildTask + +def afterTargets +(targets : List (BuildTarget α)) (action : IO PUnit) : IO BuildTask := do + let tasks ← targets.mapM (·.buildTask) + IO.mapTasks (tasks := tasks) fun xs => xs.forM IO.ofExcept *> action + +end BuildTask From ea4cbfae7308019591e4151a77bff244419df0f9 Mon Sep 17 00:00:00 2001 From: tydeu Date: Fri, 9 Jul 2021 20:49:39 -0400 Subject: [PATCH 054/696] feat: deps build in parallel + lib/bin check mtime --- Lake/Build.lean | 301 ++++++++++++++++++++++++++++-------------- Lake/BuildTarget.lean | 30 ++++- Lake/Cli.lean | 4 +- Lake/Compile.lean | 6 +- 4 files changed, 228 insertions(+), 113 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index 0c9dcc08c3..5aba4c2ca4 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -17,91 +17,117 @@ namespace Lake -- # Build Targets -abbrev FileBuildTarget := BuildTarget FilePath +abbrev FileTarget := BuildTarget FilePath + +namespace FileTarget + +def mk (file : FilePath) (maxMTime : IO.FS.SystemTime) (task : BuildTask) := + BuildTarget.mk file maxMTime task + +def pure (file : FilePath) (maxMTime : IO.FS.SystemTime) := + BuildTarget.pure file maxMTime + +end FileTarget structure LeanArtifact where oleanFile : FilePath cFile : FilePath deriving Inhabited -abbrev LeanBuildTarget := BuildTarget LeanArtifact +abbrev LeanTarget := BuildTarget LeanArtifact -namespace LeanBuildTarget +namespace LeanTarget -def oleanFile (self : LeanBuildTarget) := self.artifact.oleanFile -def oleanTarget (self : LeanBuildTarget) : FileBuildTarget := +def mk (olean c : FilePath) (maxMTime : IO.FS.SystemTime) (task : BuildTask) : LeanTarget := + BuildTarget.mk ⟨olean, c⟩ maxMTime task + +def pure (olean c : FilePath) (maxMTime : IO.FS.SystemTime) : LeanTarget := + BuildTarget.pure ⟨olean, c⟩ maxMTime + +def oleanFile (self : LeanTarget) := self.artifact.oleanFile +def oleanTarget (self : LeanTarget) : FileTarget := {self with artifact := self.oleanFile} -def cFile (self : LeanBuildTarget) := self.artifact.cFile -def cTarget (self : LeanBuildTarget) : FileBuildTarget := +def cFile (self : LeanTarget) := self.artifact.cFile +def cTarget (self : LeanTarget) : FileTarget := {self with artifact := self.cFile} -end LeanBuildTarget +end LeanTarget def buildOleanAndC (leanFile oleanFile cFile : FilePath) -(depTargets : List LeanBuildTarget) (moreDepsMTime : IO.FS.SystemTime) +(importTargets : List LeanTarget) (depsTarget : BuildTarget PUnit) (leanPath : String := "") (rootDir : FilePath := ".") (leanArgs : Array String := #[]) -: IO LeanBuildTarget := do - let artifact := ⟨oleanFile, cFile⟩ +: IO LeanTarget := do -- calculate transitive `maxMTime` let leanMData ← leanFile.metadata - let depMTimes ← depTargets.mapM (·.maxMTime) - let maxMTime := List.maximum? (leanMData.modified :: moreDepsMTime :: depMTimes) |>.get! + let impMTimes ← importTargets.mapM (·.maxMTime) + let maxMTime := List.maximum? + (leanMData.modified :: depsTarget.maxMTime :: impMTimes) |>.get! -- construct a nop target if we have an up-to-date .olean and .c try let cMTime := (← cFile.metadata).modified let oleanMTime := (← oleanFile.metadata).modified if cMTime >= maxMTime && oleanMTime >= maxMTime then - return BuildTarget.pure artifact maxMTime + return LeanTarget.pure oleanFile cFile maxMTime catch | IO.Error.noFileOrDirectory .. => pure () | e => throw e -- construct a proper target otherwise - let buildTask ← BuildTask.afterTargets depTargets do + let targets := depsTarget.withArtifact arbitrary :: importTargets + let buildTask ← BuildTask.afterTargets targets do try compileOleanAndC leanFile oleanFile cFile leanPath rootDir leanArgs catch e => -- print compile errors early IO.eprintln e throw e - return { artifact, maxMTime, buildTask } + return LeanTarget.mk oleanFile cFile maxMTime buildTask def buildO (oFile : FilePath) -(cTarget : FileBuildTarget) (leancArgs : Array String := #[]) -: IO FileBuildTarget := do +(cTarget : FileTarget) (leancArgs : Array String := #[]) +: IO FileTarget := do -- construct a nop target if we have an up-to-date .o let cMTime := cTarget.maxMTime try if (← oFile.metadata).modified >= cMTime then - return BuildTarget.pure oFile cMTime + return FileTarget.pure oFile cMTime catch | IO.Error.noFileOrDirectory .. => pure () | e => throw e -- construct a proper target otherwise - let buildTask ← cTarget.afterBuild do + let buildTask ← BuildTask.afterTarget cTarget do try compileO oFile cTarget.artifact leancArgs catch e => -- print compile errors early IO.eprintln e throw e - return {artifact := oFile, maxMTime := cMTime, buildTask} + return FileTarget.mk oFile cMTime buildTask + +abbrev PackageTarget := BuildTarget (Package × NameMap LeanTarget) + +namespace PackageTarget + +def package (self : PackageTarget) := self.artifact.1 +def moduleTargets (self : PackageTarget) : NameMap LeanTarget := + self.artifact.2 + +end PackageTarget -- # Build Modules -structure ModuleTargetContext where - package : Package - leanPath : String - buildParents : List Name := [] - -- things that should also trigger rebuilds +structure LeanTargetContext where + package : Package + leanPath : String + buildParents : List Name := [] + -- target for external dependencies -- ex. olean roots of dependencies - moreDeps : List FilePath - moreDepsMTime : IO.FS.SystemTime + depsTarget : BuildTarget PUnit -abbrev ModuleTargetM := - ReaderT ModuleTargetContext <| StateT (NameMap LeanBuildTarget) IO +abbrev LeanTargetM := + ReaderT LeanTargetContext <| StateT (NameMap LeanTarget) IO -partial def buildTargetsForModule (mod : Name) : ModuleTargetM LeanBuildTarget := do +partial def buildModule (mod : Name) : LeanTargetM LeanTarget := do let ctx ← read let pkg := ctx.package @@ -123,106 +149,177 @@ partial def buildTargetsForModule (mod : Name) : ModuleTargetM LeanBuildTarget : let directLocalImports := imports.map (·.module) |>.filter (·.getRoot == pkg.module) -- recursively build local dependencies - let depTargets ← directLocalImports.mapM fun i => + let importTargets ← directLocalImports.mapM fun i => withReader (fun ctx => { ctx with buildParents := mod :: ctx.buildParents }) <| - buildTargetsForModule i + buildModule i -- do build let cFile := pkg.modToC mod let oleanFile := pkg.modToOlean mod let target ← buildOleanAndC leanFile oleanFile cFile - depTargets ctx.moreDepsMTime ctx.leanPath pkg.dir pkg.leanArgs + importTargets ctx.depsTarget ctx.leanPath pkg.dir pkg.leanArgs modify (·.insert mod target) return target -def mkModuleTargetContext -(pkg : Package) (deps : List Package) : IO ModuleTargetContext := do - let moreDeps := deps.map (·.oleanRoot) - let moreDepsMTime := (← moreDeps.mapM (·.metadata)).map (·.modified) |>.maximum? |>.getD ⟨0, 0⟩ - let leanPath := SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) - return { package := pkg, leanPath, moreDeps, moreDepsMTime } - -def buildPackageModuleTargetMap -(pkg : Package) (deps : List Package) : IO (NameMap LeanBuildTarget) := do - let crx ← mkModuleTargetContext pkg deps - let (_, targetMap) ← buildTargetsForModule pkg.module |>.run crx |>.run {} - return targetMap - -def buildPackageModules -(pkg : Package) (deps : List Package) : IO PUnit := do - let crx ← mkModuleTargetContext pkg deps - let target ← buildTargetsForModule pkg.module |>.run crx |>.run' {} - try target.materialize catch _ => - -- actual error has already been printed within buildTask - throw <| IO.userError "Build failed." - -def buildModulesInPackage -(pkg : Package) (deps : List Package) (mods : List Name) : IO Unit := do - let ctx ← mkModuleTargetContext pkg deps - let targets ← mods.mapM buildTargetsForModule |>.run ctx |>.run' {} - let tasks ← targets.mapM (·.buildTask) - for task in tasks do - try task.await catch e => - -- actual error has already been printed within buildTask - throw <| IO.userError "Build failed." +def mkLeanTargetContext +(pkg : Package) (oleanDirs : List FilePath) (depsTarget : BuildTarget PUnit) +: LeanTargetContext := { + package:= pkg + leanPath := SearchPath.toString <| pkg.oleanDir :: oleanDirs + depsTarget +} -- # Configure/Build Packages -def buildDeps (pkg : Package) : IO (List Package) := do - let deps ← solveDeps pkg - for dep in deps do - -- build recursively - -- TODO: share build of common dependencies - let depDeps ← solveDeps dep - buildPackageModules pkg deps +def Package.buildTargetWithDepTargets +(depTargets : List PackageTarget) (self : Package) +: IO PackageTarget := do + let depsTarget ← BuildTarget.all depTargets + let depOLeanDirs := depTargets.map (·.package.oleanDir) + let ctx := mkLeanTargetContext self depOLeanDirs depsTarget + let (target, targetMap) ← buildModule self.module |>.run ctx |>.run {} + return {target with artifact := ⟨self, targetMap⟩} + +partial def Package.buildTarget (self : Package) : IO PackageTarget := do + let deps ← solveDeps self + -- build dependencies recursively + -- TODO: share build of common dependencies + let depTargets ← deps.mapM (·.buildTarget) + self.buildTargetWithDepTargets depTargets + +def Package.buildDepTargets (self : Package) : IO (List PackageTarget) := do + let deps ← solveDeps self + deps.mapM (·.buildTarget) + +def Package.buildDeps (self : Package) : IO (List Package) := do + let deps ← solveDeps self + let targets ← deps.mapM (·.buildTarget) + try targets.forM (·.materialize) catch e => + -- actual error has already been printed within the task + throw <| IO.userError "Build failed." return deps -def configure (pkg : Package) : IO Unit := do - discard <| buildDeps pkg +def configure (pkg : Package) : IO Unit := + discard pkg.buildDeps -def build (pkg : Package) : IO Unit := do - let deps ← buildDeps pkg - buildPackageModules pkg deps +def Package.build (self : Package) : IO PUnit := do + let target ← self.buildTarget + try target.materialize catch _ => + -- actual error has already been printed within the task + throw <| IO.userError "Build failed." --- # Build Package Lib/Bin +def build (pkg : Package) : IO PUnit := + pkg.build -def buildPackageOFiles -(pkg : Package) (targetMap : NameMap LeanBuildTarget) -: IO (List FilePath) := do - let oTasks ← targetMap.toList.mapM fun (mod, target) => do - let oFile := pkg.modToO mod - let target ← buildO oFile target.cTarget pkg.leancArgs - (oFile, ← target.buildTask) - oTasks.mapM fun (oFile, task) => do - task.await - oFile +-- # Build Package Lib -def buildStaticLib (pkg : Package) : IO FilePath := do - let deps ← buildDeps pkg - let targetMap ← buildPackageModuleTargetMap pkg deps - let oFiles ← buildPackageOFiles pkg targetMap - compileLib pkg.staticLibFile oFiles.toArray - pkg.staticLibFile +def PackageTarget.buildOFileTargets +(self : PackageTarget) : IO (List FileTarget) := do + self.moduleTargets.toList.mapM fun (mod, target) => do + let oFile := self.package.modToO mod + buildO oFile target.cTarget self.package.leancArgs -def buildBin (pkg : Package) : IO FilePath := do - let deps ← solveDeps pkg - let depLibs ← deps.mapM buildStaticLib - let targetMap ← buildPackageModuleTargetMap pkg deps - let oFiles ← buildPackageOFiles pkg targetMap - compileBin pkg.binFile (oFiles ++ depLibs).toArray pkg.linkArgs - pkg.binFile +def PackageTarget.buildStaticLibTarget +(self : PackageTarget) : IO FileTarget := do + let libFile := self.package.staticLibFile + -- construct a nop target if we have an up-to-date lib + let pkgMTime := self.maxMTime + try + if (← libFile.metadata).modified >= pkgMTime then + return FileTarget.pure libFile pkgMTime + catch + | IO.Error.noFileOrDirectory .. => pure () + | e => throw e + -- construct a proper target otherwise + let oFileTargets ← self.buildOFileTargets + let oFiles := oFileTargets.map (·.artifact) |>.toArray + let buildTask ← BuildTask.afterTargets oFileTargets do + try + compileStaticLib libFile oFiles + catch e => + -- print compile errors early + IO.eprintln e + throw e + return FileTarget.mk libFile pkgMTime buildTask + +def Package.buildStaticLibTarget (self : Package) : IO FileTarget := do + let target ← self.buildTarget + target.buildStaticLibTarget + +def Package.buildStaticLib (self : Package) : IO FilePath := do + let target ← self.buildStaticLibTarget + try target.materialize catch _ => + -- actual error has already been printed within the task + throw <| IO.userError "Build failed." + return target.artifact + +def buildLib (pkg : Package) : IO PUnit := + discard pkg.buildStaticLib + +-- # Build Package Bin + +def PackageTarget.buildBinTarget + (depTargets : List PackageTarget) (self : PackageTarget) +: IO FileTarget := do + let binFile := self.package.binFile + -- construct a nop target if we have an up-to-date bin + let pkgMTime := self.maxMTime + try + if (← binFile.metadata).modified >= pkgMTime then + return FileTarget.pure binFile pkgMTime + catch + | IO.Error.noFileOrDirectory .. => pure () + | e => throw e + -- construct a proper target otherwise + let oFileTargets ← self.buildOFileTargets + let oFiles := oFileTargets.map (·.artifact) |>.toArray + let libTargets ← depTargets.mapM (·.buildStaticLibTarget) + let libFiles := libTargets.map (·.artifact) |>.toArray + let buildTask ← BuildTask.afterTargets oFileTargets do + try + compileBin binFile (oFiles ++ libFiles) self.package.linkArgs + catch e => + -- print compile errors early + IO.eprintln e + throw e + return FileTarget.mk binFile pkgMTime buildTask + +def Package.buildBinTarget (self : Package) : IO FileTarget := do + let depTargets ← self.buildDepTargets + let pkgTarget ← self.buildTargetWithDepTargets depTargets + pkgTarget.buildBinTarget depTargets + +def Package.buildBin (self : Package) : IO FilePath := do + let target ← self.buildBinTarget + try target.materialize catch _ => + -- actual error has already been printed within the task + throw <| IO.userError "Build failed." + return target.artifact + +def buildBin (pkg : Package) : IO PUnit := + discard pkg.buildBin -- # Print Paths def buildImports (pkg : Package) (deps : List Package) (imports : List String := []) : IO Unit := do + -- compute context + let oleanDirs := deps.map (·.oleanDir) + let depsTarget ← BuildTarget.all (← deps.mapM (·.buildTarget)) + let ctx ← mkLeanTargetContext pkg oleanDirs depsTarget + -- build imports let imports := imports.map (·.toName) let localImports := imports.filter (·.getRoot == pkg.module) - buildModulesInPackage pkg deps localImports + let targets ← imports.mapM buildModule |>.run ctx |>.run' {} + let tasks ← targets.mapM (·.buildTask) + for task in tasks do + try task.await catch e => + -- actual error has already been printed within buildTask + throw <| IO.userError "Build failed." def printPaths (pkg : Package) (imports : List String := []) : IO Unit := do - let deps ← buildDeps pkg + let deps ← solveDeps pkg buildImports pkg deps imports IO.println <| SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) IO.println <| SearchPath.toString <| pkg.sourceDir :: deps.map (·.sourceDir) diff --git a/Lake/BuildTarget.lean b/Lake/BuildTarget.lean index f5f2062ca4..092eb5de91 100644 --- a/Lake/BuildTarget.lean +++ b/Lake/BuildTarget.lean @@ -16,6 +16,9 @@ def nop : BuildTask := def await (self : BuildTask) : IO PUnit := do IO.ofExcept (← IO.wait self) +def all (tasks : List BuildTask) : IO BuildTask := + IO.asTask (tasks.forM (·.await)) + end BuildTask instance : Inhabited BuildTask := ⟨BuildTask.nop⟩ @@ -34,8 +37,22 @@ namespace BuildTarget def pure (artifact : α) (maxMTime : IO.FS.SystemTime := ⟨0, 0⟩) : BuildTarget α := {artifact, maxMTime, buildTask := BuildTask.nop} -def afterBuild (action : IO PUnit) (self : BuildTarget α) : IO BuildTask := - IO.mapTask (fun x => IO.ofExcept x *> action) self.buildTask +def all (targets : List (BuildTarget α)) : IO (BuildTarget PUnit) := do + let depsMTime := targets.map (·.maxMTime) |>.maximum? |>.getD ⟨0, 0⟩ + let task ← BuildTask.all <| targets.map (·.buildTask) + BuildTarget.mk () depsMTime task + +def collectAll (targets : List (BuildTarget α)) : IO (BuildTarget (List α)) := do + let artifacts := targets.map (·.artifact) + let depsMTime := targets.map (·.maxMTime) |>.maximum? |>.getD ⟨0, 0⟩ + let task ← BuildTask.all <| targets.map (·.buildTask) + BuildTarget.mk artifacts depsMTime task + +def withArtifact (artifact : α) (self : BuildTarget β) : BuildTarget α := + {self with artifact := artifact} + +def discardArtifact (self : BuildTarget α) : BuildTarget PUnit := + self.withArtifact () def materialize (self : BuildTarget α) : IO PUnit := self.buildTask.await @@ -44,9 +61,10 @@ end BuildTarget namespace BuildTask -def afterTargets -(targets : List (BuildTarget α)) (action : IO PUnit) : IO BuildTask := do - let tasks ← targets.mapM (·.buildTask) - IO.mapTasks (tasks := tasks) fun xs => xs.forM IO.ofExcept *> action +def afterTarget (target : BuildTarget α) (action : IO PUnit) : IO BuildTask := + IO.mapTask (fun x => IO.ofExcept x *> action) target.buildTask + +def afterTargets (targets : List (BuildTarget α)) (action : IO PUnit) : IO BuildTask := do + IO.mapTasks (fun xs => xs.forM IO.ofExcept *> action) <| targets.map (·.buildTask) end BuildTask diff --git a/Lake/Cli.lean b/Lake/Cli.lean index 799f0680e3..001ccea6f6 100644 --- a/Lake/Cli.lean +++ b/Lake/Cli.lean @@ -103,8 +103,8 @@ def cli : (cmd : String) → (lakeArgs pkgArgs : List String) → IO Unit | "configure", [], [] => do configure (← getRootPkg) | "print-paths", imports, [] => do printPaths (← getRootPkg) imports | "build", [], [] => do build (← getRootPkg) -| "build-lib", [], [] => do discard <| buildStaticLib (← getRootPkg) -| "build-bin", [], [] => do discard <| buildBin (← getRootPkg) +| "build-lib", [], [] => do buildLib (← getRootPkg) +| "build-bin", [], [] => do buildBin (← getRootPkg) | "help", ["init"], [] => IO.println helpInit | "help", ["configure"], [] => IO.println helpConfigure | "help", ["build"], [] => IO.println helpBuild diff --git a/Lake/Compile.lean b/Lake/Compile.lean index 5e7014e2ff..655df2afad 100644 --- a/Lake/Compile.lean +++ b/Lake/Compile.lean @@ -33,15 +33,15 @@ def compileO } def compileBin -(binFile : FilePath) (oFiles : Array FilePath) (linkArgs : Array String := #[]) +(binFile : FilePath) (linkFiles : Array FilePath) (linkArgs : Array String := #[]) : IO Unit := do if let some dir := binFile.parent then IO.FS.createDirAll dir execCmd { cmd := "leanc" - args := #["-o", binFile.toString] ++ oFiles.map toString ++ linkArgs + args := #["-o", binFile.toString] ++ linkFiles.map toString ++ linkArgs } -def compileLib +def compileStaticLib (libFile : FilePath) (oFiles : Array FilePath) : IO Unit := do if let some dir := libFile.parent then IO.FS.createDirAll dir From 16534d3be6c0b3f51a9a17b4e2b78500e70439d1 Mon Sep 17 00:00:00 2001 From: tydeu Date: Fri, 9 Jul 2021 21:03:16 -0400 Subject: [PATCH 055/696] chore: update Lean version --- leanpkg.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/leanpkg.toml b/leanpkg.toml index 7c54a930ab..a7f7ffefde 100644 --- a/leanpkg.toml +++ b/leanpkg.toml @@ -1,4 +1,4 @@ [package] name = "lake" version = "2.0-pre" -lean_version = "leanprover/lean4:nightly-2021-06-14" +lean_version = "leanprover/lean4:nightly-2021-07-09" From 042353d8627478d5f7d98c3978a0f006aad41eca Mon Sep 17 00:00:00 2001 From: tydeu Date: Sat, 10 Jul 2021 12:00:16 -0400 Subject: [PATCH 056/696] feat: allow cli arguments to be passed to `package.lean` --- Lake.lean | 2 +- Lake/Cli.lean | 115 ++++++------------------------------------- Lake/Help.lean | 93 ++++++++++++++++++++++++++++++++++ Lake/LeanConfig.lean | 26 +++++++--- Lake/Package.lean | 67 +++++++++++++------------ Lake/Resolve.lean | 4 +- Lake/SearchPath.lean | 2 +- Lake/Version.lean | 11 +++++ 8 files changed, 178 insertions(+), 142 deletions(-) create mode 100644 Lake/Help.lean create mode 100644 Lake/Version.lean diff --git a/Lake.lean b/Lake.lean index 160597a07f..3fed7cb58a 100644 --- a/Lake.lean +++ b/Lake.lean @@ -13,5 +13,5 @@ def main (args : List String) : IO UInt32 := do Lake.cli cmd outerArgs innerArgs pure 0 catch e => - IO.eprintln e -- avoid "uncaught exception: ..." + IO.eprintln e -- avoid "uncaught exception: ..." pure 1 diff --git a/Lake/Cli.lean b/Lake/Cli.lean index 001ccea6f6..04d741c601 100644 --- a/Lake/Cli.lean +++ b/Lake/Cli.lean @@ -5,113 +5,26 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Lake.Init import Lake.Build +import Lake.Help import Lake.LeanConfig namespace Lake - -def version.major := 2 -def version.minor := 0 -def version.isPre := true -def versionString := s!"{version.major}.{version.minor}-pre" - -def usage := -"Lake, version " ++ versionString ++ " - -Usage: - lake - -init create a Lean package in the current directory -configure download and build dependencies -build configure and build *.olean files -build-lib configure and build a static library -build-bin configure and build a native binary executable - -See `lake help ` for more information on a specific command." - -def helpConfigure := -"Download and build dependencies - -Usage: - lake configure - -This command sets up the `build/deps` directory. - -For each (transitive) git dependency, the specified commit is checked out -into a sub-directory of `build/deps`. If there are dependencies on multiple -versions of the same package, the version materialized is undefined. No copy -is made of local dependencies." - -def helpBuild := -"Configure this package and build *.olean files - -Usage: - lake build - -This command configures the package's dependencies and then builds the package -with `lean`. Additional arguments can be passed to `lean` by setting the -`leanArgs` field in the package's configuration." - -def helpBuildLib := -"Configure this package and build a static library - -Usage: - lake build-lib - -This command configures this package's dependencies, builds the package itself, -compiles the extracted C code with `leanc`, and uses `ar` to produce a static -library in `build/lib`. - -Additional arguments can be passed to `lean` or `leanc` by setting the -`leanArgs` or `leancArgs` fields in the package's configuration." - -def helpBuildBin := -"Configure the package and build a native binary executable - -Usage: - lake build-bin - -This command configures this package's dependencies, builds the package itself, -compiles the extracted C code `leannc`, and then links the object files with -`leanc` to produce a native binary executable in `build/bin`. - -This requires a declaration of name `main` in the root namespace, which must -return `IO Unit` or `IO UInt32` (the exit code) and may accept the program's -command line arguments as a `List String` parameter. - -Additional arguments can be passed to `lean`, the `leanc` compiler, or the -`leanc` linker by setting the `leanArgs`, `leancArgs`, or `linkArgs` fields in -the package's configuration." - -def helpInit := -"Create a new Lean package in the current directory - -Usage: - lake init - -This command creates a new Lean package with the given name in the current -directory." - -def getRootPkg : IO Package := do - let cfg ← PackageConfig.fromLeanFile leanPkgFile - if cfg.leanVersion ≠ leanVersionString then +def getCwdPkg (args : List String) : IO Package := do + let pkg ← Package.fromDir "." args + if pkg.leanVersion ≠ leanVersionString then IO.eprintln $ "\nWARNING: Lean version mismatch: installed version is " ++ - leanVersionString ++ ", but package requires " ++ cfg.leanVersion ++ "\n" - return ⟨".", cfg⟩ + leanVersionString ++ ", but package requires " ++ pkg.leanVersion ++ "\n" + return pkg def cli : (cmd : String) → (lakeArgs pkgArgs : List String) → IO Unit -| "init", [name], [] => init name -| "configure", [], [] => do configure (← getRootPkg) -| "print-paths", imports, [] => do printPaths (← getRootPkg) imports -| "build", [], [] => do build (← getRootPkg) -| "build-lib", [], [] => do buildLib (← getRootPkg) -| "build-bin", [], [] => do buildBin (← getRootPkg) -| "help", ["init"], [] => IO.println helpInit -| "help", ["configure"], [] => IO.println helpConfigure -| "help", ["build"], [] => IO.println helpBuild -| "help", ["build-lib"], [] => IO.println helpBuildLib -| "help", ["build-bin"], [] => IO.println helpBuildBin -| "help", _, [] => IO.println usage -| _, _, _ => throw <| IO.userError usage +| "init", [name], [] => init name +| "configure", [], pkgArgs => do configure (← getCwdPkg pkgArgs) +| "print-paths", imports, pkgArgs => do printPaths (← getCwdPkg pkgArgs) imports +| "build", [], pkgArgs => do build (← getCwdPkg pkgArgs) +| "build-lib", [], pkgArgs => do buildLib (← getCwdPkg pkgArgs) +| "build-bin", [], pkgArgs => do buildBin (← getCwdPkg pkgArgs) +| "help", [cmd], _ => IO.println <| help cmd +| _, _, _ => throw <| IO.userError usage private def splitCmdlineArgsCore : List String → List String × List String | [] => ([], []) diff --git a/Lake/Help.lean b/Lake/Help.lean new file mode 100644 index 0000000000..315de06d30 --- /dev/null +++ b/Lake/Help.lean @@ -0,0 +1,93 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone +-/ +import Lake.Version + +namespace Lake + +def usage := +"Lake, version " ++ versionString ++ " + +Usage: + lake + +init create a Lean package in the current directory +configure download and build dependencies +build configure and build *.olean files +build-lib configure and build a static library +build-bin configure and build a native binary executable + +See `lake help ` for more information on a specific command." + +def helpConfigure := +"Download and build dependencies + +Usage: + lake configure + +This command sets up the `build/deps` directory. + +For each (transitive) git dependency, the specified commit is checked out +into a sub-directory of `build/deps`. If there are dependencies on multiple +versions of the same package, the version materialized is undefined. No copy +is made of local dependencies." + +def helpBuild := +"Configure this package and build *.olean files + +Usage: + lake build + +This command configures the package's dependencies and then builds the package +with `lean`. Additional arguments can be passed to `lean` by setting the +`leanArgs` field in the package's configuration." + +def helpBuildLib := +"Configure this package and build a static library + +Usage: + lake build-lib + +This command configures this package's dependencies, builds the package itself, +compiles the extracted C code with `leanc`, and uses `ar` to produce a static +library in `build/lib`. + +Additional arguments can be passed to `lean` or `leanc` by setting the +`leanArgs` or `leancArgs` fields in the package's configuration." + +def helpBuildBin := +"Configure the package and build a native binary executable + +Usage: + lake build-bin + +This command configures this package's dependencies, builds the package itself, +compiles the extracted C code `leannc`, and then links the object files with +`leanc` to produce a native binary executable in `build/bin`. + +This requires a declaration of name `main` in the root namespace, which must +return `IO Unit` or `IO UInt32` (the exit code) and may accept the program's +command line arguments as a `List String` parameter. + +Additional arguments can be passed to `lean`, the `leanc` compiler, or the +`leanc` linker by setting the `leanArgs`, `leancArgs`, or `linkArgs` fields in +the package's configuration." + +def helpInit := +"Create a new Lean package in the current directory + +Usage: + lake init + +This command creates a new Lean package with the given name in the current +directory." + +def help : (cmd : String) → String +| "init" => helpInit +| "configure" => helpConfigure +| "build" => helpBuild +| "build-lib" => helpBuildLib +| "build-bin" => helpBuildBin +| _ => usage diff --git a/Lake/LeanConfig.lean b/Lake/LeanConfig.lean index 6fece98b92..bb90519f06 100644 --- a/Lake/LeanConfig.lean +++ b/Lake/LeanConfig.lean @@ -12,16 +12,30 @@ namespace Lake def leanPkgFile : FilePath := "package.lean" -namespace PackageConfig +namespace Package -unsafe def fromLeanFileUnsafe (path : FilePath) : IO PackageConfig := do +unsafe def fromLeanFileUnsafe +(path : FilePath) (root : FilePath) (args : List String := []) +: IO Package := do let input ← IO.FS.readFile path let (env, ok) ← Elab.runFrontend input Options.empty path.toString `package if ok then - IO.ofExcept <| Id.run <| ExceptT.run <| - env.evalConstCheck PackageConfig Options.empty ``PackageConfig `package + let packagerE := Id.run <| ExceptT.run <| + env.evalConstCheck Packager Options.empty ``Packager `package + match packagerE with + | Except.ok packager => Package.mk root (← packager root args) + | Except.error error => + let configE := Id.run <| ExceptT.run <| + env.evalConstCheck PackageConfig Options.empty ``PackageConfig `package + match configE with + | Except.ok config => Package.mk root config + | Except.error error => throw <| IO.userError error else throw <| IO.userError <| s!"package configuration (at {path}) has errors" -@[implementedBy fromLeanFileUnsafe] -constant fromLeanFile (path : FilePath) : IO PackageConfig +unsafe def fromDirUnsafe +(path : FilePath) (args : List String := []) : IO Package := + fromLeanFileUnsafe (path / leanPkgFile) path args + +@[implementedBy fromDirUnsafe] +constant fromDir (path : FilePath) (args : List String := []) : IO Package diff --git a/Lake/Package.lean b/Lake/Package.lean index 3c37090375..c58a48165d 100644 --- a/Lake/Package.lean +++ b/Lake/Package.lean @@ -21,7 +21,8 @@ inductive Source where structure Dependency where name : String - src : Source + src : Source + args : List String := [] structure PackageConfig where name : String @@ -30,7 +31,6 @@ structure PackageConfig where leanArgs : Array String := #[] leancArgs : Array String := #[] linkArgs : Array String := #[] - timeout : Option Nat := none module : Name := name.capitalize dependencies : List Dependency := [] deriving Inhabited @@ -40,88 +40,93 @@ structure Package where config : PackageConfig deriving Inhabited +def Packager := FilePath → List String → IO PackageConfig + namespace Package -def name (self : Package) := +def name (self : Package) : String := self.config.name -def module (self : Package) := +def version (self : Package) : String := + self.config.version + +def leanVersion (self : Package) : String := + self.config.leanVersion + +def module (self : Package) : Name := self.config.module -def moduleName (self : Package) := +def moduleName (self : Package) : String := self.config.module.toString -def dependencies (self : Package) := +def dependencies (self : Package) : List Dependency := self.config.dependencies -def leanArgs (self : Package) := +def leanArgs (self : Package) : Array String := self.config.leanArgs -def leancArgs (self : Package) := +def leancArgs (self : Package) : Array String := self.config.leancArgs -def linkArgs (self : Package) := +def linkArgs (self : Package) : Array String := self.config.linkArgs -def timeout (self : Package) := - self.config.timeout - -def sourceDir (self : Package) := +def sourceDir (self : Package) : FilePath := self.dir -def sourceRoot (self : Package) := +def sourceRoot (self : Package) : FilePath := self.sourceDir / self.moduleName -def modToSource (mod : Name) (self : Package) := +def modToSource (mod : Name) (self : Package) : FilePath := Lean.modToFilePath self.sourceDir mod "lean" -def buildDir (self : Package) := +def buildDir (self : Package) : FilePath := self.dir / Lake.buildPath -def oleanDir (self : Package) := +def oleanDir (self : Package) : FilePath := self.buildDir -def oleanRoot (self : Package) := +def oleanRoot (self : Package) : FilePath := self.oleanDir / FilePath.withExtension self.moduleName "olean" -def modToOlean (mod : Name) (self : Package) := +def modToOlean (mod : Name) (self : Package) : FilePath := Lean.modToFilePath self.oleanDir mod "olean" -def tempBuildDir (self : Package) := +def tempBuildDir (self : Package) : FilePath := self.dir / tempBuildPath -def cDir (self : Package) := +def cDir (self : Package) : FilePath := self.tempBuildDir -def modToC (mod : Name) (self : Package) := +def modToC (mod : Name) (self : Package) : FilePath := Lean.modToFilePath self.cDir mod "c" -def oDir (self : Package) := +def oDir (self : Package) : FilePath := self.tempBuildDir -def modToO (mod : Name) (self : Package) := +def modToO (mod : Name) (self : Package) : FilePath := Lean.modToFilePath self.oDir mod "o" -def binDir (self : Package) := +def binDir (self : Package) : FilePath := self.buildDir / "bin" -def binName (self : Package) := +def binName (self : Package) : FilePath := self.moduleName def binFileName (self : Package) : FilePath := FilePath.withExtension self.binName FilePath.exeExtension -def binFile (self : Package) := +def binFile (self : Package) : FilePath := self.binDir / self.binFileName -def libDir (self : Package) := +def libDir (self : Package) : FilePath := self.buildDir / "lib" -def staticLibName (self : Package) := +def staticLibName (self : Package) : FilePath := self.moduleName -def staticLibFileName (self : Package) := +def staticLibFileName (self : Package) : FilePath := s!"lib{self.module}.a" -def staticLibFile (self : Package) := +def staticLibFile (self : Package) : FilePath := self.libDir / self.staticLibFileName diff --git a/Lake/Resolve.lean b/Lake/Resolve.lean index 95211eb027..bd3b168019 100644 --- a/Lake/Resolve.lean +++ b/Lake/Resolve.lean @@ -68,8 +68,8 @@ def solveDepsCore (pkg : Package) : (maxDepth : Nat) → Solver Unit let newDeps ← pkg.dependencies.filterM (notYetAssigned ·.name) for dep in newDeps do let dir ← materialize pkg.dir dep - let cfg ← PackageConfig.fromLeanFile <| dir / leanPkgFile - modify (·.insert dep.name ⟨dir, cfg⟩) + let pkg ← Package.fromDir dir dep.args + modify (·.insert dep.name pkg) for dep in newDeps do let depPkg ← resolvedPackage dep.name unless depPkg.name = dep.name do diff --git a/Lake/SearchPath.lean b/Lake/SearchPath.lean index ee073b099b..429d805d91 100644 --- a/Lake/SearchPath.lean +++ b/Lake/SearchPath.lean @@ -24,7 +24,7 @@ def getLakeHome? : IO (Option FilePath) := do /-- Initializes the search path the Lake executable - uses when intepreting package configuration files. + uses when interpreting package configuration files. In order to use the Lean stdlib (e.g., `Init`), the executable needs the search path to include the directory diff --git a/Lake/Version.lean b/Lake/Version.lean new file mode 100644 index 0000000000..5526e7cb01 --- /dev/null +++ b/Lake/Version.lean @@ -0,0 +1,11 @@ +/- +Copyright (c) 2021 Mac Malone. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Mac Malone +-/ +namespace Lake + +def version.major := 2 +def version.minor := 0 +def version.isPre := true +def versionString := s!"{version.major}.{version.minor}-pre" From 9da32ce7ebdc463bf2b88f5c4d1e6c525a717703 Mon Sep 17 00:00:00 2001 From: tydeu Date: Sat, 10 Jul 2021 12:21:52 -0400 Subject: [PATCH 057/696] chore: add Packager test --- Lake/LeanConfig.lean | 3 ++- examples/io/.gitignore | 1 + examples/io/Io.lean | 2 ++ examples/io/clean.sh | 1 + examples/io/package.lean | 8 ++++++++ examples/io/package.sh | 1 + examples/io/test.sh | 3 +++ 7 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 examples/io/.gitignore create mode 100644 examples/io/Io.lean create mode 100644 examples/io/clean.sh create mode 100644 examples/io/package.lean create mode 100644 examples/io/package.sh create mode 100644 examples/io/test.sh diff --git a/Lake/LeanConfig.lean b/Lake/LeanConfig.lean index bb90519f06..9de4e609ac 100644 --- a/Lake/LeanConfig.lean +++ b/Lake/LeanConfig.lean @@ -29,7 +29,8 @@ unsafe def fromLeanFileUnsafe env.evalConstCheck PackageConfig Options.empty ``PackageConfig `package match configE with | Except.ok config => Package.mk root config - | Except.error error => throw <| IO.userError error + | Except.error error => throw <| IO.userError <| + s!"unexpected type at 'package', `Lake.Packager` or `Lake.PackageConfig` expected" else throw <| IO.userError <| s!"package configuration (at {path}) has errors" diff --git a/examples/io/.gitignore b/examples/io/.gitignore new file mode 100644 index 0000000000..796b96d1c4 --- /dev/null +++ b/examples/io/.gitignore @@ -0,0 +1 @@ +/build diff --git a/examples/io/Io.lean b/examples/io/Io.lean new file mode 100644 index 0000000000..00ac27245d --- /dev/null +++ b/examples/io/Io.lean @@ -0,0 +1,2 @@ +def main : IO Unit := + IO.println "Hello from Io!" diff --git a/examples/io/clean.sh b/examples/io/clean.sh new file mode 100644 index 0000000000..07cbd15340 --- /dev/null +++ b/examples/io/clean.sh @@ -0,0 +1 @@ +rm -r build diff --git a/examples/io/package.lean b/examples/io/package.lean new file mode 100644 index 0000000000..58ef709beb --- /dev/null +++ b/examples/io/package.lean @@ -0,0 +1,8 @@ +import Lake.Package + +def package : Lake.Packager := fun path args => do + IO.println s!"computing io package in {path} with args {args} ..." + return { + name := "io" + version := "1.0" + } diff --git a/examples/io/package.sh b/examples/io/package.sh new file mode 100644 index 0000000000..b6449c2dd0 --- /dev/null +++ b/examples/io/package.sh @@ -0,0 +1 @@ +../../build/bin/Lake build-bin diff --git a/examples/io/test.sh b/examples/io/test.sh new file mode 100644 index 0000000000..1872494d40 --- /dev/null +++ b/examples/io/test.sh @@ -0,0 +1,3 @@ +./clean.sh +./package.sh +./build/bin/io From d1674a6ba0d2f3898f6d9dc4ab96c5b9fda48527 Mon Sep 17 00:00:00 2001 From: tydeu Date: Sat, 10 Jul 2021 12:23:20 -0400 Subject: [PATCH 058/696] refactor: rename `helloDeps` test to `deps` --- examples/{helloDeps => deps}/a/.gitignore | 0 examples/{helloDeps => deps}/a/A.lean | 0 examples/{helloDeps => deps}/a/package.lean | 0 examples/{helloDeps => deps}/b/.gitignore | 0 examples/{helloDeps => deps}/b/B.lean | 0 examples/{helloDeps => deps}/b/package.lean | 0 examples/{helloDeps => deps}/clean.sh | 0 examples/{helloDeps => deps}/foo/.gitignore | 0 examples/{helloDeps => deps}/foo/Foo.lean | 0 examples/{helloDeps => deps}/foo/Foo/Bar.lean | 0 examples/{helloDeps => deps}/foo/Foo/Baz.lean | 0 examples/{helloDeps => deps}/foo/Foo/Foo.lean | 0 examples/{helloDeps => deps}/foo/package.lean | 0 examples/{helloDeps => deps}/package.sh | 0 examples/{helloDeps => deps}/test.sh | 0 15 files changed, 0 insertions(+), 0 deletions(-) rename examples/{helloDeps => deps}/a/.gitignore (100%) rename examples/{helloDeps => deps}/a/A.lean (100%) rename examples/{helloDeps => deps}/a/package.lean (100%) rename examples/{helloDeps => deps}/b/.gitignore (100%) rename examples/{helloDeps => deps}/b/B.lean (100%) rename examples/{helloDeps => deps}/b/package.lean (100%) rename examples/{helloDeps => deps}/clean.sh (100%) rename examples/{helloDeps => deps}/foo/.gitignore (100%) rename examples/{helloDeps => deps}/foo/Foo.lean (100%) rename examples/{helloDeps => deps}/foo/Foo/Bar.lean (100%) rename examples/{helloDeps => deps}/foo/Foo/Baz.lean (100%) rename examples/{helloDeps => deps}/foo/Foo/Foo.lean (100%) rename examples/{helloDeps => deps}/foo/package.lean (100%) rename examples/{helloDeps => deps}/package.sh (100%) rename examples/{helloDeps => deps}/test.sh (100%) diff --git a/examples/helloDeps/a/.gitignore b/examples/deps/a/.gitignore similarity index 100% rename from examples/helloDeps/a/.gitignore rename to examples/deps/a/.gitignore diff --git a/examples/helloDeps/a/A.lean b/examples/deps/a/A.lean similarity index 100% rename from examples/helloDeps/a/A.lean rename to examples/deps/a/A.lean diff --git a/examples/helloDeps/a/package.lean b/examples/deps/a/package.lean similarity index 100% rename from examples/helloDeps/a/package.lean rename to examples/deps/a/package.lean diff --git a/examples/helloDeps/b/.gitignore b/examples/deps/b/.gitignore similarity index 100% rename from examples/helloDeps/b/.gitignore rename to examples/deps/b/.gitignore diff --git a/examples/helloDeps/b/B.lean b/examples/deps/b/B.lean similarity index 100% rename from examples/helloDeps/b/B.lean rename to examples/deps/b/B.lean diff --git a/examples/helloDeps/b/package.lean b/examples/deps/b/package.lean similarity index 100% rename from examples/helloDeps/b/package.lean rename to examples/deps/b/package.lean diff --git a/examples/helloDeps/clean.sh b/examples/deps/clean.sh similarity index 100% rename from examples/helloDeps/clean.sh rename to examples/deps/clean.sh diff --git a/examples/helloDeps/foo/.gitignore b/examples/deps/foo/.gitignore similarity index 100% rename from examples/helloDeps/foo/.gitignore rename to examples/deps/foo/.gitignore diff --git a/examples/helloDeps/foo/Foo.lean b/examples/deps/foo/Foo.lean similarity index 100% rename from examples/helloDeps/foo/Foo.lean rename to examples/deps/foo/Foo.lean diff --git a/examples/helloDeps/foo/Foo/Bar.lean b/examples/deps/foo/Foo/Bar.lean similarity index 100% rename from examples/helloDeps/foo/Foo/Bar.lean rename to examples/deps/foo/Foo/Bar.lean diff --git a/examples/helloDeps/foo/Foo/Baz.lean b/examples/deps/foo/Foo/Baz.lean similarity index 100% rename from examples/helloDeps/foo/Foo/Baz.lean rename to examples/deps/foo/Foo/Baz.lean diff --git a/examples/helloDeps/foo/Foo/Foo.lean b/examples/deps/foo/Foo/Foo.lean similarity index 100% rename from examples/helloDeps/foo/Foo/Foo.lean rename to examples/deps/foo/Foo/Foo.lean diff --git a/examples/helloDeps/foo/package.lean b/examples/deps/foo/package.lean similarity index 100% rename from examples/helloDeps/foo/package.lean rename to examples/deps/foo/package.lean diff --git a/examples/helloDeps/package.sh b/examples/deps/package.sh similarity index 100% rename from examples/helloDeps/package.sh rename to examples/deps/package.sh diff --git a/examples/helloDeps/test.sh b/examples/deps/test.sh similarity index 100% rename from examples/helloDeps/test.sh rename to examples/deps/test.sh From 1ccebe9b89bb57dd9d25b4ec196b78451e1e1dd3 Mon Sep 17 00:00:00 2001 From: tydeu Date: Sat, 10 Jul 2021 12:36:13 -0400 Subject: [PATCH 059/696] chore: improve shell scripts --- clean.sh | 1 + examples/deps/clean.sh | 6 +++--- examples/hello/clean.sh | 2 +- examples/io/clean.sh | 2 +- examples/test.sh | 14 ++++++++++++++ 5 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 clean.sh create mode 100644 examples/test.sh diff --git a/clean.sh b/clean.sh new file mode 100644 index 0000000000..1582321863 --- /dev/null +++ b/clean.sh @@ -0,0 +1 @@ +rm -rf build diff --git a/examples/deps/clean.sh b/examples/deps/clean.sh index d6f800d026..bb08ef93ed 100644 --- a/examples/deps/clean.sh +++ b/examples/deps/clean.sh @@ -1,3 +1,3 @@ -rm -r a/build -rm -r b/build -rm -r foo/build +rm -rf a/build +rm -rf b/build +rm -rf foo/build diff --git a/examples/hello/clean.sh b/examples/hello/clean.sh index 07cbd15340..1582321863 100644 --- a/examples/hello/clean.sh +++ b/examples/hello/clean.sh @@ -1 +1 @@ -rm -r build +rm -rf build diff --git a/examples/io/clean.sh b/examples/io/clean.sh index 07cbd15340..1582321863 100644 --- a/examples/io/clean.sh +++ b/examples/io/clean.sh @@ -1 +1 @@ -rm -r build +rm -rf build diff --git a/examples/test.sh b/examples/test.sh new file mode 100644 index 0000000000..b24edadb15 --- /dev/null +++ b/examples/test.sh @@ -0,0 +1,14 @@ +echo 'testing hello' +cd hello +./test.sh +cd .. + +echo 'testing io' +cd io +./test.sh +cd .. + +echo 'testing deps' +cd deps +./test.sh +cd .. From 9ce5fa6a6d85859e3207eb2cc8a75d06378e9d1a Mon Sep 17 00:00:00 2001 From: tydeu Date: Sat, 10 Jul 2021 13:04:18 -0400 Subject: [PATCH 060/696] refactor: generalize build error catching --- Lake/Build.lean | 83 ++++++++++++++++++++----------------------------- 1 file changed, 34 insertions(+), 49 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index 5aba4c2ca4..c3c534409e 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -54,14 +54,20 @@ def cTarget (self : LeanTarget) : FileTarget := end LeanTarget -def buildOleanAndC (leanFile oleanFile cFile : FilePath) -(importTargets : List LeanTarget) (depsTarget : BuildTarget PUnit) +def catchErrors (action : IO PUnit) : IO PUnit := do + try action catch e => + -- print compile errors early + IO.eprintln e + throw e + +def buildOleanAndC (leanFile oleanFile cFile : FilePath) +(importTargets : List LeanTarget) (depsTarget : BuildTarget PUnit) (leanPath : String := "") (rootDir : FilePath := ".") (leanArgs : Array String := #[]) : IO LeanTarget := do -- calculate transitive `maxMTime` let leanMData ← leanFile.metadata let impMTimes ← importTargets.mapM (·.maxMTime) - let maxMTime := List.maximum? + let maxMTime := List.maximum? (leanMData.modified :: depsTarget.maxMTime :: impMTimes) |>.get! -- construct a nop target if we have an up-to-date .olean and .c try @@ -74,34 +80,24 @@ def buildOleanAndC (leanFile oleanFile cFile : FilePath) | e => throw e -- construct a proper target otherwise let targets := depsTarget.withArtifact arbitrary :: importTargets - let buildTask ← BuildTask.afterTargets targets do - try - compileOleanAndC leanFile oleanFile cFile leanPath rootDir leanArgs - catch e => - -- print compile errors early - IO.eprintln e - throw e + let buildTask ← BuildTask.afterTargets targets <| catchErrors <| + compileOleanAndC leanFile oleanFile cFile leanPath rootDir leanArgs return LeanTarget.mk oleanFile cFile maxMTime buildTask -def buildO (oFile : FilePath) -(cTarget : FileTarget) (leancArgs : Array String := #[]) +def buildO (oFile : FilePath) +(cTarget : FileTarget) (leancArgs : Array String := #[]) : IO FileTarget := do -- construct a nop target if we have an up-to-date .o let cMTime := cTarget.maxMTime - try - if (← oFile.metadata).modified >= cMTime then + try + if (← oFile.metadata).modified >= cMTime then return FileTarget.pure oFile cMTime catch | IO.Error.noFileOrDirectory .. => pure () | e => throw e -- construct a proper target otherwise - let buildTask ← BuildTask.afterTarget cTarget do - try - compileO oFile cTarget.artifact leancArgs - catch e => - -- print compile errors early - IO.eprintln e - throw e + let buildTask ← BuildTask.afterTarget cTarget <| catchErrors <| + compileO oFile cTarget.artifact leancArgs return FileTarget.mk oFile cMTime buildTask abbrev PackageTarget := BuildTarget (Package × NameMap LeanTarget) @@ -156,13 +152,13 @@ partial def buildModule (mod : Name) : LeanTargetM LeanTarget := do -- do build let cFile := pkg.modToC mod let oleanFile := pkg.modToOlean mod - let target ← buildOleanAndC leanFile oleanFile cFile + let target ← buildOleanAndC leanFile oleanFile cFile importTargets ctx.depsTarget ctx.leanPath pkg.dir pkg.leanArgs modify (·.insert mod target) return target -def mkLeanTargetContext -(pkg : Package) (oleanDirs : List FilePath) (depsTarget : BuildTarget PUnit) +def mkLeanTargetContext +(pkg : Package) (oleanDirs : List FilePath) (depsTarget : BuildTarget PUnit) : LeanTargetContext := { package:= pkg leanPath := SearchPath.toString <| pkg.oleanDir :: oleanDirs @@ -213,19 +209,19 @@ def build (pkg : Package) : IO PUnit := -- # Build Package Lib -def PackageTarget.buildOFileTargets -(self : PackageTarget) : IO (List FileTarget) := do +def PackageTarget.buildOFileTargets +(self : PackageTarget) : IO (List FileTarget) := do self.moduleTargets.toList.mapM fun (mod, target) => do let oFile := self.package.modToO mod buildO oFile target.cTarget self.package.leancArgs -def PackageTarget.buildStaticLibTarget +def PackageTarget.buildStaticLibTarget (self : PackageTarget) : IO FileTarget := do let libFile := self.package.staticLibFile -- construct a nop target if we have an up-to-date lib let pkgMTime := self.maxMTime - try - if (← libFile.metadata).modified >= pkgMTime then + try + if (← libFile.metadata).modified >= pkgMTime then return FileTarget.pure libFile pkgMTime catch | IO.Error.noFileOrDirectory .. => pure () @@ -233,13 +229,8 @@ def PackageTarget.buildStaticLibTarget -- construct a proper target otherwise let oFileTargets ← self.buildOFileTargets let oFiles := oFileTargets.map (·.artifact) |>.toArray - let buildTask ← BuildTask.afterTargets oFileTargets do - try - compileStaticLib libFile oFiles - catch e => - -- print compile errors early - IO.eprintln e - throw e + let buildTask ← BuildTask.afterTargets oFileTargets <| catchErrors <| + compileStaticLib libFile oFiles return FileTarget.mk libFile pkgMTime buildTask def Package.buildStaticLibTarget (self : Package) : IO FileTarget := do @@ -258,14 +249,14 @@ def buildLib (pkg : Package) : IO PUnit := -- # Build Package Bin -def PackageTarget.buildBinTarget +def PackageTarget.buildBinTarget (depTargets : List PackageTarget) (self : PackageTarget) : IO FileTarget := do let binFile := self.package.binFile -- construct a nop target if we have an up-to-date bin let pkgMTime := self.maxMTime - try - if (← binFile.metadata).modified >= pkgMTime then + try + if (← binFile.metadata).modified >= pkgMTime then return FileTarget.pure binFile pkgMTime catch | IO.Error.noFileOrDirectory .. => pure () @@ -275,13 +266,8 @@ def PackageTarget.buildBinTarget let oFiles := oFileTargets.map (·.artifact) |>.toArray let libTargets ← depTargets.mapM (·.buildStaticLibTarget) let libFiles := libTargets.map (·.artifact) |>.toArray - let buildTask ← BuildTask.afterTargets oFileTargets do - try - compileBin binFile (oFiles ++ libFiles) self.package.linkArgs - catch e => - -- print compile errors early - IO.eprintln e - throw e + let buildTask ← BuildTask.afterTargets oFileTargets <| catchErrors <| + compileBin binFile (oFiles ++ libFiles) self.package.linkArgs return FileTarget.mk binFile pkgMTime buildTask def Package.buildBinTarget (self : Package) : IO FileTarget := do @@ -301,8 +287,8 @@ def buildBin (pkg : Package) : IO PUnit := -- # Print Paths -def buildImports -(pkg : Package) (deps : List Package) (imports : List String := []) +def buildImports +(pkg : Package) (deps : List Package) (imports : List String := []) : IO Unit := do -- compute context let oleanDirs := deps.map (·.oleanDir) @@ -323,4 +309,3 @@ def printPaths (pkg : Package) (imports : List String := []) : IO Unit := do buildImports pkg deps imports IO.println <| SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) IO.println <| SearchPath.toString <| pkg.sourceDir :: deps.map (·.sourceDir) - From 6161d7f2d9ea4903f288e27ef159912d858b52b6 Mon Sep 17 00:00:00 2001 From: tydeu Date: Sat, 10 Jul 2021 13:39:51 -0400 Subject: [PATCH 061/696] refactor: generalize `BuildTarget` traces --- Lake/Build.lean | 16 +++++------ Lake/BuildTarget.lean | 64 +++++++++++++++++++++++++++++-------------- 2 files changed, 51 insertions(+), 29 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index c3c534409e..e1be701fb3 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -17,7 +17,7 @@ namespace Lake -- # Build Targets -abbrev FileTarget := BuildTarget FilePath +abbrev FileTarget := MTimeBuildTarget FilePath namespace FileTarget @@ -34,7 +34,7 @@ structure LeanArtifact where cFile : FilePath deriving Inhabited -abbrev LeanTarget := BuildTarget LeanArtifact +abbrev LeanTarget := MTimeBuildTarget LeanArtifact namespace LeanTarget @@ -61,7 +61,7 @@ def catchErrors (action : IO PUnit) : IO PUnit := do throw e def buildOleanAndC (leanFile oleanFile cFile : FilePath) -(importTargets : List LeanTarget) (depsTarget : BuildTarget PUnit) +(importTargets : List LeanTarget) (depsTarget : MTimeBuildTarget PUnit) (leanPath : String := "") (rootDir : FilePath := ".") (leanArgs : Array String := #[]) : IO LeanTarget := do -- calculate transitive `maxMTime` @@ -100,7 +100,7 @@ def buildO (oFile : FilePath) compileO oFile cTarget.artifact leancArgs return FileTarget.mk oFile cMTime buildTask -abbrev PackageTarget := BuildTarget (Package × NameMap LeanTarget) +abbrev PackageTarget := MTimeBuildTarget (Package × NameMap LeanTarget) namespace PackageTarget @@ -118,7 +118,7 @@ structure LeanTargetContext where buildParents : List Name := [] -- target for external dependencies -- ex. olean roots of dependencies - depsTarget : BuildTarget PUnit + depsTarget : MTimeBuildTarget PUnit abbrev LeanTargetM := ReaderT LeanTargetContext <| StateT (NameMap LeanTarget) IO @@ -158,7 +158,7 @@ partial def buildModule (mod : Name) : LeanTargetM LeanTarget := do return target def mkLeanTargetContext -(pkg : Package) (oleanDirs : List FilePath) (depsTarget : BuildTarget PUnit) +(pkg : Package) (oleanDirs : List FilePath) (depsTarget : MTimeBuildTarget PUnit) : LeanTargetContext := { package:= pkg leanPath := SearchPath.toString <| pkg.oleanDir :: oleanDirs @@ -170,7 +170,7 @@ def mkLeanTargetContext def Package.buildTargetWithDepTargets (depTargets : List PackageTarget) (self : Package) : IO PackageTarget := do - let depsTarget ← BuildTarget.all depTargets + let depsTarget ← MTimeBuildTarget.all depTargets let depOLeanDirs := depTargets.map (·.package.oleanDir) let ctx := mkLeanTargetContext self depOLeanDirs depsTarget let (target, targetMap) ← buildModule self.module |>.run ctx |>.run {} @@ -292,7 +292,7 @@ def buildImports : IO Unit := do -- compute context let oleanDirs := deps.map (·.oleanDir) - let depsTarget ← BuildTarget.all (← deps.mapM (·.buildTarget)) + let depsTarget ← MTimeBuildTarget.all (← deps.mapM (·.buildTarget)) let ctx ← mkLeanTargetContext pkg oleanDirs depsTarget -- build imports let imports := imports.map (·.toName) diff --git a/Lake/BuildTarget.lean b/Lake/BuildTarget.lean index 092eb5de91..d232371c34 100644 --- a/Lake/BuildTarget.lean +++ b/Lake/BuildTarget.lean @@ -10,7 +10,7 @@ def BuildTask := Task (Except IO.Error PUnit) namespace BuildTask -def nop : BuildTask := +def nop : BuildTask := Task.pure (Except.ok ()) def await (self : BuildTask) : IO PUnit := do @@ -23,48 +23,70 @@ end BuildTask instance : Inhabited BuildTask := ⟨BuildTask.nop⟩ -structure BuildTarget (α : Type) where - artifact : α - maxMTime : IO.FS.SystemTime +structure BuildTarget (t : Type) (a : Type) where + artifact : a + trace : t buildTask : BuildTask -- manually derive `Inhabited` instance because automatic deriving fails -instance [Inhabited α] : Inhabited (BuildTarget α) := +instance [Inhabited t] [Inhabited a] : Inhabited (BuildTarget t a) := ⟨Inhabited.default, Inhabited.default, BuildTask.nop⟩ namespace BuildTarget -def pure (artifact : α) (maxMTime : IO.FS.SystemTime := ⟨0, 0⟩) : BuildTarget α := - {artifact, maxMTime, buildTask := BuildTask.nop} +def pure (artifact : a) (trace : t) : BuildTarget t a := + {artifact, trace, buildTask := BuildTask.nop} -def all (targets : List (BuildTarget α)) : IO (BuildTarget PUnit) := do - let depsMTime := targets.map (·.maxMTime) |>.maximum? |>.getD ⟨0, 0⟩ - let task ← BuildTask.all <| targets.map (·.buildTask) - BuildTarget.mk () depsMTime task +def withTrace (trace : t) (self : BuildTarget r a) : BuildTarget t a := + {self with trace := trace} -def collectAll (targets : List (BuildTarget α)) : IO (BuildTarget (List α)) := do - let artifacts := targets.map (·.artifact) - let depsMTime := targets.map (·.maxMTime) |>.maximum? |>.getD ⟨0, 0⟩ - let task ← BuildTask.all <| targets.map (·.buildTask) - BuildTarget.mk artifacts depsMTime task +def discardTrace (self : BuildTarget t a) : BuildTarget PUnit a := + self.withTrace () -def withArtifact (artifact : α) (self : BuildTarget β) : BuildTarget α := +def withArtifact (artifact : a) (self : BuildTarget t b) : BuildTarget t a := {self with artifact := artifact} -def discardArtifact (self : BuildTarget α) : BuildTarget PUnit := +def discardArtifact (self : BuildTarget t α) : BuildTarget t PUnit := self.withArtifact () -def materialize (self : BuildTarget α) : IO PUnit := +def materialize (self : BuildTarget t α) : IO PUnit := self.buildTask.await end BuildTarget namespace BuildTask -def afterTarget (target : BuildTarget α) (action : IO PUnit) : IO BuildTask := +def afterTarget (target : BuildTarget t a) (action : IO PUnit) : IO BuildTask := IO.mapTask (fun x => IO.ofExcept x *> action) target.buildTask -def afterTargets (targets : List (BuildTarget α)) (action : IO PUnit) : IO BuildTask := do +def afterTargets (targets : List (BuildTarget t a)) (action : IO PUnit) : IO BuildTask := do IO.mapTasks (fun xs => xs.forM IO.ofExcept *> action) <| targets.map (·.buildTask) end BuildTask + +abbrev MTimeBuildTarget := BuildTarget IO.FS.SystemTime +instance : OfNat IO.FS.SystemTime (nat_lit 0) := ⟨⟨0,0⟩⟩ + +namespace MTimeBuildTarget + +def maxMTime (self : MTimeBuildTarget a) := + self.trace + +def mk (artifact : a) (maxMTime : IO.FS.SystemTime := 0) (buildTask : BuildTask) : MTimeBuildTarget a := + {artifact, trace := maxMTime, buildTask} + +def pure (artifact : a) (maxMTime : IO.FS.SystemTime := 0) : MTimeBuildTarget a := + {artifact, trace := maxMTime, buildTask := BuildTask.nop} + +def all (targets : List (MTimeBuildTarget a)) : IO (MTimeBuildTarget PUnit) := do + let depsMTime := targets.map (·.maxMTime) |>.maximum? |>.getD 0 + let task ← BuildTask.all <| targets.map (·.buildTask) + return MTimeBuildTarget.mk () depsMTime task + +def collectAll (targets : List (MTimeBuildTarget a)) : IO (MTimeBuildTarget (List a)) := do + let artifacts := targets.map (·.artifact) + let depsMTime := targets.map (·.maxMTime) |>.maximum? |>.getD ⟨0, 0⟩ + let task ← BuildTask.all <| targets.map (·.buildTask) + return MTimeBuildTarget.mk artifacts depsMTime task + +end MTimeBuildTarget From 70d258049e436727a231c9e8309c2d14fdfba900 Mon Sep 17 00:00:00 2001 From: tydeu Date: Sat, 10 Jul 2021 15:23:07 -0400 Subject: [PATCH 062/696] refactor: generalize mtime checking --- Lake/Build.lean | 174 ++++++++++++++++++++---------------------- Lake/BuildTarget.lean | 43 ++++++++--- 2 files changed, 118 insertions(+), 99 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index e1be701fb3..7d3c3c8669 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -34,6 +34,11 @@ structure LeanArtifact where cFile : FilePath deriving Inhabited +protected def LeanArtifact.getMTime (self : LeanArtifact) : IO MTime := do + return max (← getMTime self.oleanFile) (← getMTime self.cFile) + +instance : GetMTime LeanArtifact := ⟨LeanArtifact.getMTime⟩ + abbrev LeanTarget := MTimeBuildTarget LeanArtifact namespace LeanTarget @@ -54,52 +59,6 @@ def cTarget (self : LeanTarget) : FileTarget := end LeanTarget -def catchErrors (action : IO PUnit) : IO PUnit := do - try action catch e => - -- print compile errors early - IO.eprintln e - throw e - -def buildOleanAndC (leanFile oleanFile cFile : FilePath) -(importTargets : List LeanTarget) (depsTarget : MTimeBuildTarget PUnit) -(leanPath : String := "") (rootDir : FilePath := ".") (leanArgs : Array String := #[]) -: IO LeanTarget := do - -- calculate transitive `maxMTime` - let leanMData ← leanFile.metadata - let impMTimes ← importTargets.mapM (·.maxMTime) - let maxMTime := List.maximum? - (leanMData.modified :: depsTarget.maxMTime :: impMTimes) |>.get! - -- construct a nop target if we have an up-to-date .olean and .c - try - let cMTime := (← cFile.metadata).modified - let oleanMTime := (← oleanFile.metadata).modified - if cMTime >= maxMTime && oleanMTime >= maxMTime then - return LeanTarget.pure oleanFile cFile maxMTime - catch - | IO.Error.noFileOrDirectory .. => pure () - | e => throw e - -- construct a proper target otherwise - let targets := depsTarget.withArtifact arbitrary :: importTargets - let buildTask ← BuildTask.afterTargets targets <| catchErrors <| - compileOleanAndC leanFile oleanFile cFile leanPath rootDir leanArgs - return LeanTarget.mk oleanFile cFile maxMTime buildTask - -def buildO (oFile : FilePath) -(cTarget : FileTarget) (leancArgs : Array String := #[]) -: IO FileTarget := do - -- construct a nop target if we have an up-to-date .o - let cMTime := cTarget.maxMTime - try - if (← oFile.metadata).modified >= cMTime then - return FileTarget.pure oFile cMTime - catch - | IO.Error.noFileOrDirectory .. => pure () - | e => throw e - -- construct a proper target otherwise - let buildTask ← BuildTask.afterTarget cTarget <| catchErrors <| - compileO oFile cTarget.artifact leancArgs - return FileTarget.mk oFile cMTime buildTask - abbrev PackageTarget := MTimeBuildTarget (Package × NameMap LeanTarget) namespace PackageTarget @@ -110,6 +69,51 @@ def moduleTargets (self : PackageTarget) : NameMap LeanTarget := end PackageTarget +-- # Build Components + +def catchErrors (action : IO PUnit) : IO PUnit := do + try action catch e => + -- print compile errors early + IO.eprintln e + throw e + +def skipIfNewer [GetMTime a] +(artifact : a) (depMTime : MTime) (build : IO BuildTask) +: IO (MTimeBuildTarget a) := do + -- construct a nop target if we have an up-to-date file + try + if (← getMTime artifact) >= depMTime then + return MTimeBuildTarget.pure artifact depMTime + catch + | IO.Error.noFileOrDirectory .. => pure () + | e => throw e + -- otherwise construct a proper target + return MTimeBuildTarget.mk artifact depMTime (← build) + +def fetchLeanTarget (leanFile oleanFile cFile : FilePath) +(importTargets : List LeanTarget) (depsTarget : MTimeBuildTarget PUnit) +(leanPath : String := "") (rootDir : FilePath := ".") (leanArgs : Array String := #[]) +: IO LeanTarget := do + -- calculate max dependency `MTime` + let leanMTime ← getMTime leanFile + let importMTimes := importTargets.map (·.mtime) + let depMTime := MTime.listMax <| leanMTime :: depsTarget.mtime :: importMTimes + -- construct a nop target if we have an up-to-date .olean and .c + let depTargets := depsTarget.withArtifact arbitrary :: importTargets + skipIfNewer ⟨oleanFile, cFile⟩ depMTime <| + BuildTask.afterTargets depTargets <| catchErrors <| + compileOleanAndC leanFile oleanFile cFile leanPath rootDir leanArgs + +def buildO (oFile : FilePath) +(cTarget : FileTarget) (leancArgs : Array String := #[]) : IO BuildTask := + BuildTask.afterTarget cTarget <| catchErrors <| + compileO oFile cTarget.artifact leancArgs + +def fetchOFileTarget (oFile : FilePath) +(cTarget : FileTarget) (leancArgs : Array String := #[]) : IO FileTarget := + -- construct a nop target if we have an up-to-date .o + skipIfNewer oFile cTarget.mtime <| buildO oFile cTarget leancArgs + -- # Build Modules structure LeanTargetContext where @@ -152,7 +156,7 @@ partial def buildModule (mod : Name) : LeanTargetM LeanTarget := do -- do build let cFile := pkg.modToC mod let oleanFile := pkg.modToOlean mod - let target ← buildOleanAndC leanFile oleanFile cFile + let target ← fetchLeanTarget leanFile oleanFile cFile importTargets ctx.depsTarget ctx.leanPath pkg.dir pkg.leanArgs modify (·.insert mod target) return target @@ -209,81 +213,71 @@ def build (pkg : Package) : IO PUnit := -- # Build Package Lib -def PackageTarget.buildOFileTargets +def PackageTarget.fetchOFileTargets (self : PackageTarget) : IO (List FileTarget) := do self.moduleTargets.toList.mapM fun (mod, target) => do let oFile := self.package.modToO mod - buildO oFile target.cTarget self.package.leancArgs + fetchOFileTarget oFile target.cTarget self.package.leancArgs -def PackageTarget.buildStaticLibTarget -(self : PackageTarget) : IO FileTarget := do - let libFile := self.package.staticLibFile - -- construct a nop target if we have an up-to-date lib - let pkgMTime := self.maxMTime - try - if (← libFile.metadata).modified >= pkgMTime then - return FileTarget.pure libFile pkgMTime - catch - | IO.Error.noFileOrDirectory .. => pure () - | e => throw e - -- construct a proper target otherwise - let oFileTargets ← self.buildOFileTargets +def PackageTarget.buildStaticLib +(self : PackageTarget) : IO BuildTask := do + let oFileTargets ← self.fetchOFileTargets let oFiles := oFileTargets.map (·.artifact) |>.toArray - let buildTask ← BuildTask.afterTargets oFileTargets <| catchErrors <| - compileStaticLib libFile oFiles - return FileTarget.mk libFile pkgMTime buildTask + BuildTask.afterTargets oFileTargets <| catchErrors <| + compileStaticLib self.package.staticLibFile oFiles -def Package.buildStaticLibTarget (self : Package) : IO FileTarget := do +def PackageTarget.fetchStaticLibTarget +(self : PackageTarget) : IO FileTarget := do + -- construct a nop target if we have an up-to-date lib + skipIfNewer self.package.staticLibFile self.mtime self.buildStaticLib + +def Package.fetchStaticLibTarget (self : Package) : IO FileTarget := do let target ← self.buildTarget - target.buildStaticLibTarget + target.fetchStaticLibTarget -def Package.buildStaticLib (self : Package) : IO FilePath := do - let target ← self.buildStaticLibTarget +def Package.fetchStaticLib (self : Package) : IO FilePath := do + let target ← self.fetchStaticLibTarget try target.materialize catch _ => -- actual error has already been printed within the task throw <| IO.userError "Build failed." return target.artifact def buildLib (pkg : Package) : IO PUnit := - discard pkg.buildStaticLib + discard pkg.fetchStaticLib -- # Build Package Bin -def PackageTarget.buildBinTarget - (depTargets : List PackageTarget) (self : PackageTarget) -: IO FileTarget := do +def PackageTarget.buildBin +(depTargets : List PackageTarget) (self : PackageTarget) +: IO BuildTask := do let binFile := self.package.binFile - -- construct a nop target if we have an up-to-date bin - let pkgMTime := self.maxMTime - try - if (← binFile.metadata).modified >= pkgMTime then - return FileTarget.pure binFile pkgMTime - catch - | IO.Error.noFileOrDirectory .. => pure () - | e => throw e - -- construct a proper target otherwise - let oFileTargets ← self.buildOFileTargets + let oFileTargets ← self.fetchOFileTargets let oFiles := oFileTargets.map (·.artifact) |>.toArray - let libTargets ← depTargets.mapM (·.buildStaticLibTarget) + let libTargets ← depTargets.mapM (·.fetchStaticLibTarget) let libFiles := libTargets.map (·.artifact) |>.toArray let buildTask ← BuildTask.afterTargets oFileTargets <| catchErrors <| compileBin binFile (oFiles ++ libFiles) self.package.linkArgs - return FileTarget.mk binFile pkgMTime buildTask + return buildTask -def Package.buildBinTarget (self : Package) : IO FileTarget := do +def PackageTarget.fetchBinTarget +(depTargets : List PackageTarget) (self : PackageTarget) : IO FileTarget := + -- construct a nop target if we have an up-to-date bin + skipIfNewer self.package.binFile self.mtime <| self.buildBin depTargets + +def Package.fetchBinTarget (self : Package) : IO FileTarget := do let depTargets ← self.buildDepTargets let pkgTarget ← self.buildTargetWithDepTargets depTargets - pkgTarget.buildBinTarget depTargets + pkgTarget.fetchBinTarget depTargets -def Package.buildBin (self : Package) : IO FilePath := do - let target ← self.buildBinTarget +def Package.fetchBin (self : Package) : IO FilePath := do + let target ← self.fetchBinTarget try target.materialize catch _ => -- actual error has already been printed within the task throw <| IO.userError "Build failed." return target.artifact def buildBin (pkg : Package) : IO PUnit := - discard pkg.buildBin + discard pkg.fetchBin -- # Print Paths diff --git a/Lake/BuildTarget.lean b/Lake/BuildTarget.lean index d232371c34..bfbe4946db 100644 --- a/Lake/BuildTarget.lean +++ b/Lake/BuildTarget.lean @@ -64,28 +64,53 @@ def afterTargets (targets : List (BuildTarget t a)) (action : IO PUnit) : IO Bui end BuildTask -abbrev MTimeBuildTarget := BuildTarget IO.FS.SystemTime -instance : OfNat IO.FS.SystemTime (nat_lit 0) := ⟨⟨0,0⟩⟩ +-- # MTime Build Targets + +section +open IO.FS (SystemTime) + +def MTime := SystemTime + +instance : Inhabited MTime := ⟨⟨0,0⟩⟩ +instance : OfNat MTime (nat_lit 0) := ⟨⟨0,0⟩⟩ +instance : BEq MTime := inferInstanceAs (BEq SystemTime) +instance : Repr MTime := inferInstanceAs (Repr SystemTime) +instance : Ord MTime := inferInstanceAs (Ord SystemTime) +instance : LT MTime := ltOfOrd +instance : LE MTime := leOfOrd +end + +def MTime.listMax (mtimes : List MTime) := mtimes.maximum?.getD 0 + +class GetMTime (α) where + getMTime : α → IO MTime + +export GetMTime (getMTime) + +instance : GetMTime System.FilePath where + getMTime file := do (← file.metadata).modified + +abbrev MTimeBuildTarget := BuildTarget MTime namespace MTimeBuildTarget -def maxMTime (self : MTimeBuildTarget a) := +def mtime (self : MTimeBuildTarget a) := self.trace -def mk (artifact : a) (maxMTime : IO.FS.SystemTime := 0) (buildTask : BuildTask) : MTimeBuildTarget a := - {artifact, trace := maxMTime, buildTask} +def mk (artifact : a) (mtime : MTime := 0) (buildTask : BuildTask) : MTimeBuildTarget a := + {artifact, trace := mtime, buildTask} -def pure (artifact : a) (maxMTime : IO.FS.SystemTime := 0) : MTimeBuildTarget a := - {artifact, trace := maxMTime, buildTask := BuildTask.nop} +def pure (artifact : a) (mtime : MTime := 0) : MTimeBuildTarget a := + {artifact, trace := mtime, buildTask := BuildTask.nop} def all (targets : List (MTimeBuildTarget a)) : IO (MTimeBuildTarget PUnit) := do - let depsMTime := targets.map (·.maxMTime) |>.maximum? |>.getD 0 + let depsMTime := MTime.listMax <| targets.map (·.mtime) let task ← BuildTask.all <| targets.map (·.buildTask) return MTimeBuildTarget.mk () depsMTime task def collectAll (targets : List (MTimeBuildTarget a)) : IO (MTimeBuildTarget (List a)) := do let artifacts := targets.map (·.artifact) - let depsMTime := targets.map (·.maxMTime) |>.maximum? |>.getD ⟨0, 0⟩ + let depsMTime := MTime.listMax <| targets.map (·.mtime) let task ← BuildTask.all <| targets.map (·.buildTask) return MTimeBuildTarget.mk artifacts depsMTime task From b290c1ad28dde7d2a7b38612640f735599c08649 Mon Sep 17 00:00:00 2001 From: tydeu Date: Sat, 10 Jul 2021 21:19:01 -0400 Subject: [PATCH 063/696] refactor: generalized `buildModule` and cleaned up ``printPaths` --- Lake/Build.lean | 186 +++++++++++++++++++++++++++++------------------- 1 file changed, 113 insertions(+), 73 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index 7d3c3c8669..538fb37644 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -114,61 +114,97 @@ def fetchOFileTarget (oFile : FilePath) -- construct a nop target if we have an up-to-date .o skipIfNewer oFile cTarget.mtime <| buildO oFile cTarget leancArgs +-- # Topological Builder + +/-- A recursive object fetcher. -/ +def RecFetch.{u,v,w} (k : Type u) (o : Type v) (m : Type v → Type w) := + k → (k → m o) → m o + +/-- Monad transformer for an RBMap-based topological builder. -/ +abbrev RBTopT.{u,v} (k : Type u) (o : Type u) (cmp) (m : Type u → Type v) := + StateT (Std.RBMap k o cmp) <| ExceptT (List k) m + +/-- An RBMap-based topological fetch. -/ +def RBTopFetch.{u,v} (k : Type u) (o : Type u) (cmp) (m : Type u → Type v) := + RecFetch k o (RBTopT (cmp := cmp) k o m) + +/-- Recursively builds a RBMao of key-object pairs topologically. -/ +partial def buildRBTopCore +{k o} {cmp} {m : Type u → Type u} [BEq k] [Inhabited o] [Monad m] +(parents : List k) (key : k) (fetch : RecFetch k o (RBTopT (cmp := cmp) k o m)) +: RBTopT (cmp := cmp) k o m o := do + -- detect cyclic builds + if parents.contains key then + throw <| key :: (parents.partition (· != key)).1 ++ [key] + + -- return previous output if already built + if let some output := (← get).find? key then + return output + + -- build the key recursively + let output ← fetch key fun k => + buildRBTopCore (key :: parents) k fetch + + -- save output (to prevent repeated builds of the same key) + modify (·.insert key output) + return output + +def buildRBTop +{k o} {cmp} {m : Type u → Type u} [BEq k] [Inhabited o] [Monad m] +(key : k) (fetch : RecFetch k o (RBTopT (cmp := cmp) k o m)) := + buildRBTopCore [] key fetch + -- # Build Modules -structure LeanTargetContext where - package : Package - leanPath : String - buildParents : List Name := [] - -- target for external dependencies - -- ex. olean roots of dependencies - depsTarget : MTimeBuildTarget PUnit +def parseDirectLocalImports (root : Name) (leanFile : FilePath) : IO (List Name) := do + let contents ← IO.FS.readFile leanFile + let (imports, _, _) ← Elab.parseImports contents leanFile.toString + imports.map (·.module) |>.filter (·.getRoot == root) +abbrev RecFetchLeanTarget (m) := + RecFetch Name LeanTarget m + +/- + `depsTarget` is used for external dependencies + the module builder must wait to finish building before it can start + ex. olean roots of dependencies +-/ +def RecFetchLeanTarget.mk +(pkg : Package) (oleanDirs : List FilePath) (depsTarget : MTimeBuildTarget PUnit) +{m} [Monad m] [MonadLiftT IO m] : RecFetch Name LeanTarget m := + let leanPath := SearchPath.toString <| pkg.oleanDir :: oleanDirs + fun mod fetch => do + let leanFile := pkg.modToSource mod + let imports ← parseDirectLocalImports pkg.module leanFile + let importTargets ← imports.mapM fetch + fetchLeanTarget leanFile (pkg.modToOlean mod) (pkg.modToC mod) + importTargets depsTarget leanPath pkg.dir pkg.leanArgs + +/- + Equivalent to `RBTopT (cmp := Name.quickCmp) Name LeanTarget IO` + Phrased this way to uses `NameMap` +-/ abbrev LeanTargetM := - ReaderT LeanTargetContext <| StateT (NameMap LeanTarget) IO + StateT (NameMap LeanTarget) <| ExceptT (List Name) IO -partial def buildModule (mod : Name) : LeanTargetM LeanTarget := do - let ctx ← read - let pkg := ctx.package +abbrev RecLeanTargetM := + ReaderT (RecFetchLeanTarget LeanTargetM) LeanTargetM - -- detect cyclic imports - if ctx.buildParents.contains mod then - let cycle := mod :: (ctx.buildParents.partition (· != mod)).1 ++ [mod] +def buildModule (mod : Name) : RecLeanTargetM LeanTarget := + fun fetch => buildRBTop mod fetch + +def RecLeanTargetM.run +(pkg : Package) (oleanDirs : List FilePath) +(depsTarget : MTimeBuildTarget PUnit) (self : RecLeanTargetM α) +: IO (α × NameMap LeanTarget) := do + let fetch := RecFetchLeanTarget.mk pkg oleanDirs depsTarget + let res ← ReaderT.run self fetch |>.run {} |>.run + match res with + | Except.ok res => res + | Except.error cycle => let cycle := cycle.map (s!" {·}") throw <| IO.userError s!"import cycle detected:\n{"\n".intercalate cycle}" - -- return previous result if already visited - if let some target := (← get).find? mod then - return target - - -- deduce lean file - let leanFile := pkg.modToSource mod - - -- parse imports - let (imports, _, _) ← Elab.parseImports (← IO.FS.readFile leanFile) leanFile.toString - let directLocalImports := imports.map (·.module) |>.filter (·.getRoot == pkg.module) - - -- recursively build local dependencies - let importTargets ← directLocalImports.mapM fun i => - withReader (fun ctx => { ctx with buildParents := mod :: ctx.buildParents }) <| - buildModule i - - -- do build - let cFile := pkg.modToC mod - let oleanFile := pkg.modToOlean mod - let target ← fetchLeanTarget leanFile oleanFile cFile - importTargets ctx.depsTarget ctx.leanPath pkg.dir pkg.leanArgs - modify (·.insert mod target) - return target - -def mkLeanTargetContext -(pkg : Package) (oleanDirs : List FilePath) (depsTarget : MTimeBuildTarget PUnit) -: LeanTargetContext := { - package:= pkg - leanPath := SearchPath.toString <| pkg.oleanDir :: oleanDirs - depsTarget -} - -- # Configure/Build Packages def Package.buildTargetWithDepTargets @@ -176,8 +212,8 @@ def Package.buildTargetWithDepTargets : IO PackageTarget := do let depsTarget ← MTimeBuildTarget.all depTargets let depOLeanDirs := depTargets.map (·.package.oleanDir) - let ctx := mkLeanTargetContext self depOLeanDirs depsTarget - let (target, targetMap) ← buildModule self.module |>.run ctx |>.run {} + let (target, targetMap) ← buildModule self.module + |>.run self depOLeanDirs depsTarget return {target with artifact := ⟨self, targetMap⟩} partial def Package.buildTarget (self : Package) : IO PackageTarget := do @@ -211,6 +247,35 @@ def Package.build (self : Package) : IO PUnit := do def build (pkg : Package) : IO PUnit := pkg.build +-- # Print Paths + +def Package.buildModuleTargetsWithDeps +(deps : List Package) (mods : List Name) (self : Package) +: IO (List LeanTarget) := do + let oleanDirs := deps.map (·.oleanDir) + let depsTarget ← MTimeBuildTarget.all (← deps.mapM (·.buildTarget)) + let (targets, _) ← mods.mapM buildModule |>.run self oleanDirs depsTarget + targets + +def Package.buildModulesWithDeps +(deps : List Package) (mods : List Name) (self : Package) +: IO PUnit := do + let targets ← self.buildModuleTargetsWithDeps deps mods + let tasks ← targets.mapM (·.buildTask) + for task in tasks do + try task.await catch e => + -- actual error has already been printed within target + throw <| IO.userError "Build failed." + +def printPaths (pkg : Package) (imports : List String := []) : IO Unit := do + let deps ← solveDeps pkg + unless imports.isEmpty do + let imports := imports.map (·.toName) + let localImports := imports.filter (·.getRoot == pkg.module) + pkg.buildModulesWithDeps deps localImports + IO.println <| SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) + IO.println <| SearchPath.toString <| pkg.sourceDir :: deps.map (·.sourceDir) + -- # Build Package Lib def PackageTarget.fetchOFileTargets @@ -278,28 +343,3 @@ def Package.fetchBin (self : Package) : IO FilePath := do def buildBin (pkg : Package) : IO PUnit := discard pkg.fetchBin - --- # Print Paths - -def buildImports -(pkg : Package) (deps : List Package) (imports : List String := []) -: IO Unit := do - -- compute context - let oleanDirs := deps.map (·.oleanDir) - let depsTarget ← MTimeBuildTarget.all (← deps.mapM (·.buildTarget)) - let ctx ← mkLeanTargetContext pkg oleanDirs depsTarget - -- build imports - let imports := imports.map (·.toName) - let localImports := imports.filter (·.getRoot == pkg.module) - let targets ← imports.mapM buildModule |>.run ctx |>.run' {} - let tasks ← targets.mapM (·.buildTask) - for task in tasks do - try task.await catch e => - -- actual error has already been printed within buildTask - throw <| IO.userError "Build failed." - -def printPaths (pkg : Package) (imports : List String := []) : IO Unit := do - let deps ← solveDeps pkg - buildImports pkg deps imports - IO.println <| SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) - IO.println <| SearchPath.toString <| pkg.sourceDir :: deps.map (·.sourceDir) From 3ad82dcc4223c8c49a082e780da40724de5aeb20 Mon Sep 17 00:00:00 2001 From: tydeu Date: Sat, 10 Jul 2021 22:40:34 -0400 Subject: [PATCH 064/696] chore: minor code cleanup --- Lake/Build.lean | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index 538fb37644..127c0d8b0e 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -142,8 +142,8 @@ partial def buildRBTopCore return output -- build the key recursively - let output ← fetch key fun k => - buildRBTopCore (key :: parents) k fetch + let output ← fetch key fun depKey => + buildRBTopCore (key :: parents) depKey fetch -- save output (to prevent repeated builds of the same key) modify (·.insert key output) @@ -171,7 +171,7 @@ abbrev RecFetchLeanTarget (m) := -/ def RecFetchLeanTarget.mk (pkg : Package) (oleanDirs : List FilePath) (depsTarget : MTimeBuildTarget PUnit) -{m} [Monad m] [MonadLiftT IO m] : RecFetch Name LeanTarget m := +{m} [Monad m] [MonadLiftT IO m] : RecFetchLeanTarget m := let leanPath := SearchPath.toString <| pkg.oleanDir :: oleanDirs fun mod fetch => do let leanFile := pkg.modToSource mod From d4e3a4f79e74bfcdf34837f4f1a027840036f5e8 Mon Sep 17 00:00:00 2001 From: tydeu Date: Sat, 10 Jul 2021 23:02:31 -0400 Subject: [PATCH 065/696] fix: bin build now properly waits for dep libs --- Lake/Build.lean | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index 127c0d8b0e..fe61325470 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -315,14 +315,12 @@ def buildLib (pkg : Package) : IO PUnit := def PackageTarget.buildBin (depTargets : List PackageTarget) (self : PackageTarget) : IO BuildTask := do - let binFile := self.package.binFile let oFileTargets ← self.fetchOFileTargets let oFiles := oFileTargets.map (·.artifact) |>.toArray let libTargets ← depTargets.mapM (·.fetchStaticLibTarget) let libFiles := libTargets.map (·.artifact) |>.toArray - let buildTask ← BuildTask.afterTargets oFileTargets <| catchErrors <| - compileBin binFile (oFiles ++ libFiles) self.package.linkArgs - return buildTask + BuildTask.afterTargets (oFileTargets ++ libTargets) <| catchErrors <| + compileBin self.package.binFile (oFiles ++ libFiles) self.package.linkArgs def PackageTarget.fetchBinTarget (depTargets : List PackageTarget) (self : PackageTarget) : IO FileTarget := From 3b3beec0d4fadae8fa8e56bdcfabd5f9506a44ae Mon Sep 17 00:00:00 2001 From: tydeu Date: Sun, 11 Jul 2021 18:42:56 -0400 Subject: [PATCH 066/696] refactor: clean up `buildRBTop` and related code --- Lake/Build.lean | 118 +++++++++++++++++++++++++++--------------------- 1 file changed, 67 insertions(+), 51 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index fe61325470..3974923cdf 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -116,43 +116,54 @@ def fetchOFileTarget (oFile : FilePath) -- # Topological Builder +open Std + /-- A recursive object fetcher. -/ def RecFetch.{u,v,w} (k : Type u) (o : Type v) (m : Type v → Type w) := k → (k → m o) → m o -/-- Monad transformer for an RBMap-based topological builder. -/ +/-- A exception plus state monad transformer (i.e., `StateT` + `ExceptT`). -/ +abbrev EStateT (ε σ m) := + StateT σ <| ExceptT ε m + +def EStateT.run (state : σ) (self : EStateT ε σ m α) : m (Except ε (α × σ)) := + StateT.run self state |>.run + +def EStateT.run' [Monad m] (state : σ) (self : EStateT ε σ m α) : m (Except ε α) := + StateT.run' self state |>.run + +/-- + Monad transformer for an RBMap-based topological walk. + If a cycle is detected, the list keys traversed is thrown. +-/ abbrev RBTopT.{u,v} (k : Type u) (o : Type u) (cmp) (m : Type u → Type v) := - StateT (Std.RBMap k o cmp) <| ExceptT (List k) m + EStateT (List k) (RBMap k o cmp) m -/-- An RBMap-based topological fetch. -/ -def RBTopFetch.{u,v} (k : Type u) (o : Type u) (cmp) (m : Type u → Type v) := - RecFetch k o (RBTopT (cmp := cmp) k o m) - -/-- Recursively builds a RBMao of key-object pairs topologically. -/ +/-- Auxiliary function for `buildRBTop`. -/ partial def buildRBTopCore {k o} {cmp} {m : Type u → Type u} [BEq k] [Inhabited o] [Monad m] -(parents : List k) (key : k) (fetch : RecFetch k o (RBTopT (cmp := cmp) k o m)) -: RBTopT (cmp := cmp) k o m o := do +(parents : List k) (fetch : RecFetch k o (RBTopT k o cmp m)) +(key : k) : RBTopT k o cmp m o := do -- detect cyclic builds if parents.contains key then throw <| key :: (parents.partition (· != key)).1 ++ [key] - -- return previous output if already built if let some output := (← get).find? key then return output - -- build the key recursively - let output ← fetch key fun depKey => - buildRBTopCore (key :: parents) depKey fetch - - -- save output (to prevent repeated builds of the same key) + let output ← fetch key (buildRBTopCore (key :: parents) fetch) + -- save the output (to prevent repeated builds of the same key) modify (·.insert key output) return output -def buildRBTop -{k o} {cmp} {m : Type u → Type u} [BEq k] [Inhabited o] [Monad m] -(key : k) (fetch : RecFetch k o (RBTopT (cmp := cmp) k o m)) := - buildRBTopCore [] key fetch +/-- + Recursively constructs an `RBMao` of key-object pairs by + fetching objects topologically (i.e., via a deep-first search with + memoization). Called a suspending scheduler in "Build systems à la carte". +-/ +def buildRBTop {k o} {cmp} {m} [BEq k] [Inhabited o] [Monad m] +(fetch : RecFetch k o (RBTopT k o cmp m)) (key : k) : RBTopT k o cmp m o := + buildRBTopCore [] fetch key -- # Build Modules @@ -161,17 +172,17 @@ def parseDirectLocalImports (root : Name) (leanFile : FilePath) : IO (List Name) let (imports, _, _) ← Elab.parseImports contents leanFile.toString imports.map (·.module) |>.filter (·.getRoot == root) -abbrev RecFetchLeanTarget (m) := - RecFetch Name LeanTarget m - /- - `depsTarget` is used for external dependencies - the module builder must wait to finish building before it can start - ex. olean roots of dependencies + Produces a recursive module target fetcher that + builds the target module after waiting for `depsTarget` and + its direct local imports (relative to `pkg`) to build. + + The module is built with the configuration from `pkg` and + a `LEAN_PATH` that includes `oleanDirs`. -/ -def RecFetchLeanTarget.mk +def fetchAfterDirectLocalImports (pkg : Package) (oleanDirs : List FilePath) (depsTarget : MTimeBuildTarget PUnit) -{m} [Monad m] [MonadLiftT IO m] : RecFetchLeanTarget m := +{m} [Monad m] [MonadLiftT IO m] : RecFetch Name LeanTarget m := let leanPath := SearchPath.toString <| pkg.oleanDir :: oleanDirs fun mod fetch => do let leanFile := pkg.modToSource mod @@ -181,30 +192,39 @@ def RecFetchLeanTarget.mk importTargets depsTarget leanPath pkg.dir pkg.leanArgs /- - Equivalent to `RBTopT (cmp := Name.quickCmp) Name LeanTarget IO` - Phrased this way to uses `NameMap` + Equivalent to `RBTopT (cmp := Name.quickCmp) Name LeanTarget IO`. + Phrased this way to use `NameMap`. -/ abbrev LeanTargetM := - StateT (NameMap LeanTarget) <| ExceptT (List Name) IO + EStateT (List Name) (NameMap LeanTarget) IO + +abbrev LeanTargetFetch := + RecFetch Name LeanTarget LeanTargetM abbrev RecLeanTargetM := - ReaderT (RecFetchLeanTarget LeanTargetM) LeanTargetM + ReaderT (RecFetch Name LeanTarget LeanTargetM) LeanTargetM -def buildModule (mod : Name) : RecLeanTargetM LeanTarget := - fun fetch => buildRBTop mod fetch - -def RecLeanTargetM.run -(pkg : Package) (oleanDirs : List FilePath) -(depsTarget : MTimeBuildTarget PUnit) (self : RecLeanTargetM α) -: IO (α × NameMap LeanTarget) := do - let fetch := RecFetchLeanTarget.mk pkg oleanDirs depsTarget - let res ← ReaderT.run self fetch |>.run {} |>.run - match res with - | Except.ok res => res +def throwOnCycle (mx : IO (Except (List Name) α)) : IO α := + mx >>= fun + | Except.ok a => a | Except.error cycle => let cycle := cycle.map (s!" {·}") throw <| IO.userError s!"import cycle detected:\n{"\n".intercalate cycle}" +def Package.buildModuleTargetDAG +(oleanDirs : List FilePath) (depsTarget : MTimeBuildTarget PUnit) +(self : Package) : IO (LeanTarget × NameMap LeanTarget) := do + let fetch := fetchAfterDirectLocalImports self oleanDirs depsTarget + throwOnCycle <| buildRBTop fetch self.module |>.run {} + +def Package.buildModuleTargets +(mods : List Name) (oleanDirs : List FilePath) +(depsTarget : MTimeBuildTarget PUnit) (self : Package) +: IO (List LeanTarget) := do + let fetch : LeanTargetFetch := + fetchAfterDirectLocalImports self oleanDirs depsTarget + throwOnCycle <| mods.mapM (buildRBTop fetch) |>.run' {} + -- # Configure/Build Packages def Package.buildTargetWithDepTargets @@ -212,8 +232,7 @@ def Package.buildTargetWithDepTargets : IO PackageTarget := do let depsTarget ← MTimeBuildTarget.all depTargets let depOLeanDirs := depTargets.map (·.package.oleanDir) - let (target, targetMap) ← buildModule self.module - |>.run self depOLeanDirs depsTarget + let (target, targetMap) ← self.buildModuleTargetDAG depOLeanDirs depsTarget return {target with artifact := ⟨self, targetMap⟩} partial def Package.buildTarget (self : Package) : IO PackageTarget := do @@ -254,18 +273,15 @@ def Package.buildModuleTargetsWithDeps : IO (List LeanTarget) := do let oleanDirs := deps.map (·.oleanDir) let depsTarget ← MTimeBuildTarget.all (← deps.mapM (·.buildTarget)) - let (targets, _) ← mods.mapM buildModule |>.run self oleanDirs depsTarget - targets + self.buildModuleTargets mods oleanDirs depsTarget def Package.buildModulesWithDeps (deps : List Package) (mods : List Name) (self : Package) : IO PUnit := do let targets ← self.buildModuleTargetsWithDeps deps mods - let tasks ← targets.mapM (·.buildTask) - for task in tasks do - try task.await catch e => - -- actual error has already been printed within target - throw <| IO.userError "Build failed." + try targets.forM (·.materialize) catch e => + -- actual error has already been printed within target + throw <| IO.userError "Build failed." def printPaths (pkg : Package) (imports : List String := []) : IO Unit := do let deps ← solveDeps pkg From c08812e9e1a0c9bd3a6b7f279c79147306c78482 Mon Sep 17 00:00:00 2001 From: tydeu Date: Mon, 12 Jul 2021 19:40:37 -0400 Subject: [PATCH 067/696] refactor: merge `fetchLeanTarget` into `fetchAfterDirectLocalImports` --- Lake/Build.lean | 37 +++++++++++++++++-------------------- Lake/BuildTarget.lean | 12 +++++++++--- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index 3974923cdf..fc90a0303d 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -78,8 +78,9 @@ def catchErrors (action : IO PUnit) : IO PUnit := do throw e def skipIfNewer [GetMTime a] -(artifact : a) (depMTime : MTime) (build : IO BuildTask) -: IO (MTimeBuildTarget a) := do +(artifact : a) (depMTime : MTime) +{m} [Monad m] [MonadLiftT IO m] [MonadExceptOf IO.Error m] +(build : m BuildTask) : m (MTimeBuildTarget a) := do -- construct a nop target if we have an up-to-date file try if (← getMTime artifact) >= depMTime then @@ -90,20 +91,6 @@ def skipIfNewer [GetMTime a] -- otherwise construct a proper target return MTimeBuildTarget.mk artifact depMTime (← build) -def fetchLeanTarget (leanFile oleanFile cFile : FilePath) -(importTargets : List LeanTarget) (depsTarget : MTimeBuildTarget PUnit) -(leanPath : String := "") (rootDir : FilePath := ".") (leanArgs : Array String := #[]) -: IO LeanTarget := do - -- calculate max dependency `MTime` - let leanMTime ← getMTime leanFile - let importMTimes := importTargets.map (·.mtime) - let depMTime := MTime.listMax <| leanMTime :: depsTarget.mtime :: importMTimes - -- construct a nop target if we have an up-to-date .olean and .c - let depTargets := depsTarget.withArtifact arbitrary :: importTargets - skipIfNewer ⟨oleanFile, cFile⟩ depMTime <| - BuildTask.afterTargets depTargets <| catchErrors <| - compileOleanAndC leanFile oleanFile cFile leanPath rootDir leanArgs - def buildO (oFile : FilePath) (cTarget : FileTarget) (leancArgs : Array String := #[]) : IO BuildTask := BuildTask.afterTarget cTarget <| catchErrors <| @@ -182,14 +169,24 @@ def parseDirectLocalImports (root : Name) (leanFile : FilePath) : IO (List Name) -/ def fetchAfterDirectLocalImports (pkg : Package) (oleanDirs : List FilePath) (depsTarget : MTimeBuildTarget PUnit) -{m} [Monad m] [MonadLiftT IO m] : RecFetch Name LeanTarget m := +{m} [Monad m] [MonadLiftT IO m] [MonadExceptOf IO.Error m] : RecFetch Name LeanTarget m := let leanPath := SearchPath.toString <| pkg.oleanDir :: oleanDirs fun mod fetch => do let leanFile := pkg.modToSource mod let imports ← parseDirectLocalImports pkg.module leanFile - let importTargets ← imports.mapM fetch - fetchLeanTarget leanFile (pkg.modToOlean mod) (pkg.modToC mod) - importTargets depsTarget leanPath pkg.dir pkg.leanArgs + -- calculate max dependency `MTime` + let leanMTime ← getMTime leanFile + let importMTimes ← imports.mapM fun i => getMTime <| pkg.modToSource i + let depMTime := MTime.listMax <| leanMTime :: depsTarget.mtime :: importMTimes + -- construct target + let cFile := pkg.modToC mod + let oleanFile := pkg.modToOlean mod + -- we fetch the import targets even if a rebuild is not required + -- because other build processes (ex. `.o`) rely on the map being complete + let importTasks ← imports.mapM fun i => fetch i >>= (·.buildTask) + skipIfNewer ⟨oleanFile, cFile⟩ depMTime <| + BuildTask.afterTasks (depsTarget.buildTask :: importTasks) <| catchErrors <| + compileOleanAndC leanFile oleanFile cFile leanPath pkg.dir pkg.leanArgs /- Equivalent to `RBTopT (cmp := Name.quickCmp) Name LeanTarget IO`. diff --git a/Lake/BuildTarget.lean b/Lake/BuildTarget.lean index bfbe4946db..1db9e8c900 100644 --- a/Lake/BuildTarget.lean +++ b/Lake/BuildTarget.lean @@ -19,6 +19,12 @@ def await (self : BuildTask) : IO PUnit := do def all (tasks : List BuildTask) : IO BuildTask := IO.asTask (tasks.forM (·.await)) +def afterTask (task : BuildTask) (action : IO PUnit) : IO BuildTask := + IO.mapTask (fun x => IO.ofExcept x *> action) task + +def afterTasks (tasks : List BuildTask) (action : IO PUnit) : IO BuildTask := + IO.mapTasks (fun xs => xs.forM IO.ofExcept *> action) <| tasks + end BuildTask instance : Inhabited BuildTask := ⟨BuildTask.nop⟩ @@ -57,10 +63,10 @@ end BuildTarget namespace BuildTask def afterTarget (target : BuildTarget t a) (action : IO PUnit) : IO BuildTask := - IO.mapTask (fun x => IO.ofExcept x *> action) target.buildTask + afterTask target.buildTask action -def afterTargets (targets : List (BuildTarget t a)) (action : IO PUnit) : IO BuildTask := do - IO.mapTasks (fun xs => xs.forM IO.ofExcept *> action) <| targets.map (·.buildTask) +def afterTargets (targets : List (BuildTarget t a)) (action : IO PUnit) : IO BuildTask := + afterTasks (targets.map (·.buildTask)) action end BuildTask From 511f34fd5377c62170a16f46ff60786ee72c8408 Mon Sep 17 00:00:00 2001 From: tydeu Date: Mon, 12 Jul 2021 21:15:08 -0400 Subject: [PATCH 068/696] refactor: simplify `Compile.lean` --- Lake/Compile.lean | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/Lake/Compile.lean b/Lake/Compile.lean index 655df2afad..32828dc094 100644 --- a/Lake/Compile.lean +++ b/Lake/Compile.lean @@ -8,44 +8,43 @@ import Lake.Proc namespace Lake open System -def compileOleanAndC -(leanFile oleanFile cFile : FilePath) +def createParentDirs (path : FilePath) : IO Unit := + if let some dir := path.parent then IO.FS.createDirAll dir else pure () + +def compileOleanAndC (leanFile oleanFile cFile : FilePath) (leanPath : String := "") (rootDir : FilePath := ".") (leanArgs : Array String := #[]) : IO Unit := do - if let some dir := cFile.parent then IO.FS.createDirAll dir - if let some dir := oleanFile.parent then IO.FS.createDirAll dir + createParentDirs cFile + createParentDirs oleanFile execCmd { cmd := "lean" args := leanArgs ++ #[ - "-R", rootDir.toString, "-o", oleanFile.toString, "-c", + "-R", rootDir.toString, "-o", oleanFile.toString, "-c", cFile.toString, leanFile.toString ] env := #[("LEAN_PATH", leanPath)] } -def compileO -(oFile cFile : FilePath) (leancArgs : Array String := #[]) -: IO Unit := do - if let some dir := oFile.parent then IO.FS.createDirAll dir +def compileO (oFile cFile : FilePath) +(leancArgs : Array String := #[]) : IO Unit := do + createParentDirs oFile execCmd { cmd := "leanc" args := #["-c", "-o", oFile.toString, cFile.toString] ++ leancArgs } -def compileBin -(binFile : FilePath) (linkFiles : Array FilePath) (linkArgs : Array String := #[]) -: IO Unit := do - if let some dir := binFile.parent then IO.FS.createDirAll dir - execCmd { - cmd := "leanc" - args := #["-o", binFile.toString] ++ linkFiles.map toString ++ linkArgs - } - def compileStaticLib -(libFile : FilePath) (oFiles : Array FilePath) -: IO Unit := do - if let some dir := libFile.parent then IO.FS.createDirAll dir +(libFile : FilePath) (oFiles : Array FilePath) : IO Unit := do + createParentDirs libFile execCmd { cmd := "ar" args := #["rcs", libFile.toString] ++ oFiles.map toString } + +def compileBin (binFile : FilePath) +(linkFiles : Array FilePath) (linkArgs : Array String := #[]) : IO Unit := do + createParentDirs binFile + execCmd { + cmd := "leanc" + args := #["-o", binFile.toString] ++ linkFiles.map toString ++ linkArgs + } From 4844f8c4590f69e60ab089af84a6dd4e43edcf09 Mon Sep 17 00:00:00 2001 From: tydeu Date: Tue, 13 Jul 2021 13:40:05 -0400 Subject: [PATCH 069/696] refactor: once again use to lean target mtime in fetch --- Lake/Build.lean | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index fc90a0303d..df0793cb51 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -174,16 +174,17 @@ def fetchAfterDirectLocalImports fun mod fetch => do let leanFile := pkg.modToSource mod let imports ← parseDirectLocalImports pkg.module leanFile + -- we fetch the import targets even if a rebuild is not required + -- because other build processes (ex. `.o`) rely on the map being complete + let importTargets ← imports.mapM fetch -- calculate max dependency `MTime` let leanMTime ← getMTime leanFile - let importMTimes ← imports.mapM fun i => getMTime <| pkg.modToSource i + let importMTimes ← importTargets.map (·.mtime) let depMTime := MTime.listMax <| leanMTime :: depsTarget.mtime :: importMTimes -- construct target let cFile := pkg.modToC mod let oleanFile := pkg.modToOlean mod - -- we fetch the import targets even if a rebuild is not required - -- because other build processes (ex. `.o`) rely on the map being complete - let importTasks ← imports.mapM fun i => fetch i >>= (·.buildTask) + let importTasks := importTargets.map (·.buildTask) skipIfNewer ⟨oleanFile, cFile⟩ depMTime <| BuildTask.afterTasks (depsTarget.buildTask :: importTasks) <| catchErrors <| compileOleanAndC leanFile oleanFile cFile leanPath pkg.dir pkg.leanArgs From 115fdbea98d22e996c9a05413d421d1ad984199e Mon Sep 17 00:00:00 2001 From: tydeu Date: Tue, 13 Jul 2021 15:29:59 -0400 Subject: [PATCH 070/696] refactor: simplify mtime checking code --- Lake/Build.lean | 36 +++++++++++++++++++----------------- Lake/BuildTarget.lean | 1 + 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index df0793cb51..ce8cb65f74 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -69,6 +69,25 @@ def moduleTargets (self : PackageTarget) : NameMap LeanTarget := end PackageTarget +-- # MTime Checking + +/-- Check if the artifact's `MTIme` is at least `depMTime`. -/ +def checkIfNewer [GetMTime a] (artifact : a) (depMTime : MTime) : IO Bool := do + try (← getMTime artifact) >= depMTime catch _ => false + +/-- + Construct a no-op target if the given artifact is up-to-date. + Otherwise, construct a target with the given build task. +-/ +def skipIfNewer [GetMTime a] +(artifact : a) (depMTime : MTime) +{m} [Monad m] [MonadLiftT IO m] [MonadExceptOf IO.Error m] +(build : m BuildTask) : m (MTimeBuildTarget a) := do + if (← checkIfNewer artifact depMTime) then + MTimeBuildTarget.pure artifact depMTime + else + MTimeBuildTarget.mk artifact depMTime (← build) + -- # Build Components def catchErrors (action : IO PUnit) : IO PUnit := do @@ -77,20 +96,6 @@ def catchErrors (action : IO PUnit) : IO PUnit := do IO.eprintln e throw e -def skipIfNewer [GetMTime a] -(artifact : a) (depMTime : MTime) -{m} [Monad m] [MonadLiftT IO m] [MonadExceptOf IO.Error m] -(build : m BuildTask) : m (MTimeBuildTarget a) := do - -- construct a nop target if we have an up-to-date file - try - if (← getMTime artifact) >= depMTime then - return MTimeBuildTarget.pure artifact depMTime - catch - | IO.Error.noFileOrDirectory .. => pure () - | e => throw e - -- otherwise construct a proper target - return MTimeBuildTarget.mk artifact depMTime (← build) - def buildO (oFile : FilePath) (cTarget : FileTarget) (leancArgs : Array String := #[]) : IO BuildTask := BuildTask.afterTarget cTarget <| catchErrors <| @@ -98,7 +103,6 @@ def buildO (oFile : FilePath) def fetchOFileTarget (oFile : FilePath) (cTarget : FileTarget) (leancArgs : Array String := #[]) : IO FileTarget := - -- construct a nop target if we have an up-to-date .o skipIfNewer oFile cTarget.mtime <| buildO oFile cTarget leancArgs -- # Topological Builder @@ -307,7 +311,6 @@ def PackageTarget.buildStaticLib def PackageTarget.fetchStaticLibTarget (self : PackageTarget) : IO FileTarget := do - -- construct a nop target if we have an up-to-date lib skipIfNewer self.package.staticLibFile self.mtime self.buildStaticLib def Package.fetchStaticLibTarget (self : Package) : IO FileTarget := do @@ -338,7 +341,6 @@ def PackageTarget.buildBin def PackageTarget.fetchBinTarget (depTargets : List PackageTarget) (self : PackageTarget) : IO FileTarget := - -- construct a nop target if we have an up-to-date bin skipIfNewer self.package.binFile self.mtime <| self.buildBin depTargets def Package.fetchBinTarget (self : Package) : IO FileTarget := do diff --git a/Lake/BuildTarget.lean b/Lake/BuildTarget.lean index 1db9e8c900..fb58264fe8 100644 --- a/Lake/BuildTarget.lean +++ b/Lake/BuildTarget.lean @@ -29,6 +29,7 @@ end BuildTask instance : Inhabited BuildTask := ⟨BuildTask.nop⟩ + structure BuildTarget (t : Type) (a : Type) where artifact : a trace : t From 758021f03afd4bf9bfe6e42ce1de4c64fadce412 Mon Sep 17 00:00:00 2001 From: tydeu Date: Tue, 13 Jul 2021 20:11:15 -0400 Subject: [PATCH 071/696] feat: add hash checking for builds --- Lake/Build.lean | 123 +++++++++++++++++++++++++++--------------- Lake/BuildTarget.lean | 16 +++++- Lake/Package.lean | 3 ++ 3 files changed, 98 insertions(+), 44 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index ce8cb65f74..19eebaf24b 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -34,47 +34,81 @@ structure LeanArtifact where cFile : FilePath deriving Inhabited +structure LeanTrace where + hash : Hash + mtime : MTime + deriving Inhabited + protected def LeanArtifact.getMTime (self : LeanArtifact) : IO MTime := do return max (← getMTime self.oleanFile) (← getMTime self.cFile) instance : GetMTime LeanArtifact := ⟨LeanArtifact.getMTime⟩ -abbrev LeanTarget := MTimeBuildTarget LeanArtifact +abbrev LeanTarget a := BuildTarget LeanTrace a namespace LeanTarget -def mk (olean c : FilePath) (maxMTime : IO.FS.SystemTime) (task : BuildTask) : LeanTarget := - BuildTarget.mk ⟨olean, c⟩ maxMTime task +def hash (self : LeanTarget a) := self.trace.hash +def mtime (self : LeanTarget a) := self.trace.mtime -def pure (olean c : FilePath) (maxMTime : IO.FS.SystemTime) : LeanTarget := - BuildTarget.pure ⟨olean, c⟩ maxMTime - -def oleanFile (self : LeanTarget) := self.artifact.oleanFile -def oleanTarget (self : LeanTarget) : FileTarget := - {self with artifact := self.oleanFile} - -def cFile (self : LeanTarget) := self.artifact.cFile -def cTarget (self : LeanTarget) : FileTarget := - {self with artifact := self.cFile} +def all (targets : List (LeanTarget a)) : IO (LeanTarget PUnit) := do + let hash := Hash.foldList 0 <| targets.map (·.hash) + let mtime := MTime.listMax <| targets.map (·.mtime) + let task ← BuildTask.all <| targets.map (·.buildTask) + return BuildTarget.mk () ⟨hash, mtime⟩ task end LeanTarget -abbrev PackageTarget := MTimeBuildTarget (Package × NameMap LeanTarget) +abbrev ModuleTarget := LeanTarget LeanArtifact + +namespace ModuleTarget + +def mk (olean c : FilePath) (hash : Hash) (mtime : IO.FS.SystemTime) (task : BuildTask) : ModuleTarget := + BuildTarget.mk ⟨olean, c⟩ ⟨hash, mtime⟩ task + +def pure (olean c : FilePath) (hash : Hash) (mtime : IO.FS.SystemTime) : ModuleTarget := + BuildTarget.pure ⟨olean, c⟩ ⟨hash, mtime⟩ + +def oleanFile (self : ModuleTarget) := self.artifact.oleanFile +def oleanTarget (self : ModuleTarget) : FileTarget := + {self with artifact := self.oleanFile, trace := self.mtime} + +def cFile (self : ModuleTarget) := self.artifact.cFile +def cTarget (self : ModuleTarget) : FileTarget := + {self with artifact := self.cFile, trace := self.mtime} + +end ModuleTarget + +abbrev PackageTarget := LeanTarget (Package × NameMap ModuleTarget) namespace PackageTarget def package (self : PackageTarget) := self.artifact.1 -def moduleTargets (self : PackageTarget) : NameMap LeanTarget := +def moduleTargets (self : PackageTarget) : NameMap ModuleTarget := self.artifact.2 end PackageTarget --- # MTime Checking +-- # Trace Checking + +/-- Check if `hash` matches that in the file give file. -/ +def checkIfSameHash (hash : Hash) (file : FilePath) : IO Bool := + try + let contents ← IO.FS.readFile file + match contents.toNat? with + | some h => Hash.ofNat h == hash + | none => false + catch _ => + false /-- Check if the artifact's `MTIme` is at least `depMTime`. -/ def checkIfNewer [GetMTime a] (artifact : a) (depMTime : MTime) : IO Bool := do try (← getMTime artifact) >= depMTime catch _ => false +/-- Construct a no-op build task if the given condition holds, otherwise perform `build`. -/ +def skipIf [Pure m] (cond : Bool) (build : m BuildTask) : m BuildTask := do + if cond then pure BuildTask.nop else build + /-- Construct a no-op target if the given artifact is up-to-date. Otherwise, construct a target with the given build task. @@ -83,10 +117,8 @@ def skipIfNewer [GetMTime a] (artifact : a) (depMTime : MTime) {m} [Monad m] [MonadLiftT IO m] [MonadExceptOf IO.Error m] (build : m BuildTask) : m (MTimeBuildTarget a) := do - if (← checkIfNewer artifact depMTime) then - MTimeBuildTarget.pure artifact depMTime - else - MTimeBuildTarget.mk artifact depMTime (← build) + MTimeBuildTarget.mk artifact depMTime <| ← + skipIf (← checkIfNewer artifact depMTime) build -- # Build Components @@ -158,11 +190,6 @@ def buildRBTop {k o} {cmp} {m} [BEq k] [Inhabited o] [Monad m] -- # Build Modules -def parseDirectLocalImports (root : Name) (leanFile : FilePath) : IO (List Name) := do - let contents ← IO.FS.readFile leanFile - let (imports, _, _) ← Elab.parseImports contents leanFile.toString - imports.map (·.module) |>.filter (·.getRoot == root) - /- Produces a recursive module target fetcher that builds the target module after waiting for `depsTarget` and @@ -172,39 +199,49 @@ def parseDirectLocalImports (root : Name) (leanFile : FilePath) : IO (List Name) a `LEAN_PATH` that includes `oleanDirs`. -/ def fetchAfterDirectLocalImports -(pkg : Package) (oleanDirs : List FilePath) (depsTarget : MTimeBuildTarget PUnit) -{m} [Monad m] [MonadLiftT IO m] [MonadExceptOf IO.Error m] : RecFetch Name LeanTarget m := +(pkg : Package) (oleanDirs : List FilePath) (depsTarget : LeanTarget PUnit) +{m} [Monad m] [MonadLiftT IO m] [MonadExceptOf IO.Error m] : RecFetch Name ModuleTarget m := let leanPath := SearchPath.toString <| pkg.oleanDir :: oleanDirs fun mod fetch => do let leanFile := pkg.modToSource mod - let imports ← parseDirectLocalImports pkg.module leanFile + let contents ← IO.FS.readFile leanFile + -- parse direct local imports + let (imports, _, _) ← Elab.parseImports contents leanFile.toString + let imports := imports.map (·.module) |>.filter (·.getRoot == pkg.module) -- we fetch the import targets even if a rebuild is not required -- because other build processes (ex. `.o`) rely on the map being complete let importTargets ← imports.mapM fetch - -- calculate max dependency `MTime` + -- calculate trace + let leanHash := hash contents let leanMTime ← getMTime leanFile + let importHashes ← importTargets.map (·.hash) let importMTimes ← importTargets.map (·.mtime) - let depMTime := MTime.listMax <| leanMTime :: depsTarget.mtime :: importMTimes + let fullHash := Hash.foldList leanHash (depsTarget.hash :: importHashes) + let maxMTime := MTime.listMax (leanMTime :: depsTarget.mtime :: importMTimes) + let hashFile := pkg.modToHashFile mod + let sameHash ← checkIfSameHash fullHash hashFile + let mtime := ite sameHash 0 maxMTime -- construct target let cFile := pkg.modToC mod let oleanFile := pkg.modToOlean mod let importTasks := importTargets.map (·.buildTask) - skipIfNewer ⟨oleanFile, cFile⟩ depMTime <| - BuildTask.afterTasks (depsTarget.buildTask :: importTasks) <| catchErrors <| + BuildTarget.mk ⟨oleanFile, cFile⟩ ⟨fullHash, mtime⟩ <| ← skipIf sameHash <| + BuildTask.afterTasks (depsTarget.buildTask :: importTasks) <| catchErrors do compileOleanAndC leanFile oleanFile cFile leanPath pkg.dir pkg.leanArgs + IO.FS.writeFile hashFile (toString fullHash) /- - Equivalent to `RBTopT (cmp := Name.quickCmp) Name LeanTarget IO`. + Equivalent to `RBTopT (cmp := Name.quickCmp) Name ModuleTarget IO`. Phrased this way to use `NameMap`. -/ abbrev LeanTargetM := - EStateT (List Name) (NameMap LeanTarget) IO + EStateT (List Name) (NameMap ModuleTarget) IO abbrev LeanTargetFetch := - RecFetch Name LeanTarget LeanTargetM + RecFetch Name ModuleTarget LeanTargetM abbrev RecLeanTargetM := - ReaderT (RecFetch Name LeanTarget LeanTargetM) LeanTargetM + ReaderT (RecFetch Name ModuleTarget LeanTargetM) LeanTargetM def throwOnCycle (mx : IO (Except (List Name) α)) : IO α := mx >>= fun @@ -214,15 +251,15 @@ def throwOnCycle (mx : IO (Except (List Name) α)) : IO α := throw <| IO.userError s!"import cycle detected:\n{"\n".intercalate cycle}" def Package.buildModuleTargetDAG -(oleanDirs : List FilePath) (depsTarget : MTimeBuildTarget PUnit) -(self : Package) : IO (LeanTarget × NameMap LeanTarget) := do +(oleanDirs : List FilePath) (depsTarget : LeanTarget PUnit) +(self : Package) : IO (ModuleTarget × NameMap ModuleTarget) := do let fetch := fetchAfterDirectLocalImports self oleanDirs depsTarget throwOnCycle <| buildRBTop fetch self.module |>.run {} def Package.buildModuleTargets (mods : List Name) (oleanDirs : List FilePath) -(depsTarget : MTimeBuildTarget PUnit) (self : Package) -: IO (List LeanTarget) := do +(depsTarget : LeanTarget PUnit) (self : Package) +: IO (List ModuleTarget) := do let fetch : LeanTargetFetch := fetchAfterDirectLocalImports self oleanDirs depsTarget throwOnCycle <| mods.mapM (buildRBTop fetch) |>.run' {} @@ -232,7 +269,7 @@ def Package.buildModuleTargets def Package.buildTargetWithDepTargets (depTargets : List PackageTarget) (self : Package) : IO PackageTarget := do - let depsTarget ← MTimeBuildTarget.all depTargets + let depsTarget ← LeanTarget.all depTargets let depOLeanDirs := depTargets.map (·.package.oleanDir) let (target, targetMap) ← self.buildModuleTargetDAG depOLeanDirs depsTarget return {target with artifact := ⟨self, targetMap⟩} @@ -272,9 +309,9 @@ def build (pkg : Package) : IO PUnit := def Package.buildModuleTargetsWithDeps (deps : List Package) (mods : List Name) (self : Package) -: IO (List LeanTarget) := do +: IO (List ModuleTarget) := do let oleanDirs := deps.map (·.oleanDir) - let depsTarget ← MTimeBuildTarget.all (← deps.mapM (·.buildTarget)) + let depsTarget ← LeanTarget.all (← deps.mapM (·.buildTarget)) self.buildModuleTargets mods oleanDirs depsTarget def Package.buildModulesWithDeps diff --git a/Lake/BuildTarget.lean b/Lake/BuildTarget.lean index fb58264fe8..cb89baffc5 100644 --- a/Lake/BuildTarget.lean +++ b/Lake/BuildTarget.lean @@ -71,7 +71,21 @@ def afterTargets (targets : List (BuildTarget t a)) (action : IO PUnit) : IO Bui end BuildTask --- # MTime Build Targets +-- # Hash Traces + +section +def Hash := UInt64 + +instance : OfNat Hash n := inferInstanceAs (OfNat UInt64 n) +instance : Inhabited Hash := inferInstanceAs (Inhabited UInt64) +instance : BEq Hash := inferInstanceAs (BEq UInt64) +end + +def Hash.ofNat (n : Nat) := UInt64.ofNat n +def Hash.foldList (init : Hash) (hashes : List Hash) := + List.foldl mixHash init hashes + +-- # MTime Traces section open IO.FS (SystemTime) diff --git a/Lake/Package.lean b/Lake/Package.lean index c58a48165d..9b35732ad2 100644 --- a/Lake/Package.lean +++ b/Lake/Package.lean @@ -89,6 +89,9 @@ def oleanDir (self : Package) : FilePath := def oleanRoot (self : Package) : FilePath := self.oleanDir / FilePath.withExtension self.moduleName "olean" +def modToHashFile (mod : Name) (self : Package) : FilePath := + Lean.modToFilePath self.oleanDir mod "hash" + def modToOlean (mod : Name) (self : Package) : FilePath := Lean.modToFilePath self.oleanDir mod "olean" From 3ef381bb6cb68ee6eeb49ae8a6ae3174db9dcbf8 Mon Sep 17 00:00:00 2001 From: tydeu Date: Wed, 14 Jul 2021 12:46:07 -0400 Subject: [PATCH 072/696] refactor: merge `Proc` into `Compile` and cleanup `Build` --- Lake/Build.lean | 48 +++++++++++++++---------------------------- Lake/BuildTarget.lean | 36 ++++++++++++++++++++++---------- Lake/Compile.lean | 39 +++++++++++++++++++++++------------ Lake/Git.lean | 30 +++++++++++++++++++-------- Lake/Init.lean | 1 - Lake/Proc.lean | 18 ---------------- 6 files changed, 90 insertions(+), 82 deletions(-) delete mode 100644 Lake/Proc.lean diff --git a/Lake/Build.lean b/Lake/Build.lean index 19eebaf24b..ee90697f7a 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -21,10 +21,10 @@ abbrev FileTarget := MTimeBuildTarget FilePath namespace FileTarget -def mk (file : FilePath) (maxMTime : IO.FS.SystemTime) (task : BuildTask) := +def mk (file : FilePath) (maxMTime : MTime) (task : BuildTask) := BuildTarget.mk file maxMTime task -def pure (file : FilePath) (maxMTime : IO.FS.SystemTime) := +def pure (file : FilePath) (maxMTime : MTime) := BuildTarget.pure file maxMTime end FileTarget @@ -122,16 +122,9 @@ def skipIfNewer [GetMTime a] -- # Build Components -def catchErrors (action : IO PUnit) : IO PUnit := do - try action catch e => - -- print compile errors early - IO.eprintln e - throw e - def buildO (oFile : FilePath) -(cTarget : FileTarget) (leancArgs : Array String := #[]) : IO BuildTask := - BuildTask.afterTarget cTarget <| catchErrors <| - compileO oFile cTarget.artifact leancArgs +(cTarget : BuildTarget t FilePath) (leancArgs : Array String := #[]) : IO BuildTask := + afterTarget cTarget <| compileO oFile cTarget.artifact leancArgs def fetchOFileTarget (oFile : FilePath) (cTarget : FileTarget) (leancArgs : Array String := #[]) : IO FileTarget := @@ -225,8 +218,8 @@ def fetchAfterDirectLocalImports let cFile := pkg.modToC mod let oleanFile := pkg.modToOlean mod let importTasks := importTargets.map (·.buildTask) - BuildTarget.mk ⟨oleanFile, cFile⟩ ⟨fullHash, mtime⟩ <| ← skipIf sameHash <| - BuildTask.afterTasks (depsTarget.buildTask :: importTasks) <| catchErrors do + BuildTarget.mk ⟨oleanFile, cFile⟩ ⟨fullHash, mtime⟩ <| ← + skipIf sameHash <| afterTaskList (depsTarget.buildTask :: importTasks) do compileOleanAndC leanFile oleanFile cFile leanPath pkg.dir pkg.leanArgs IO.FS.writeFile hashFile (toString fullHash) @@ -234,14 +227,11 @@ def fetchAfterDirectLocalImports Equivalent to `RBTopT (cmp := Name.quickCmp) Name ModuleTarget IO`. Phrased this way to use `NameMap`. -/ -abbrev LeanTargetM := +abbrev ModuleTargetM := EStateT (List Name) (NameMap ModuleTarget) IO -abbrev LeanTargetFetch := - RecFetch Name ModuleTarget LeanTargetM - -abbrev RecLeanTargetM := - ReaderT (RecFetch Name ModuleTarget LeanTargetM) LeanTargetM +abbrev ModuleTargetFetch := + RecFetch Name ModuleTarget ModuleTargetM def throwOnCycle (mx : IO (Except (List Name) α)) : IO α := mx >>= fun @@ -260,7 +250,7 @@ def Package.buildModuleTargets (mods : List Name) (oleanDirs : List FilePath) (depsTarget : LeanTarget PUnit) (self : Package) : IO (List ModuleTarget) := do - let fetch : LeanTargetFetch := + let fetch : ModuleTargetFetch := fetchAfterDirectLocalImports self oleanDirs depsTarget throwOnCycle <| mods.mapM (buildRBTop fetch) |>.run' {} @@ -337,22 +327,19 @@ def PackageTarget.fetchOFileTargets (self : PackageTarget) : IO (List FileTarget) := do self.moduleTargets.toList.mapM fun (mod, target) => do let oFile := self.package.modToO mod - fetchOFileTarget oFile target.cTarget self.package.leancArgs + fetchOFileTarget (oFile) target.cTarget self.package.leancArgs def PackageTarget.buildStaticLib (self : PackageTarget) : IO BuildTask := do let oFileTargets ← self.fetchOFileTargets let oFiles := oFileTargets.map (·.artifact) |>.toArray - BuildTask.afterTargets oFileTargets <| catchErrors <| - compileStaticLib self.package.staticLibFile oFiles + oFileTargets >> compileStaticLib self.package.staticLibFile oFiles -def PackageTarget.fetchStaticLibTarget -(self : PackageTarget) : IO FileTarget := do +def PackageTarget.fetchStaticLibTarget (self : PackageTarget) : IO FileTarget := do skipIfNewer self.package.staticLibFile self.mtime self.buildStaticLib def Package.fetchStaticLibTarget (self : Package) : IO FileTarget := do - let target ← self.buildTarget - target.fetchStaticLibTarget + (← self.buildTarget).fetchStaticLibTarget def Package.fetchStaticLib (self : Package) : IO FilePath := do let target ← self.fetchStaticLibTarget @@ -370,11 +357,10 @@ def PackageTarget.buildBin (depTargets : List PackageTarget) (self : PackageTarget) : IO BuildTask := do let oFileTargets ← self.fetchOFileTargets - let oFiles := oFileTargets.map (·.artifact) |>.toArray let libTargets ← depTargets.mapM (·.fetchStaticLibTarget) - let libFiles := libTargets.map (·.artifact) |>.toArray - BuildTask.afterTargets (oFileTargets ++ libTargets) <| catchErrors <| - compileBin self.package.binFile (oFiles ++ libFiles) self.package.linkArgs + let linkTargets := oFileTargets ++ libTargets + let linkFiles := linkTargets.map (·.artifact) |>.toArray + linkTargets >> compileBin self.package.binFile linkFiles self.package.linkArgs def PackageTarget.fetchBinTarget (depTargets : List PackageTarget) (self : PackageTarget) : IO FileTarget := diff --git a/Lake/BuildTarget.lean b/Lake/BuildTarget.lean index cb89baffc5..538a37d2c8 100644 --- a/Lake/BuildTarget.lean +++ b/Lake/BuildTarget.lean @@ -6,6 +6,8 @@ Authors: Mac Malone namespace Lake +-- # Build Task + def BuildTask := Task (Except IO.Error PUnit) namespace BuildTask @@ -13,22 +15,32 @@ namespace BuildTask def nop : BuildTask := Task.pure (Except.ok ()) +def spawn (action : IO PUnit) (prio := Task.Priority.dedicated) : IO BuildTask := + IO.asTask action prio + def await (self : BuildTask) : IO PUnit := do IO.ofExcept (← IO.wait self) def all (tasks : List BuildTask) : IO BuildTask := IO.asTask (tasks.forM (·.await)) -def afterTask (task : BuildTask) (action : IO PUnit) : IO BuildTask := - IO.mapTask (fun x => IO.ofExcept x *> action) task - -def afterTasks (tasks : List BuildTask) (action : IO PUnit) : IO BuildTask := - IO.mapTasks (fun xs => xs.forM IO.ofExcept *> action) <| tasks - end BuildTask instance : Inhabited BuildTask := ⟨BuildTask.nop⟩ +def afterTask (task : BuildTask) (action : IO PUnit) : IO BuildTask := + IO.mapTask (fun x => IO.ofExcept x *> action) task + +def afterTaskList (tasks : List BuildTask) (action : IO PUnit) : IO BuildTask := + IO.mapTasks (fun xs => xs.forM IO.ofExcept *> action) <| tasks + +instance : HAndThen BuildTask (IO PUnit) (IO BuildTask) := + ⟨afterTask⟩ + +instance : HAndThen (List BuildTask) (IO PUnit) (IO BuildTask) := + ⟨afterTaskList⟩ + +-- # Build Target structure BuildTarget (t : Type) (a : Type) where artifact : a @@ -61,15 +73,17 @@ def materialize (self : BuildTarget t α) : IO PUnit := end BuildTarget -namespace BuildTask - def afterTarget (target : BuildTarget t a) (action : IO PUnit) : IO BuildTask := afterTask target.buildTask action -def afterTargets (targets : List (BuildTarget t a)) (action : IO PUnit) : IO BuildTask := - afterTasks (targets.map (·.buildTask)) action +def afterTargetList (targets : List (BuildTarget t a)) (action : IO PUnit) : IO BuildTask := + afterTaskList (targets.map (·.buildTask)) action -end BuildTask +instance : HAndThen (BuildTarget t a) (IO PUnit) (IO BuildTask) := + ⟨afterTarget⟩ + +instance : HAndThen (List (BuildTarget t a)) (IO PUnit) (IO BuildTask) := + ⟨afterTargetList⟩ -- # Hash Traces diff --git a/Lake/Compile.lean b/Lake/Compile.lean index 32828dc094..234bb0bd74 100644 --- a/Lake/Compile.lean +++ b/Lake/Compile.lean @@ -1,22 +1,35 @@ /- -Copyright (c) 2021 Mac Malone. All rights reserved. +Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. -Authors: Mac Malone +Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ -import Lake.Proc +import Lake.BuildTarget namespace Lake open System -def createParentDirs (path : FilePath) : IO Unit := - if let some dir := path.parent then IO.FS.createDirAll dir else pure () +def createParentDirs (path : FilePath) : IO PUnit := do + if let some dir := path.parent then + try IO.FS.createDirAll dir catch e => IO.eprintln e + +def proc (args : IO.Process.SpawnArgs) : IO PUnit := do + let envStr := String.join <| args.env.toList.map fun (k, v) => s!"{k}={v.getD ""} " + let cmdStr := " ".intercalate (args.cmd :: args.args.toList) + IO.println <| "> " ++ envStr ++ + match args.cwd with + | some cwd => s!"{cmdStr} # in directory {cwd}" + | none => cmdStr + let child ← IO.Process.spawn args + let exitCode ← child.wait + if exitCode != 0 then + IO.eprintln s!"external command exited with status {exitCode}" def compileOleanAndC (leanFile oleanFile cFile : FilePath) (leanPath : String := "") (rootDir : FilePath := ".") (leanArgs : Array String := #[]) -: IO Unit := do +: IO PUnit := do createParentDirs cFile createParentDirs oleanFile - execCmd { + proc { cmd := "lean" args := leanArgs ++ #[ "-R", rootDir.toString, "-o", oleanFile.toString, "-c", @@ -26,25 +39,25 @@ def compileOleanAndC (leanFile oleanFile cFile : FilePath) } def compileO (oFile cFile : FilePath) -(leancArgs : Array String := #[]) : IO Unit := do +(leancArgs : Array String := #[]) : IO PUnit := do createParentDirs oFile - execCmd { + proc { cmd := "leanc" args := #["-c", "-o", oFile.toString, cFile.toString] ++ leancArgs } def compileStaticLib -(libFile : FilePath) (oFiles : Array FilePath) : IO Unit := do +(libFile : FilePath) (oFiles : Array FilePath) : IO PUnit := do createParentDirs libFile - execCmd { + proc { cmd := "ar" args := #["rcs", libFile.toString] ++ oFiles.map toString } def compileBin (binFile : FilePath) -(linkFiles : Array FilePath) (linkArgs : Array String := #[]) : IO Unit := do +(linkFiles : Array FilePath) (linkArgs : Array String := #[]) : IO PUnit := do createParentDirs binFile - execCmd { + proc { cmd := "leanc" args := #["-o", binFile.toString] ++ linkFiles.map toString ++ linkArgs } diff --git a/Lake/Git.lean b/Lake/Git.lean index d6afd45501..bde51c16cb 100644 --- a/Lake/Git.lean +++ b/Lake/Git.lean @@ -3,7 +3,6 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ -import Lake.Proc import Lake.LeanVersion open System @@ -17,23 +16,38 @@ def defaultRevision : Option String → String | none => upstreamBranch | some branch => branch +def execGit (args : Array String) (repo : Option FilePath := none) : IO PUnit := do + let child ← IO.Process.spawn { + cmd := "git", args, cwd := repo, + stdout := IO.Process.Stdio.null, stderr := IO.Process.Stdio.null + } + let exitCode ← child.wait + if exitCode != 0 then + throw <| IO.userError <| "git exited with code " ++ toString exitCode + +def captureGit (args : Array String) (repo : Option FilePath := none) : IO String := do + let out ← IO.Process.output {cmd := "git", args, cwd := repo} + if out.exitCode != 0 then + throw <| IO.userError <| "git exited with code " ++ toString out.exitCode + return out.stdout + def clone (url : String) (dir : FilePath) := - execCmd {cmd := "git", args := #["clone", url, dir.toString]} + execGit #["clone", url, dir.toString] def quietInit (repo : Option FilePath := none) := - execCmd {cmd := "git", args := #["init", "-q"]} + execGit #["init", "-q"] repo def fetch (repo : Option FilePath := none) := - execCmd {cmd := "git", args := #["fetch"], cwd := repo} + execGit #["fetch"] repo def checkoutBranch (branch : String) (repo : Option FilePath := none) := - execCmd {cmd := "git", args := #["checkout", "-B", branch]} + execGit #["checkout", "-B", branch] repo def checkoutDetach (hash : String) (repo : Option FilePath := none) := - execCmd {cmd := "git", args := #["checkout", "--detach", hash], cwd := repo} + execGit #["checkout", "--detach", hash] repo def parseRevision (rev : String) (repo : Option FilePath := none) : IO String := do - let rev ← IO.Process.run {cmd := "git", args := #["rev-parse", "-q", "--verify", rev], cwd := repo} + let rev ← captureGit #["rev-parse", "-q", "--verify", rev] repo rev.trim -- remove newline at end def headRevision (repo : Option FilePath := none) : IO String := @@ -44,7 +58,7 @@ def parseOriginRevision (rev : String) (repo : Option FilePath := none) : IO Str <|> throw (IO.userError s!"cannot find revision {rev} in repository {repo}") def latestOriginRevision (branch : Option String) (repo : Option FilePath := none) : IO String := do - discard <| IO.Process.run {cmd := "git", args := #["fetch"], cwd := repo} + execGit #["fetch"] repo parseOriginRevision (defaultRevision branch) repo def revisionExists (rev : String) (repo : Option FilePath := none) : IO Bool := do diff --git a/Lake/Init.lean b/Lake/Init.lean index 3685453121..7ad5181028 100644 --- a/Lake/Init.lean +++ b/Lake/Init.lean @@ -4,7 +4,6 @@ Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Lake.Git -import Lake.Proc import Lake.LeanConfig namespace Lake diff --git a/Lake/Proc.lean b/Lake/Proc.lean deleted file mode 100644 index 83d8db043c..0000000000 --- a/Lake/Proc.lean +++ /dev/null @@ -1,18 +0,0 @@ -/- -Copyright (c) 2017 Microsoft Corporation. All rights reserved. -Released under Apache 2.0 license as described in the file LICENSE. -Authors: Gabriel Ebner, Sebastian Ullrich --/ -namespace Lake - -def execCmd (args : IO.Process.SpawnArgs) : IO Unit := do - let envstr := String.join <| args.env.toList.map fun (k, v) => s!"{k}={v.getD ""} " - let cmdstr := " ".intercalate (args.cmd :: args.args.toList) - IO.eprintln <| "> " ++ envstr ++ - match args.cwd with - | some cwd => s!"{cmdstr} # in directory {cwd}" - | none => cmdstr - let child ← IO.Process.spawn args - let exitCode ← child.wait - if exitCode != 0 then - throw <| IO.userError <| s!"external command exited with status {exitCode}" From e040804678cc536fdee91eee38f2d10def9ba9b3 Mon Sep 17 00:00:00 2001 From: tydeu Date: Wed, 14 Jul 2021 13:35:42 -0400 Subject: [PATCH 073/696] refactor: split task and trace from target into separate files --- Lake/BuildTarget.lean | 84 ++++--------------------------------------- Lake/BuildTask.lean | 58 ++++++++++++++++++++++++++++++ Lake/BuildTrace.lean | 43 ++++++++++++++++++++++ Lake/Compile.lean | 1 - 4 files changed, 108 insertions(+), 78 deletions(-) create mode 100644 Lake/BuildTask.lean create mode 100644 Lake/BuildTrace.lean diff --git a/Lake/BuildTarget.lean b/Lake/BuildTarget.lean index 538a37d2c8..ae1e8287dc 100644 --- a/Lake/BuildTarget.lean +++ b/Lake/BuildTarget.lean @@ -3,43 +3,11 @@ Copyright (c) 2021 Mac Malone. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Mac Malone -/ +import Lake.BuildTask +import Lake.BuildTrace namespace Lake --- # Build Task - -def BuildTask := Task (Except IO.Error PUnit) - -namespace BuildTask - -def nop : BuildTask := - Task.pure (Except.ok ()) - -def spawn (action : IO PUnit) (prio := Task.Priority.dedicated) : IO BuildTask := - IO.asTask action prio - -def await (self : BuildTask) : IO PUnit := do - IO.ofExcept (← IO.wait self) - -def all (tasks : List BuildTask) : IO BuildTask := - IO.asTask (tasks.forM (·.await)) - -end BuildTask - -instance : Inhabited BuildTask := ⟨BuildTask.nop⟩ - -def afterTask (task : BuildTask) (action : IO PUnit) : IO BuildTask := - IO.mapTask (fun x => IO.ofExcept x *> action) task - -def afterTaskList (tasks : List BuildTask) (action : IO PUnit) : IO BuildTask := - IO.mapTasks (fun xs => xs.forM IO.ofExcept *> action) <| tasks - -instance : HAndThen BuildTask (IO PUnit) (IO BuildTask) := - ⟨afterTask⟩ - -instance : HAndThen (List BuildTask) (IO PUnit) (IO BuildTask) := - ⟨afterTaskList⟩ - -- # Build Target structure BuildTarget (t : Type) (a : Type) where @@ -73,11 +41,11 @@ def materialize (self : BuildTarget t α) : IO PUnit := end BuildTarget -def afterTarget (target : BuildTarget t a) (action : IO PUnit) : IO BuildTask := - afterTask target.buildTask action +def afterTarget (target : BuildTarget t a) (act : IO PUnit) : IO BuildTask := + afterTask target.buildTask act -def afterTargetList (targets : List (BuildTarget t a)) (action : IO PUnit) : IO BuildTask := - afterTaskList (targets.map (·.buildTask)) action +def afterTargetList (targets : List (BuildTarget t a)) (act : IO PUnit) : IO BuildTask := + afterTaskList (targets.map (·.buildTask)) act instance : HAndThen (BuildTarget t a) (IO PUnit) (IO BuildTask) := ⟨afterTarget⟩ @@ -85,45 +53,7 @@ instance : HAndThen (BuildTarget t a) (IO PUnit) (IO BuildTask) := instance : HAndThen (List (BuildTarget t a)) (IO PUnit) (IO BuildTask) := ⟨afterTargetList⟩ --- # Hash Traces - -section -def Hash := UInt64 - -instance : OfNat Hash n := inferInstanceAs (OfNat UInt64 n) -instance : Inhabited Hash := inferInstanceAs (Inhabited UInt64) -instance : BEq Hash := inferInstanceAs (BEq UInt64) -end - -def Hash.ofNat (n : Nat) := UInt64.ofNat n -def Hash.foldList (init : Hash) (hashes : List Hash) := - List.foldl mixHash init hashes - --- # MTime Traces - -section -open IO.FS (SystemTime) - -def MTime := SystemTime - -instance : Inhabited MTime := ⟨⟨0,0⟩⟩ -instance : OfNat MTime (nat_lit 0) := ⟨⟨0,0⟩⟩ -instance : BEq MTime := inferInstanceAs (BEq SystemTime) -instance : Repr MTime := inferInstanceAs (Repr SystemTime) -instance : Ord MTime := inferInstanceAs (Ord SystemTime) -instance : LT MTime := ltOfOrd -instance : LE MTime := leOfOrd -end - -def MTime.listMax (mtimes : List MTime) := mtimes.maximum?.getD 0 - -class GetMTime (α) where - getMTime : α → IO MTime - -export GetMTime (getMTime) - -instance : GetMTime System.FilePath where - getMTime file := do (← file.metadata).modified +-- # MTIme Build Target abbrev MTimeBuildTarget := BuildTarget MTime diff --git a/Lake/BuildTask.lean b/Lake/BuildTask.lean new file mode 100644 index 0000000000..5e55d90cb1 --- /dev/null +++ b/Lake/BuildTask.lean @@ -0,0 +1,58 @@ +/- +Copyright (c) 2021 Mac Malone. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Mac Malone +-/ + +namespace Lake + +instance : Monad Task where + map := Task.map + pure := Task.pure + bind := Task.bind + +abbrev ETask ε α := ExceptT ε Task α + +abbrev IOTask α := ETask IO.Error α + +namespace IOTask + +def spawn (act : IO α) (prio := Task.Priority.default) : IO (IOTask α) := + IO.asTask act prio + +def await (self : IOTask α) : IO α := do + IO.ofExcept (← IO.wait self) + +def collectAll (tasks : List (IOTask α)) : IO (IOTask (List α)) := + IO.asTask (tasks.mapM (·.await)) + +end IOTask + +def BuildTask := IOTask PUnit + +namespace BuildTask + +def nop : BuildTask := + pure () + +def spawn (act : IO PUnit) (prio := Task.Priority.dedicated) : IO BuildTask := + IO.asTask act prio + +def all (tasks : List BuildTask) (prio := Task.Priority.default) : IO BuildTask := + IO.asTask (tasks.forM (·.await)) prio + +end BuildTask + +instance : Inhabited BuildTask := ⟨BuildTask.nop⟩ + +def afterTask (task : BuildTask) (act : IO PUnit) : IO BuildTask := + IO.mapTask (fun x => IO.ofExcept x *> act) task + +def afterTaskList (tasks : List BuildTask) (act : IO PUnit) : IO BuildTask := + IO.mapTasks (fun xs => xs.forM IO.ofExcept *> act) <| tasks + +instance : HAndThen BuildTask (IO PUnit) (IO BuildTask) := + ⟨afterTask⟩ + +instance : HAndThen (List BuildTask) (IO PUnit) (IO BuildTask) := + ⟨afterTaskList⟩ diff --git a/Lake/BuildTrace.lean b/Lake/BuildTrace.lean new file mode 100644 index 0000000000..ea82cfa477 --- /dev/null +++ b/Lake/BuildTrace.lean @@ -0,0 +1,43 @@ +/- +Copyright (c) 2021 Mac Malone. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Mac Malone +-/ + +namespace Lake + +-- # Hash Traces + +def Hash := UInt64 + +instance : OfNat Hash n := inferInstanceAs (OfNat UInt64 n) +instance : Inhabited Hash := inferInstanceAs (Inhabited UInt64) +instance : BEq Hash := inferInstanceAs (BEq UInt64) + +def Hash.ofNat (n : Nat) := UInt64.ofNat n +def Hash.foldList (init : Hash) (hashes : List Hash) := + List.foldl mixHash init hashes + +-- # Modification Time Traces + +open IO.FS (SystemTime) + +def MTime := SystemTime + +instance : Inhabited MTime := ⟨⟨0,0⟩⟩ +instance : OfNat MTime (nat_lit 0) := ⟨⟨0,0⟩⟩ +instance : BEq MTime := inferInstanceAs (BEq SystemTime) +instance : Repr MTime := inferInstanceAs (Repr SystemTime) +instance : Ord MTime := inferInstanceAs (Ord SystemTime) +instance : LT MTime := ltOfOrd +instance : LE MTime := leOfOrd + +def MTime.listMax (mtimes : List MTime) := mtimes.maximum?.getD 0 + +class GetMTime (α) where + getMTime : α → IO MTime + +export GetMTime (getMTime) + +instance : GetMTime System.FilePath where + getMTime file := do (← file.metadata).modified diff --git a/Lake/Compile.lean b/Lake/Compile.lean index 234bb0bd74..5e27372a84 100644 --- a/Lake/Compile.lean +++ b/Lake/Compile.lean @@ -3,7 +3,6 @@ Copyright (c) 2017 Microsoft Corporation. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ -import Lake.BuildTarget namespace Lake open System From 31fa37dbfefc7b726718fe81f257ab6e418743bb Mon Sep 17 00:00:00 2001 From: tydeu Date: Wed, 14 Jul 2021 14:53:52 -0400 Subject: [PATCH 074/696] refactor: remove `Monad Task` instance (for now) --- Lake/BuildTask.lean | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Lake/BuildTask.lean b/Lake/BuildTask.lean index 5e55d90cb1..0451022306 100644 --- a/Lake/BuildTask.lean +++ b/Lake/BuildTask.lean @@ -6,11 +6,6 @@ Authors: Mac Malone namespace Lake -instance : Monad Task where - map := Task.map - pure := Task.pure - bind := Task.bind - abbrev ETask ε α := ExceptT ε Task α abbrev IOTask α := ETask IO.Error α @@ -33,7 +28,7 @@ def BuildTask := IOTask PUnit namespace BuildTask def nop : BuildTask := - pure () + Task.pure (Except.ok ()) def spawn (act : IO PUnit) (prio := Task.Priority.dedicated) : IO BuildTask := IO.asTask act prio From f8a31011a6502389839316f9216727e26f7cfc63 Mon Sep 17 00:00:00 2001 From: tydeu Date: Thu, 15 Jul 2021 12:21:52 -0400 Subject: [PATCH 075/696] feat: add more package configuration settings --- Lake/Build.lean | 60 +++++++++--------------------------- Lake/BuildTarget.lean | 39 ++++++++++++++++++++++++ Lake/BuildTrace.lean | 7 +++++ Lake/Package.lean | 71 +++++++++++++++++++++++++++---------------- Lake/Resolve.lean | 14 ++++----- 5 files changed, 112 insertions(+), 79 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index ee90697f7a..76be65c8b1 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -17,49 +17,17 @@ namespace Lake -- # Build Targets -abbrev FileTarget := MTimeBuildTarget FilePath - -namespace FileTarget - -def mk (file : FilePath) (maxMTime : MTime) (task : BuildTask) := - BuildTarget.mk file maxMTime task - -def pure (file : FilePath) (maxMTime : MTime) := - BuildTarget.pure file maxMTime - -end FileTarget - -structure LeanArtifact where +structure ModuleArtifact where oleanFile : FilePath cFile : FilePath deriving Inhabited -structure LeanTrace where - hash : Hash - mtime : MTime - deriving Inhabited - -protected def LeanArtifact.getMTime (self : LeanArtifact) : IO MTime := do +protected def ModuleArtifact.getMTime (self : ModuleArtifact) : IO MTime := do return max (← getMTime self.oleanFile) (← getMTime self.cFile) -instance : GetMTime LeanArtifact := ⟨LeanArtifact.getMTime⟩ +instance : GetMTime ModuleArtifact := ⟨ModuleArtifact.getMTime⟩ -abbrev LeanTarget a := BuildTarget LeanTrace a - -namespace LeanTarget - -def hash (self : LeanTarget a) := self.trace.hash -def mtime (self : LeanTarget a) := self.trace.mtime - -def all (targets : List (LeanTarget a)) : IO (LeanTarget PUnit) := do - let hash := Hash.foldList 0 <| targets.map (·.hash) - let mtime := MTime.listMax <| targets.map (·.mtime) - let task ← BuildTask.all <| targets.map (·.buildTask) - return BuildTarget.mk () ⟨hash, mtime⟩ task - -end LeanTarget - -abbrev ModuleTarget := LeanTarget LeanArtifact +abbrev ModuleTarget := LeanTarget ModuleArtifact namespace ModuleTarget @@ -196,11 +164,11 @@ def fetchAfterDirectLocalImports {m} [Monad m] [MonadLiftT IO m] [MonadExceptOf IO.Error m] : RecFetch Name ModuleTarget m := let leanPath := SearchPath.toString <| pkg.oleanDir :: oleanDirs fun mod fetch => do - let leanFile := pkg.modToSource mod + let leanFile := pkg.modToSrc mod let contents ← IO.FS.readFile leanFile -- parse direct local imports let (imports, _, _) ← Elab.parseImports contents leanFile.toString - let imports := imports.map (·.module) |>.filter (·.getRoot == pkg.module) + let imports := imports.map (·.module) |>.filter (·.getRoot == pkg.moduleRoot) -- we fetch the import targets even if a rebuild is not required -- because other build processes (ex. `.o`) rely on the map being complete let importTargets ← imports.mapM fetch @@ -244,7 +212,7 @@ def Package.buildModuleTargetDAG (oleanDirs : List FilePath) (depsTarget : LeanTarget PUnit) (self : Package) : IO (ModuleTarget × NameMap ModuleTarget) := do let fetch := fetchAfterDirectLocalImports self oleanDirs depsTarget - throwOnCycle <| buildRBTop fetch self.module |>.run {} + throwOnCycle <| buildRBTop fetch self.moduleRoot |>.run {} def Package.buildModuleTargets (mods : List Name) (oleanDirs : List FilePath) @@ -259,9 +227,10 @@ def Package.buildModuleTargets def Package.buildTargetWithDepTargets (depTargets : List PackageTarget) (self : Package) : IO PackageTarget := do - let depsTarget ← LeanTarget.all depTargets - let depOLeanDirs := depTargets.map (·.package.oleanDir) - let (target, targetMap) ← self.buildModuleTargetDAG depOLeanDirs depsTarget + let depsTarget ← LeanTarget.all <| + self.moreDepsTarget.withArtifact arbitrary :: depTargets + let oLeanDirs := depTargets.map (·.package.oleanDir) + let (target, targetMap) ← self.buildModuleTargetDAG oLeanDirs depsTarget return {target with artifact := ⟨self, targetMap⟩} partial def Package.buildTarget (self : Package) : IO PackageTarget := do @@ -301,7 +270,8 @@ def Package.buildModuleTargetsWithDeps (deps : List Package) (mods : List Name) (self : Package) : IO (List ModuleTarget) := do let oleanDirs := deps.map (·.oleanDir) - let depsTarget ← LeanTarget.all (← deps.mapM (·.buildTarget)) + let depsTarget ← LeanTarget.all <| + self.moreDepsTarget.withArtifact arbitrary :: (← deps.mapM (·.buildTarget)) self.buildModuleTargets mods oleanDirs depsTarget def Package.buildModulesWithDeps @@ -316,10 +286,10 @@ def printPaths (pkg : Package) (imports : List String := []) : IO Unit := do let deps ← solveDeps pkg unless imports.isEmpty do let imports := imports.map (·.toName) - let localImports := imports.filter (·.getRoot == pkg.module) + let localImports := imports.filter (·.getRoot == pkg.moduleRoot) pkg.buildModulesWithDeps deps localImports IO.println <| SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) - IO.println <| SearchPath.toString <| pkg.sourceDir :: deps.map (·.sourceDir) + IO.println <| SearchPath.toString <| pkg.srcDir :: deps.map (·.srcDir) -- # Build Package Lib diff --git a/Lake/BuildTarget.lean b/Lake/BuildTarget.lean index ae1e8287dc..6becb2cc70 100644 --- a/Lake/BuildTarget.lean +++ b/Lake/BuildTarget.lean @@ -21,9 +21,15 @@ instance [Inhabited t] [Inhabited a] : Inhabited (BuildTarget t a) := namespace BuildTarget +def nil [Inhabited t] : BuildTarget t PUnit := + ⟨(), Inhabited.default, BuildTask.nop⟩ + def pure (artifact : a) (trace : t) : BuildTarget t a := {artifact, trace, buildTask := BuildTask.nop} +def opaque (trace : t) (task : BuildTask) : BuildTarget t PUnit := + ⟨(), trace, task⟩ + def withTrace (trace : t) (self : BuildTarget r a) : BuildTarget t a := {self with trace := trace} @@ -80,3 +86,36 @@ def collectAll (targets : List (MTimeBuildTarget a)) : IO (MTimeBuildTarget (Lis return MTimeBuildTarget.mk artifacts depsMTime task end MTimeBuildTarget + +-- # File Target + +open System + +abbrev FileTarget := MTimeBuildTarget FilePath + +namespace FileTarget + +def mk (file : FilePath) (maxMTime : MTime) (task : BuildTask) := + BuildTarget.mk file maxMTime task + +def pure (file : FilePath) (maxMTime : MTime) := + BuildTarget.pure file maxMTime + +end FileTarget + +-- # Lean Target + +abbrev LeanTarget a := BuildTarget LeanTrace a + +namespace LeanTarget + +def hash (self : LeanTarget a) := self.trace.hash +def mtime (self : LeanTarget a) := self.trace.mtime + +def all (targets : List (LeanTarget a)) : IO (LeanTarget PUnit) := do + let hash := Hash.foldList 0 <| targets.map (·.hash) + let mtime := MTime.listMax <| targets.map (·.mtime) + let task ← BuildTask.all <| targets.map (·.buildTask) + return BuildTarget.mk () ⟨hash, mtime⟩ task + +end LeanTarget diff --git a/Lake/BuildTrace.lean b/Lake/BuildTrace.lean index ea82cfa477..5710f483aa 100644 --- a/Lake/BuildTrace.lean +++ b/Lake/BuildTrace.lean @@ -41,3 +41,10 @@ export GetMTime (getMTime) instance : GetMTime System.FilePath where getMTime file := do (← file.metadata).modified + +-- # Combined Trace + +structure LeanTrace where + hash : Hash + mtime : MTime + deriving Inhabited diff --git a/Lake/Package.lean b/Lake/Package.lean index 9b35732ad2..604381c6a4 100644 --- a/Lake/Package.lean +++ b/Lake/Package.lean @@ -6,14 +6,19 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone import Lean.Data.Name import Lean.Elab.Import import Lake.LeanVersion +import Lake.BuildTarget open Lean System namespace Lake -def buildPath : FilePath := "build" -def tempBuildPath := buildPath / "temp" -def depsPath := buildPath / "deps" +def defaultSrcDir : FilePath := "." +def defaultBuildDir : FilePath := "build" +def defaultBinDir := defaultBuildDir / "bin" +def defaultLibDir := defaultBuildDir / "lib" +def defaultOleanDir := defaultBuildDir / "lib" +def defaultIrDir := defaultBuildDir / "ir" +def defaultDepsDir : FilePath := "lean_packages" inductive Source where | path (dir : FilePath) : Source @@ -27,12 +32,21 @@ structure Dependency where structure PackageConfig where name : String version : String + moduleRoot : Name := name.capitalize leanVersion : String := leanVersionString leanArgs : Array String := #[] leancArgs : Array String := #[] linkArgs : Array String := #[] - module : Name := name.capitalize + srcDir : FilePath := defaultSrcDir + oleanDir : FilePath := defaultOleanDir + irDir : FilePath := defaultIrDir + binDir : FilePath := defaultBinDir + binName : String := name + libDir : FilePath := defaultLibDir + libName : String := name.capitalize + depsDir : FilePath := defaultDepsDir dependencies : List Dependency := [] + moreDepsTarget : BuildTarget LeanTrace PUnit := BuildTarget.nil deriving Inhabited structure Package where @@ -53,15 +67,18 @@ def version (self : Package) : String := def leanVersion (self : Package) : String := self.config.leanVersion -def module (self : Package) : Name := - self.config.module +def moduleRoot (self : Package) : Name := + self.config.moduleRoot -def moduleName (self : Package) : String := - self.config.module.toString +def moduleRootName (self : Package) : String := + self.config.moduleRoot.toString def dependencies (self : Package) : List Dependency := self.config.dependencies +def moreDepsTarget (self : Package) : BuildTarget LeanTrace PUnit := + self.config.moreDepsTarget + def leanArgs (self : Package) : Array String := self.config.leanArgs @@ -71,23 +88,23 @@ def leancArgs (self : Package) : Array String := def linkArgs (self : Package) : Array String := self.config.linkArgs -def sourceDir (self : Package) : FilePath := - self.dir +def depsDir (self : Package) : FilePath := + self.dir / self.config.depsDir -def sourceRoot (self : Package) : FilePath := - self.sourceDir / self.moduleName +def srcDir (self : Package) : FilePath := + self.dir / self.config.srcDir -def modToSource (mod : Name) (self : Package) : FilePath := - Lean.modToFilePath self.sourceDir mod "lean" +def srcRoot (self : Package) : FilePath := + self.srcDir / FilePath.withExtension self.moduleRootName "lean" -def buildDir (self : Package) : FilePath := - self.dir / Lake.buildPath +def modToSrc (mod : Name) (self : Package) : FilePath := + Lean.modToFilePath self.srcDir mod "lean" def oleanDir (self : Package) : FilePath := - self.buildDir + self.dir / self.config.oleanDir def oleanRoot (self : Package) : FilePath := - self.oleanDir / FilePath.withExtension self.moduleName "olean" + self.oleanDir / FilePath.withExtension self.moduleRootName "olean" def modToHashFile (mod : Name) (self : Package) : FilePath := Lean.modToFilePath self.oleanDir mod "hash" @@ -95,26 +112,26 @@ def modToHashFile (mod : Name) (self : Package) : FilePath := def modToOlean (mod : Name) (self : Package) : FilePath := Lean.modToFilePath self.oleanDir mod "olean" -def tempBuildDir (self : Package) : FilePath := - self.dir / tempBuildPath +def irDir (self : Package) : FilePath := + self.dir / self.config.irDir def cDir (self : Package) : FilePath := - self.tempBuildDir + self.irDir def modToC (mod : Name) (self : Package) : FilePath := Lean.modToFilePath self.cDir mod "c" def oDir (self : Package) : FilePath := - self.tempBuildDir + self.irDir def modToO (mod : Name) (self : Package) : FilePath := Lean.modToFilePath self.oDir mod "o" def binDir (self : Package) : FilePath := - self.buildDir / "bin" + self.dir / self.config.binDir def binName (self : Package) : FilePath := - self.moduleName + self.config.binName def binFileName (self : Package) : FilePath := FilePath.withExtension self.binName FilePath.exeExtension @@ -123,13 +140,13 @@ def binFile (self : Package) : FilePath := self.binDir / self.binFileName def libDir (self : Package) : FilePath := - self.buildDir / "lib" + self.dir / self.config.libDir def staticLibName (self : Package) : FilePath := - self.moduleName + self.config.libName def staticLibFileName (self : Package) : FilePath := - s!"lib{self.module}.a" + s!"lib{self.moduleRoot}.a" def staticLibFile (self : Package) : FilePath := self.libDir / self.staticLibFileName diff --git a/Lake/Resolve.lean b/Lake/Resolve.lean index bd3b168019..b39588c760 100644 --- a/Lake/Resolve.lean +++ b/Lake/Resolve.lean @@ -26,15 +26,15 @@ def materializeGit let hash ← parseOriginRevision rev dir checkoutDetach hash dir -def materialize (pkgDir : FilePath) (dep : Dependency) : IO FilePath := +def materialize (pkg : Package) (dep : Dependency) : IO FilePath := match dep.src with | Source.path dir => do - let depdir := pkgDir / dir - depdir + let depDir := pkg.dir / dir + depDir | Source.git url rev branch => do - let depdir := pkgDir / depsPath / dep.name - materializeGit dep.name depdir url rev branch - depdir + let depDir := pkg.depsDir / dep.name + materializeGit dep.name depDir url rev branch + depDir def Assignment := List (String × Package) @@ -67,7 +67,7 @@ def solveDepsCore (pkg : Package) : (maxDepth : Nat) → Solver Unit | maxDepth + 1 => do let newDeps ← pkg.dependencies.filterM (notYetAssigned ·.name) for dep in newDeps do - let dir ← materialize pkg.dir dep + let dir ← materialize pkg dep let pkg ← Package.fromDir dir dep.args modify (·.insert dep.name pkg) for dep in newDeps do From 3b2c91f39657897cbb08e7cd9c57c971e78f88e8 Mon Sep 17 00:00:00 2001 From: tydeu Date: Thu, 15 Jul 2021 12:40:23 -0400 Subject: [PATCH 076/696] refactor: functions for building a specified module root --- Lake/Build.lean | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Lake/Build.lean b/Lake/Build.lean index 76be65c8b1..1b88558023 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -208,11 +208,15 @@ def throwOnCycle (mx : IO (Except (List Name) α)) : IO α := let cycle := cycle.map (s!" {·}") throw <| IO.userError s!"import cycle detected:\n{"\n".intercalate cycle}" -def Package.buildModuleTargetDAG -(oleanDirs : List FilePath) (depsTarget : LeanTarget PUnit) +def Package.buildModuleTargetDAGFor +(mod : Name) (oleanDirs : List FilePath) (depsTarget : LeanTarget PUnit) (self : Package) : IO (ModuleTarget × NameMap ModuleTarget) := do let fetch := fetchAfterDirectLocalImports self oleanDirs depsTarget - throwOnCycle <| buildRBTop fetch self.moduleRoot |>.run {} + throwOnCycle <| buildRBTop fetch mod |>.run {} + +def Package.buildModuleTargetDAG +(oleanDirs : List FilePath) (depsTarget : LeanTarget PUnit) (self : Package) := + self.buildModuleTargetDAGFor self.moduleRoot oleanDirs depsTarget def Package.buildModuleTargets (mods : List Name) (oleanDirs : List FilePath) @@ -224,15 +228,19 @@ def Package.buildModuleTargets -- # Configure/Build Packages -def Package.buildTargetWithDepTargets -(depTargets : List PackageTarget) (self : Package) +def Package.buildTargetWithDepTargetsFor +(mod : Name) (depTargets : List PackageTarget) (self : Package) : IO PackageTarget := do let depsTarget ← LeanTarget.all <| self.moreDepsTarget.withArtifact arbitrary :: depTargets let oLeanDirs := depTargets.map (·.package.oleanDir) - let (target, targetMap) ← self.buildModuleTargetDAG oLeanDirs depsTarget + let (target, targetMap) ← self.buildModuleTargetDAGFor mod oLeanDirs depsTarget return {target with artifact := ⟨self, targetMap⟩} +def Package.buildTargetWithDepTargets +(depTargets : List PackageTarget) (self : Package) : IO PackageTarget := + self.buildTargetWithDepTargetsFor self.moduleRoot depTargets + partial def Package.buildTarget (self : Package) : IO PackageTarget := do let deps ← solveDeps self -- build dependencies recursively From b14eef6e06e096ed050fc7e3cd8a630c25e9eaa5 Mon Sep 17 00:00:00 2001 From: tydeu Date: Thu, 15 Jul 2021 12:50:54 -0400 Subject: [PATCH 077/696] refactor: split out top / lib / bin build from `Build.lean` --- Lake/Build.lean | 122 +-------------------------------------------- Lake/BuildBin.lean | 79 +++++++++++++++++++++++++++++ Lake/BuildTop.lean | 60 ++++++++++++++++++++++ Lake/Cli.lean | 1 + 4 files changed, 141 insertions(+), 121 deletions(-) create mode 100644 Lake/BuildBin.lean create mode 100644 Lake/BuildTop.lean diff --git a/Lake/Build.lean b/Lake/Build.lean index 1b88558023..f90283186e 100644 --- a/Lake/Build.lean +++ b/Lake/Build.lean @@ -6,6 +6,7 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone import Lean.Data.Name import Lean.Elab.Import import Lake.BuildTarget +import Lake.BuildTop import Lake.Resolve import Lake.Package import Lake.Compile @@ -88,67 +89,6 @@ def skipIfNewer [GetMTime a] MTimeBuildTarget.mk artifact depMTime <| ← skipIf (← checkIfNewer artifact depMTime) build --- # Build Components - -def buildO (oFile : FilePath) -(cTarget : BuildTarget t FilePath) (leancArgs : Array String := #[]) : IO BuildTask := - afterTarget cTarget <| compileO oFile cTarget.artifact leancArgs - -def fetchOFileTarget (oFile : FilePath) -(cTarget : FileTarget) (leancArgs : Array String := #[]) : IO FileTarget := - skipIfNewer oFile cTarget.mtime <| buildO oFile cTarget leancArgs - --- # Topological Builder - -open Std - -/-- A recursive object fetcher. -/ -def RecFetch.{u,v,w} (k : Type u) (o : Type v) (m : Type v → Type w) := - k → (k → m o) → m o - -/-- A exception plus state monad transformer (i.e., `StateT` + `ExceptT`). -/ -abbrev EStateT (ε σ m) := - StateT σ <| ExceptT ε m - -def EStateT.run (state : σ) (self : EStateT ε σ m α) : m (Except ε (α × σ)) := - StateT.run self state |>.run - -def EStateT.run' [Monad m] (state : σ) (self : EStateT ε σ m α) : m (Except ε α) := - StateT.run' self state |>.run - -/-- - Monad transformer for an RBMap-based topological walk. - If a cycle is detected, the list keys traversed is thrown. --/ -abbrev RBTopT.{u,v} (k : Type u) (o : Type u) (cmp) (m : Type u → Type v) := - EStateT (List k) (RBMap k o cmp) m - -/-- Auxiliary function for `buildRBTop`. -/ -partial def buildRBTopCore -{k o} {cmp} {m : Type u → Type u} [BEq k] [Inhabited o] [Monad m] -(parents : List k) (fetch : RecFetch k o (RBTopT k o cmp m)) -(key : k) : RBTopT k o cmp m o := do - -- detect cyclic builds - if parents.contains key then - throw <| key :: (parents.partition (· != key)).1 ++ [key] - -- return previous output if already built - if let some output := (← get).find? key then - return output - -- build the key recursively - let output ← fetch key (buildRBTopCore (key :: parents) fetch) - -- save the output (to prevent repeated builds of the same key) - modify (·.insert key output) - return output - -/-- - Recursively constructs an `RBMao` of key-object pairs by - fetching objects topologically (i.e., via a deep-first search with - memoization). Called a suspending scheduler in "Build systems à la carte". --/ -def buildRBTop {k o} {cmp} {m} [BEq k] [Inhabited o] [Monad m] -(fetch : RecFetch k o (RBTopT k o cmp m)) (key : k) : RBTopT k o cmp m o := - buildRBTopCore [] fetch key - -- # Build Modules /- @@ -298,63 +238,3 @@ def printPaths (pkg : Package) (imports : List String := []) : IO Unit := do pkg.buildModulesWithDeps deps localImports IO.println <| SearchPath.toString <| pkg.oleanDir :: deps.map (·.oleanDir) IO.println <| SearchPath.toString <| pkg.srcDir :: deps.map (·.srcDir) - --- # Build Package Lib - -def PackageTarget.fetchOFileTargets -(self : PackageTarget) : IO (List FileTarget) := do - self.moduleTargets.toList.mapM fun (mod, target) => do - let oFile := self.package.modToO mod - fetchOFileTarget (oFile) target.cTarget self.package.leancArgs - -def PackageTarget.buildStaticLib -(self : PackageTarget) : IO BuildTask := do - let oFileTargets ← self.fetchOFileTargets - let oFiles := oFileTargets.map (·.artifact) |>.toArray - oFileTargets >> compileStaticLib self.package.staticLibFile oFiles - -def PackageTarget.fetchStaticLibTarget (self : PackageTarget) : IO FileTarget := do - skipIfNewer self.package.staticLibFile self.mtime self.buildStaticLib - -def Package.fetchStaticLibTarget (self : Package) : IO FileTarget := do - (← self.buildTarget).fetchStaticLibTarget - -def Package.fetchStaticLib (self : Package) : IO FilePath := do - let target ← self.fetchStaticLibTarget - try target.materialize catch _ => - -- actual error has already been printed within the task - throw <| IO.userError "Build failed." - return target.artifact - -def buildLib (pkg : Package) : IO PUnit := - discard pkg.fetchStaticLib - --- # Build Package Bin - -def PackageTarget.buildBin -(depTargets : List PackageTarget) (self : PackageTarget) -: IO BuildTask := do - let oFileTargets ← self.fetchOFileTargets - let libTargets ← depTargets.mapM (·.fetchStaticLibTarget) - let linkTargets := oFileTargets ++ libTargets - let linkFiles := linkTargets.map (·.artifact) |>.toArray - linkTargets >> compileBin self.package.binFile linkFiles self.package.linkArgs - -def PackageTarget.fetchBinTarget -(depTargets : List PackageTarget) (self : PackageTarget) : IO FileTarget := - skipIfNewer self.package.binFile self.mtime <| self.buildBin depTargets - -def Package.fetchBinTarget (self : Package) : IO FileTarget := do - let depTargets ← self.buildDepTargets - let pkgTarget ← self.buildTargetWithDepTargets depTargets - pkgTarget.fetchBinTarget depTargets - -def Package.fetchBin (self : Package) : IO FilePath := do - let target ← self.fetchBinTarget - try target.materialize catch _ => - -- actual error has already been printed within the task - throw <| IO.userError "Build failed." - return target.artifact - -def buildBin (pkg : Package) : IO PUnit := - discard pkg.fetchBin diff --git a/Lake/BuildBin.lean b/Lake/BuildBin.lean new file mode 100644 index 0000000000..8da99d0802 --- /dev/null +++ b/Lake/BuildBin.lean @@ -0,0 +1,79 @@ +/- +Copyright (c) 2021 Mac Malone. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Mac Malone +-/ +import Lake.Build + +open System +namespace Lake + +-- # Build `.o` Files + +def buildO (oFile : FilePath) +(cTarget : BuildTarget t FilePath) (leancArgs : Array String := #[]) : IO BuildTask := + afterTarget cTarget <| compileO oFile cTarget.artifact leancArgs + +def fetchOFileTarget (oFile : FilePath) +(cTarget : FileTarget) (leancArgs : Array String := #[]) : IO FileTarget := + skipIfNewer oFile cTarget.mtime <| buildO oFile cTarget leancArgs + +-- # Build Package Lib + +def PackageTarget.fetchOFileTargets +(self : PackageTarget) : IO (List FileTarget) := do + self.moduleTargets.toList.mapM fun (mod, target) => do + let oFile := self.package.modToO mod + fetchOFileTarget (oFile) target.cTarget self.package.leancArgs + +def PackageTarget.buildStaticLib +(self : PackageTarget) : IO BuildTask := do + let oFileTargets ← self.fetchOFileTargets + let oFiles := oFileTargets.map (·.artifact) |>.toArray + oFileTargets >> compileStaticLib self.package.staticLibFile oFiles + +def PackageTarget.fetchStaticLibTarget (self : PackageTarget) : IO FileTarget := do + skipIfNewer self.package.staticLibFile self.mtime self.buildStaticLib + +def Package.fetchStaticLibTarget (self : Package) : IO FileTarget := do + (← self.buildTarget).fetchStaticLibTarget + +def Package.fetchStaticLib (self : Package) : IO FilePath := do + let target ← self.fetchStaticLibTarget + try target.materialize catch _ => + -- actual error has already been printed within the task + throw <| IO.userError "Build failed." + return target.artifact + +def buildLib (pkg : Package) : IO PUnit := + discard pkg.fetchStaticLib + +-- # Build Package Bin + +def PackageTarget.buildBin +(depTargets : List PackageTarget) (self : PackageTarget) +: IO BuildTask := do + let oFileTargets ← self.fetchOFileTargets + let libTargets ← depTargets.mapM (·.fetchStaticLibTarget) + let linkTargets := oFileTargets ++ libTargets + let linkFiles := linkTargets.map (·.artifact) |>.toArray + linkTargets >> compileBin self.package.binFile linkFiles self.package.linkArgs + +def PackageTarget.fetchBinTarget +(depTargets : List PackageTarget) (self : PackageTarget) : IO FileTarget := + skipIfNewer self.package.binFile self.mtime <| self.buildBin depTargets + +def Package.fetchBinTarget (self : Package) : IO FileTarget := do + let depTargets ← self.buildDepTargets + let pkgTarget ← self.buildTargetWithDepTargets depTargets + pkgTarget.fetchBinTarget depTargets + +def Package.fetchBin (self : Package) : IO FilePath := do + let target ← self.fetchBinTarget + try target.materialize catch _ => + -- actual error has already been printed within the task + throw <| IO.userError "Build failed." + return target.artifact + +def buildBin (pkg : Package) : IO PUnit := + discard pkg.fetchBin diff --git a/Lake/BuildTop.lean b/Lake/BuildTop.lean new file mode 100644 index 0000000000..d621549a40 --- /dev/null +++ b/Lake/BuildTop.lean @@ -0,0 +1,60 @@ +/- +Copyright (c) 2017 Microsoft Corporation. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone +-/ +import Std.Data.RBMap + +open Std +namespace Lake + +-- # Topological Builder + +open Std + +/-- A recursive object fetcher. -/ +def RecFetch.{u,v,w} (k : Type u) (o : Type v) (m : Type v → Type w) := + k → (k → m o) → m o + +/-- A exception plus state monad transformer (i.e., `StateT` + `ExceptT`). -/ +abbrev EStateT (ε σ m) := + StateT σ <| ExceptT ε m + +def EStateT.run (state : σ) (self : EStateT ε σ m α) : m (Except ε (α × σ)) := + StateT.run self state |>.run + +def EStateT.run' [Monad m] (state : σ) (self : EStateT ε σ m α) : m (Except ε α) := + StateT.run' self state |>.run + +/-- + Monad transformer for an RBMap-based topological walk. + If a cycle is detected, the list keys traversed is thrown. +-/ +abbrev RBTopT.{u,v} (k : Type u) (o : Type u) (cmp) (m : Type u → Type v) := + EStateT (List k) (RBMap k o cmp) m + +/-- Auxiliary function for `buildRBTop`. -/ +partial def buildRBTopCore +{k o} {cmp} {m : Type u → Type u} [BEq k] [Inhabited o] [Monad m] +(parents : List k) (fetch : RecFetch k o (RBTopT k o cmp m)) +(key : k) : RBTopT k o cmp m o := do + -- detect cyclic builds + if parents.contains key then + throw <| key :: (parents.partition (· != key)).1 ++ [key] + -- return previous output if already built + if let some output := (← get).find? key then + return output + -- build the key recursively + let output ← fetch key (buildRBTopCore (key :: parents) fetch) + -- save the output (to prevent repeated builds of the same key) + modify (·.insert key output) + return output + +/-- + Recursively constructs an `RBMao` of key-object pairs by + fetching objects topologically (i.e., via a deep-first search with + memoization). Called a suspending scheduler in "Build systems à la carte". +-/ +def buildRBTop {k o} {cmp} {m} [BEq k] [Inhabited o] [Monad m] +(fetch : RecFetch k o (RBTopT k o cmp m)) (key : k) : RBTopT k o cmp m o := + buildRBTopCore [] fetch key diff --git a/Lake/Cli.lean b/Lake/Cli.lean index 04d741c601..b26c9a64c5 100644 --- a/Lake/Cli.lean +++ b/Lake/Cli.lean @@ -5,6 +5,7 @@ Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone -/ import Lake.Init import Lake.Build +import Lake.BuildBin import Lake.Help import Lake.LeanConfig From 93c954397622a569eba014abe9eb2cf0341f2358 Mon Sep 17 00:00:00 2001 From: tydeu Date: Thu, 15 Jul 2021 13:04:31 -0400 Subject: [PATCH 078/696] feat: add convenience functions for constructing a `LeanTrace` --- Lake/BuildTrace.lean | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Lake/BuildTrace.lean b/Lake/BuildTrace.lean index 5710f483aa..7aa9b9523b 100644 --- a/Lake/BuildTrace.lean +++ b/Lake/BuildTrace.lean @@ -48,3 +48,13 @@ structure LeanTrace where hash : Hash mtime : MTime deriving Inhabited + +namespace LeanTrace + +def fromHash (hash : Hash) : LeanTrace := + LeanTrace.mk hash 0 + +def fromMTime (mtime : MTime) : LeanTrace := + LeanTrace.mk 0 mtime + +end LeanTrace From 9e5505b6ca77dbcfb06655d3139ec2fe1ee3555b Mon Sep 17 00:00:00 2001 From: tydeu Date: Sat, 17 Jul 2021 13:02:39 -0400 Subject: [PATCH 079/696] feat: add `new` and `run` commands --- Lake/Cli.lean | 9 ++++++ Lake/Help.lean | 78 ++++++++++++++++++++++++++++++++--------------- Lake/Init.lean | 24 ++++++++++----- Lake/Package.lean | 9 +++++- 4 files changed, 86 insertions(+), 34 deletions(-) diff --git a/Lake/Cli.lean b/Lake/Cli.lean index b26c9a64c5..41f1074607 100644 --- a/Lake/Cli.lean +++ b/Lake/Cli.lean @@ -10,6 +10,7 @@ import Lake.Help import Lake.LeanConfig namespace Lake + def getCwdPkg (args : List String) : IO Package := do let pkg ← Package.fromDir "." args if pkg.leanVersion ≠ leanVersionString then @@ -17,8 +18,16 @@ def getCwdPkg (args : List String) : IO Package := do leanVersionString ++ ", but package requires " ++ pkg.leanVersion ++ "\n" return pkg +def Package.run (script : String) (args : List String) (self : Package) : IO PUnit := + if let some script := self.scripts.find? script then + script args + else + self.scripts.forM fun name _ => IO.println name + def cli : (cmd : String) → (lakeArgs pkgArgs : List String) → IO Unit +| "new", [name], [] => new name | "init", [name], [] => init name +| "run", [script], args => do (← getCwdPkg []).run script args | "configure", [], pkgArgs => do configure (← getCwdPkg pkgArgs) | "print-paths", imports, pkgArgs => do printPaths (← getCwdPkg pkgArgs) imports | "build", [], pkgArgs => do build (← getCwdPkg pkgArgs) diff --git a/Lake/Help.lean b/Lake/Help.lean index 315de06d30..41eb0bd979 100644 --- a/Lake/Help.lean +++ b/Lake/Help.lean @@ -13,7 +13,9 @@ def usage := Usage: lake +new create a Lean package in a new directory init create a Lean package in the current directory +run