From 0853d40daee12eea0730e6c51547604af5db331c Mon Sep 17 00:00:00 2001 From: Mac Malone Date: Sat, 29 Mar 2025 18:06:28 -0400 Subject: [PATCH] feat: lake: per-target external libraries (#7716) This PR adds the `moreLinkObjs` and `moreLinkLibs` options for Lean packages, libraries, and executables. These serves as functional replacements for `extern_lib` and provided additional flexibility. External libraries applied to the whole package and were necessarily static. This options are configured on a per-target basis and support shared-only libraries. **Breaking change:** `precompileModules` now only loads modules of the current library individually. Modules of other libraries are loaded together via that library's shared library. --- src/lake/Lake/Build/Actions.lean | 7 +- src/lake/Lake/Build/Common.lean | 95 +++++----- src/lake/Lake/Build/Data.lean | 16 +- src/lake/Lake/Build/Executable.lean | 25 ++- src/lake/Lake/Build/ExternLib.lean | 39 ++++- src/lake/Lake/Build/Facets.lean | 6 +- src/lake/Lake/Build/Imports.lean | 17 +- src/lake/Lake/Build/InputFile.lean | 4 +- src/lake/Lake/Build/Job/Register.lean | 1 + src/lake/Lake/Build/Key.lean | 1 + src/lake/Lake/Build/Library.lean | 53 ++++-- src/lake/Lake/Build/Module.lean | 162 ++++++++++++------ src/lake/Lake/Build/Target/Basic.lean | 7 +- src/lake/Lake/Build/Target/Fetch.lean | 31 +++- src/lake/Lake/CLI/Main.lean | 4 +- src/lake/Lake/CLI/Serve.lean | 4 +- src/lake/Lake/CLI/Translate/Lean.lean | 9 +- src/lake/Lake/CLI/Translate/Toml.lean | 7 +- src/lake/Lake/Config/ConfigTarget.lean | 3 + src/lake/Lake/Config/Dynlib.lean | 6 +- src/lake/Lake/Config/InputFile.lean | 26 +++ src/lake/Lake/Config/LeanConfig.lean | 15 +- src/lake/Lake/Config/LeanLib.lean | 30 +++- src/lake/Lake/Config/LeanLibConfig.lean | 7 +- src/lake/Lake/Config/Module.lean | 1 - src/lake/Lake/Config/Monad.lean | 4 + src/lake/Lake/Config/Package.lean | 25 ++- src/lake/Lake/Config/Workspace.lean | 8 +- src/lake/Lake/DSL/Targets.lean | 4 +- src/lake/Lake/Load/Toml.lean | 7 +- src/lake/examples/ffi/app/Test.lean | 1 + src/lake/examples/ffi/lib/c/ffi.cpp | 9 - src/lake/examples/ffi/lib/c/ffi_shared.cpp | 8 + src/lake/examples/ffi/lib/c/ffi_static.c | 5 + src/lake/examples/ffi/lib/lakefile.lean | 49 +++++- src/lake/examples/ffi/lib/lean/FFI.lean | 4 +- src/lake/examples/ffi/lib/lean/FFI/Fn.lean | 2 - .../examples/ffi/lib/lean/FFI/Shared.lean | 2 + .../lib/lean/FFI/{Add.lean => Static.lean} | 0 src/lake/examples/ffi/test.sh | 9 +- 40 files changed, 496 insertions(+), 217 deletions(-) delete mode 100644 src/lake/examples/ffi/lib/c/ffi.cpp create mode 100644 src/lake/examples/ffi/lib/c/ffi_shared.cpp create mode 100644 src/lake/examples/ffi/lib/c/ffi_static.c delete mode 100644 src/lake/examples/ffi/lib/lean/FFI/Fn.lean create mode 100644 src/lake/examples/ffi/lib/lean/FFI/Shared.lean rename src/lake/examples/ffi/lib/lean/FFI/{Add.lean => Static.lean} (100%) diff --git a/src/lake/Lake/Build/Actions.lean b/src/lake/Lake/Build/Actions.lean index e913d1202a..a36c46efda 100644 --- a/src/lake/Lake/Build/Actions.lean +++ b/src/lake/Lake/Build/Actions.lean @@ -4,6 +4,7 @@ Released under Apache 2.0 license as described in the file LICENSE. Authors: Gabriel Ebner, Sebastian Ullrich, Mac Malone, Siddharth Bhat -/ prelude +import Lake.Config.Dynlib import Lake.Util.Proc import Lake.Util.NativeLib import Lake.Util.IO @@ -21,7 +22,7 @@ def compileLeanModule (leanFile : FilePath) (oleanFile? ileanFile? cFile? bcFile?: Option FilePath) (leanPath : SearchPath := []) (rootDir : FilePath := ".") - (dynlibs : Array FilePath := #[]) (plugins : Array FilePath := #[]) + (dynlibs plugins : Array Dynlib := #[]) (leanArgs : Array String := #[]) (lean : FilePath := "lean") : LogIO Unit := do let mut args := leanArgs ++ @@ -39,9 +40,9 @@ def compileLeanModule createParentDirs bcFile args := args ++ #["-b", bcFile.toString] for dynlib in dynlibs do - args := args ++ #["--load-dynlib", dynlib.toString] + args := args ++ #["--load-dynlib", dynlib.path.toString] for plugin in plugins do - args := args ++ #["--plugin", plugin.toString] + args := args ++ #["--plugin", plugin.path.toString] args := args.push "--json" withLogErrorPos do let out ← rawProc { diff --git a/src/lake/Lake/Build/Common.lean b/src/lake/Lake/Build/Common.lean index 305d63c2df..66087354b2 100644 --- a/src/lake/Lake/Build/Common.lean +++ b/src/lake/Lake/Build/Common.lean @@ -6,6 +6,7 @@ Authors: Mac Malone prelude import Lake.Config.Monad import Lake.Util.JsonObject +import Lake.Build.Target.Fetch import Lake.Build.Actions import Lake.Build.Job @@ -382,75 +383,77 @@ def buildStaticLib compileStaticLib libFile oFiles (← getLeanAr) thin return libFile +private def mkLinkObjArgs + (objs : Array FilePath) (libs : Array Dynlib) : Array String +:= Id.run do + let mut args := #[] + for obj in objs do + args := args.push obj.toString + for lib in libs do + if let some dir := lib.dir? then + args := args.push s!"-L{dir}" + args := args.push s!"-l{lib.name}" + return args + /-- Build a shared library by linking the results of `linkJobs` using the Lean toolchain's C compiler. -/ +def buildSharedLib + (libName : String) (libFile : FilePath) + (linkObjs : Array (Job FilePath)) (linkLibs : Array (Job Dynlib)) + (weakArgs traceArgs : Array String := #[]) (linker := "c++") + (extraDepTrace : JobM _ := pure BuildTrace.nil) + (plugin := false) +: SpawnM (Job Dynlib) := + (Job.collectArray linkObjs).bindM fun objs => do + (Job.collectArray linkLibs).mapM (sync := true) fun libs => do + addPureTrace traceArgs + addPlatformTrace -- shared libraries are platform-dependent artifacts + addTrace (← extraDepTrace) + buildFileUnlessUpToDate' libFile do + let args := mkLinkObjArgs objs libs ++ weakArgs ++ traceArgs + compileSharedLib libFile args linker + return {name := libName, path := libFile, plugin} + +/-- +Build a shared library by linking the results of `linkJobs` +using `linker`. +-/ def buildLeanSharedLib - (libFile : FilePath) (linkJobs : Array (Job FilePath)) - (weakArgs traceArgs : Array String := #[]) -: SpawnM (Job FilePath) := - (Job.collectArray linkJobs).mapM fun links => do + (libName : String) (libFile : FilePath) + (linkObjs : Array (Job FilePath)) (linkLibs : Array (Job Dynlib)) + (weakArgs traceArgs : Array String := #[]) (plugin := false) +: SpawnM (Job Dynlib) := + (Job.collectArray linkObjs).bindM fun objs => do + (Job.collectArray linkLibs).mapM (sync := true) fun libs => do addLeanTrace addPureTrace traceArgs addPlatformTrace -- shared libraries are platform-dependent artifacts buildFileUnlessUpToDate' libFile do let lean ← getLeanInstall - let args := links.map toString ++ weakArgs ++ traceArgs ++ lean.ccLinkSharedFlags + let args := mkLinkObjArgs objs libs ++ + weakArgs ++ traceArgs ++ lean.ccLinkSharedFlags compileSharedLib libFile args lean.cc - return libFile + return {name := libName, path := libFile, plugin} /-- Build an executable by linking the results of `linkJobs` using the Lean toolchain's linker. -/ def buildLeanExe - (exeFile : FilePath) (linkJobs : Array (Job FilePath)) + (exeFile : FilePath) + (linkObjs : Array (Job FilePath)) (linkLibs : Array (Job Dynlib)) (weakArgs traceArgs : Array String := #[]) (sharedLean : Bool := false) : SpawnM (Job FilePath) := - (Job.collectArray linkJobs).mapM fun links => do + (Job.collectArray linkObjs).bindM fun objs => do + (Job.collectArray linkLibs).mapM (sync := true) fun libs => do addLeanTrace addPureTrace traceArgs addPlatformTrace -- executables are platform-dependent artifacts buildFileUnlessUpToDate' exeFile do let lean ← getLeanInstall - let args := links.map toString ++ weakArgs ++ traceArgs ++ lean.ccLinkFlags sharedLean + let args := mkLinkObjArgs objs libs ++ + weakArgs ++ traceArgs ++ lean.ccLinkFlags sharedLean compileExe exeFile args lean.cc return exeFile - -/-- -Build a shared library from a static library using `leanc` -using the Lean toolchain's linker. --/ -def buildLeanSharedLibOfStatic - (staticLibJob : Job FilePath) - (weakArgs traceArgs : Array String := #[]) -: SpawnM (Job FilePath) := - staticLibJob.mapM fun staticLib => do - addLeanTrace - addPureTrace traceArgs - addPlatformTrace -- shared libraries are platform-dependent artifacts - let dynlib := staticLib.withExtension sharedLibExt - buildFileUnlessUpToDate' dynlib do - let lean ← getLeanInstall - let baseArgs := - if System.Platform.isOSX then - #[s!"-Wl,-force_load,{staticLib}"] - else - #["-Wl,--whole-archive", staticLib.toString, "-Wl,--no-whole-archive"] - let args := baseArgs ++ weakArgs ++ traceArgs ++ lean.ccLinkSharedFlags - compileSharedLib dynlib args lean.cc - return dynlib - -/-- Construct a `Dynlib` object for a shared library target. -/ -def computeDynlibOfShared (sharedLibTarget : Job FilePath) : SpawnM (Job Dynlib) := - sharedLibTarget.mapM fun sharedLib => do - if let some stem := sharedLib.fileStem then - if Platform.isWindows then - return {path := sharedLib, name := stem} - else if stem.startsWith "lib" then - return {path := sharedLib, name := stem.drop 3} - else - error s!"shared library `{sharedLib}` does not start with `lib`; this is not supported on Unix" - else - error s!"shared library `{sharedLib}` has no file name" diff --git a/src/lake/Lake/Build/Data.lean b/src/lake/Lake/Build/Data.lean index cee9f040f4..289a94a7dd 100644 --- a/src/lake/Lake/Build/Data.lean +++ b/src/lake/Lake/Build/Data.lean @@ -6,6 +6,7 @@ Authors: Mac Malone prelude import Lake.Build.Key import Lake.Util.Family +import Lake.Config.Dynlib open Lean namespace Lake @@ -28,7 +29,7 @@ class DataKind (α : Type u) where /-- The name which describes `α`. -/ name : Name /-- Proof that `α` is the data type described by `name`. -/ - wf : α = DataType name + wf : ¬ name.isAnonymous ∧ α = DataType name /-- Tries to synthesize a `Name` descriptor of a data type. @@ -54,7 +55,7 @@ theorem OptDataKind.eq_data_type instance [DataKind α] : OptDataKind α where name := DataKind.name α - wf _ := DataKind.wf + wf _ := DataKind.wf.2 instance : CoeOut (OptDataKind α) Lean.Name := ⟨(·.name)⟩ instance : ToString (OptDataKind α) := ⟨(·.name.toString)⟩ @@ -177,10 +178,13 @@ scoped macro (name := dataTypeDecl) : command => do let fam := mkCIdentFrom (← getRef) ``DataType let kindName := Name.quoteFrom kind kind.getId - let id := mkIdentFrom kind (canonical := true) <| - kind.getId.modifyBase (kind.getId ++ ·) - `($[$doc?]? family_def $id : $fam $kindName := $ty - instance : DataKind $ty := ⟨$kindName, by simp⟩) + `($[$doc?]? family_def $kind : $fam $kindName := $ty + instance : DataKind $ty := ⟨$kindName, by simp [Name.isAnonymous]⟩) + +data_type unit : Unit +data_type bool : Bool +data_type file_path : System.FilePath +data_type dynlib : Dynlib /-- Internal macro for declaring new facet within Lake. -/ scoped macro (name := builtinFacetCommand) diff --git a/src/lake/Lake/Build/Executable.lean b/src/lake/Lake/Build/Executable.lean index 1a773e6c6c..5150b9db8b 100644 --- a/src/lake/Lake/Build/Executable.lean +++ b/src/lake/Lake/Build/Executable.lean @@ -20,17 +20,26 @@ def LeanExe.recBuildExe (self : LeanExe) : FetchM (Job FilePath) := so that errors in the import block of transitive imports will not kill this job before the root is built. -/ - let mut linkJobs := #[] - for facet in self.root.nativeFacets self.supportInterpreter do - linkJobs := linkJobs.push <| ← facet.fetch self.root + let mut objJobs := #[] + let mut libJobs := #[] + let shouldExport := self.supportInterpreter + for facet in self.root.nativeFacets shouldExport do + objJobs := objJobs.push <| ← facet.fetch self.root let imports ← (← self.root.transImports.fetch).await for mod in imports do - for facet in mod.nativeFacets self.supportInterpreter do - linkJobs := linkJobs.push <| ← facet.fetch mod + for facet in mod.nativeFacets shouldExport do + objJobs := objJobs.push <| ← facet.fetch mod + let libs := imports.foldl (·.insert ·.lib) OrdHashSet.empty |>.toArray + for lib in libs do + for link in lib.moreLinkObjs do + objJobs := objJobs.push <| ← link.fetchIn lib.pkg + for link in lib.moreLinkLibs do + libJobs := libJobs.push <| ← link.fetchIn lib.pkg let deps := (← (← self.pkg.transDeps.fetch).await).push self.pkg - for dep in deps do for lib in dep.externLibs do - linkJobs := linkJobs.push <| ← lib.static.fetch - buildLeanExe self.file linkJobs self.weakLinkArgs self.linkArgs self.sharedLean + for dep in deps do + for lib in dep.externLibs do + objJobs := objJobs.push <| ← lib.static.fetch + buildLeanExe self.file objJobs libJobs self.weakLinkArgs self.linkArgs self.sharedLean /-- The facet configuration for the builtin `LeanExe.exeFacet`. -/ def LeanExe.exeFacetConfig : LeanExeFacetConfig exeFacet := diff --git a/src/lake/Lake/Build/ExternLib.lean b/src/lake/Lake/Build/ExternLib.lean index 77b48d820d..02f4ce6734 100644 --- a/src/lake/Lake/Build/ExternLib.lean +++ b/src/lake/Lake/Build/ExternLib.lean @@ -10,7 +10,7 @@ import Lake.Build.Common Build function definitions for external libraries. -/ -open System (FilePath) +open System namespace Lake @@ -22,6 +22,30 @@ def ExternLib.recBuildStatic (lib : ExternLib) : FetchM (Job FilePath) := def ExternLib.staticFacetConfig : ExternLibFacetConfig staticFacet := mkFacetJobConfig recBuildStatic +/-- +Build a shared library from a static library using `leanc` +using the Lean toolchain's linker. +-/ +def buildLeanSharedLibOfStatic + (staticLibJob : Job FilePath) + (weakArgs traceArgs : Array String := #[]) +: SpawnM (Job FilePath) := + staticLibJob.mapM fun staticLib => do + addLeanTrace + addPureTrace traceArgs + addPlatformTrace -- shared libraries are platform-dependent artifacts + let dynlib := staticLib.withExtension sharedLibExt + buildFileUnlessUpToDate' dynlib do + let lean ← getLeanInstall + let baseArgs := + if System.Platform.isOSX then + #[s!"-Wl,-force_load,{staticLib}"] + else + #["-Wl,--whole-archive", staticLib.toString, "-Wl,--no-whole-archive"] + let args := baseArgs ++ weakArgs ++ traceArgs ++ lean.ccLinkSharedFlags + compileSharedLib dynlib args lean.cc + return dynlib + def ExternLib.recBuildShared (lib : ExternLib) : FetchM (Job FilePath) := withRegisterJob s!"{lib.staticTargetName.toString}:shared" do buildLeanSharedLibOfStatic (← lib.static.fetch) lib.linkArgs @@ -30,6 +54,19 @@ def ExternLib.recBuildShared (lib : ExternLib) : FetchM (Job FilePath) := def ExternLib.sharedFacetConfig : ExternLibFacetConfig sharedFacet := mkFacetJobConfig recBuildShared +/-- Construct a `Dynlib` object for a shared library target. -/ +def computeDynlibOfShared (sharedLibTarget : Job FilePath) : SpawnM (Job Dynlib) := + sharedLibTarget.mapM fun sharedLib => do + if let some stem := sharedLib.fileStem then + if Platform.isWindows then + return {path := sharedLib, name := stem} + else if stem.startsWith "lib" then + return {path := sharedLib, name := stem.drop 3} + else + error s!"shared library `{sharedLib}` does not start with `lib`; this is not supported on Unix" + else + error s!"shared library `{sharedLib}` has no file name" + def ExternLib.recComputeDynlib (lib : ExternLib) : FetchM (Job Dynlib) := do withRegisterJob s!"{lib.staticTargetName.toString}:dynlib" do computeDynlibOfShared (← lib.shared.fetch) diff --git a/src/lake/Lake/Build/Facets.lean b/src/lake/Lake/Build/Facets.lean index bcd502b326..225a50cec2 100644 --- a/src/lake/Lake/Build/Facets.lean +++ b/src/lake/Lake/Build/Facets.lean @@ -23,8 +23,8 @@ open Lean hiding SearchPath namespace Lake structure ModuleDeps where - dynlibs : Array FilePath := #[] - plugins : Array FilePath := #[] + dynlibs : Array Dynlib := #[] + plugins : Array Dynlib := #[] deriving Inhabited, Repr /-! ## Module Facets -/ @@ -168,7 +168,7 @@ much better for distribution. builtin_facet staticExportFacet @ static.export : LeanLib => FilePath /-- A Lean library's shared artifact. -/ -builtin_facet shared : LeanLib => FilePath +builtin_facet shared : LeanLib => Dynlib /-- A Lean library's `extraDepTargets` mixed with its package's. -/ builtin_facet extraDep : LeanLib => Unit diff --git a/src/lake/Lake/Build/Imports.lean b/src/lake/Lake/Build/Imports.lean index 5a8ad5cc69..feb1b3ae92 100644 --- a/src/lake/Lake/Build/Imports.lean +++ b/src/lake/Lake/Build/Imports.lean @@ -23,25 +23,22 @@ def buildImportsAndDeps (leanFile : FilePath) (imports : Array Module) : FetchM (Job ModuleDeps) := do withRegisterJob s!"setup ({leanFile})" do + let root ← getRootPackage if imports.isEmpty then -- build the package's (and its dependencies') `extraDepTarget` - (← getRootPackage).extraDep.fetch <&> (·.map fun _ => {}) + root.extraDep.fetch <&> (·.map fun _ => {}) else -- build local imports from list let modJob := Job.mixArray <| ← imports.mapM (·.olean.fetch) let precompileImports ← (← computePrecompileImportsAux leanFile imports).await let pkgs := precompileImports.foldl (·.insert ·.pkg) OrdPackageSet.empty |>.toArray let externLibsJob ← fetchExternLibs pkgs - let modLibsJob ← Job.collectArray <$> - precompileImports.mapM (·.dynlib.fetch) - let dynlibsJob ← (← getRootPackage).dynlibs.fetch - let pluginsJob ← (← getRootPackage).plugins.fetch + let impLibsJob ← fetchImportLibs imports + let dynlibsJob ← root.dynlibs.fetchIn root + let pluginsJob ← root.plugins.fetchIn root modJob.bindM fun _ => - modLibsJob.bindM fun modLibs => + impLibsJob.bindM fun impLibs => dynlibsJob.bindM fun dynlibs => pluginsJob.bindM fun plugins => externLibsJob.mapM fun externLibs => do - -- NOTE: Lean wants the external library symbols before module symbols - let dynlibs := (externLibs ++ dynlibs).map (·.path) - let plugins := (modLibs ++ plugins).map (·.path) - return {dynlibs, plugins} + return computeModuleDeps impLibs externLibs dynlibs plugins diff --git a/src/lake/Lake/Build/InputFile.lean b/src/lake/Lake/Build/InputFile.lean index a0891eceea..39d77f5843 100644 --- a/src/lake/Lake/Build/InputFile.lean +++ b/src/lake/Lake/Build/InputFile.lean @@ -19,7 +19,7 @@ namespace Lake private def InputFile.recFetch (t : InputFile) : FetchM (Job FilePath) := withRegisterJob s!"{t.name}" do - inputFile t.config.path t.config.text + inputFile t.path t.text /-- The facet configuration for the builtin `ExternLib.staticFacet`. -/ def InputFile.defaultFacetConfig : KFacetConfig InputFile.facetKind defaultFacet := @@ -37,7 +37,7 @@ def InputFile.initFacetConfigs : DNameMap (KFacetConfig InputFile.facetKind) := private def InputDir.recFetch (t : InputDir) : FetchM (Job (Array FilePath)) := withRegisterJob s!"{t.name}" do - inputDir t.config.path t.config.text t.config.filter.matches + inputDir t.path t.text t.filter /-- The facet configuration for the builtin `ExternLib.staticFacet`. -/ def InputDir.defaultFacetConfig : KFacetConfig InputDir.facetKind defaultFacet := diff --git a/src/lake/Lake/Build/Job/Register.lean b/src/lake/Lake/Build/Job/Register.lean index a418994438..eacb89dca0 100644 --- a/src/lake/Lake/Build/Job/Register.lean +++ b/src/lake/Lake/Build/Job/Register.lean @@ -24,6 +24,7 @@ Preserves information that downstream jobs want to depend on while resetting job-local information that should not be inherited by downstream jobs. -/ def Job.renew (self : Job α) : Job α := + have : OptDataKind α := self.kind self.mapResult (sync := true) fun | .ok a s => .ok a s.renew | .error _ s => .error 0 s.renew diff --git a/src/lake/Lake/Build/Key.lean b/src/lake/Lake/Build/Key.lean index a8d48a856a..a3f936ec14 100644 --- a/src/lake/Lake/Build/Key.lean +++ b/src/lake/Lake/Build/Key.lean @@ -96,6 +96,7 @@ def PartialBuildKey.toString : (self : PartialBuildKey) → String | .facet t f => if f.isAnonymous then toString t else s!"{toString t}:{f}" instance : ToString PartialBuildKey := ⟨PartialBuildKey.toString⟩ +instance : Repr PartialBuildKey := inferInstanceAs (Repr BuildKey) namespace BuildKey diff --git a/src/lake/Lake/Build/Library.lean b/src/lake/Lake/Build/Library.lean index 029b1bbbbc..b6c5cee547 100644 --- a/src/lake/Lake/Build/Library.lean +++ b/src/lake/Lake/Build/Library.lean @@ -6,14 +6,14 @@ Authors: Mac Malone prelude import Lake.Build.Common import Lake.Build.Targets -import Lake.Build.Target.Fetch /-! # Library Facet Builds Build function definitions for a library's builtin facets. -/ +open System Lean + namespace Lake -open System (FilePath) /-! ## Build Lean & Static Lib -/ @@ -21,7 +21,9 @@ open System (FilePath) Collect the local modules of a library. That is, the modules from `getModuleArray` plus their local transitive imports. -/ -partial def LeanLib.recCollectLocalModules (self : LeanLib) : FetchM (Job (Array Module)) := ensureJob do +partial def LeanLib.recCollectLocalModules + (self : LeanLib) : FetchM (Job (Array Module)) +:= ensureJob do let mut mods := #[] let mut modSet := ModuleSet.empty for mod in (← self.getModuleArray) do @@ -35,7 +37,7 @@ where modSet := modSet.insert root let imps ← (← root.imports.fetch).await for mod in imps do - if self.isLocalModule mod.name then + if mod.lib.name = self.name then (mods, modSet) ← go mod mods modSet mods := mods.push root return (mods, modSet) @@ -45,7 +47,8 @@ def LeanLib.modulesFacetConfig : LibraryFacetConfig modulesFacet := mkFacetJobConfig LeanLib.recCollectLocalModules (buildable := false) protected def LeanLib.recBuildLean -(self : LeanLib) : FetchM (Job Unit) := do + (self : LeanLib) : FetchM (Job Unit) +:= do let mods ← (← self.modules.fetch).await mods.foldlM (init := Job.nil) fun job mod => do return job.mix <| ← mod.leanArts.fetch @@ -55,7 +58,8 @@ def LeanLib.leanArtsFacetConfig : LibraryFacetConfig leanArtsFacet := mkFacetJobConfig LeanLib.recBuildLean @[specialize] protected def LeanLib.recBuildStatic -(self : LeanLib) (shouldExport : Bool) : FetchM (Job FilePath) := do + (self : LeanLib) (shouldExport : Bool) : FetchM (Job FilePath) +:= do let suffix := if (← getIsVerbose) then if shouldExport then " (with exports)" else " (without exports)" @@ -65,13 +69,14 @@ def LeanLib.leanArtsFacetConfig : LibraryFacetConfig leanArtsFacet := let mods ← (← self.modules.fetch).await let oJobs ← mods.flatMapM fun mod => mod.nativeFacets shouldExport |>.mapM (·.fetch mod) + let moreOJobs ← self.moreLinkObjs.mapM (·.fetchIn self.pkg) let libFile := if shouldExport then self.staticExportLibFile else self.staticLibFile /- Static libraries with explicit exports are built as thin libraries. The Lean build itself requires a thin static library with exported symbols as part of its build process on Windows. It does not distribute this library. -/ - buildStaticLib libFile oJobs (thin := shouldExport) + buildStaticLib libFile (oJobs ++ moreOJobs) (thin := shouldExport) /-- The `LibraryFacetConfig` for the builtin `staticFacet`. -/ def LeanLib.staticFacetConfig : LibraryFacetConfig staticFacet := @@ -83,15 +88,37 @@ def LeanLib.staticExportFacetConfig : LibraryFacetConfig staticExportFacet := /-! ## Build Shared Lib -/ -protected def LeanLib.recBuildShared -(self : LeanLib) : FetchM (Job FilePath) := do +protected def LeanLib.recBuildShared (self : LeanLib) : FetchM (Job Dynlib) := do withRegisterJob s!"{self.name}:shared" do let mods ← (← self.modules.fetch).await - let oJobs ← mods.flatMapM fun mod => + let objJobs ← mods.flatMapM fun mod => mod.nativeFacets true |>.mapM (·.fetch mod) - let pkgs := mods.foldl (·.insert ·.pkg) OrdPackageSet.empty |>.toArray - let externJobs ← pkgs.flatMapM (·.externLibs.mapM (·.shared.fetch)) - buildLeanSharedLib self.sharedLibFile (oJobs ++ externJobs) self.weakLinkArgs self.linkArgs + let objJobs ← self.moreLinkObjs.foldlM (init := objJobs) + (·.push <$> ·.fetchIn self.pkg) + let libJobs ← id do + -- Fetch dependnecy dynlibs + -- for platforms that must link to them (e.g., Windows) + if Platform.isWindows then + let imps ← mods.foldlM (init := OrdModuleSet.empty) fun imps mod => do + return imps.appendArray (← (← mod.transImports.fetch).await) + let s := (NameSet.empty.insert self.name, #[]) + let (_, jobs) ← imps.foldlM (init := s) fun (libs, jobs) imp => do + if libs.contains imp.lib.name then + return (libs, jobs) + else + let job ← imp.lib.shared.fetch + let moreJobs ← imp.lib.moreLinkLibs.mapM (·.fetchIn self.pkg) + let jobs := jobs.push job ++ moreJobs + return (libs.insert imp.lib.name, jobs) + let jobs ← self.moreLinkLibs.foldlM + (·.push <$> ·.fetchIn self.pkg) jobs + let jobs ← self.pkg.externLibs.foldlM + (·.push <$> ·.dynlib.fetch) jobs + return jobs + else + return #[] + buildLeanSharedLib self.libName self.sharedLibFile objJobs libJobs + self.weakLinkArgs self.linkArgs (plugin := self.roots.size == 1) /-- The `LibraryFacetConfig` for the builtin `sharedFacet`. -/ def LeanLib.sharedFacetConfig : LibraryFacetConfig sharedFacet := diff --git a/src/lake/Lake/Build/Module.lean b/src/lake/Lake/Build/Module.lean index c7c0129bf1..d51d0b76a9 100644 --- a/src/lake/Lake/Build/Module.lean +++ b/src/lake/Lake/Build/Module.lean @@ -14,17 +14,18 @@ import Lake.Build.Target Build function definitions for a module's builtin facets. -/ -open System +open System Lean namespace Lake /-- Compute library directories and build external library Jobs of the given packages. -/ +@[deprecated "Deprecated without replacement" (since := "2025-03-28")] def recBuildExternDynlibs (pkgs : Array Package) : FetchM (Array (Job Dynlib) × Array FilePath) := do let mut libDirs := #[] let mut jobs : Array (Job Dynlib) := #[] for pkg in pkgs do - libDirs := libDirs.push pkg.nativeLibDir + libDirs := libDirs.push pkg.sharedLibDir jobs := jobs.append <| ← pkg.externLibs.mapM (·.dynlib.fetch) return (jobs, libDirs) @@ -104,6 +105,72 @@ def Module.precompileImportsFacetConfig : ModuleFacetConfig precompileImportsFac def fetchExternLibs (pkgs : Array Package) : FetchM (Job (Array Dynlib)) := Job.collectArray <$> pkgs.flatMapM (·.externLibs.mapM (·.dynlib.fetch)) +private def Module.fetchImportLibsCore + (self : Module) (imps : Array Module) + (init : NameSet × Array (Job Dynlib)): FetchM (NameSet × Array (Job Dynlib)) +:= imps.foldlM (init := init) fun (libs, jobs) imp => do + if libs.contains imp.lib.name then + return (libs, jobs) + else if self.lib.name = imp.lib.name then + let job ← imp.dynlib.fetch + return (libs, jobs.push job) + else + + let jobs ← imp.lib.moreLinkLibs.foldlM (init := jobs) + (·.push <$> ·.fetchIn imp.pkg) + -- Lean wants the external library symbols before module symbols. + let jobs ← jobs.push <$> imp.lib.shared.fetch + return (libs.insert imp.lib.name, jobs) + +/-- +Computes the transitive dynamic libraries of a module's imports. +Modules from the same library are loaded individually, while modules +from other libraries are loaded as part of the whole library. +-/ +@[inline] private def Module.fetchImportLibs + (self : Module) (imps : Array Module) : FetchM (Job (Array Dynlib)) +:= do + let s ← self.fetchImportLibsCore imps ({}, #[]) + return Job.collectArray s.2 + +/-- Fetch the dynlibs of a list of imports. **For internal use.** -/ +@[inline] def fetchImportLibs + (mods : Array Module) : FetchM (Job (Array Dynlib)) +:= do + let (_, jobs) ← mods.foldlM (init := ({}, #[])) fun s mod => do + let precompileImports ← + if mod.shouldPrecompile + then mod.transImports.fetch + else mod.precompileImports.fetch + let precompileImports ← precompileImports.await + mod.fetchImportLibsCore precompileImports s + let jobs ← mods.foldlM (init := jobs) fun jobs mod => do + if mod.shouldPrecompile + then jobs.push <$> mod.dynlib.fetch + else return jobs + return Job.collectArray jobs + +def computeModuleDeps + (impLibs : Array Dynlib) (externLibs : Array Dynlib) + (dynlibs : Array Dynlib) (plugins : Array Dynlib) +: ModuleDeps := Id.run do + /- + Requirements: + * Lean wants the external library symbols before module symbols. + * Unix requires the file extension of the dynlib. + * For some reason, building from the Lean server requires full paths. + Everything else loads fine with just the augmented library path. + * Linux needs the augmented path to resolve nested dependencies in dynlibs. + -/ + let mut plugins := plugins + let mut dynlibs := externLibs ++ dynlibs + for impLib in impLibs do + if impLib.plugin then + plugins := plugins.push impLib + else + dynlibs := dynlibs.push impLib + return {dynlibs, plugins} + /-- Recursively build a module's dependencies, including: * Transitive local imports @@ -126,36 +193,25 @@ def Module.recBuildDeps (mod : Module) : FetchM (Job ModuleDeps) := ensureJob do let precompileImports ← if mod.shouldPrecompile then mod.transImports.fetch else mod.precompileImports.fetch let precompileImports ← precompileImports.await - let modLibJobs ← precompileImports.mapM (·.dynlib.fetch) - let modLibsJob := Job.collectArray modLibJobs + let importLibsJob ← mod.fetchImportLibs precompileImports let pkgs := precompileImports.foldl (·.insert ·.pkg) OrdPackageSet.empty let pkgs := if mod.shouldPrecompile then pkgs.insert mod.pkg else pkgs let externLibsJob ← fetchExternLibs pkgs.toArray - let dynlibsJob ← mod.dynlibs.fetch - let pluginsJob ← mod.plugins.fetch + let dynlibsJob ← mod.dynlibs.fetchIn mod.pkg + let pluginsJob ← mod.plugins.fetchIn mod.pkg extraDepJob.bindM fun _ => do - importJob.bindM fun _ => do + importJob.bindM (sync := true) fun _ => do let depTrace ← takeTrace - modLibsJob.bindM fun modLibs => do - externLibsJob.bindM fun externLibs => do - dynlibsJob.bindM fun dynlibs => do - pluginsJob.mapM fun plugins => do + importLibsJob.bindM (sync := true) fun impLibs => do + externLibsJob.bindM (sync := true) fun externLibs => do + dynlibsJob.bindM (sync := true) fun dynlibs => do + pluginsJob.mapM (sync := true) fun plugins => do match mod.platformIndependent with | none => addTrace depTrace | some false => addTrace depTrace; addPlatformTrace | some true => setTrace depTrace - /- - Requirements: - * Lean wants the external library symbols before module symbols. - * Unix requires the file extension of the dynlib. - * For some reason, building from the Lean server requires full paths. - Everything else loads fine with just the augmented library path. - * Linux needs the augmented path to resolve nested dependencies in dynlibs. - -/ - let dynlibs := externLibs.map (·.path) ++ dynlibs.map (·.path) - let plugins := modLibs.map (·.path) ++ plugins.map (·.path) - return {dynlibs, plugins} + return computeModuleDeps impLibs externLibs dynlibs plugins /-- The `ModuleFacetConfig` for the builtin `depsFacet`. -/ def Module.depsFacetConfig : ModuleFacetConfig depsFacet := @@ -315,41 +371,39 @@ def Module.recBuildDynlib (mod : Module) : FetchM (Job Dynlib) := withRegisterJob s!"{mod.name}:dynlib" do -- Fetch object files - let linkJobs ← mod.nativeFacets true |>.mapM (·.fetch mod) - let linksJob := Job.collectArray linkJobs + let modLinkJobs ← mod.nativeFacets true |>.mapM (·.fetch mod) + let modLinksJob := Job.collectArray modLinkJobs + + -- Build dynlib + let buildDynlib (moreLibs : Array Dynlib) := + modLinksJob.mapM fun modLinks => do + addLeanTrace + addPlatformTrace -- shared libraries are platform-dependent artifacts + addPureTrace mod.linkArgs + buildFileUnlessUpToDate' mod.dynlibFile do + let lean ← getLeanInstall + let args := + modLinks.map toString ++ + moreLibs.map (·.path.toString) ++ + mod.weakLinkArgs ++ mod.linkArgs ++ lean.ccLinkSharedFlags + compileSharedLib mod.dynlibFile args lean.cc + return ⟨mod.dynlibFile, mod.dynlibName, true⟩ -- Fetch dependencies' dynlibs -- for platforms that must link to them (e.g., Windows) - let (modLibsJob, externLibsJob) ← id do - if Platform.isWindows then - let transImports ← (← mod.transImports.fetch).await - let modLibsJob ← Job.collectArray <$> transImports.mapM (·.dynlib.fetch) - let pkgs := transImports.foldl (·.insert ·.pkg) - OrdPackageSet.empty |>.insert mod.pkg |>.toArray - let externLibsJob ← fetchExternLibs pkgs - return (modLibsJob, externLibsJob) - else - return (Job.pure #[], Job.pure #[]) - - -- Build dynlib - linksJob.bindM fun links => do - modLibsJob.bindM fun modLibs => do - externLibsJob.mapM fun externLibs => do - addLeanTrace - addPlatformTrace -- shared libraries are platform-dependent artifacts - addPureTrace mod.linkArgs - buildFileUnlessUpToDate' mod.dynlibFile do - let lean ← getLeanInstall - let args := links.map toString - let args := - if Platform.isWindows then - args ++ (modLibs ++ externLibs).map (·.path.toString) - else - args - let args := args ++ - mod.weakLinkArgs ++ mod.linkArgs ++ lean.ccLinkSharedFlags - compileSharedLib mod.dynlibFile args lean.cc - return ⟨mod.dynlibFile, mod.dynlibName⟩ + if Platform.isWindows then + let imps ← (← mod.transImports.fetch).await + let impLibs ← mod.fetchImportLibs imps + let dynlibsJob ← (mod.dynlibs ++ mod.plugins).fetchIn mod.pkg + let pkgs := imps.foldl (·.insert ·.pkg) OrdPackageSet.empty + |>.insert mod.pkg |>.toArray + let externLibsJob ← fetchExternLibs pkgs + impLibs.bindM fun impLibs => + dynlibsJob.bindM (sync := true) fun dynlibs => + externLibsJob.bindM (sync := true) fun externLibs => + buildDynlib (impLibs ++ externLibs ++ dynlibs) + else + buildDynlib #[] /-- The `ModuleFacetConfig` for the builtin `dynlibFacet`. -/ def Module.dynlibFacetConfig : ModuleFacetConfig dynlibFacet := diff --git a/src/lake/Lake/Build/Target/Basic.lean b/src/lake/Lake/Build/Target/Basic.lean index 669d9a72ef..bc5346f5ca 100644 --- a/src/lake/Lake/Build/Target/Basic.lean +++ b/src/lake/Lake/Build/Target/Basic.lean @@ -16,11 +16,9 @@ open Std namespace Lake -/-- A Lake target that is known to produce an output of a specific type. -/ +/-- A Lake target that is expected to produce an output of a specific type. -/ structure Target (α : Type) where - key : BuildKey - [data_def : FamilyDef BuildData key α] - deriving Repr + key : PartialBuildKey protected def Target.repr (x : Target α) (prec : Nat) : Format := let indent := if prec >= max_prec then 1 else 2 @@ -29,6 +27,7 @@ protected def Target.repr (x : Target α) (prec : Nat) : Format := instance : Repr (Target α) := ⟨Target.repr⟩ instance : ToString (Target α) := ⟨(·.key.toString)⟩ +instance : Coe PartialBuildKey (Target α) := ⟨Target.mk⟩ /-- Shorthand for `Array (Target α)` that supports diff --git a/src/lake/Lake/Build/Target/Fetch.lean b/src/lake/Lake/Build/Target/Fetch.lean index ec1e5f9579..13b427a325 100644 --- a/src/lake/Lake/Build/Target/Fetch.lean +++ b/src/lake/Lake/Build/Target/Fetch.lean @@ -12,7 +12,7 @@ open Lean namespace Lake variable (defaultPkg : Package) (root : BuildKey) in -private def PartialBuildKey.fetchInCore +private def PartialBuildKey.fetchInCoreAux (self : PartialBuildKey) (facetless : Bool := false) : FetchM ((key : BuildKey) × Job (BuildData key)) := do match self with @@ -48,7 +48,7 @@ private def PartialBuildKey.fetchInCore let job ← (pkg.target target).fetch return ⟨key, cast (by simp) job⟩ | .facet target shortFacet => - let ⟨key, job⟩ ← PartialBuildKey.fetchInCore target false + let ⟨key, job⟩ ← PartialBuildKey.fetchInCoreAux target false let kind := job.kind if h : kind.isAnonymous then error s!"invalid target '{root}': targets of opaque data kinds do not support facets" @@ -67,6 +67,11 @@ where | error s!"invalid target '{root}': package '{name}' not found in workspace" return pkg +@[inline] private def PartialBuildKey.fetchInCore + (defaultPkg : Package) (self : PartialBuildKey) +: FetchM ((key : BuildKey) × Job (BuildData key)) := + fetchInCoreAux defaultPkg self self true + /-- Fetches the target specified by this key, resolving gaps as needed. @@ -78,7 +83,7 @@ Fetches the target specified by this key, resolving gaps as needed. rather than their configuration. -/ @[inline] def PartialBuildKey.fetchIn (defaultPkg : Package) (self : PartialBuildKey) : FetchM OpaqueJob := - (·.2.toOpaque) <$> fetchInCore defaultPkg self self true + (·.2.toOpaque) <$> fetchInCore defaultPkg self variable (root : BuildKey) in private def BuildKey.fetchCore @@ -109,8 +114,20 @@ private def BuildKey.fetchCore (self : BuildKey) [FamilyOut BuildData self α] : FetchM (Job α) := cast (by simp) <| fetchCore self self -@[inline] protected def Target.fetch (self : Target α) : FetchM (Job α) := do - have := self.data_def; self.key.fetch +protected def Target.fetchIn + [DataKind α] (defaultPkg : Package) (self : Target α) : FetchM (Job α) +:= do + let ⟨_, job⟩ ← self.key.fetchInCore defaultPkg + have ⟨kind, ⟨h_anon, h_kind⟩⟩ := inferInstanceAs (DataKind α) + if h : job.kind.name = kind then + have h := by + have h_job := h ▸ job.kind.wf + rw [h_job h_anon, h_kind] + return cast h job + else + let actual := if job.kind.name.isAnonymous then "unknown" else s!"'{job.kind.name}'" + error s!"type mismtach in target '{self.key}': expected '{kind}', got {actual}" -protected def TargetArray.fetch (self : TargetArray α) : FetchM (Job (Array α)) := do - Job.collectArray <$> self.mapM (·.fetch) +protected def TargetArray.fetchIn + [DataKind α] (defaultPkg : Package) (self : TargetArray α) : FetchM (Job (Array α)) +:= Job.collectArray <$> self.mapM (·.fetchIn defaultPkg) diff --git a/src/lake/Lake/CLI/Main.lean b/src/lake/Lake/CLI/Main.lean index 897f914849..06fbafff31 100644 --- a/src/lake/Lake/CLI/Main.lean +++ b/src/lake/Lake/CLI/Main.lean @@ -522,8 +522,8 @@ protected def lean : CliM PUnit := do let spawnArgs := { args := #[leanFile] ++ - dynlibs.map (s!"--load-dynlib={·}") ++ - plugins.map (s!"--plugin={·}") ++ + dynlibs.map (s!"--load-dynlib={·.path}") ++ + plugins.map (s!"--plugin={·.path}") ++ ws.root.moreLeanArgs ++ opts.subArgs cmd := ws.lakeEnv.lean.lean.toString env := ws.augmentedEnvVars diff --git a/src/lake/Lake/CLI/Serve.lean b/src/lake/Lake/CLI/Serve.lean index fc36f47316..1491d7de71 100644 --- a/src/lake/Lake/CLI/Serve.lean +++ b/src/lake/Lake/CLI/Serve.lean @@ -65,8 +65,8 @@ def setupFile let paths : LeanPaths := { oleanPath := ws.leanPath srcPath := ws.leanSrcPath - loadDynlibPaths := dynlibs - pluginPaths := plugins + loadDynlibPaths := dynlibs.map (·.path) + pluginPaths := plugins.map (·.path) } let setupOptions : LeanOptions ← do let some moduleName ← searchModuleNameOfFileName path ws.leanSrcPath diff --git a/src/lake/Lake/CLI/Translate/Lean.lean b/src/lake/Lake/CLI/Translate/Lean.lean index a8bfe50181..49ad2ca059 100644 --- a/src/lake/Lake/CLI/Translate/Lean.lean +++ b/src/lake/Lake/CLI/Translate/Lean.lean @@ -204,7 +204,7 @@ protected def PathPatDescr.toLean? (p : PathPatDescr) : Option Term := instance : ToLean? PathPatDescr := ⟨PathPatDescr.toLean?⟩ -@[inline] protected def PartialBuildKey.toLean (k : BuildKey) : Term := +@[inline] protected def PartialBuildKey.toLean (k : PartialBuildKey) : Term := go k [] where go k (fs : List Name) := Unhygienic.run do @@ -232,6 +232,7 @@ where facets.toArray.map fun f => Unhygienic.run `(facetSuffix|:$(mkIdent f)) instance : ToLean PartialBuildKey := ⟨PartialBuildKey.toLean⟩ +instance : ToLean (Target α) := ⟨(·.key.toLean)⟩ /-! ## Dependency Configuration Encoder -/ @@ -281,17 +282,15 @@ local macro "gen_lean_encoders%" : command => do let cmds := #[] -- Targets let cmds ← genMkDeclFields cmds ``LeanConfig false - (exclude := #[`dynlibs, `plugins]) let cmds ← genMkDeclFields cmds ``LeanLibConfig true - (exclude := #[`nativeFacets, `dynlibs, `plugins]) + (exclude := #[`nativeFacets]) let cmds ← genMkDeclFields cmds ``LeanExeConfig true - (exclude := #[`nativeFacets, `dynlibs, `plugins]) + (exclude := #[`nativeFacets]) let cmds ← genMkDeclFields cmds ``InputFileConfig true let cmds ← genMkDeclFields cmds ``InputDirConfig true -- Package let cmds ← genMkDeclFields cmds ``WorkspaceConfig false let cmds ← genMkDeclFields cmds ``PackageConfig true - (exclude := #[`nativeFacets, `dynlibs, `plugins]) return ⟨mkNullNode cmds⟩ gen_lean_encoders% diff --git a/src/lake/Lake/CLI/Translate/Toml.lean b/src/lake/Lake/CLI/Translate/Toml.lean index 9ab8f66a95..306c07bc0f 100644 --- a/src/lake/Lake/CLI/Translate/Toml.lean +++ b/src/lake/Lake/CLI/Translate/Toml.lean @@ -118,6 +118,7 @@ instance : EncodeField (LeanLibConfig n) `defaultFacets (Array Name) := ⟨encod instance : ToToml BuildKey := ⟨(toToml ·.toString)⟩ instance : ToToml PartialBuildKey := ⟨(toToml ·.toString)⟩ +instance : ToToml (Target α) := ⟨(toToml ·.key.toString)⟩ /-! ## Dependency Configuration Encoders -/ @@ -164,17 +165,15 @@ local macro "gen_toml_encoders%" : command => do let cmds := #[] -- Targets let cmds ← genToToml cmds ``LeanConfig false - (exclude := #[`dynlibs, `plugins]) let cmds ← genToToml cmds ``LeanLibConfig true - (exclude := #[`nativeFacets, `dynlibs, `plugins]) + (exclude := #[`nativeFacets]) let cmds ← genToToml cmds ``LeanExeConfig true - (exclude := #[`nativeFacets, `dynlibs, `plugins]) + (exclude := #[`nativeFacets]) let cmds ← genToToml cmds ``InputFileConfig true let cmds ← genToToml cmds ``InputDirConfig true -- Package let cmds ← genToToml cmds ``WorkspaceConfig false let cmds ← genToToml cmds ``PackageConfig true - (exclude := #[`nativeFacets, `dynlibs, `plugins]) return ⟨mkNullNode cmds⟩ gen_toml_encoders% diff --git a/src/lake/Lake/Config/ConfigTarget.lean b/src/lake/Lake/Config/ConfigTarget.lean index 7e7af93d14..446da811ec 100644 --- a/src/lake/Lake/Config/ConfigTarget.lean +++ b/src/lake/Lake/Config/ConfigTarget.lean @@ -22,6 +22,9 @@ structure ConfigTarget (kind : Name) where /-- The target's user-defined configuration. -/ config : ConfigType kind pkg.name name +instance : Hashable (ConfigTarget k) := ⟨(hash ·.name)⟩ +instance : BEq (ConfigTarget k) := ⟨(·.name == ·.name)⟩ + @[simp] axiom OpaqueConfigTarget.def {k : Name} : OpaqueConfigTarget k = ConfigTarget k @[inline] def PConfigDecl.mkConfigTarget (pkg : Package) (self : PConfigDecl pkg.name) : ConfigTarget self.kind := diff --git a/src/lake/Lake/Config/Dynlib.lean b/src/lake/Lake/Config/Dynlib.lean index ef22f1ed82..46b10ac83f 100644 --- a/src/lake/Lake/Config/Dynlib.lean +++ b/src/lake/Lake/Config/Dynlib.lean @@ -10,12 +10,15 @@ open System Lean namespace Lake -/-- A dynamic/shared library for linking. -/ +/-- A dynamic/shared library artifact for linking. -/ structure Dynlib where /-- Library file path. -/ path : FilePath /-- Library name without platform-specific prefix/suffix (for `-l`). -/ name : String + /-- Whether this library can be loaded as a plugin. -/ + plugin := false + deriving Inhabited, Repr /-- Optional library directory (for `-L`). -/ def Dynlib.dir? (self : Dynlib) : Option FilePath := @@ -23,3 +26,4 @@ def Dynlib.dir? (self : Dynlib) : Option FilePath := instance : ToText Dynlib := ⟨(·.path.toString)⟩ instance : ToJson Dynlib := ⟨(·.path.toString)⟩ +instance : Coe Dynlib FilePath := ⟨(·.path)⟩ diff --git a/src/lake/Lake/Config/InputFile.lean b/src/lake/Lake/Config/InputFile.lean index 855e425193..1618720f87 100644 --- a/src/lake/Lake/Config/InputFile.lean +++ b/src/lake/Lake/Config/InputFile.lean @@ -12,5 +12,31 @@ open Lean System /-- An input file -- its package plus its configuration. -/ abbrev InputFile := ConfigTarget InputFile.configKind +/-- +The full path to the input file. +That is, the package's `dir` joined with the file's `path`. +-/ +@[inline] def InputFile.path (self : InputFile) : FilePath := + self.pkg.dir / self.config.path + +/-- Whether this is a text file. -/ +@[inline] def InputFile.text (self : InputFile) : Bool := + self.config.text + /-- An input directory -- its package plus its configuration. -/ abbrev InputDir := ConfigTarget InputDir.configKind + +/-- +The full path to the input file. +That is, the package's `dir` joined with the file's `path`. +-/ +@[inline] def InputDir.path (self : InputDir) : FilePath := + self.pkg.dir / self.config.path + +/-- Whether this directory contains text files. -/ +@[inline] def InputDir.text (self : InputDir) : Bool := + self.config.text + +/-- The file inclusion filter function for the input directory. -/ +@[inline] def InputDir.filter (self : InputDir) : FilePath → Bool := + self.config.filter.matches diff --git a/src/lake/Lake/Config/LeanConfig.lean b/src/lake/Lake/Config/LeanConfig.lean index a75f6dde0b..286217c6ef 100644 --- a/src/lake/Lake/Config/LeanConfig.lean +++ b/src/lake/Lake/Config/LeanConfig.lean @@ -183,15 +183,26 @@ configuration LeanConfig where -/ weakLeancArgs : Array String := #[] /-- + Additional target objects to use when linking (both static and shared). + These will come *after* the paths of native facets. + -/ + moreLinkObjs : TargetArray FilePath := #[] + /-- + Additional target libraries to pass to `leanc` when linking + (e.g., for shared libraries or binary executables). + These will come *after* the paths of other link objects. + -/ + moreLinkLibs : TargetArray Dynlib := #[] + /-- Additional arguments to pass to `leanc` when linking (e.g., for shared libraries or binary executables). These will come *after* the paths of - external libraries. + the linked objects. -/ moreLinkArgs : Array String := #[] /-- Additional arguments to pass to `leanc` when linking (e.g., for shared libraries or binary executables). These will come *after* the paths of - external libraries. + the linked objects. Unlike `moreLinkArgs`, these arguments do not affect the trace of the build result, so they can be changed without triggering a rebuild. diff --git a/src/lake/Lake/Config/LeanLib.lean b/src/lake/Lake/Config/LeanLib.lean index 817e9d7934..027d96ae31 100644 --- a/src/lake/Lake/Config/LeanLib.lean +++ b/src/lake/Lake/Config/LeanLib.lean @@ -49,17 +49,21 @@ The names of the library's root modules @[inline] def isBuildableModule (mod : Name) (self : LeanLib) : Bool := self.config.isBuildableModule mod +/-- The name of the library artifact. -/ +@[inline] def libName (self : LeanLib) : String := + self.config.libName + /-- The file name of the library's static binary (i.e., its `.a`) -/ @[inline] def staticLibFileName (self : LeanLib) : FilePath := nameToStaticLib self.config.libName /-- The path to the static library in the package's `libDir`. -/ @[inline] def staticLibFile (self : LeanLib) : FilePath := - self.pkg.nativeLibDir / self.staticLibFileName + self.pkg.staticLibDir / self.staticLibFileName /-- The path to the static library (with exported symbols) in the package's `libDir`. -/ @[inline] def staticExportLibFile (self : LeanLib) : FilePath := - self.pkg.nativeLibDir / self.staticLibFileName.addExtension "export" + self.pkg.staticLibDir / self.staticLibFileName.addExtension "export" /-- The file name of the library's shared binary (i.e., its `dll`, `dylib`, or `so`) . -/ @[inline] def sharedLibFileName (self : LeanLib) : FilePath := @@ -67,7 +71,7 @@ The names of the library's root modules /-- The path to the shared library in the package's `libDir`. -/ @[inline] def sharedLibFile (self : LeanLib) : FilePath := - self.pkg.nativeLibDir / self.sharedLibFileName + self.pkg.sharedLibDir / self.sharedLibFileName /-- The library's `extraDepTargets` configuration. -/ @[inline] def extraDepTargets (self : LeanLib) := @@ -166,21 +170,35 @@ and then the library's `moreLeancArgs`. /-- The arguments to weakly pass to `leanc` when compiling the library's Lean-produced C files. -That is, the package's `weakLeancArgs` plus the library's `weakLeancArgs`. +That is, the package's `weakLeancArgs` plus the library's `weakLeancArgs`. -/ @[inline] def weakLeancArgs (self : LeanLib) : Array String := self.pkg.weakLeancArgs ++ self.config.weakLeancArgs +/-- +Additionl target objects to pass to `ar` when linking the static library. +That is, the package's `moreLinkObjs` plus the library's `moreLinkObjs`. +-/ +@[inline] def moreLinkObjs (self : LeanLib) : TargetArray FilePath := + self.pkg.moreLinkObjs ++ self.config.moreLinkObjs + +/- +Additionl target libraries to are linked to the shared library. +That is, the package's `moreLinkLibs` plus the library's `moreLinkLibs`. +-/ +@[inline] def moreLinkLibs (self : LeanLib) : TargetArray Dynlib := + self.pkg.moreLinkLibs ++ self.config.moreLinkLibs + /-- The arguments to pass to `leanc` when linking the shared library. -That is, the package's `moreLinkArgs` plus the library's `moreLinkArgs`. +That is, the package's `moreLinkArgs` plus the library's `moreLinkArgs`. -/ @[inline] def linkArgs (self : LeanLib) : Array String := self.pkg.moreLinkArgs ++ self.config.moreLinkArgs /-- The arguments to weakly pass to `leanc` when linking the shared library. -That is, the package's `weakLinkArgs` plus the library's `weakLinkArgs`. +That is, the package's `weakLinkArgs` plus the library's `weakLinkArgs`. -/ @[inline] def weakLinkArgs (self : LeanLib) : Array String := self.pkg.weakLinkArgs ++ self.config.weakLinkArgs diff --git a/src/lake/Lake/Config/LeanLibConfig.lean b/src/lake/Lake/Config/LeanLibConfig.lean index 8e10aba7dd..a1f22abfd2 100644 --- a/src/lake/Lake/Config/LeanLibConfig.lean +++ b/src/lake/Lake/Config/LeanLibConfig.lean @@ -4,6 +4,7 @@ Released under Apache 2.0 license as described in the file LICENSE. Authors: Mac Malone -/ prelude +import Lean.Compiler.NameMangling import Lake.Util.Casing import Lake.Build.Facets import Lake.Config.InstallPath @@ -42,11 +43,11 @@ configuration LeanLibConfig (name : Name) extends LeanConfig where globs : Array Glob := roots.map Glob.one /-- - The name of the library. + The name of the library artifact. Used as a base for the file names of its static and dynamic binaries. - Defaults to the name of the target. + Defaults to the mangled name of the target. -/ - libName : String := name.toString (escape := false) + libName : String := name.mangle "" /-- An `Array` of target names to build before the library's modules. -/ extraDepTargets : Array Name := #[] diff --git a/src/lake/Lake/Config/Module.lean b/src/lake/Lake/Config/Module.lean index 75d9c5b5dd..291a4037fb 100644 --- a/src/lake/Lake/Config/Module.lean +++ b/src/lake/Lake/Config/Module.lean @@ -8,7 +8,6 @@ import Lake.Build.Trace import Lake.Config.LeanLib import Lake.Config.OutFormat import Lake.Util.OrdHashSet -import Lean.Compiler.NameMangling namespace Lake open Lean System diff --git a/src/lake/Lake/Config/Monad.lean b/src/lake/Lake/Config/Monad.lean index 923586f07a..0828ba5ccc 100644 --- a/src/lake/Lake/Config/Monad.lean +++ b/src/lake/Lake/Config/Monad.lean @@ -231,6 +231,10 @@ variable [Functor m] @[inline] def getLeanCc? : m (Option String) := (·.leanCc?) <$> getLeanInstall +/-- Get the flags required to link shared libraries using the detected Lean installation. -/ +@[inline] def getLeanLinkSharedFlags : m (Array String) := + (·.ccLinkSharedFlags) <$> getLeanInstall + /-! ### Lake Install Helpers -/ /-- Get the detected Lake installation. -/ diff --git a/src/lake/Lake/Config/Package.lean b/src/lake/Lake/Config/Package.lean index 54fb39d0fb..9bcd259288 100644 --- a/src/lake/Lake/Config/Package.lean +++ b/src/lake/Lake/Config/Package.lean @@ -554,6 +554,14 @@ namespace Package @[inline] def weakLeancArgs (self : Package) : Array String := self.config.weakLeancArgs +/-- The package's `moreLinkObjs` configuration. -/ +@[inline] def moreLinkObjs (self : Package) : TargetArray FilePath := + self.config.moreLinkObjs + +/-- The package's `moreLinkLibs` configuration. -/ +@[inline] def moreLinkLibs (self : Package) : TargetArray Dynlib := + self.config.moreLinkLibs + /-- The package's `moreLinkArgs` configuration. -/ @[inline] def moreLinkArgs (self : Package) : Array String := self.config.moreLinkArgs @@ -574,8 +582,23 @@ namespace Package @[inline] def leanLibDir (self : Package) : FilePath := self.buildDir / self.config.leanLibDir +/-- +Where static libraries for the package are located. +The package's `buildDir` joined with its `nativeLibDir` configuration. +-/ +@[inline] def staticLibDir (self : Package) : FilePath := + self.buildDir / self.config.nativeLibDir + +/-- +Where shared libraries for the package are located. +The package's `buildDir` joined with its `nativeLibDir` configuration. +-/ +@[inline] def sharedLibDir (self : Package) : FilePath := + self.buildDir / self.config.nativeLibDir + /-- The package's `buildDir` joined with its `nativeLibDir` configuration. -/ -@[inline] def nativeLibDir (self : Package) : FilePath := +@[deprecated "Use staticLibDir or sharedLibDir instead." (since := "2025-03-29")] +def nativeLibDir (self : Package) : FilePath := self.buildDir / self.config.nativeLibDir /-- The package's `buildDir` joined with its `binDir` configuration. -/ diff --git a/src/lake/Lake/Config/Workspace.lean b/src/lake/Lake/Config/Workspace.lean index f8421a4a39..153cdfaa87 100644 --- a/src/lake/Lake/Config/Workspace.lean +++ b/src/lake/Lake/Config/Workspace.lean @@ -176,14 +176,18 @@ The workspace's shared library path (e.g., for `--load-dynlib`). This is added to the `sharedLibPathEnvVar` by `lake env`. -/ def sharedLibPath (self : Workspace) : SearchPath := - self.packages.foldr (fun pkg dirs => pkg.nativeLibDir :: dirs) [] + self.packages.foldr (fun pkg dirs => pkg.sharedLibDir :: dirs) [] /-- The detected `PATH` of the environment augmented with the workspace's `binDir` and Lean and Lake installations' `binDir`. +On Windows, also adds the workspace shared library path. -/ def augmentedPath (self : Workspace) : SearchPath := - self.binPath ++ self.lakeEnv.path + if Platform.isWindows then + self.binPath ++ self.sharedLibPath ++ self.lakeEnv.path + else + self.binPath ++ self.lakeEnv.path /-- The detected `LEAN_PATH` of the environment augmented with diff --git a/src/lake/Lake/DSL/Targets.lean b/src/lake/Lake/DSL/Targets.lean index 9a9f89edd3..afab66dc71 100644 --- a/src/lake/Lake/DSL/Targets.lean +++ b/src/lake/Lake/DSL/Targets.lean @@ -153,8 +153,8 @@ abbrev mkTargetDecl (f : NPackage pkgName → FetchM (Job α)) : TargetDecl := let cfg := mkTargetJobConfig fun pkg => do - withRegisterJob (pkg.target target |>.key.toSimpleString) - (f pkg) + withRegisterJob (pkg.target target |>.key.toSimpleString) do + f pkg .mk (.mk pkgName target .anonymous (.mk cfg) (by simp [Name.isAnonymous])) rfl /-- diff --git a/src/lake/Lake/Load/Toml.lean b/src/lake/Lake/Load/Toml.lean index 10907c8ba1..7b51722ab0 100644 --- a/src/lake/Lake/Load/Toml.lean +++ b/src/lake/Lake/Load/Toml.lean @@ -286,6 +286,7 @@ def PartialBuildKey.decodeToml (v : Value) : EDecodeM PartialBuildKey := do | .error e => throwDecodeErrorAt v.ref e instance : DecodeToml PartialBuildKey := ⟨PartialBuildKey.decodeToml⟩ +instance : DecodeToml (Target α) := ⟨(Target.mk <$> PartialBuildKey.decodeToml ·)⟩ instance : DecodeField (LeanLibConfig n) `defaultFacets where decodeField := decodeFieldCore `defaultFacets (decodeFacets LeanLib.facetKind) @@ -360,17 +361,15 @@ local macro "gen_toml_decoders%" : command => do let cmds := #[] -- Targets let cmds ← genDecodeToml cmds ``LeanConfig false - (exclude := #[`dynlibs, `plugins]) let cmds ← genDecodeToml cmds ``LeanLibConfig true - (exclude := #[`nativeFacets, `dynlibs, `plugins]) + (exclude := #[`nativeFacets]) let cmds ← genDecodeToml cmds ``LeanExeConfig true - (exclude := #[`nativeFacets, `dynlibs, `plugins]) + (exclude := #[`nativeFacets]) let cmds ← genDecodeToml cmds ``InputFileConfig true let cmds ← genDecodeToml cmds ``InputDirConfig true -- Package let cmds ← genDecodeToml cmds ``WorkspaceConfig false let cmds ← genDecodeToml cmds ``PackageConfig true - (exclude := #[`dynlibs, `plugins]) return ⟨mkNullNode cmds⟩ gen_toml_decoders% diff --git a/src/lake/examples/ffi/app/Test.lean b/src/lake/examples/ffi/app/Test.lean index 5131ef7b8a..4b6bccad8c 100644 --- a/src/lake/examples/ffi/app/Test.lean +++ b/src/lake/examples/ffi/app/Test.lean @@ -1,3 +1,4 @@ import FFI #eval myAdd 3 4 +#eval myLeanFun diff --git a/src/lake/examples/ffi/lib/c/ffi.cpp b/src/lake/examples/ffi/lib/c/ffi.cpp deleted file mode 100644 index bd9e26e0c6..0000000000 --- a/src/lake/examples/ffi/lib/c/ffi.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include - -extern "C" uint32_t my_add(uint32_t a, uint32_t b) { - return a + b; -} - -extern "C" lean_obj_res my_lean_fun() { - return lean_io_result_mk_ok(lean_box(0)); -} diff --git a/src/lake/examples/ffi/lib/c/ffi_shared.cpp b/src/lake/examples/ffi/lib/c/ffi_shared.cpp new file mode 100644 index 0000000000..767a0bdef2 --- /dev/null +++ b/src/lake/examples/ffi/lib/c/ffi_shared.cpp @@ -0,0 +1,8 @@ +#include +#include + +extern "C" lean_obj_res my_lean_fun() { + std::string ss = "hi"; + lean_obj_res result = lean_mk_string(ss.c_str()); + return lean_io_result_mk_ok(result); +} diff --git a/src/lake/examples/ffi/lib/c/ffi_static.c b/src/lake/examples/ffi/lib/c/ffi_static.c new file mode 100644 index 0000000000..e09ff2a861 --- /dev/null +++ b/src/lake/examples/ffi/lib/c/ffi_static.c @@ -0,0 +1,5 @@ +#include + +uint32_t my_add(uint32_t a, uint32_t b) { + return a + b; +} diff --git a/src/lake/examples/ffi/lib/lakefile.lean b/src/lake/examples/ffi/lib/lakefile.lean index 18d3af36f9..2e21d6b7d5 100644 --- a/src/lake/examples/ffi/lib/lakefile.lean +++ b/src/lake/examples/ffi/lib/lakefile.lean @@ -4,19 +4,50 @@ open System Lake DSL package ffi where srcDir := "lean" -lean_lib FFI - @[default_target] lean_exe test where root := `Main -target ffi.o pkg : FilePath := do - let oFile := pkg.buildDir / "c" / "ffi.o" - let srcJob ← inputTextFile <| pkg.dir / "c" / "ffi.cpp" +lean_lib FFI + +/-! ## Static C FFI Library -/ + +input_file ffi_static.c where + path := "c" / "ffi_static.c" + text := true + +target ffi_static.o pkg : FilePath := do + let srcJob ← ffi_static.c.fetch + let oFile := pkg.buildDir / "c" / "ffi_static.o" + buildO oFile srcJob #[] #["-fPIC"] "cc" + +target libleanffi_static pkg : FilePath := do + let ffiO ← ffi_static.o.fetch + let name := nameToStaticLib "leanffi" + buildStaticLib (pkg.staticLibDir / name) #[ffiO] + +lean_lib FFI.Static where + moreLinkObjs := #[libleanffi_static] + +/-! ## Shared C++ FFI Library -/ + +input_file ffi_shared.cpp where + path := "c" / "ffi_shared.cpp" + text := true + +target ffi_shared.o pkg : FilePath := do + let srcJob ← ffi_shared.cpp.fetch + let oFile := pkg.buildDir / "c" / "ffi_shared.o" let weakArgs := #["-I", (← getLeanIncludeDir).toString] buildO oFile srcJob weakArgs #["-fPIC"] "c++" getLeanTrace -extern_lib libleanffi pkg := do - let ffiO ← ffi.o.fetch - let name := nameToStaticLib "leanffi" - buildStaticLib (pkg.nativeLibDir / name) #[ffiO] +target libleanffi_shared pkg : Dynlib := do + let libName := "leanffi" + let ffiO ← ffi_shared.o.fetch + let leanArgs ← getLeanLinkSharedFlags + buildSharedLib libName (pkg.sharedLibDir / nameToSharedLib libName) + #[ffiO] #[] #[] leanArgs "c++" getLeanTrace + +lean_lib FFI.Shared where + moreLinkLibs := #[libleanffi_shared] + moreLinkArgs := #["-lstdc++"] diff --git a/src/lake/examples/ffi/lib/lean/FFI.lean b/src/lake/examples/ffi/lib/lean/FFI.lean index 35c838bcd4..c7f9851b55 100644 --- a/src/lake/examples/ffi/lib/lean/FFI.lean +++ b/src/lake/examples/ffi/lib/lean/FFI.lean @@ -1,2 +1,2 @@ -import FFI.Fn -import FFI.Add +import FFI.Static +import FFI.Shared diff --git a/src/lake/examples/ffi/lib/lean/FFI/Fn.lean b/src/lake/examples/ffi/lib/lean/FFI/Fn.lean deleted file mode 100644 index ec3d7d4960..0000000000 --- a/src/lake/examples/ffi/lib/lean/FFI/Fn.lean +++ /dev/null @@ -1,2 +0,0 @@ -@[extern "my_lean_fun"] -opaque myLeanFun : IO PUnit diff --git a/src/lake/examples/ffi/lib/lean/FFI/Shared.lean b/src/lake/examples/ffi/lib/lean/FFI/Shared.lean new file mode 100644 index 0000000000..bfd4b61960 --- /dev/null +++ b/src/lake/examples/ffi/lib/lean/FFI/Shared.lean @@ -0,0 +1,2 @@ +@[extern "my_lean_fun"] +opaque myLeanFun : IO String diff --git a/src/lake/examples/ffi/lib/lean/FFI/Add.lean b/src/lake/examples/ffi/lib/lean/FFI/Static.lean similarity index 100% rename from src/lake/examples/ffi/lib/lean/FFI/Add.lean rename to src/lake/examples/ffi/lib/lean/FFI/Static.lean diff --git a/src/lake/examples/ffi/test.sh b/src/lake/examples/ffi/test.sh index 042d0642c1..9484811875 100755 --- a/src/lake/examples/ffi/test.sh +++ b/src/lake/examples/ffi/test.sh @@ -5,14 +5,17 @@ LAKE=${LAKE:-../../.lake/build/bin/lake} ./clean.sh +$LAKE -d app build -v +$LAKE -d lib build -v + +$LAKE exe -d app app +$LAKE exe -d lib test + # Tests that a non-precompiled build does not load anything as a dynlib/plugin # https://github.com/leanprover/lean4/issues/4565 $LAKE -d app build -v | (grep --color -E 'load-dynlib|plugin' && exit 1 || true) $LAKE -d lib build -v | (grep --color -E 'load-dynlib|plugin' && exit 1 || true) -./app/.lake/build/bin/app -./lib/.lake/build/bin/test - # Tests the successful precompilation of an `extern_lib` # Also tests a module with `precompileModules` always precompiles its imports $LAKE -d app build Test