From ea8e415645fcf82e6f7a28e6ed5ecf78c511f645 Mon Sep 17 00:00:00 2001 From: Beanow <497556+Beanow@users.noreply.github.com> Date: Sun, 1 Dec 2024 11:28:15 +0100 Subject: [PATCH 1/4] feat: add script to build compiler from source --- bin/build-forked-compiler | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100755 bin/build-forked-compiler diff --git a/bin/build-forked-compiler b/bin/build-forked-compiler new file mode 100755 index 0000000..880fd09 --- /dev/null +++ b/bin/build-forked-compiler @@ -0,0 +1,13 @@ +#!/bin/sh + +set -eu + +readonly repo_root=$(dirname $(dirname $(realpath $0))) + +# You must have cloned the `gleam-lang/gleam` git repository for this command. +# This gives you a choice of compilers without having to go into git submodules. +GLEAM_DIR=${GLEAM_DIR:?"Please clone gleam-lang/gleam and set the GLEAM_DIR env to the repository root"} + +rm -fr "${repo_root}/wasm-compiler" +mkdir "${repo_root}/wasm-compiler" +wasm-pack build --release --target web --out-dir "${repo_root}/wasm-compiler" "${GLEAM_DIR}/compiler-wasm" From 8f2897e9c0bf57fcd5756c1714826910e3aade48 Mon Sep 17 00:00:00 2001 From: Beanow <497556+Beanow@users.noreply.github.com> Date: Sun, 1 Dec 2024 13:29:38 +0100 Subject: [PATCH 2/4] feat: use precompiled dependencies rather than submodules --- gleam.toml | 2 + manifest.toml | 4 + src/playground.gleam | 172 ++++++++++++++++--------------------------- static/compiler.js | 4 + static/worker.js | 34 ++++++--- 5 files changed, 94 insertions(+), 122 deletions(-) diff --git a/gleam.toml b/gleam.toml index f08d54e..431945b 100644 --- a/gleam.toml +++ b/gleam.toml @@ -19,6 +19,8 @@ simplifile = ">= 2.2.0 and < 3.0.0" snag = "~> 0.2" htmb = "~> 1.1" filepath = ">= 1.0.0 and < 2.0.0" +globlin = ">= 2.0.2 and < 3.0.0" +globlin_fs = ">= 2.0.0 and < 3.0.0" [dev-dependencies] gleeunit = ">= 1.0.0 and < 2.0.0" diff --git a/manifest.toml b/manifest.toml index 40bdd15..4b02d80 100644 --- a/manifest.toml +++ b/manifest.toml @@ -5,6 +5,8 @@ packages = [ { name = "filepath", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "EFB6FF65C98B2A16378ABC3EE2B14124168C0CE5201553DE652E2644DCFDB594" }, { name = "gleam_stdlib", version = "0.40.0", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "86606B75A600BBD05E539EB59FABC6E307EEEA7B1E5865AFB6D980A93BCB2181" }, { name = "gleeunit", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "F7A7228925D3EE7D0813C922E062BFD6D7E9310F0BEE585D3A42F3307E3CFD13" }, + { name = "globlin", version = "2.0.2", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "globlin", source = "hex", outer_checksum = "393E3421E4DA269B0E6025D69DA0F2D3DDD8517500F6BA2AE3C4024FA5F0B498" }, + { name = "globlin_fs", version = "2.0.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib", "globlin", "simplifile"], otp_app = "globlin_fs", source = "hex", outer_checksum = "2A84CE81FD7958B967EF39CC234AFB64DAB20169D0EF9B9C3943CD3C5B561182" }, { name = "htmb", version = "1.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "htmb", source = "hex", outer_checksum = "30D448F0E15DFCF7283AAAC2F351D77B9D54E318219C9FDDB1877572B67C27B7" }, { name = "simplifile", version = "2.2.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "0DFABEF7DC7A9E2FF4BB27B108034E60C81BEBFCB7AB816B9E7E18ED4503ACD8" }, { name = "snag", version = "0.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "snag", source = "hex", outer_checksum = "54D32E16E33655346AA3E66CBA7E191DE0A8793D2C05284E3EFB90AD2CE92BCC" }, @@ -14,6 +16,8 @@ packages = [ filepath = { version = ">= 1.0.0 and < 2.0.0" } gleam_stdlib = { version = ">= 0.34.0 and < 2.0.0" } gleeunit = { version = ">= 1.0.0 and < 2.0.0" } +globlin = { version = ">= 2.0.2 and < 3.0.0" } +globlin_fs = { version = ">= 2.0.0 and < 3.0.0" } htmb = { version = "~> 1.1" } simplifile = { version = ">= 2.2.0 and < 3.0.0" } snag = { version = "~> 0.2" } diff --git a/src/playground.gleam b/src/playground.gleam index 5fa39c2..c1a3264 100644 --- a/src/playground.gleam +++ b/src/playground.gleam @@ -4,6 +4,8 @@ import gleam/list import gleam/result import gleam/string import gleam/string_builder +import globlin +import globlin_fs import htmb.{type Html, h} import playground/html.{ ScriptOptions, html_dangerous_inline_script, html_link, html_meta, @@ -23,6 +25,8 @@ const meta_image = "https://gleam.run/images/og-image.png" const meta_url = "https://play.gleam.run" +const available_packages = ["gleam_stdlib", "globlin", "filepath"] + // Paths const static = "static" @@ -31,13 +35,7 @@ const public = "public" const public_precompiled = "public/precompiled" -const prelude = "build/dev/javascript/prelude.mjs" - -const stdlib_compiled = "build/dev/javascript/gleam_stdlib/gleam" - -const stdlib_sources = "build/packages/gleam_stdlib/src/gleam" - -const stdlib_external = "build/packages/gleam_stdlib/src" +const compiled_lib = "build/dev/javascript" const compiler_wasm = "./wasm-compiler" @@ -53,8 +51,8 @@ pub fn main() { pub fn main() { let result = { use _ <- result.try(reset_output()) - use _ <- result.try(make_prelude_available()) - use _ <- result.try(make_stdlib_available()) + use _ <- result.try(ensure_directory(public_precompiled)) + use _ <- result.try(make_packages_available(available_packages)) use _ <- result.try(copy_wasm_compiler()) let page_html = @@ -62,7 +60,6 @@ pub fn main() { |> htmb.render_page("html") |> string_builder.to_string - use _ <- result.try(ensure_directory(public)) let path = filepath.join(public, "index.html") use _ <- result.try(write_text(path, page_html)) @@ -90,127 +87,82 @@ fn write_text(path: String, text: String) -> snag.Result(Nil) { |> file_error("Failed to write " <> path) } -fn copy_wasm_compiler() -> snag.Result(Nil) { - use compiler_wasm_exists <- result.try( - simplifile.is_directory(compiler_wasm) - |> file_error("Failed to check compiler-wasm directory"), - ) - use <- require(compiler_wasm_exists, "compiler-wasm must have been compiled") - - simplifile.copy_directory(compiler_wasm, public <> "/compiler") - |> file_error("Failed to copy compiler-wasm") -} - -fn make_prelude_available() -> snag.Result(Nil) { - use _ <- result.try( - simplifile.create_directory_all(public_precompiled) - |> file_error("Failed to make " <> public_precompiled), - ) - - simplifile.copy_file(prelude, public_precompiled <> "/gleam.mjs") - |> file_error("Failed to copy prelude.mjs") -} - -fn make_stdlib_available() -> snag.Result(Nil) { - use files <- result.try( - simplifile.read_directory(stdlib_sources) - |> file_error("Failed to read stdlib directory"), - ) - - let modules = - files - |> list.filter(fn(file) { string.ends_with(file, ".gleam") }) - |> list.map(string.replace(_, ".gleam", "")) - - use _ <- result.try( - generate_stdlib_bundle(modules) - |> snag.context("Failed to generate stdlib.js bundle"), - ) - +pub fn make_packages_available(packages: List(String)) -> snag.Result(Nil) { + // Set up prelude use _ <- result.try( - copy_compiled_stdlib(modules) - |> snag.context("Failed to copy precompiled stdlib modules"), + copy_lib_files(["prelude.mjs", "gleam_version"]) + |> snag.context("Failed to copy lib prelude"), ) + // Recursive directory copies for packages use _ <- result.try( - copy_stdlib_externals() - |> snag.context("Failed to copy stdlib external files"), + copy_lib_dirs(packages) + |> snag.context("Failed to copy lib packages"), ) - Ok(Nil) + // Walk the lib directory to enumerate them in a manifest. + generate_lib_manifest() } -fn copy_stdlib_externals() -> snag.Result(Nil) { - use files <- result.try( - simplifile.read_directory(stdlib_external) - |> file_error("Failed to read stdlib external directory"), - ) - let files = list.filter(files, string.ends_with(_, ".mjs")) - +fn copy_lib_files(files: List(String)) -> snag.Result(Nil) { list.try_each(files, fn(file) { - let from = stdlib_external <> "/" <> file - let to = public_precompiled <> "/" <> file - simplifile.copy_file(from, to) - |> file_error("Failed to copy stdlib external file " <> from) + simplifile.copy_file( + filepath.join(compiled_lib, file), + filepath.join(public_precompiled, file), + ) + |> file_error("Failed to copy file " <> file) }) } -fn copy_compiled_stdlib(modules: List(String)) -> snag.Result(Nil) { - use stdlib_dir_exists <- result.try( - simplifile.is_directory(stdlib_compiled) - |> file_error("Failed to check stdlib directory"), - ) - use <- require( - stdlib_dir_exists, - "Project must have been compiled for JavaScript", - ) +fn copy_lib_dirs(packages: List(String)) -> snag.Result(Nil) { + list.try_each(packages, fn(package) { + simplifile.copy_directory( + filepath.join(compiled_lib, package), + filepath.join(public_precompiled, package), + ) + |> file_error("Failed to copy directory " <> package) + }) +} - let dest = public_precompiled <> "/gleam" - use _ <- result.try( - simplifile.create_directory_all(dest) - |> file_error("Failed to make " <> dest), +fn generate_lib_manifest() -> snag.Result(Nil) { + let assert Ok(pattern) = globlin.new_pattern("**/*") + + use cwd <- result.try( + simplifile.current_directory() + |> file_error("Finding current directory"), ) - use _ <- result.try( - list.try_each(modules, fn(name) { - let from = stdlib_compiled <> "/" <> name <> ".mjs" - let to = dest <> "/" <> name <> ".mjs" - simplifile.copy_file(from, to) - |> file_error("Failed to copy stdlib module " <> from) - }), + let abs_dir = filepath.join(cwd, public_precompiled) + use files <- result.try( + globlin_fs.glob_from( + pattern, + directory: abs_dir, + returning: globlin_fs.RegularFiles, + ) + |> file_error("Walking lib files"), ) - Ok(Nil) + files + // Make sure we turn the matched absolute paths back to relative ones. + |> list.map(string.drop_left(_, string.length(abs_dir) + 1)) + |> list.sort(string.compare) + // Export them as a const JS array literal. + |> string.join("',\n '") + |> string.append("export const files = [\n '", _) + |> string.append("'\n]\n") + |> simplifile.write(public_precompiled <> ".js", _) + |> file_error("Failed to write lib manifest") } -fn generate_stdlib_bundle(modules: List(String)) -> snag.Result(Nil) { - use entries <- result.try( - list.try_map(modules, fn(name) { - let path = stdlib_sources <> "/" <> name <> ".gleam" - use code <- result.try( - simplifile.read(path) - |> file_error("Failed to read stdlib module " <> path), - ) - let name = string.replace(name, ".gleam", "") - let code = - code - |> string.replace("\\", "\\\\") - |> string.replace("`", "\\`") - |> string.split("\n") - |> list.filter(fn(line) { !string.starts_with(string.trim(line), "//") }) - |> list.filter(fn(line) { line != "" }) - |> string.join("\n") - - Ok(" \"gleam/" <> name <> "\": `" <> code <> "`") - }), +fn copy_wasm_compiler() -> snag.Result(Nil) { + use compiler_wasm_exists <- result.try( + simplifile.is_directory(compiler_wasm) + |> file_error("Failed to check compiler-wasm directory"), ) + use <- require(compiler_wasm_exists, "compiler-wasm must have been compiled") - entries - |> string.join(",\n") - |> string.append("export default {\n", _) - |> string.append("\n}\n") - |> simplifile.write(public <> "/stdlib.js", _) - |> file_error("Failed to write stdlib.js") + simplifile.copy_directory(compiler_wasm, public <> "/compiler") + |> file_error("Failed to copy compiler-wasm") } fn reset_output() -> snag.Result(Nil) { diff --git a/static/compiler.js b/static/compiler.js index 021992c..d087e9a 100644 --- a/static/compiler.js +++ b/static/compiler.js @@ -53,6 +53,10 @@ class Project { return this.#id; } + writeFileBytes(fileName, content) { + compiler.wasm.write_file_bytes(this.#id, fileName, content); + } + writeModule(moduleName, code) { compiler.wasm.write_module(this.#id, moduleName, code); } diff --git a/static/worker.js b/static/worker.js index 08c8527..adbbcd9 100644 --- a/static/worker.js +++ b/static/worker.js @@ -1,11 +1,24 @@ import initGleamCompiler from "./compiler.js"; -import stdlib from "./stdlib.js"; +import { files as libFiles } from "./precompiled.js"; const compiler = await initGleamCompiler(); const project = compiler.newProject(); -for (const [name, code] of Object.entries(stdlib)) { - project.writeModule(name, code); +function libUrl(file) { + const url = new URL(import.meta.url); + url.pathname = file ? `precompiled/${file}` : "precompiled"; + url.hash = ""; + url.search = ""; + return url.toString(); +} + +// Write all files from /lib ahead of time. +// Use binary because we also need capnp cache files here. +for (const file of libFiles) { + const url = libUrl(file); + const res = await fetch(url); + const bytes = await res.bytes(); + project.writeFileBytes(`/lib/${file}`, bytes); } // Monkey patch console.log to keep a copy of the output @@ -17,15 +30,12 @@ console.log = (...args) => { }; async function loadProgram(js) { - const url = new URL(import.meta.url); - url.pathname = ""; - url.hash = ""; - url.search = ""; - const href = url.toString(); - const js1 = js.replaceAll( - /from\s+"\.\/(.+)"/g, - `from "${href}precompiled/$1"`, - ); + const href = libUrl(); + const js1 = js + // Importing a dependency uses `../{packageName}/{module}.mjs` + .replaceAll(/from\s+"\.\.\/(.+)"/g, `from "${href}/$1"`) + // The root package depending on prelude `./gleam.mjs`. + .replaceAll(/from\s+"\.\/gleam\.mjs\"/g, `from "${href}/prelude.mjs"`); const js2 = btoa(unescape(encodeURIComponent(js1))); const module = await import("data:text/javascript;base64," + js2); return module.main; From bf5b82dec9028f4d01dd8b8484a9e8585444368b Mon Sep 17 00:00:00 2001 From: Beanow <497556+Beanow@users.noreply.github.com> Date: Sun, 1 Dec 2024 14:15:32 +0100 Subject: [PATCH 3/4] fix: copy static after packages are made available --- src/playground.gleam | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/playground.gleam b/src/playground.gleam index c1a3264..99ec643 100644 --- a/src/playground.gleam +++ b/src/playground.gleam @@ -53,6 +53,10 @@ pub fn main() { use _ <- result.try(reset_output()) use _ <- result.try(ensure_directory(public_precompiled)) use _ <- result.try(make_packages_available(available_packages)) + use _ <- result.try( + simplifile.copy_directory(static, public) + |> file_error("Failed to copy static directory"), + ) use _ <- result.try(copy_wasm_compiler()) let page_html = @@ -176,15 +180,10 @@ fn reset_output() -> snag.Result(Nil) { |> file_error("Failed to read public directory"), ) - use _ <- result.try( - files - |> list.map(string.append(public <> "/", _)) - |> simplifile.delete_all - |> file_error("Failed to delete public directory"), - ) - - simplifile.copy_directory(static, public) - |> file_error("Failed to copy static directory") + files + |> list.map(string.append(public <> "/", _)) + |> simplifile.delete_all + |> file_error("Failed to delete public directory") } fn require( From 3e2c449eb0d6ea99feedbb8ee06a31b7f8a382c2 Mon Sep 17 00:00:00 2001 From: Beanow <497556+Beanow@users.noreply.github.com> Date: Sun, 1 Dec 2024 14:32:03 +0100 Subject: [PATCH 4/4] feat: declare precompiled packages as dependencies --- src/playground.gleam | 34 ++++++++++++++++++++++------------ static/worker.js | 23 ++++++++++++++++++++++- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/src/playground.gleam b/src/playground.gleam index 99ec643..561dc04 100644 --- a/src/playground.gleam +++ b/src/playground.gleam @@ -25,7 +25,7 @@ const meta_image = "https://gleam.run/images/og-image.png" const meta_url = "https://play.gleam.run" -const available_packages = ["gleam_stdlib", "globlin", "filepath"] +const available_packages = ["filepath", "gleam_stdlib", "globlin"] // Paths @@ -105,7 +105,7 @@ pub fn make_packages_available(packages: List(String)) -> snag.Result(Nil) { ) // Walk the lib directory to enumerate them in a manifest. - generate_lib_manifest() + generate_lib_manifest(packages) } fn copy_lib_files(files: List(String)) -> snag.Result(Nil) { @@ -128,7 +128,7 @@ fn copy_lib_dirs(packages: List(String)) -> snag.Result(Nil) { }) } -fn generate_lib_manifest() -> snag.Result(Nil) { +fn generate_lib_manifest(packages: List(String)) -> snag.Result(Nil) { let assert Ok(pattern) = globlin.new_pattern("**/*") use cwd <- result.try( @@ -146,15 +146,25 @@ fn generate_lib_manifest() -> snag.Result(Nil) { |> file_error("Walking lib files"), ) - files - // Make sure we turn the matched absolute paths back to relative ones. - |> list.map(string.drop_left(_, string.length(abs_dir) + 1)) - |> list.sort(string.compare) - // Export them as a const JS array literal. - |> string.join("',\n '") - |> string.append("export const files = [\n '", _) - |> string.append("'\n]\n") - |> simplifile.write(public_precompiled <> ".js", _) + let files = + files + // Make sure we turn the matched absolute paths back to relative ones. + |> list.map(string.drop_left(_, string.length(abs_dir) + 1)) + |> list.sort(string.compare) + // Export them as a const JS array literal. + |> string.join("',\n '") + |> string.append("export const files = [\n '", _) + |> string.append("'\n];\n") + + let packages = + packages + |> list.sort(string.compare) + // Export them as a const JS array literal. + |> string.join("', '") + |> string.append("export const packages = ['", _) + |> string.append("'];\n") + + simplifile.write(public_precompiled <> ".js", string.append(packages, files)) |> file_error("Failed to write lib manifest") } diff --git a/static/worker.js b/static/worker.js index adbbcd9..4b37c90 100644 --- a/static/worker.js +++ b/static/worker.js @@ -1,5 +1,8 @@ import initGleamCompiler from "./compiler.js"; -import { files as libFiles } from "./precompiled.js"; +import { + files as libFiles, + packages as availablePackages, +} from "./precompiled.js"; const compiler = await initGleamCompiler(); const project = compiler.newProject(); @@ -21,6 +24,24 @@ for (const file of libFiles) { project.writeFileBytes(`/lib/${file}`, bytes); } +// Ensures precompiled libraries are direct dependencies. +// The compiler will warn about transitive dependencies, but not check the version requirements. +function configWithDependencies(deps) { + const conf = ` + name = "library" + version = "1.0.0" + [dependencies] + ${deps.map((dep) => `${dep} = "0.0.0"`).join("\n")} + `; + return new TextEncoder().encode(conf); +} + +// TODO: packages could be filtered to restore transitive dependency warnings. +project.writeFileBytes( + "/gleam.toml", + configWithDependencies(availablePackages) +); + // Monkey patch console.log to keep a copy of the output let logged = ""; const log = console.log;