Fix playground infrastructure for ReScript 11 and update CONTRIBUTING instructions

Author: ryyppy

CONTRIBUTING.md

@@ -196,8 +196,6 @@ This is usually the file you want to create to test certain compile behavior wit
 
 ## Contribute to the ReScript Playground Bundle
 
-> Note: These instructions are designed for building the 4.06 based version of ReScript (ReScript v6).
-
 The "Playground bundle" is a JS version of the ReScript compiler; including all necessary dependency files (stdlib / belt etc). It is useful for building tools where you want to compile and execute arbitrary ReScript code in the browser.
 
 The ReScript source code is compiled with a tool called [JSOO (js_of_ocaml)](https://ocsigen.org/js_of_ocaml/4.0.0/manual/overview), which uses OCaml bytecode to compile to JavaScript and is part of the bigger OCaml ecosystem.
@@ -210,99 +208,64 @@ opam install js_of_ocaml.4.0.0
 
 ### Building the Bundle
 
-The entry point of the JSOO bundle is located in `jscomp/main/jsoo_playground_main.ml`, the code for packing the compiler into a single compiler file is located in `jscomp/snapshot.ninja`, and the script for running JSOO can be found in `scripts/repl.js`. A full clean build can be done like this:
-
-```
-# We create a target directory for storing the bundle / stdlib files
-mkdir playground && mkdir playground/stdlib
-
-# We build the ReScript source code and also the bytecode for the JSOO entrypoint
-node scripts/ninja.js config && node scripts/ninja.js build
-
-# Now we run the repl.js script which will create all the required artifacts in the `./playground` directory
-node scripts/repl.js
-```
-
-In case you want to build the project with our default third party packages (like `@rescript/react`), prepare the `playground-bundling` project and then run `repl.js` with `BUILD_THIRD_PARTY` enabled:
+The entry point of the JSOO bundle is located in `jscomp/jsoo/jsoo_playground_main.ml`, the compiler and its relevant runtime cmij files can be built via make:
 
-```
-# Prepare the `playground-bundling` project to allow building of the third party cmij packages
-npm link
-cd packages/playground-bundling
-npm install
-npm link rescript
-
-BUILD_THIRD_PARTY=true node scripts/repl.js
+```sh
+make playground
+make playground-cmijs
 ```
 
-_Troubleshooting: if ninja build step failed with `Error: cannot find file '+runtime.js'`, make sure `ocamlfind` is installed with `opam install ocamlfind`._
+Note that building the cmijs is based on the dependencies defined in `packages/playground-bundling/package.json`. In case you want to build some different version of e.g. `@rescript/react` or just want to add a new package, change the definition within the `package.json` file and re-run `make playground-cmijs` again.
 
 After a successful compilation, you will find following files in your project:
 
 - `playground/compiler.js` -> This is the ReScript compiler, which binds the ReScript API to the `window` object.
-- `playground/stdlib/*.js` -> All the ReScript runtime files.
 - `playground/packages` -> Contains third party deps with cmij.js files (as defined in `packages/playground-bundling/bsconfig.json`)
+- `playground/compilerCmij.js` -> The compiler base cmij containing all the relevant core modules (`Js`, `Belt`, `Pervasives`, etc.)
 
-You can now use the `compiler.js` file either directly by using a `<script src="/path/to/compiler.js"/>` inside a html file, use a browser bundler infrastructure to optimize it, or you can even use it with `nodejs`:
+You can now use the `compiler.js` file either directly by using a `<script src="/path/to/compiler.js"/>` and `<script src="/path/to/packages/compilerCmij.js"/>` inside a html file, use a browser bundler infrastructure to optimize it, or use `nodejs` to run it on a command line:
 
 ```
 $ node
 > require("./compiler.js");
+> require("./packages/compilerCmij.js")
 > let compiler = rescript_compiler.make()
 > let result = compiler.rescript.compile(`Js.log(Sys.ocaml_version)`);
 > eval(result.js_code);
 4.06.2+BS
 ```
 
-You can also run `node playground/playground_test.js` for a quick sanity check to see if all the build artifacts are working together correctly.
-
-### Playground JS bundle API
+### Testing the Playground bundle
 
-As soon as the bundle is loaded, you will get access to the functions exposed in [`jsoo_playground_main.ml`](jscomp/main/jsoo_playground_main.ml). Best way to check out the API is by inspecting a compiler instance it either in node, or in the browser:
-
-```
-$ node
-require('./compiler.js')
-
-> let compiler = rescript_compiler.make()
-> console.log(compiler)
-```
+Run `node playground/playground_test.js` for a quick sanity check to see if all the build artifacts are working together correctly. When releasing the playground bundle, the test will always be executed before publishing to catch regressions.
 
 ### Working on the Playground JS API
 
 Whenever you are modifying any files in the ReScript compiler, or in the `jsoo_playground_main.ml` file, you'll need to rebuild the source and recreate the JS bundle.
 
-```sh
-node scripts/ninja.js config && node scripts/ninja.js build
-node scripts/repl.js
 ```
+make playground
 
-**.cmj files in the Web**
-
-A `.cmj` file contains compile information and JS package information of ReScript build artifacts (your `.res / .ml` modules) and are generated on build (`scripts/ninja.js build`).
-
-A `.cmi` file is an [OCaml originated file extension](https://waleedkhan.name/blog/ocaml-file-extensions/) and contains all interface information of a certain module without any implementation.
-
-In this repo, these files usually sit right next to each compiled `.ml` / `.res` file. The structure of a `.cmj` file is defined in [js_cmj_format.ml](jscomp/core/js_cmj_format.ml). You can run a tool called `./jscomp/bin/cmjdump.exe [some-file.cmj]` to inspect the contents of given `.cmj` file.
-
-`.cmj` files are required to compile modules (this includes modules like RescriptReact). ReScript includes a subset of modules by default, which can be found in `jscomp/stdlib-406` and `jscomp/others`. You can also find those modules listed in the JSOO call in `scripts/repl.js`. As you probably noticed, the generated `playground` files are all plain `.js`, so how are the `cmj` / `cmi` files embedded?
-
-JSOO offers an `build-fs` subcommand that takes a list of `.cmi` and `.cmj` files and creates a `cmij.js` file that can be loaded by the JS runtime **after** the `compiler.js` bundle has been loaded (either via a `require()` call in Node, or via `<link/>` directive in an HTML file). Since we are shipping our playground with third party modules like `RescriptReact`, we created a utility directory `packages/playground-bundling` that comes with a utility script to do the `cmij.js` file creation for us. Check out `packages/playground-bundling/scripts/generate_cmijs.js` for details.
+# optionally run your test / arbitrary node script to verify your changes
+node playground/playground_test.js
+```
 
 ### Publishing the Playground Bundle on our KeyCDN
 
 > Note: If you want to publish from your local machine, make sure to set the `KEYCDN_USER` and `KEYCDN_PASSWORD` environment variables accordingly (credentials currently managed by @ryyppy). Our CI servers / GH Action servers are already pre-configured with the right env variable values.
 
 Our `compiler.js` and third-party packages bundles are hosted on [KeyCDN](https://www.keycdn.com) and uploaded via FTPS.
 
-After a successful bundle build, run our upload script to publish the build artifacts to our server:
+The full release can be executed with the following make script:
 
 ```
-playground/upload_bundle.sh
+make playground-release
 ```
 
 The script will automatically detect the ReScript version from the `compiler.js` bundle and automatically create the correct directory structure on our CDN ftp server.
 
+Note that there's currently still a manual step involved on [rescript-lang.org](https://rescript-lang.org) to make the uploaded playground version publicly available.
+
 ## Contribute to the API Reference
 
 The API reference is generated from doc comments in the source code. [Here](https://github.com/rescript-lang/rescript-compiler/blob/99650/jscomp/others/js_re.mli#L146-L161)'s a good example.

Makefile

@@ -53,6 +53,21 @@ lib: build node_modules/.bin/semver
 artifacts: lib
 	./scripts/makeArtifactList.js
 
+# Builds the core playground bundle (without the relevant cmijs files for the runtime)
+playground:
+	dune build --profile browser
+	cp ./_build/default/jscomp/jsoo/jsoo_playground_main.bc.js playground/compiler.js
+
+# Creates all the relevant core and third party cmij files to side-load together with the playground bundle
+playground-cmijs: artifacts
+	node packages/playground-bundling/scripts/generate_cmijs.js
+
+# Builds the playground, runs some e2e tests and releases the playground to the
+# CDN (requires KEYCDN_USER and KEYCDN_PASSWORD set in the env variables)
+playground-release: playground playground-cmijs
+	node playground/playground_test.js
+	sh playground/upload_bundle.sh
+
 format:
 	dune build @fmt --auto-promote
 
@@ -70,4 +85,4 @@ clean-all: clean clean-gentype
 
 .DEFAULT_GOAL := build
 
-.PHONY: build watch ninja bench dce test test-syntax test-syntax-roundtrip test-gentype test-all lib artifacts format checkformat clean-gentype clean clean-all
+.PHONY: build watch ninja bench dce test test-syntax test-syntax-roundtrip test-gentype test-all lib playground playground-cmijs test-playground playground-release artifacts format checkformat clean-gentype clean clean-all

packages/playground-bundling/scripts/generate_cmijs.js

@@ -5,16 +5,12 @@
  * project. Or in other words: You need to build cmij files with the same
  * rescript version as the compiler bundle.
  *
- * This script extracts all cmi / cmj files of all dependencies listed in the
- * project root's bsconfig.json, creates cmij.js files for each library and
- * puts them in the compiler playground directory.
+ * This script extracts all cmi / cmj files of the rescript/lib/ocaml and all
+ * dependencies listed in the project root's bsconfig.json, creates cmij.js
+ * files for each library and puts them in the compiler playground directory.
  *
  * The cmij files are representing the marshaled dependencies that can be used with the ReScript
  * playground bundle.
- *
- * This script is intended to be called by the `repl.js` script, but it can be independently run
- * by either running it within the compiler repo, or by providing a proper `PLAYGROUND_DIR` environment
- * flag pointing to the target folder the artifacts will be created.
  */
 
 const child_process = require("child_process");
@@ -23,13 +19,20 @@ const path = require("path");
 
 const bsconfig = require("../bsconfig.json");
 
-const PLAYGROUND_DIR =
-  process.env.PLAYGROUND ||
-  path.join(__dirname, "..", "..", "..", "playground");
+const RESCRIPT_COMPILER_ROOT_DIR = path.join(__dirname, "..", "..", "..");
+const PLAYGROUND_DIR = path.join(RESCRIPT_COMPILER_ROOT_DIR, "playground");
 
+// The playground-bundling root dir
 const PROJECT_ROOT_DIR = path.join(__dirname, "..");
+
+// Final target output directory where all the cmijs will be stored
 const PACKAGES_DIR = path.join(PLAYGROUND_DIR, "packages");
 
+// Making sure this directory exists, since it's not checked in to git
+if (!fs.existsSync(PACKAGES_DIR)) {
+  fs.mkdirSync(PACKAGES_DIR, { recursive: true });
+}
+
 const config = {
   cwd: PROJECT_ROOT_DIR,
   encoding: "utf8",
@@ -43,6 +46,9 @@ function e(cmd) {
   console.log(`<<<<<<`);
 }
 
+e(`npm install`);
+e(`npm link ${RESCRIPT_COMPILER_ROOT_DIR}`);
+
 const packages = bsconfig["bs-dependencies"];
 
 // We need to build the compiler's builtin modules as a separate cmij.
@@ -63,35 +69,40 @@ function buildCompilerCmij() {
   );
 }
 
-packages.forEach(function installLib(package) {
-  const libOcamlFolder = path.join(
-    PROJECT_ROOT_DIR,
-    "node_modules",
-    package,
-    "lib",
-    "ocaml"
-  );
-  const libEs6Folder = path.join(
-    PROJECT_ROOT_DIR,
-    "node_modules",
-    package,
-    "lib",
-    "es6"
-  );
-  const outputFolder = path.join(PACKAGES_DIR, package);
+function buildThirdPartyCmijs() {
+  packages.forEach(function installLib(package) {
+    const libOcamlFolder = path.join(
+      PROJECT_ROOT_DIR,
+      "node_modules",
+      package,
+      "lib",
+      "ocaml"
+    );
+    const libEs6Folder = path.join(
+      PROJECT_ROOT_DIR,
+      "node_modules",
+      package,
+      "lib",
+      "es6"
+    );
+    const outputFolder = path.join(PACKAGES_DIR, package);
 
-  const cmijFile = path.join(outputFolder, `cmij.js`);
+    const cmijFile = path.join(outputFolder, `cmij.js`);
 
-  if (!fs.existsSync(PLAYGROUND_DIR)) {
-    console.error(`PLAYGROUND_DIR "${PLAYGROUND_DIR}" does not exist`);
-    process.exit(1);
-  }
+    if (!fs.existsSync(PLAYGROUND_DIR)) {
+      console.error(`PLAYGROUND_DIR "${PLAYGROUND_DIR}" does not exist`);
+      process.exit(1);
+    }
 
-  if (!fs.existsSync(outputFolder)) {
-    fs.mkdirSync(outputFolder, { recursive: true });
-  }
-  e(`find ${libEs6Folder} -name '*.js' -exec cp {} ${outputFolder} \\;`);
-  e(
-    `find ${libOcamlFolder} -name "*.cmi" -or -name "*.cmj" | xargs -n1 basename | xargs js_of_ocaml build-fs -o ${cmijFile} -I ${libOcamlFolder}`
-  );
-});
+    if (!fs.existsSync(outputFolder)) {
+      fs.mkdirSync(outputFolder, { recursive: true });
+    }
+    e(`find ${libEs6Folder} -name '*.js' -exec cp {} ${outputFolder} \\;`);
+    e(
+      `find ${libOcamlFolder} -name "*.cmi" -or -name "*.cmj" | xargs -n1 basename | xargs js_of_ocaml build-fs -o ${cmijFile} -I ${libOcamlFolder}`
+    );
+  });
+}
+
+buildCompilerCmij();
+buildThirdPartyCmijs();

playground/playground_test.js

@@ -1,4 +1,5 @@
 require("./compiler.js")
+require("./packages/compilerCmij.js")
 require("./packages/@rescript/react/cmij.js")
 
 let compiler = rescript_compiler.make()