Upgrade to Sqlite 3.36.0 (#478)

* Run `npm ci` automatically after the devcontainer is created.
This reduces the chance that another dev will forget to do this.

* Upgrade to Emscripten 2.0.29
Changes required:
* Defined EM_NODE_JS environment variable to get rid of
a warning that appears if the NODE environment variable
is set, but EM_NODE_JS is not.
* EXTRA_EXPORTED_RUNTIME_METHODS is now EXPORTED_RUNTIME_METHODS
* No longer pass the `-s LINKABLE=1` option to emcc when compiling. (This is a linktime setting and emcc
warns about not using a linktime setting when compiling now).

* Upgrade to Sqlite 3.36.0
Sqlite now publishes the hash as a SHA3 instead of SHA1,
necessitating the installation of the sha3sum command.

Author: Taytay

.devcontainer/Dockerfile

@@ -24,7 +24,7 @@ FROM mcr.microsoft.com/vscode/devcontainers/typescript-node:0-${VARIANT}
 # Install EMSDK to /emsdk just like the EMSDK Dockerfile: https://github.com/emscripten-core/emsdk/blob/master/docker/Dockerfile
 ENV EMSDK /emsdk
 # We pin EMSDK to 2.0.15 rather than 'latest' so that everyone is using the same compiler version
-ENV EMSCRIPTEN_VERSION 2.0.15
+ENV EMSCRIPTEN_VERSION 2.0.29
 
 RUN git clone https://github.com/emscripten-core/emsdk.git $EMSDK
 
@@ -36,7 +36,7 @@ RUN echo "## Install Emscripten" \
 # Copied directly from https://github.com/emscripten-core/emsdk/blob/master/docker/Dockerfile
 RUN cd ${EMSDK} \
     && echo "## Generate standard configuration" \
-    && ./emsdk activate $EMSCRIPTEN_VERSION \
+    && ./emsdk activate ${EMSCRIPTEN_VERSION} \
     && chmod 777 ${EMSDK}/upstream/emscripten \
     && chmod -R 777 ${EMSDK}/upstream/emscripten/cache \
     && echo "int main() { return 0; }" > hello.c \
@@ -47,6 +47,7 @@ RUN cd ${EMSDK} \
 ENV PATH $EMSDK:$EMSDK/upstream/emscripten/:$PATH
 
 # Cleanup Emscripten installation and strip some symbols
+# Copied directly from https://github.com/emscripten-core/emsdk/blob/master/docker/Dockerfile
 RUN echo "## Aggressive optimization: Remove debug symbols" \
     && cd ${EMSDK} && . ./emsdk_env.sh \
     # Remove debugging symbols from embedded node (extra 7MB)
@@ -60,6 +61,12 @@ RUN echo "## Aggressive optimization: Remove debug symbols" \
     && echo "## Done"
 
 RUN echo ". /emsdk/emsdk_env.sh" >> /etc/bash.bashrc
+# We must set the EM_NODE_JS environment variable for a somewhat silly reason
+# We run our build scripts with `npm run`, which sets the NODE environment variable as it runs.
+# The EMSDK picks up on that environment variable and gives a deprecation warning: warning: honoring legacy environment variable `NODE`.  Please switch to using `EM_NODE_JS` instead`
+# So, we are going to put this environment variable here explicitly to avoid the deprecation warning.
+RUN echo 'export EM_NODE_JS="$EMSDK_NODE"' >> /etc/bash.bashrc
+
 # END EMSDK
 # --------------------------------------------------------------------
 
@@ -117,6 +124,9 @@ RUN apt-get update \
         wget \
         xdg-utils
 
+# Installs the command "sha3sum", which is used check the download integrity of sqlite source.
+RUN apt-get install -y libdigest-sha3-perl
+
 # We set this env variable (RUN_WORKER_TEST_WITHOUT_PUPPETEER_SANDBOX=1) this to tell our sql.js test harness to run Puppeteer without the sandbox.
 # Otherwise, when we instantiate Puppeteer, we get this error: 
 # Puppeteer can't start due to a sandbox error. (Details follow.)

.devcontainer/devcontainer.json

@@ -18,7 +18,9 @@
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],
 	// Use 'postCreateCommand' to run commands after the container is created.
-	// "postCreateCommand": "yarn install",
+	// We use `npm ci` instead of `npm install` because we want to respect the lockfile and ONLY the lockfile.
+	// That way, our devcontainer is more reproducible. --Taytay
+	"postCreateCommand": "npm ci",
 	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "node"
 }

Makefile

@@ -6,9 +6,9 @@
 
 # I got this handy makefile syntax from : https://github.com/mandel59/sqlite-wasm (MIT License) Credited in LICENSE
 # To use another version of Sqlite, visit https://www.sqlite.org/download.html and copy the appropriate values here:
-SQLITE_AMALGAMATION = sqlite-amalgamation-3350000
-SQLITE_AMALGAMATION_ZIP_URL = https://www.sqlite.org/2021/sqlite-amalgamation-3350000.zip
-SQLITE_AMALGAMATION_ZIP_SHA1 = ba64bad885c9f51df765a9624700747e7bf21b79
+SQLITE_AMALGAMATION = sqlite-amalgamation-3360000
+SQLITE_AMALGAMATION_ZIP_URL = https://www.sqlite.org/2021/sqlite-amalgamation-3360000.zip
+SQLITE_AMALGAMATION_ZIP_SHA3 = d25609210ec93b3c8c7da66a03cf82e2c9868cfbd2d7d866982861855e96f972
 
 # Note that extension-functions.c hasn't been updated since 2010-02-06, so likely doesn't need to be updated
 EXTENSION_FUNCTIONS = extension-functions.c
@@ -35,7 +35,7 @@ EMFLAGS = \
 	-s RESERVED_FUNCTION_POINTERS=64 \
 	-s ALLOW_TABLE_GROWTH=1 \
 	-s EXPORTED_FUNCTIONS=@src/exported_functions.json \
-	-s EXTRA_EXPORTED_RUNTIME_METHODS=@src/exported_runtime_methods.json \
+	-s EXPORTED_RUNTIME_METHODS=@src/exported_runtime_methods.json \
 	-s SINGLE_FILE=0 \
 	-s NODEJS_CATCH_EXIT=0 \
 	-s NODEJS_CATCH_REJECTION=0
@@ -154,7 +154,7 @@ out/sqlite3.bc: sqlite-src/$(SQLITE_AMALGAMATION)
 out/extension-functions.bc: sqlite-src/$(SQLITE_AMALGAMATION)
 	mkdir -p out
 	# Generate llvm bitcode
-	$(EMCC) $(CFLAGS) -s LINKABLE=1 -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@
+	$(EMCC) $(CFLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@
 
 # TODO: This target appears to be unused. If we re-instatate it, we'll need to add more files inside of the JS folder
 # module.tar.gz: test package.json AUTHORS README.md dist/sql-asm.js
@@ -175,8 +175,8 @@ sqlite-src: sqlite-src/$(SQLITE_AMALGAMATION) sqlite-src/$(SQLITE_AMALGAMATION)/
 
 sqlite-src/$(SQLITE_AMALGAMATION): cache/$(SQLITE_AMALGAMATION).zip sqlite-src/$(SQLITE_AMALGAMATION)/$(EXTENSION_FUNCTIONS)
 	mkdir -p sqlite-src/$(SQLITE_AMALGAMATION)
-	echo '$(SQLITE_AMALGAMATION_ZIP_SHA1)  ./cache/$(SQLITE_AMALGAMATION).zip' > cache/check.txt
-	sha1sum -c cache/check.txt
+	echo '$(SQLITE_AMALGAMATION_ZIP_SHA3)  ./cache/$(SQLITE_AMALGAMATION).zip' > cache/check.txt
+	sha3sum -c cache/check.txt
 	# We don't delete the sqlite_amalgamation folder. That's a job for clean
 	# Also, the extension functions get copied here, and if we get the order of these steps wrong,
 	# this step could remove the extension functions, and that's not what we want