separate static and dynamic builds

Author: stevensJourney

.gitignore

@@ -11,6 +11,7 @@
 
 # PowerSync Rust Core
 libpowersync*.wasm
+powersync-libs
 
 /cache
 /debug

Makefile

@@ -6,9 +6,7 @@ EXTENSION_FUNCTIONS = extension-functions.c
 EXTENSION_FUNCTIONS_URL = https://www.sqlite.org/contrib/download/extension-functions.c?get=25
 EXTENSION_FUNCTIONS_SHA3 = ee39ddf5eaa21e1d0ebcbceeab42822dd0c4f82d8039ce173fd4814807faabfa
 
-
-
-# source files
+# WA-SQLite source files
 CFILES = \
 	sqlite3.c \
 	extension-functions.c \
@@ -19,6 +17,9 @@ CFILES = \
 	libvfs.c \
 	$(CFILES_EXTRA)
 
+POWERSYNC_CFILES = \
+	setup_powersync.c
+
 JSFILES = \
 	src/libauthorizer.js \
 	src/libfunction.js \
@@ -28,8 +29,14 @@ JSFILES = \
 vpath %.c src
 vpath %.c deps
 vpath %.c deps/$(SQLITE_VERSION)
+vpath %.c powersync-static
 
+POWERSYNC_EXPORTED_FUNCTIONS = powersync-static/powersync_exported_functions.json
 EXPORTED_FUNCTIONS = src/exported_functions.json
+
+# EMCC does not support multiple exports files. Need to combine them temporarily
+COMBINED_EXPORTED_FUNCTIONS = tmp/combined_exports.json
+
 EXPORTED_RUNTIME_METHODS = src/extra_exported_runtime_methods.json
 ASYNCIFY_IMPORTS = src/asyncify_imports.json
 ASYNCIFY_EXPORTS = src/asyncify_exports.json
@@ -38,6 +45,10 @@ ASYNCIFY_EXPORTS = src/asyncify_exports.json
 OBJ_FILES_DEBUG = $(patsubst %.c,tmp/obj/debug/%.o,$(CFILES))
 OBJ_FILES_DIST = $(patsubst %.c,tmp/obj/dist/%.o,$(CFILES))
 
+POWERSYNC_OBJ_FILES_DEBUG = $(patsubst %.c,tmp/powersync-obj/debug/%.o,$(POWERSYNC_CFILES))
+POWERSYNC_OBJ_FILES_DIST = $(patsubst %.c,tmp/powersync-obj/dist/%.o,$(POWERSYNC_CFILES))
+POWERSYNC_STATIC_FILES = powersync-libs/libpowersync-wasm.a
+
 # build options
 EMCC ?= emcc
 
@@ -82,6 +93,10 @@ EMFLAGS_INTERFACES = \
 	-s EXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS) \
 	-s EXPORTED_RUNTIME_METHODS=@$(EXPORTED_RUNTIME_METHODS)
 
+COMBINED_EMFLAGS_INTERFACES = \
+	-s EXPORTED_FUNCTIONS=@$(COMBINED_EXPORTED_FUNCTIONS) \
+	-s EXPORTED_RUNTIME_METHODS=@$(EXPORTED_RUNTIME_METHODS)
+
 EMFLAGS_LIBRARIES = \
 	--js-library src/libadapters.js \
 	--js-library src/libtableupdates.js \
@@ -150,12 +165,19 @@ cache/$(EXTENSION_FUNCTIONS):
 clean-deps:
 	rm -rf deps
 
+# Combine WA-SQLite and PowerSync exported functions
+$(EXPORTED_FUNCTIONS): $(EXPORTED_FUNCTIONS) $(POWERSYNC_EXPORTED_FUNCTIONS)
+	jq -s 'add' $(EXPORTED_FUNCTIONS) $(POWERSYNC_EXPORTED_FUNCTIONS) > $(COMBINED_EXPORTED_FUNCTIONS)
+
 deps/$(SQLITE_VERSION)/sqlite3.h deps/$(SQLITE_VERSION)/sqlite3.c:
 	mkdir -p cache/$(SQLITE_VERSION)
 	curl -LsS $(SQLITE_TARBALL_URL) | tar -xzf - -C cache/$(SQLITE_VERSION)/ --strip-components=1
 	mkdir -p deps/$(SQLITE_VERSION)
 	(cd deps/$(SQLITE_VERSION); ../../cache/$(SQLITE_VERSION)/configure --enable-all && make sqlite3.c)
 
+# Download static files from PowerSync Core repository
+$(POWERSYNC_STATIC_FILES):
+	node ./scripts/download-core-build.js
 
 ifeq ($(shell uname), Darwin)
     OPENSSL_CHECK_CMD := openssl dgst -sha3-256 -r cache/$(EXTENSION_FUNCTIONS) | sed -e 's/ .*//' > deps/sha3
@@ -187,6 +209,7 @@ deps/$(EXTENSION_FUNCTIONS): cache/$(EXTENSION_FUNCTIONS)
 	rm -rf deps/sha3 $@
 	sed -i '' "/#define COMPILE_SQLITE_EXTENSIONS_AS_LOADABLE_MODULE/d" cache/$(EXTENSION_FUNCTIONS)
 	cp cache/$(EXTENSION_FUNCTIONS) $@
+
 ## tmp
 .PHONY: clean-tmp
 clean-tmp:
@@ -200,42 +223,70 @@ tmp/obj/dist/%.o: %.c
 	mkdir -p tmp/obj/dist
 	$(EMCC) $(CFLAGS_DIST) $(WASQLITE_DEFINES) $^ -c -o $@
 
+# Build PowerSync specific C files
+tmp/powersync-obj/debug/%.o: %.c
+	mkdir -p tmp/powersync-obj/debug
+	$(EMCC) $(CFLAGS_DEBUG) $(WASQLITE_DEFINES) $^ -c -o $@
+
+# Build PowerSync specific C files
+tmp/powersync-obj/dist/%.o: %.c
+	mkdir -p tmp/powersync-obj/dist
+	$(EMCC) $(CFLAGS_DIST) $(WASQLITE_DEFINES) $^ -c -o $@
+
 
 ## debug
 .PHONY: clean-debug
 clean-debug:
 	rm -rf debug
 
 .PHONY: debug
-debug: debug/wa-sqlite.mjs debug/wa-sqlite-async.mjs debug/wa-sqlite-jspi.mjs debug/wa-sqlite-main.mjs debug/wa-sqlite-main-async.mjs
+debug: debug/wa-sqlite.mjs debug/wa-sqlite-async.mjs debug/wa-sqlite-jspi.mjs debug/wa-sqlite-dynamic-main.mjs debug/wa-sqlite-dynamic-main-async.mjs
 
-# TODO link static
-debug/wa-sqlite.mjs: $(OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+# Statically links PowerSync Core
+debug/wa-sqlite.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(POWERSYNC_STATIC_FILES)
 	mkdir -p debug
 	$(EMCC) $(EMFLAGS_DEBUG) \
-	  $(EMFLAGS_INTERFACES) \
+	  $(COMBINED_EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
-	  $(OBJ_FILES_DEBUG) -o $@
+	  $(POWERSYNC_STATIC_FILES) \
+	  $(OBJ_FILES_DEBUG) \
+	  $(POWERSYNC_OBJ_FILES_DEBUG) -o $@
 
-# TODO link static
-debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG)  $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+# Statically links PowerSync Core
+debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG)  $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES)
 	mkdir -p debug
 	$(EMCC) $(EMFLAGS_DEBUG) \
-	  $(EMFLAGS_INTERFACES) \
+	  $(COMBINED_EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
 	  $(EMFLAGS_ASYNCIFY_DEBUG) \
+	  $(POWERSYNC_STATIC_FILES) \
+	  $(POWERSYNC_OBJ_FILES_DEBUG) \
+	  $(OBJ_FILES_DEBUG) -o $@
+
+# Statically links PowerSync Core
+debug/wa-sqlite-jspi.mjs: $(OBJ_FILES_DEBUG) $(POWERSYNC_OBJ_FILES_DEBUG) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES)
+	mkdir -p debug
+	$(EMCC) $(EMFLAGS_DEBUG) $(EMFLAGS_DYNAMIC)  \
+	  $(COMBINED_EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(EMFLAGS_JSPI) \
+	  $(POWERSYNC_STATIC_FILES) \
+	  $(POWERSYNC_OBJ_FILES_DEBUG) \
 	  $(OBJ_FILES_DEBUG) -o $@
 
-# Main module distributable
-debug/wa-sqlite-main.mjs: $(OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+
+# Dynamic main module
+# Exported functions are omitted here since everything is currently exported
+debug/wa-sqlite-dynamic-main.mjs: $(OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
 	mkdir -p debug
 	$(EMCC) $(EMFLAGS_DEBUG) $(EMFLAGS_DYNAMIC) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
 	  $(OBJ_FILES_DEBUG) -o $@
 
-# Main module distributable
-debug/wa-sqlite-main-async.mjs: $(OBJ_FILES_DEBUG)  $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+# Dynamic main module
+# Exported functions are omitted here since everything is currently exported
+debug/wa-sqlite-dynamic-main-async.mjs: $(OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
 	mkdir -p debug
 	$(EMCC) $(EMFLAGS_DEBUG) $(EMFLAGS_DYNAMIC) \
 	  $(EMFLAGS_INTERFACES) \
@@ -264,62 +315,65 @@ debug/wa-sqlite-main-async.mjs: $(OBJ_FILES_DEBUG)  $(EXPORTED_FUNCTIONS) $(EXPO
 # 		$(RS_WASM_TGT_DIR)/debug/deps/*.bc \
 # 	  $(OBJ_FILES_DEBUG_FTS) *.o -o $@
 
-# TODO link static
-debug/wa-sqlite-jspi.mjs: $(OBJ_FILES_DEBUG) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
-	mkdir -p debug
-	$(EMCC) $(EMFLAGS_DEBUG) $(EMFLAGS_DYNAMIC)  \
-	  $(EMFLAGS_INTERFACES) \
-	  $(EMFLAGS_LIBRARIES) \
-	  $(EMFLAGS_JSPI) \
-	  $(OBJ_FILES_DEBUG) -o $@
 
 ## dist
 .PHONY: clean-dist
 clean-dist:
 	rm -rf dist
 
 .PHONY: dist
-dist: dist/wa-sqlite.mjs dist/wa-sqlite-async.mjs dist/wa-sqlite-jspi.mjs dist/wa-sqlite-main.mjs dist/wa-sqlite-async-main.mjs
+dist: dist/wa-sqlite.mjs dist/wa-sqlite-async.mjs dist/wa-sqlite-jspi.mjs dist/wa-sqlite-dynamic-main.mjs dist/wa-sqlite-async-dynamic-main.mjs
 
-# TODO link static
-dist/wa-sqlite.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+# Statically links PowerSync Core
+dist/wa-sqlite.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(POWERSYNC_STATIC_FILES)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) \
-	  $(EMFLAGS_INTERFACES) \
+	  $(COMBINED_EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
+	  $(POWERSYNC_STATIC_FILES) \
+	  $(POWERSYNC_OBJ_FILES_DIST) \
 	  $(OBJ_FILES_DIST)  -o $@
 
-# TODO link static
-dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+# Statically links PowerSync Core
+dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) \
-	  $(EMFLAGS_INTERFACES) \
+	  $(COMBINED_EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
 	  $(EMFLAGS_ASYNCIFY_DIST) \
+	  $(POWERSYNC_STATIC_FILES) \
+	  $(POWERSYNC_OBJ_FILES_DIST) \
+	  $(OBJ_FILES_DIST) -o $@
+
+# Statically links PowerSync Core
+dist/wa-sqlite-jspi.mjs: $(OBJ_FILES_DIST) $(POWERSYNC_OBJ_FILES_DIST) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS) $(POWERSYNC_STATIC_FILES)
+	mkdir -p dist
+	$(EMCC) $(EMFLAGS_DIST) \
+	  $(COMBINED_EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(EMFLAGS_JSPI) \
+	  $(POWERSYNC_STATIC_FILES) \
+	  $(POWERSYNC_OBJ_FILES_DIST) \
 	  $(OBJ_FILES_DIST) -o $@
 
-dist/wa-sqlite-main.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+# Dynamic main module
+# Exported functions are omitted here since everything is currently exported
+dist/wa-sqlite-dynamic-main.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) $(EMFLAGS_DYNAMIC) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
 	  $(OBJ_FILES_DIST)  -o $@
 
-dist/wa-sqlite-async-main.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+# Dynamic main module
+# Exported functions are omitted here since everything is currently exported
+dist/wa-sqlite-async-dynamic-main.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) $(EMFLAGS_DYNAMIC) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
 	  $(EMFLAGS_ASYNCIFY_DIST) \
 	  $(OBJ_FILES_DIST) -o $@
 
-# TODO link static
-dist/wa-sqlite-jspi.mjs: $(OBJ_FILES_DIST) $(JSFILES) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
-	mkdir -p dist
-	$(EMCC) $(EMFLAGS_DIST) \
-	  $(EMFLAGS_INTERFACES) \
-	  $(EMFLAGS_LIBRARIES) \
-	  $(EMFLAGS_JSPI) \
-	  $(OBJ_FILES_DIST) -o $@
 
 FORCE:

README.md

@@ -51,11 +51,13 @@ make -B
 
 ### PowerSync Core
 
-To update the PowerSync SQLite core see the [script](./scripts/powersync-core.js). Edit the `CORE_VERSION` and run the script with
+The PowerSync Rust SQLite extension is used both statically and dynamically in this package. The static core binary is downloaded in the `Makefile` during compilation. The dynamic WASM files are not commited to this repository's `dist` folder, instead the WASM files are downloaded as a `postinstall` script.
 
-```bash
-node scripts/powersync-core.js
-```
+Update the `powersync-version` file in order to update the PowerSync core version.
+
+### Usage with PowerSync
+
+This fork of WA-SQLite can be used with PowerSync either statically or dynamically. See the [demo worker](./demo/demo-worker.js) for usage.
 
 ## API
 Javascript wrappers for core SQLITE C API functions (and some others) are provided. Some convenience functions are also provided to reduce boilerplate. Here is sample code to load the library and call the API:

demo/demo-worker.js

@@ -9,12 +9,14 @@ const BUILDS = new Map([
   // ['default', '../debug/wa-sqlite.mjs'],
   // ['asyncify', '../debug/wa-sqlite-async.mjs'],
   // ['jspi', '../debug/wa-sqlite-jspi.mjs'],
+  // Dynamic builds
+  ['default-dynamic', '../dist/wa-sqlite-dynamic-main.mjs'],
+  ['asyncify-dynamic', '../dist/wa-sqlite-async-dynamic-main.mjs'],
 ]);
 
 const EXT_WASM = new Map([
-  ['default', 'libpowersync.wasm'],
-  ['asyncify', 'libpowersync-async.wasm'],
-  ['jspi', 'libpowersync-jspi.wasm'],
+  ['default-dynamic', 'libpowersync.wasm'],
+  ['asyncify-dynamic', 'libpowersync-async.wasm'],
 ]);
 
 /**
@@ -83,7 +85,6 @@ maybeReset().then(async () => {
   const buildName = searchParams.get('build') || BUILDS.keys().next().value;
   const configName = searchParams.get('config') || VFS_CONFIGS.keys().next().value;
   const config = VFS_CONFIGS.get(configName);
-  const extWasm = EXT_WASM.get(buildName);
 
   const dbName = searchParams.get('dbName') ?? 'hello';
   const vfsName = searchParams.get('vfsName') ?? config.vfsName ?? 'demo';
@@ -102,17 +103,24 @@ maybeReset().then(async () => {
 
   const sqlite3 = SQLite.Factory(module);
 
-  // Load the extension library into this scope
-  let extScope = {};
-  await module.loadDynamicLibrary(
-    extWasm,
-    { loadAsync: true },
-    extScope
-  );
 
-  // This calls sqlite3_auto_extension(sqlite3_powersync_init).
-  // For generic extensions, we'd need to call sqlite3_auto_extension directly.
-  extScope.powersync_init_static();
+  if (buildName.endsWith('-dynamic')) {
+    const extWasm = EXT_WASM.get(buildName);
+    // Load the extension library into this scope
+    let extScope = {};
+    await module.loadDynamicLibrary(
+      extWasm,
+      { loadAsync: true },
+      extScope
+    );
+
+    // This calls sqlite3_auto_extension(sqlite3_powersync_init).
+    // For generic extensions, we'd need to call sqlite3_auto_extension directly.
+    extScope.powersync_init_static();
+  } else {
+    // This is not part of the Sqlite3 API
+    module.ccall('setup_powersync', 'int', []);
+  }
 
   if (config.vfsModule) {
     // Create the VFS and register it as the default file system.