initial rust extension loading

stevensJourney · stevensJourney · commit 8479cc38a284 · 2023-11-29T15:58:18.000-07:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,13 @@
+# @journeyapps/wa-sqlite
+
+## 0.0.2
+
+### Patch Changes
+
+- 1cce650: Fixed Typescript declaration files.
+
+## 0.0.1
+
+### Patch Changes
+
+- 9bb4bb4: Initial loading of PowerSync Rust SQLite extension.
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,10 @@
+FROM mcr.microsoft.com/devcontainers/base:bullseye
+
+# Need Chrome: 
+RUN sudo apt update 
+RUN sudo apt install chromium clang default-jre -y
+
+
+COPY ./scripts/docker-setup.sh /tmp/setup.sh
+
+RUN sudo /bin/bash /tmp/setup.sh
diff --git a/Makefile b/Makefile
@@ -29,6 +29,13 @@ ASYNCIFY_IMPORTS = src/asyncify_imports.json
 OBJ_FILES_DEBUG = $(patsubst %.c,tmp/obj/debug/%.o,$(CFILES))
 OBJ_FILES_DIST = $(patsubst %.c,tmp/obj/dist/%.o,$(CFILES))
 
+RS_LIB = powersync
+RS_LIB_DIR = ./powersync-sqlite-core
+RS_WASM_TGT = wasm32-unknown-emscripten
+RS_WASM_TGT_DIR = ${RS_LIB_DIR}/target/$(RS_WASM_TGT)
+RS_RELEASE_BC = $(RS_WASM_TGT_DIR)/wasm/deps/$(RS_LIB).bc
+RS_DEBUG_BC = $(RS_WASM_TGT_DIR)/debug/deps/$(RS_LIB).bc
+
 # build options
 EMCC ?= emcc
 
@@ -66,6 +73,7 @@ EMFLAGS_LIBRARIES = \
 	--js-library src/libauthorizer.js \
 	--js-library src/libfunction.js \
 	--js-library src/libmodule.js \
+	--js-library src/libtableupdates.js \
 	--js-library src/libprogress.js \
 	--js-library src/libvfs.js
 
@@ -150,6 +158,16 @@ tmp/obj/dist/%.o: %.c
 	mkdir -p tmp/obj/dist
 	$(EMCC) $(CFLAGS_DIST) $(WASQLITE_DEFINES) $^ -c -o $@
 
+$(RS_DEBUG_BC): FORCE
+	mkdir -p tmp/bc/dist
+	cd $(RS_LIB_DIR); \
+	RUSTFLAGS="--emit=llvm-bc -C linker=/bin/true" cargo build -p powersync_loadable --profile wasm --no-default-features --features "powersync_core/static powersync_core/omit_load_extension sqlite_nostd/static sqlite_nostd/omit_load_extension" -Z build-std=panic_abort,core,alloc --target $(RS_WASM_TGT)
+
+$(RS_RELEASE_BC): FORCE
+	mkdir -p tmp/bc/dist
+	cd $(RS_LIB_DIR); \
+	RUSTFLAGS="--emit=llvm-bc -C linker=/bin/true" cargo build -p powersync_loadable --profile wasm --no-default-features --features "powersync_core/static powersync_core/omit_load_extension sqlite_nostd/static sqlite_nostd/omit_load_extension" -Z build-std=panic_abort,core,alloc --target $(RS_WASM_TGT)
+
 
 ## debug
 .PHONY: clean-debug
@@ -159,20 +177,22 @@ clean-debug:
 .PHONY: debug
 debug: debug/wa-sqlite.mjs debug/wa-sqlite-async.mjs
 
-debug/wa-sqlite.mjs: $(OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+debug/wa-sqlite.mjs: $(OBJ_FILES_DEBUG) $(RS_DEBUG_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
 	mkdir -p debug
 	$(EMCC) $(EMFLAGS_DEBUG) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
-	  $(OBJ_FILES_DEBUG) -o $@
+		$(RS_WASM_TGT_DIR)/debug/deps/*.bc \
+	  $(OBJ_FILES_DEBUG) *.o -o $@
 
-debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG) $(RS_DEBUG_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
 	mkdir -p debug
 	$(EMCC) $(EMFLAGS_DEBUG) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
 	  $(EMFLAGS_ASYNCIFY_DEBUG) \
-	  $(OBJ_FILES_DEBUG) -o $@
+		$(RS_WASM_TGT_DIR)/debug/deps/*.bc \
+	  $(OBJ_FILES_DEBUG) *.o -o $@
 
 ## dist
 .PHONY: clean-dist
@@ -182,17 +202,22 @@ clean-dist:
 .PHONY: dist
 dist: dist/wa-sqlite.mjs dist/wa-sqlite-async.mjs
 
-dist/wa-sqlite.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+dist/wa-sqlite.mjs: $(OBJ_FILES_DIST) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
-	  $(OBJ_FILES_DIST) -o $@
+		$(RS_WASM_TGT_DIR)/wasm/deps/*.bc \
+	  $(OBJ_FILES_DIST)  -o $@
 
-dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
 	  $(EMFLAGS_ASYNCIFY_DIST) \
-	  $(OBJ_FILES_DIST) -o $@
+		$(CFLAGS_DIST) \
+		$(RS_WASM_TGT_DIR)/wasm/deps/*.bc \
+	  $(OBJ_FILES_DIST)  -o $@
+
+FORCE: ;
diff --git a/README.md b/README.md
@@ -13,14 +13,12 @@ The primary motivation for this project is to enable additions to SQLite with on
 Note that earlier versions of the project only provided pre-built artifacts in the
 "buildless" branch; that branch will no longer be maintained.
 
-Minor build customization (e.g. changing build defines or flags) can be done with [make arguments](https://github.com/rhashimoto/wa-sqlite/discussions/128), and the helper project [sqwab](https://github.com/rhashimoto/sqwab) can be used to build without a local build environment.
-
-If you do want to build yourself, here are the prerequisites:
+If you do want to build - e.g. you want to change build flags, use a specific EMSDK version, or modify wa-sqlite itself - here are the prerequisites:
 
 * Building on Debian Linux is known to work, compatibility with other platforms is unknown.
 * `yarn` - If you use a different package manager (e.g. `npm`) then file paths in the demo will need adjustment.
 * [Emscripten SDK](https://emscripten.org/docs/getting_started/downloads.html) 3.1.25+.
-* `curl`, `make`, `openssl`, `sed`, `tclsh`, `unzip`
+* `curl`, `make`, `openssl`, `sed`, `unzip`
 
 Here are the build steps:
 * Make sure `emcc` works.
@@ -31,6 +29,38 @@ Here are the build steps:
 
 The default build produces ES6 modules + WASM, [synchronous and asynchronous](https://github.com/rhashimoto/wa-sqlite/issues/7) (using Asyncify) in `dist/`.
 
+## JouneyApps instructions
+
+Note as per above that this is known to compile under Debian. 
+
+MacOS initially complained about OpenSSL config. Compiling on MacOS may be possible with additional config, but this is currently unknown.
+
+Development has been done using VSCode's [development container](https://code.visualstudio.com/docs/devcontainers/containers) feature.
+
+```bash
+git clone [this repo]
+```
+
+
+```bash
+git submodule init
+```
+
+```bash
+git submodule update --recursive
+```
+
+```bash
+yarn install
+
+```
+
+
+```bash 
+make -B
+```
+
+
 ## API
 Javascript wrappers for core SQLITE C API functions (and some others) are provided. Some convenience functions are also provided to reduce boilerplate. Here's sample code to load the library and call the API:
 
@@ -76,4 +106,4 @@ For convenience, if any text region is selected in the editor, only that region
 ## License
 MIT License as of February 10, 2023, changed by generous sponsors
 [Fleet Device Management](https://fleetdm.com/) and [Reflect](https://reflect.app/).
-Existing licensees may continue under the GPLv3 or switch to the new license.
+Existing licensees may continue under the GPLv3 or switch to the new license.
diff --git a/package.json b/package.json
@@ -1,6 +1,13 @@
 {
-  "name": "wa-sqlite",
-  "version": "0.9.9",
+  "name": "@journeyapps/wa-sqlite",
+  "version": "0.0.2",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/powersync-ja/wa-sqlite.git"
+  },
   "type": "module",
   "main": "src/sqlite-api.js",
   "types": "src/types/index.d.ts",
@@ -15,12 +22,13 @@
   ],
   "scripts": {
     "build-docs": "typedoc",
-    "prepack": "make",
+    "release": "yarn changeset publish",
     "start": "web-dev-server --node-resolve",
     "test": "karma start karma.conf.cjs --browsers ChromeHeadless",
     "test-manual": "karma start karma.conf.cjs --auto-watch --no-single-run --reporters progress"
   },
   "devDependencies": {
+    "@changesets/cli": "^2.26.2",
     "@web/dev-server": "^0.1.13",
     "comlink": "^4.4.1",
     "jasmine-core": "^4.5.0",
diff --git a/powersync-sqlite-core b/powersync-sqlite-core
@@ -0,0 +1 @@
+Subproject commit 53223d621dbd77789eb1ac162a4c0abd4fdb4043
diff --git a/scripts/docker-setup.sh b/scripts/docker-setup.sh
@@ -0,0 +1,10 @@
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+source ~/.bashrc
+rustup component add rust-src --toolchain nightly-2023-08-28-aarch64-unknown-linux-gnu -y
+
+
+# Need NVM:
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash
+source ~/.bashrc
+nvm install 18.12.0 && nvm use 18.12.0
+npm install -g yarn
diff --git a/src/exported_functions.json b/src/exported_functions.json
@@ -53,5 +53,8 @@
   "_sqlite3_value_int64",
   "_sqlite3_value_text",
   "_sqlite3_value_type",
-  "_sqlite3_vfs_find"
+  "_sqlite3_vfs_find",
+  "_sqlite3_last_insert_rowid",
+  "_setup_powersync",
+  "_register_table_update_hook"
 ]
diff --git a/src/libmodule.c b/src/libmodule.c
@@ -32,6 +32,18 @@ extern int modRollback(sqlite3_vtab *pVTab);
 //                        void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
 //                        void **ppArg);
 extern int modRename(sqlite3_vtab *pVtab, const char *zNew);
+extern void onTableChangeCallback(sqlite3*, int, const char *, int);
+
+EMSCRIPTEN_KEEPALIVE
+void on_tables_changed(void *db, int opType, char const *dbName,
+                        char const *tableName, sqlite3_int64 rowId) {
+  onTableChangeCallback((sqlite3 *) db, opType, tableName, rowId);
+}
+
+void register_table_update_hook(sqlite3 *db) {
+    sqlite3_update_hook(db, on_tables_changed,
+                      (void *)(db));
+}
 
 static int xCreate(
   sqlite3* db,
@@ -65,6 +77,7 @@ static int xConnect(
   return result;
 }
 
+
 static int xOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor) {
   *ppCursor = (sqlite3_vtab_cursor*)sqlite3_malloc(sizeof(sqlite3_vtab_cursor));
   return modOpen(pVTab, *ppCursor);
diff --git a/src/libtableupdates.js b/src/libtableupdates.js
@@ -0,0 +1,7 @@
+mergeInto(LibraryManager.library, {
+    onTableChangeCallback: function(db, optType, tableName, rowId) {
+        // This is exposed globally since exporting from this module caused WASM compilation errors
+        const fn = globalThis['__onTablesChanged'];
+        fn?.(db, optType, tableName, rowId);
+    }
+});
diff --git a/src/libvfs.c b/src/libvfs.c
@@ -23,6 +23,11 @@ extern int vfsOpen(sqlite3_vfs* vfs, const char *zName, sqlite3_file* file, int
 extern int vfsDelete(sqlite3_vfs* vfs, const char *zName, int syncDir);
 extern int vfsAccess(sqlite3_vfs* vfs, const char *zName, int flags, int *pResOut);
 
+// This is undefined in the WASM linker step if not specified
+extern int __rust_no_alloc_shim_is_unstable = 0;
+extern int sqlite3_powersync_init(sqlite3 *db, char **pzErrMsg,
+                           const sqlite3_api_routines *pApi);
+
 // Glue functions to pass 64-bit integers by pointer.
 static int xRead(sqlite3_file* file, void* pData, int iAmt, sqlite3_int64 iOffset) {
   return vfsRead(file, pData, iAmt, &iOffset);
@@ -119,4 +124,8 @@ void* EMSCRIPTEN_KEEPALIVE getSqliteFree() {
 int main() {
   sqlite3_initialize();
   return 0;
+}
+
+int setup_powersync() {
+   return sqlite3_auto_extension((void (*)(void)) &sqlite3_powersync_init);
 }
diff --git a/src/sqlite-api.js b/src/sqlite-api.js
diff --git a/src/types/globals.d.ts b/src/types/globals.d.ts
diff --git a/src/types/index.d.ts b/src/types/index.d.ts
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Subproject commit 53223d621dbd77789eb1ac162a4c0abd4fdb4043`