Merge remote-tracking branch 'origin/main' into concurrent-storage-batches

Author: rkistner

.changeset/thin-snails-compete.md

@@ -0,0 +1,15 @@
+---
+'@powersync/service-core': minor
+'@powersync/service-module-mssql': minor
+'@powersync/service-module-postgres-storage': patch
+'@powersync/service-module-mongodb-storage': patch
+'@powersync/service-module-postgres': patch
+'@powersync/service-errors': patch
+'@powersync/service-module-mysql': patch
+'@powersync/service-image': patch
+---
+
+- First iteration of MSSQL replication using Change Data Capture (CDC). 
+- Supports resumable snapshot replication
+- Uses CDC polling for replication
+

.github/workflows/test.yml

@@ -297,3 +297,96 @@ jobs:
 
       - name: Test Storage
         run: pnpm --filter='./modules/module-mongodb-storage' test
+
+  run-mssql-tests:
+    name: MSSQL Test
+    runs-on: ubuntu-latest
+    needs: run-core-tests
+
+    env:
+      MSSQL_SA_PASSWORD: 321strong_ROOT_password
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mssql-version: [2022, 2025]
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Start MSSQL
+        run: |
+          docker run \
+            --name MSSQLTestDatabase \
+            --health-cmd="/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P \"${{ env.MSSQL_SA_PASSWORD }}\" -Q \"SELECT 1;\" || exit 1" \
+            --health-interval 5s \
+            --health-timeout 3s \
+            --health-retries 30 \
+            -e ACCEPT_EULA=Y \
+            -e MSSQL_SA_PASSWORD=${{ env.MSSQL_SA_PASSWORD }} \
+            -e MSSQL_PID=Developer \
+            -e MSSQL_AGENT_ENABLED=true \
+            -p 1433:1433 \
+            -d mcr.microsoft.com/mssql/server:${{ matrix.mssql-version }}-latest
+
+      - name: Wait for MSSQL to be healthy
+        run: |
+          timeout 120 bash -c 'until docker inspect --format="{{.State.Health.Status}}" MSSQLTestDatabase | grep -q "healthy"; do sleep 2; done'
+
+      - name: Initialize MSSQL database
+        run: |
+          docker run \
+            --rm \
+            --network host \
+            -e MSSQL_SA_PASSWORD=${{ env.MSSQL_SA_PASSWORD }} \
+            -v ${{ github.workspace }}/modules/module-mssql/ci/init-mssql.sql:/scripts/init-mssql.sql:ro \
+            mcr.microsoft.com/mssql/server:${{ matrix.mssql-version }}-latest \
+            /bin/bash -c "/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P \"${{ env.MSSQL_SA_PASSWORD }}\" -v DATABASE=powersync -v DB_USER=sa -i /scripts/init-mssql.sql"
+
+      # The mongodb-github-action below doesn't use the Docker credentials for the pull.
+      # We pre-pull, so that the image is cached.
+      - name: Pre-pull Mongo image
+        run: docker pull mongo:8.0
+
+      - name: Start MongoDB
+        uses: supercharge/mongodb-github-action@1.12.0
+        with:
+          mongodb-version: '8.0'
+          mongodb-replica-set: test-rs
+
+      - name: Start PostgreSQL (Storage)
+        run: |
+          docker run \
+            --health-cmd pg_isready \
+            --health-interval 10s \
+            --health-timeout 5s \
+            --health-retries 5 \
+            -e POSTGRES_PASSWORD=postgres \
+            -e POSTGRES_DB=powersync_storage_test \
+            -p 5431:5432 \
+            -d postgres:18
+
+      - name: Enable Corepack
+        run: corepack enable
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: '.nvmrc'
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Build
+        shell: bash
+        run: pnpm build
+
+      - name: Test Replication
+        run: pnpm --filter='./modules/module-mssql' test

modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts

@@ -146,6 +146,10 @@ export class MongoBucketBatch
     return this.last_checkpoint_lsn;
   }
 
+  get noCheckpointBeforeLsn() {
+    return this.no_checkpoint_before_lsn;
+  }
+
   async flush(options?: storage.BatchBucketFlushOptions): Promise<storage.FlushedResult | null> {
     let result: storage.FlushedResult | null = null;
     // One flush may be split over multiple transactions.

modules/module-mssql/.npmignore

@@ -0,0 +1,2 @@
+dev
+ci

modules/module-mssql/LICENSE

@@ -0,0 +1,67 @@
+# Functional Source License, Version 1.1, ALv2 Future License
+
+## Abbreviation
+
+FSL-1.1-ALv2
+
+## Notice
+
+Copyright 2023-2025 Journey Mobile, Inc.
+
+## Terms and Conditions
+
+### Licensor ("We")
+
+The party offering the Software under these Terms and Conditions.
+
+### The Software
+
+The "Software" is each version of the software that we make available under these Terms and Conditions, as indicated by our inclusion of these Terms and Conditions with the Software.
+
+### License Grant
+
+Subject to your compliance with this License Grant and the Patents, Redistribution and Trademark clauses below, we hereby grant you the right to use, copy, modify, create derivative works, publicly perform, publicly display and redistribute the Software for any Permitted Purpose identified below.
+
+### Permitted Purpose
+
+A Permitted Purpose is any purpose other than a Competing Use. A Competing Use means making the Software available to others in a commercial product or service that:
+
+1. substitutes for the Software;
+2. substitutes for any other product or service we offer using the Software that exists as of the date we make the Software available; or
+3. offers the same or substantially similar functionality as the Software.
+
+Permitted Purposes specifically include using the Software:
+
+1. for your internal use and access;
+2. for non-commercial education;
+3. for non-commercial research; and
+4. in connection with professional services that you provide to a licensee using the Software in accordance with these Terms and Conditions.
+
+### Patents
+
+To the extent your use for a Permitted Purpose would necessarily infringe our patents, the license grant above includes a license under our patents. If you make a claim against any party that the Software infringes or contributes to the infringement of any patent, then your patent license to the Software ends immediately.
+
+### Redistribution
+
+The Terms and Conditions apply to all copies, modifications and derivatives of the Software.
+If you redistribute any copies, modifications or derivatives of the Software, you must include a copy of or a link to these Terms and Conditions and not remove any copyright notices provided in or with the Software.
+
+### Disclaimer
+
+THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT.
+IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE.
+
+### Trademarks
+
+Except for displaying the License Details and identifying us as the origin of the Software, you have no right under these Terms and Conditions to use our trademarks, trade names, service marks or product names.
+
+## Grant of Future License
+
+We hereby irrevocably grant you an additional license to use the Software under the Apache License, Version 2.0 that is effective on the second anniversary of the date we make the Software available. On or after that date, you may use the Software under the Apache License, Version 2.0, in which case the following will apply:
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

modules/module-mssql/README.md

@@ -0,0 +1,3 @@
+# PowerSync MSSQL Module
+
+MSSQL replication module for PowerSync

modules/module-mssql/ci/init-mssql.sql

@@ -0,0 +1,51 @@
+-- Create database (idempotent)
+IF DB_ID('$(DATABASE)') IS NULL
+BEGIN
+    CREATE DATABASE [$(DATABASE)];
+END
+GO
+
+-- Enable CDC at the database level (idempotent)
+USE [$(DATABASE)];
+IF (SELECT is_cdc_enabled FROM sys.databases WHERE name = '$(DATABASE)') = 0
+BEGIN
+    EXEC sys.sp_cdc_enable_db;
+END
+GO
+
+-- Create PowerSync checkpoints table
+-- Powersync requires this table to ensure regular checkpoints appear in CDC
+IF OBJECT_ID('dbo._powersync_checkpoints', 'U') IS NULL
+BEGIN
+    CREATE TABLE dbo._powersync_checkpoints (
+        id INT IDENTITY PRIMARY KEY,
+        last_updated DATETIME NOT NULL DEFAULT (GETDATE())
+);
+END
+
+GRANT INSERT, UPDATE ON dbo._powersync_checkpoints TO [$(DB_USER)];
+GO
+
+-- Enable CDC for the powersync checkpoints table
+IF NOT EXISTS (SELECT 1 FROM cdc.change_tables WHERE source_object_id = OBJECT_ID(N'dbo._powersync_checkpoints'))
+    BEGIN
+    EXEC sys.sp_cdc_enable_table
+        @source_schema = N'dbo',
+        @source_name   = N'_powersync_checkpoints',
+        @role_name     = N'cdc_reader',
+        @supports_net_changes = 0;
+END
+GO
+
+-- Wait until capture job exists - usually takes a few seconds after enabling CDC on a table for the first time
+DECLARE @tries int = 10;
+WHILE @tries > 0 AND NOT EXISTS (SELECT 1 FROM msdb.dbo.cdc_jobs WHERE job_type = N'capture')
+BEGIN
+    WAITFOR DELAY '00:00:01';
+    SET @tries -= 1;
+END;
+
+-- Set the CDC capture job polling interval to 0 seconds (default is 5 seconds).
+-- At the cost of higher CPU usage, this minimizes latency for change capture.
+EXEC sys.sp_cdc_change_job @job_type = N'capture', @pollinginterval = 0;
+GO

modules/module-mssql/dev/.env.template

@@ -0,0 +1,4 @@
+ROOT_PASSWORD=321strong_ROOT_password
+DATABASE=powersync
+DB_USER=powersync_user
+DB_USER_PASSWORD=321strong_POWERSYNC_password

modules/module-mssql/dev/README.md

@@ -0,0 +1,82 @@
+# MSSQL Dev Database
+
+This directory contains Docker Compose configuration for running a local MSSQL Server instance with CDC (Change Data Capture) enabled for development and testing. The image used is the 2022 Edition of SQL Server. 2025 can also be used, but has issues on Mac OS X 26 Tahoe due to this issue: https://github.com/microsoft/mssql-docker/issues/942
+
+## Prerequisites
+
+- Docker and Docker Compose installed
+- A `.env` file in this directory see the `.env.template` for required variables
+
+## Environment Variables
+
+```bash
+ROOT_PASSWORD=
+DATABASE=
+DB_USER=
+DB_USER_PASSWORD=
+```
+
+**Note:** The `ROOT_PASSWORD` and `DB_USER_PASSWORD` must meet SQL Server password complexity requirements (at least 8 characters, including uppercase, lowercase, numbers, and special characters).
+
+## Usage
+
+### Starting the Database
+
+From the `dev` directory, run:
+
+```bash
+docker compose up -d
+```
+
+This will:
+1. Start the MSSQL Server container (`mssql-dev`)
+2. Wait for the database to be healthy
+3. Automatically run the setup container (`mssql-dev-setup`) which executes `init.sql`
+
+### Stopping the Database
+
+```bash
+docker compose down
+```
+
+To also remove the data volume:
+
+```bash
+docker compose down -v
+```
+
+### Viewing Logs
+
+```bash
+docker compose logs -f
+```
+
+## What `init.sql` Does
+
+The initialization script (`init.sql`) performs the following setup steps:
+
+1. **Database Creation**: Creates the application database (if it doesn't exist)
+2. **CDC Setup**: Enables Change Data Capture at the database level
+3. **User Creation**: Creates a SQL Server login and database user with appropriate permissions
+4. **Create PowerSync Checkpoints table**: Creates the required `_powersync_checkpoints` table.
+5. **Demo Tables**: Creates sample tables (`lists` and `todos`) for testing (optional examples)
+6. **CDC Table Enablement**: Enables CDC tracking on the demo tables
+7. **Permissions**: Grants `db_datareader` and `cdc_reader` roles to the application user
+8. **Sample Data**: Inserts initial test data into the `lists` table
+
+All operations are idempotent, so you can safely re-run the setup without errors. The demo tables section (steps 5–7) serves as an example of how to enable CDC on your own tables.
+
+## Connection Details
+
+- **Host**: `localhost`
+- **Port**: `1433`
+- **SA Login**: `sa` / `{ROOT_PASSWORD}`
+- **App Login**: `{DB_USER}` / `{DB_USER_PASSWORD}`
+- **Database**: `{DATABASE}`
+
+## Troubleshooting
+
+- If the setup container fails, check logs: `docker compose logs mssql-dev-setup`
+- Ensure your `.env` file exists and contains all required variables
+- The database container may take 30–60 seconds to become healthy on the first startup
+- If you encounter connection issues, verify the container is running: `docker compose ps`

modules/module-mssql/dev/docker-compose.yaml

@@ -0,0 +1,39 @@
+name: mssql-dev
+services:
+  mssql-dev:
+    platform: linux/amd64
+    image: mcr.microsoft.com/mssql/server:2022-latest  # 2025 Can also be used, but not on Mac 26 Tahoe due to this issue: https://github.com/microsoft/mssql-docker/issues/942
+    container_name: mssql-dev
+    ports:
+      - "1433:1433"
+    environment:
+      ACCEPT_EULA: "Y"
+      MSSQL_SA_PASSWORD: "${ROOT_PASSWORD}"
+      MSSQL_PID: "Developer"
+      MSSQL_AGENT_ENABLED: "true"   # required for CDC capture/cleanup jobs
+    volumes:
+      - data:/var/opt/mssql
+    healthcheck:
+      test: [ "CMD-SHELL", "/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P \"$${MSSQL_SA_PASSWORD}\" -Q \"SELECT 1;\" || exit 1" ]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+
+  mssql-dev-setup:
+    platform: linux/amd64
+    image: mcr.microsoft.com/mssql/server:2022-latest
+    container_name: mssql-dev-setup
+    depends_on:
+      mssql-dev:
+        condition: service_healthy
+    environment:
+      MSSQL_SA_PASSWORD: "${ROOT_PASSWORD}"
+      DATABASE: "${DATABASE}"
+      DB_USER: "${DB_USER}"
+      DB_USER_PASSWORD: "${DB_USER_PASSWORD}"
+    volumes:
+      - ./init.sql:/scripts/init.sql:ro
+    entrypoint: ["/bin/bash", "-lc", "/opt/mssql-tools18/bin/sqlcmd -C -S mssql-dev,1433 -U sa -P \"$${MSSQL_SA_PASSWORD}\" -i /scripts/init.sql && echo '✅ MSSQL init done'"]
+
+volumes:
+  data: