Merge remote-tracking branch 'origin/main' into granular-sync-rules

Author: rkistner

.changeset/nasty-snakes-double.md

@@ -0,0 +1,6 @@
+---
+'@powersync/service-core': patch
+'@powersync/service-image': patch
+---
+
+- Rework dynamic module loading, fixing startup issues for migration and compact jobs in 1.18.0 / 1.18.1

.changeset/short-pumpkins-retire.md

@@ -0,0 +1,11 @@
+---
+'@powersync/service-image': minor
+'@powersync/service-module-postgres': patch
+'@powersync/service-module-mongodb': patch
+'@powersync/service-module-mysql': patch
+'@powersync/service-module-mssql': patch
+'@powersync/service-sync-rules': patch
+'@powersync/service-jpgwire': patch
+---
+
+Add the `timestamp_max_precision` option for sync rules. It can be set to `seconds`, `milliseconds` or `microseconds` to restrict the precision of synced datetime values.

.changeset/ten-walls-cough.md

@@ -0,0 +1,6 @@
+---
+'@powersync/service-module-postgres': patch
+'@powersync/service-jpgwire': patch
+---
+
+Update `pgwire` to version `0.8.1`.

libs/lib-postgres/src/db/connection/AbstractPostgresConnection.ts

@@ -69,13 +69,11 @@ export abstract class AbstractPostgresConnection<Listener = {}> extends framewor
   }
 
   async *streamRows<T>(...args: pgwire.Statement[]): AsyncIterableIterator<T[]> {
-    let columns: Array<keyof T> = [];
+    let columns: Array<pgwire.ColumnDescription> = [];
 
     for await (const chunk of this.stream(...args)) {
       if (chunk.tag == 'RowDescription') {
-        columns = chunk.payload.map((c, index) => {
-          return c.name as keyof T;
-        });
+        columns = chunk.payload;
         continue;
       }
 
@@ -86,7 +84,7 @@ export abstract class AbstractPostgresConnection<Listener = {}> extends framewor
       yield chunk.rows.map((row) => {
         let q: Partial<T> = {};
         for (const [index, c] of columns.entries()) {
-          q[c] = row[index];
+          q[c.name as keyof T] = row.decodeWithoutCustomTypes(index);
         }
         return q as T;
       });

modules/module-mongodb/src/replication/MongoRelation.ts

@@ -8,7 +8,9 @@ import {
   CustomSqliteValue,
   SqliteInputRow,
   SqliteInputValue,
-  DateTimeValue
+  DateTimeValue,
+  DateTimeSourceOptions,
+  TimeValuePrecision
 } from '@powersync/service-sync-rules';
 
 import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
@@ -69,7 +71,7 @@ export function toMongoSyncRulesValue(data: any): SqliteInputValue {
     return data.toHexString();
   } else if (data instanceof Date) {
     const isoString = data.toISOString();
-    return new DateTimeValue(isoString);
+    return new DateTimeValue(isoString, undefined, mongoTimeOptions);
   } else if (data instanceof mongo.Binary) {
     return new Uint8Array(data.buffer);
   } else if (data instanceof mongo.Long) {
@@ -122,7 +124,7 @@ function filterJsonData(data: any, context: CompatibilityContext, depth = 0): an
     return data;
   } else if (data instanceof Date) {
     const isoString = data.toISOString();
-    return new DateTimeValue(isoString).toSqliteValue(context);
+    return new DateTimeValue(isoString, undefined, mongoTimeOptions).toSqliteValue(context);
   } else if (data instanceof mongo.ObjectId) {
     return data.toHexString();
   } else if (data instanceof mongo.UUID) {
@@ -195,3 +197,8 @@ export async function createCheckpoint(
     await session.endSession();
   }
 }
+
+const mongoTimeOptions: DateTimeSourceOptions = {
+  subSecondPrecision: TimeValuePrecision.milliseconds,
+  defaultSubSecondPrecision: TimeValuePrecision.milliseconds
+};

modules/module-mongodb/test/src/mongo_test.test.ts

@@ -4,7 +4,8 @@ import {
   CompatibilityContext,
   CompatibilityEdition,
   SqliteInputRow,
-  SqlSyncRules
+  SqlSyncRules,
+  TimeValuePrecision
 } from '@powersync/service-sync-rules';
 import { describe, expect, test } from 'vitest';
 
@@ -555,11 +556,23 @@ bucket_definitions:
         noFraction: '2023-03-06 13:47:01.000Z'
       });
 
-      const newFormat = applyRowContext(row, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS));
+      const newFormat = applyRowContext(row, new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS }));
       expect(newFormat).toMatchObject({
         fraction: '2023-03-06T13:47:01.123Z',
         noFraction: '2023-03-06T13:47:01.000Z'
       });
+
+      const reducedPrecisionFormat = applyRowContext(
+        row,
+        new CompatibilityContext({
+          edition: CompatibilityEdition.SYNC_STREAMS,
+          maxTimeValuePrecision: TimeValuePrecision.seconds
+        })
+      );
+      expect(reducedPrecisionFormat).toMatchObject({
+        fraction: '2023-03-06T13:47:01Z',
+        noFraction: '2023-03-06T13:47:01Z'
+      });
     } finally {
       await client.close();
     }

modules/module-mssql/src/common/mssqls-to-sqlite.ts

@@ -1,10 +1,14 @@
 import sql from 'mssql';
 import {
   DatabaseInputRow,
+  DateTimeSourceOptions,
+  DateTimeValue,
   ExpressionType,
   SQLITE_FALSE,
   SQLITE_TRUE,
   SqliteInputRow,
+  TimeValue,
+  TimeValuePrecision,
   toSyncRulesRow
 } from '@powersync/service-sync-rules';
 import { MSSQLUserDefinedType } from '../types/mssql-data-types.js';
@@ -32,14 +36,25 @@ export function toSqliteInputRow(row: any, columns: sql.IColumnMetadata): Sqlite
           result[key] = toISODateString(row[key] as Date);
           break;
         case sql.TYPES.Time:
-          result[key] = toISOTimeString(row[key] as Date);
+          result[key] = toISOTimeValue(row[key] as DateWithNanosecondsDelta);
           break;
         case sql.TYPES.DateTime:
         case sql.TYPES.DateTime2:
         case sql.TYPES.SmallDateTime:
         case sql.TYPES.DateTimeOffset: // The offset is lost when the driver converts to Date. This needs to be handled in the sql query.
-          const date = row[key] as Date;
-          result[key] = isNaN(date.getTime()) ? null : date.toISOString();
+          const date = row[key] as DateWithNanosecondsDelta;
+          if (isNaN(date.getTime())) {
+            result[key] = null;
+            break;
+          }
+
+          const originalFormat = date.toISOString();
+          const withNanos =
+            originalFormat.substring(0, originalFormat.length - 1) + // Drop the trailing Z
+            subMillisecondSuffix(date) + // Expand the .xyz millisecond part to include nanoseconds
+            'Z';
+
+          result[key] = new DateTimeValue(withNanos, withNanos, msSqlTimeSourceOptions);
           break;
         case sql.TYPES.Binary:
         case sql.TYPES.VarBinary:
@@ -74,13 +89,33 @@ function toISODateString(date: Date): string | null {
   return isNaN(date.getTime()) ? null : date.toISOString().split('T')[0];
 }
 
+// https://github.com/tediousjs/tedious/blob/0c256f186600d7230aec05553ebad209bed81acc/src/value-parser.ts#L668-L670
+interface DateWithNanosecondsDelta extends Date {
+  nanosecondsDelta?: number;
+}
+
 /**
  *  MSSQL time format is HH:mm:ss[.nnnnnnn]
- * @param date
- * @returns
  */
-function toISOTimeString(date: Date): string | null {
-  return isNaN(date.getTime()) ? null : date.toISOString().split('T')[1].replace('Z', '');
+function toISOTimeValue(date: DateWithNanosecondsDelta): TimeValue | null {
+  if (isNaN(date.getTime())) {
+    return null;
+  }
+
+  const withMillis = date.toISOString().split('T')[1].replace('Z', '') + subMillisecondSuffix(date);
+  return TimeValue.parse(withMillis, msSqlTimeSourceOptions);
+}
+
+function subMillisecondSuffix(value: DateWithNanosecondsDelta): string {
+  if (value.nanosecondsDelta != null) {
+    // nanosecondsDelta is actually measured in seconds. It will always be smaller than 1ms though, because that delta
+    // is stored on the datetime itself. We truncate here as a precaution.
+    const actualNanos = Math.round(value.nanosecondsDelta * Math.pow(10, 9)) % Math.pow(10, 6);
+
+    return actualNanos.toString().padStart(6, '0');
+  } else {
+    return '000000';
+  }
 }
 
 /**
@@ -156,3 +191,9 @@ export function CDCToSqliteRow(options: CDCRowToSqliteRowOptions): SqliteInputRo
   }
   return toSqliteInputRow(filteredRow, columns);
 }
+
+const msSqlTimeSourceOptions: DateTimeSourceOptions = {
+  // We actually only have 100ns precision (7 digits), but all the options are SI prefixes.
+  subSecondPrecision: TimeValuePrecision.nanoseconds,
+  defaultSubSecondPrecision: TimeValuePrecision.nanoseconds
+};

modules/module-mssql/test/src/mssql-to-sqlite.test.ts

@@ -1,4 +1,10 @@
-import { SQLITE_TRUE, SqliteInputRow } from '@powersync/service-sync-rules';
+import {
+  applyRowContext,
+  CompatibilityContext,
+  SQLITE_TRUE,
+  SqliteInputRow,
+  TimeValuePrecision
+} from '@powersync/service-sync-rules';
 import { afterAll, beforeEach, describe, expect, test } from 'vitest';
 import { clearTestDb, createUpperCaseUUID, TEST_CONNECTION_OPTIONS, waitForPendingCDCChanges } from './util.js';
 import { CDCToSqliteRow, toSqliteInputRow } from '@module/common/mssqls-to-sqlite.js';
@@ -43,10 +49,10 @@ describe('MSSQL Data Types Tests', () => {
         
         date_col DATE,
         datetime_col DATETIME,
-        datetime2_col DATETIME2(6),
+        datetime2_col DATETIME2(7),
         smalldatetime_col SMALLDATETIME,
         datetimeoffset_col DATETIMEOFFSET(3),
-        time_col TIME(6),
+        time_col TIME(7),
         
         char_col CHAR(10),
         varchar_col VARCHAR(255),
@@ -214,7 +220,14 @@ describe('MSSQL Data Types Tests', () => {
 
   test('Date types mappings', async () => {
     const beforeLSN = await getLatestLSN(connectionManager);
-    const testDate = new Date('2023-03-06T15:47:00.000Z');
+    const testDate = new Date('2023-03-06T15:47:00.123Z');
+    // This adds 0.4567 milliseconds to the JS date, see https://github.com/tediousjs/tedious/blob/0c256f186600d7230aec05553ebad209bed81acc/src/data-types/datetime2.ts#L74.
+    // Note that there's a typo in tedious there. When reading dates, the property is actually called nanosecondsDelta.
+    // This is only relevant when binding datetime values, so only in this test.
+    Object.defineProperty(testDate, 'nanosecondDelta', {
+      enumerable: false,
+      value: 0.0004567
+    });
     await connectionManager.query(
       `
       INSERT INTO ${escapeIdentifier(connectionManager.schema)}.test_data(
@@ -235,9 +248,9 @@ describe('MSSQL Data Types Tests', () => {
       [
         { name: 'date_col', type: sql.Date, value: testDate },
         { name: 'datetime_col', type: sql.DateTime, value: testDate },
-        { name: 'datetime2_col', type: sql.DateTime2(6), value: testDate },
+        { name: 'datetime2_col', type: sql.DateTime2(7), value: testDate },
         { name: 'smalldatetime_col', type: sql.SmallDateTime, value: testDate },
-        { name: 'time_col', type: sql.Time(6), value: testDate }
+        { name: 'time_col', type: sql.Time(7), value: testDate }
       ]
     );
     await waitForPendingCDCChanges(beforeLSN, connectionManager);
@@ -246,14 +259,32 @@ describe('MSSQL Data Types Tests', () => {
     const replicatedRows = await getReplicatedRows(connectionManager, 'test_data');
     const expectedResult = {
       date_col: '2023-03-06',
-      datetime_col: '2023-03-06T15:47:00.000Z',
-      datetime2_col: '2023-03-06T15:47:00.000Z',
-      smalldatetime_col: '2023-03-06T15:47:00.000Z',
-      time_col: '15:47:00.000'
+      datetime_col: '2023-03-06T15:47:00.123000000Z',
+      datetime2_col: '2023-03-06T15:47:00.123456700Z',
+      smalldatetime_col: '2023-03-06T15:47:00.000000000Z',
+      time_col: '15:47:00.123456700'
     };
 
-    expect(databaseRows[0]).toMatchObject(expectedResult);
-    expect(replicatedRows[0]).toMatchObject(expectedResult);
+    expect(applyRowContext(databaseRows[0], CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toMatchObject(
+      expectedResult
+    );
+    expect(applyRowContext(replicatedRows[0], CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toMatchObject(
+      expectedResult
+    );
+
+    const restrictedPrecisionResult = {
+      date_col: '2023-03-06',
+      datetime_col: '2023-03-06T15:47:00.123000Z',
+      datetime2_col: '2023-03-06T15:47:00.123456Z',
+      smalldatetime_col: '2023-03-06T15:47:00.000000Z',
+      time_col: '15:47:00.123456'
+    };
+    const restrictedPrecision = new CompatibilityContext({
+      edition: 2,
+      maxTimeValuePrecision: TimeValuePrecision.microseconds
+    });
+    expect(applyRowContext(databaseRows[0], restrictedPrecision)).toMatchObject(restrictedPrecisionResult);
+    expect(applyRowContext(replicatedRows[0], restrictedPrecision)).toMatchObject(restrictedPrecisionResult);
   });
 
   test('Date types edge cases mappings', async () => {
@@ -277,18 +308,22 @@ describe('MSSQL Data Types Tests', () => {
     await waitForPendingCDCChanges(beforeLSN, connectionManager);
 
     const expectedResults = [
-      { datetime2_col: '0001-01-01T00:00:00.000Z' },
-      { datetime2_col: '9999-12-31T23:59:59.999Z' },
-      { datetime_col: '1753-01-01T00:00:00.000Z' },
-      { datetime_col: '9999-12-31T23:59:59.997Z' }
+      { datetime2_col: '0001-01-01T00:00:00.000000000Z' },
+      { datetime2_col: '9999-12-31T23:59:59.999000000Z' },
+      { datetime_col: '1753-01-01T00:00:00.000000000Z' },
+      { datetime_col: '9999-12-31T23:59:59.997000000Z' }
     ];
 
     const databaseRows = await getDatabaseRows(connectionManager, 'test_data');
     const replicatedRows = await getReplicatedRows(connectionManager, 'test_data');
 
     for (let i = 0; i < expectedResults.length; i++) {
-      expect(databaseRows[i]).toMatchObject(expectedResults[i]);
-      expect(replicatedRows[i]).toMatchObject(expectedResults[i]);
+      expect(applyRowContext(databaseRows[i], CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toMatchObject(
+        expectedResults[i]
+      );
+      expect(applyRowContext(replicatedRows[i], CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toMatchObject(
+        expectedResults[i]
+      );
     }
   });
 
@@ -302,15 +337,19 @@ describe('MSSQL Data Types Tests', () => {
     await waitForPendingCDCChanges(beforeLSN, connectionManager);
 
     const expectedResult = {
-      datetimeoffset_col: '2023-03-06T10:47:00.000Z' // Converted to UTC
+      datetimeoffset_col: '2023-03-06T10:47:00.000000000Z' // Converted to UTC
     };
 
     const databaseRows = await getDatabaseRows(connectionManager, 'test_data');
     const replicatedRows = await getReplicatedRows(connectionManager, 'test_data');
 
     // Note: The driver converts DateTimeOffset to Date, which incorporates the timezone offset which is then represented in UTC.
-    expect(databaseRows[0]).toMatchObject(expectedResult);
-    expect(replicatedRows[0]).toMatchObject(expectedResult);
+    expect(applyRowContext(databaseRows[0], CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toMatchObject(
+      expectedResult
+    );
+    expect(applyRowContext(replicatedRows[0], CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toMatchObject(
+      expectedResult
+    );
   });
 
   test('UniqueIdentifier type mapping', async () => {

modules/module-mysql/package.json

@@ -33,7 +33,7 @@
     "@powersync/service-sync-rules": "workspace:*",
     "@powersync/service-types": "workspace:*",
     "@powersync/service-jsonbig": "workspace:*",
-    "@powersync/mysql-zongji": "^0.5.0",
+    "@powersync/mysql-zongji": "^0.6.0",
     "async": "^3.2.4",
     "mysql2": "^3.11.0",
     "node-sql-parser": "^5.3.9",