From ca716a79d0f118fa2c4332d9b5e86cab6e3851f7 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 8 Dec 2025 13:48:16 +0200 Subject: [PATCH 01/33] Initial split of data vs parameter sources. --- packages/sync-rules/src/BucketSource.ts | 67 ++++++++++++++----- .../sync-rules/src/SqlBucketDescriptor.ts | 4 +- packages/sync-rules/src/SqlSyncRules.ts | 41 ++++++++---- .../src/schema-generators/SchemaGenerator.ts | 2 +- packages/sync-rules/src/streams/stream.ts | 4 +- .../sync-rules/test/src/sync_rules.test.ts | 39 ++++++----- 6 files changed, 102 insertions(+), 55 deletions(-) diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 6131d78d7..d81aec21e 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -15,7 +15,55 @@ import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, Source * There are two ways to define bucket sources: Via sync rules made up of parameter and data queries, and via stream * definitions that only consist of a single query. */ -export interface BucketSource { +export interface BucketDataSource { + readonly name: string; + readonly type: BucketSourceType; + + readonly subscribedToByDefault: boolean; + + /** + * Given a row as it appears in a table that affects sync data, return buckets, logical table names and transformed + * data for rows to add to buckets. + */ + evaluateRow(options: EvaluateRowOptions): EvaluationResult[]; + + /** + * Reports {@link BucketParameterQuerier}s resolving buckets that a specific stream request should have access to. + * + * @param result The target array to insert queriers and errors into. + * @param options Options, including parameters that may affect the buckets loaded by this source. + */ + pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void; + + getSourceTables(): Set; + + /** Whether the table possibly affects the contents of buckets resolved by this source. */ + tableSyncsData(table: SourceTableInterface): boolean; + + /** + * Given a static schema, infer all logical tables and associated columns that appear in buckets defined by this + * source. + * + * This is use to generate the client-side schema. + */ + resolveResultSets(schema: SourceSchema, tables: Record>): void; + + debugWriteOutputTables(result: Record): void; + + debugRepresentation(): any; +} + +/** + * An interface declaring + * + * - which buckets the sync service should create when processing change streams from the database. + * - how data in source tables maps to data in buckets (e.g. when we're not selecting all columns). + * - which buckets a given connection has access to. + * + * There are two ways to define bucket sources: Via sync rules made up of parameter and data queries, and via stream + * definitions that only consist of a single query. + */ +export interface BucketParameterSource { readonly name: string; readonly type: BucketSourceType; @@ -30,12 +78,6 @@ export interface BucketSource { */ evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[]; - /** - * Given a row as it appears in a table that affects sync data, return buckets, logical table names and transformed - * data for rows to add to buckets. - */ - evaluateRow(options: EvaluateRowOptions): EvaluationResult[]; - /** * Reports {@link BucketParameterQuerier}s resolving buckets that a specific stream request should have access to. * @@ -57,17 +99,6 @@ export interface BucketSource { /** Whether the table possibly affects the buckets resolved by this source. */ tableSyncsParameters(table: SourceTableInterface): boolean; - /** Whether the table possibly affects the contents of buckets resolved by this source. */ - tableSyncsData(table: SourceTableInterface): boolean; - - /** - * Given a static schema, infer all logical tables and associated columns that appear in buckets defined by this - * source. - * - * This is use to generate the client-side schema. - */ - resolveResultSets(schema: SourceSchema, tables: Record>): void; - debugWriteOutputTables(result: Record): void; debugRepresentation(): any; diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index c121b9038..3a1b734e2 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -1,6 +1,6 @@ import { BucketInclusionReason, ResolvedBucket } from './BucketDescription.js'; import { BucketParameterQuerier, mergeBucketParameterQueriers, PendingQueriers } from './BucketParameterQuerier.js'; -import { BucketSource, BucketSourceType, ResultSetDescription } from './BucketSource.js'; +import { BucketDataSource, BucketParameterSource, BucketSourceType, ResultSetDescription } from './BucketSource.js'; import { ColumnDefinition } from './ExpressionType.js'; import { IdSequence } from './IdSequence.js'; import { SourceTableInterface } from './SourceTableInterface.js'; @@ -32,7 +32,7 @@ export interface QueryParseResult { errors: SqlRuleError[]; } -export class SqlBucketDescriptor implements BucketSource { +export class SqlBucketDescriptor implements BucketDataSource, BucketParameterSource { name: string; bucketParameters?: string[]; diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 6c741d0bf..b69c3c8a8 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -1,11 +1,14 @@ import { isScalar, LineCounter, parseDocument, Scalar, YAMLMap, YAMLSeq } from 'yaml'; import { isValidPriority } from './BucketDescription.js'; import { BucketParameterQuerier, mergeBucketParameterQueriers, QuerierError } from './BucketParameterQuerier.js'; +import { BucketDataSource, BucketParameterSource } from './BucketSource.js'; +import { CompatibilityContext, CompatibilityEdition, CompatibilityOption } from './compatibility.js'; import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js'; import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; import { validateSyncRulesSchema } from './json_schema.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { QueryParseResult, SqlBucketDescriptor } from './SqlBucketDescriptor.js'; +import { syncStreamFromSql } from './streams/from_sql.js'; import { TablePattern } from './TablePattern.js'; import { BucketIdTransformer, @@ -28,9 +31,6 @@ import { StreamParseOptions, SyncRules } from './types.js'; -import { BucketSource } from './BucketSource.js'; -import { syncStreamFromSql } from './streams/from_sql.js'; -import { CompatibilityContext, CompatibilityEdition, CompatibilityOption } from './compatibility.js'; import { applyRowContext } from './utils.js'; const ACCEPT_POTENTIALLY_DANGEROUS_QUERIES = Symbol('ACCEPT_POTENTIALLY_DANGEROUS_QUERIES'); @@ -94,7 +94,9 @@ export interface GetBucketParameterQuerierResult { } export class SqlSyncRules implements SyncRules { - bucketSources: BucketSource[] = []; + bucketDataSources: BucketDataSource[] = []; + bucketParameterSources: BucketParameterSource[] = []; + eventDescriptors: SqlEventDescriptor[] = []; compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; @@ -245,7 +247,8 @@ export class SqlSyncRules implements SyncRules { return descriptor.addDataQuery(q, queryOptions, compatibility); }); } - rules.bucketSources.push(descriptor); + rules.bucketDataSources.push(descriptor); + rules.bucketParameterSources.push(descriptor); } for (const entry of streamMap?.items ?? []) { @@ -270,7 +273,8 @@ export class SqlSyncRules implements SyncRules { if (data instanceof Scalar) { rules.withScalar(data, (q) => { const [parsed, errors] = syncStreamFromSql(key, q, queryOptions); - rules.bucketSources.push(parsed); + rules.bucketDataSources.push(parsed); + rules.bucketParameterSources.push(parsed); return { parsed: true, errors @@ -412,7 +416,7 @@ export class SqlSyncRules implements SyncRules { }; let rawResults: EvaluationResult[] = []; - for (let source of this.bucketSources) { + for (let source of this.bucketDataSources) { rawResults.push(...source.evaluateRow(resolvedOptions)); } @@ -438,7 +442,7 @@ export class SqlSyncRules implements SyncRules { row: SqliteRow ): { results: EvaluatedParameters[]; errors: EvaluationError[] } { let rawResults: EvaluatedParametersResult[] = []; - for (let source of this.bucketSources) { + for (let source of this.bucketParameterSources) { rawResults.push(...source.evaluateParameterRow(table, row)); } @@ -460,7 +464,7 @@ export class SqlSyncRules implements SyncRules { const errors: QuerierError[] = []; const pending = { queriers, errors }; - for (const source of this.bucketSources) { + for (const source of this.bucketParameterSources) { if ( (source.subscribedToByDefault && resolvedOptions.hasDefaultStreams) || source.name in resolvedOptions.streams @@ -474,12 +478,18 @@ export class SqlSyncRules implements SyncRules { } hasDynamicBucketQueries() { - return this.bucketSources.some((s) => s.hasDynamicBucketQueries()); + return this.bucketParameterSources.some((s) => s.hasDynamicBucketQueries()); } getSourceTables(): TablePattern[] { const sourceTables = new Map(); - for (const bucket of this.bucketSources) { + for (const bucket of this.bucketDataSources) { + for (const r of bucket.getSourceTables()) { + const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`; + sourceTables.set(key, r); + } + } + for (const bucket of this.bucketParameterSources) { for (const r of bucket.getSourceTables()) { const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`; sourceTables.set(key, r); @@ -513,16 +523,19 @@ export class SqlSyncRules implements SyncRules { } tableSyncsData(table: SourceTableInterface): boolean { - return this.bucketSources.some((b) => b.tableSyncsData(table)); + return this.bucketDataSources.some((b) => b.tableSyncsData(table)); } tableSyncsParameters(table: SourceTableInterface): boolean { - return this.bucketSources.some((b) => b.tableSyncsParameters(table)); + return this.bucketParameterSources.some((b) => b.tableSyncsParameters(table)); } debugGetOutputTables() { let result: Record = {}; - for (let bucket of this.bucketSources) { + for (let bucket of this.bucketDataSources) { + bucket.debugWriteOutputTables(result); + } + for (let bucket of this.bucketParameterSources) { bucket.debugWriteOutputTables(result); } return result; diff --git a/packages/sync-rules/src/schema-generators/SchemaGenerator.ts b/packages/sync-rules/src/schema-generators/SchemaGenerator.ts index e257f5726..84619ec9d 100644 --- a/packages/sync-rules/src/schema-generators/SchemaGenerator.ts +++ b/packages/sync-rules/src/schema-generators/SchemaGenerator.ts @@ -10,7 +10,7 @@ export abstract class SchemaGenerator { protected getAllTables(source: SqlSyncRules, schema: SourceSchema) { let tables: Record> = {}; - for (let descriptor of source.bucketSources) { + for (let descriptor of source.bucketDataSources) { descriptor.resolveResultSets(schema, tables); } diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index 01070b74d..b132dd676 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -1,7 +1,7 @@ import { BaseSqlDataQuery } from '../BaseSqlDataQuery.js'; import { BucketInclusionReason, BucketPriority, DEFAULT_BUCKET_PRIORITY } from '../BucketDescription.js'; import { BucketParameterQuerier, PendingQueriers } from '../BucketParameterQuerier.js'; -import { BucketSource, BucketSourceType, ResultSetDescription } from '../BucketSource.js'; +import { BucketDataSource, BucketParameterSource, BucketSourceType, ResultSetDescription } from '../BucketSource.js'; import { ColumnDefinition } from '../ExpressionType.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { GetQuerierOptions, RequestedStream } from '../SqlSyncRules.js'; @@ -18,7 +18,7 @@ import { } from '../types.js'; import { StreamVariant } from './variant.js'; -export class SyncStream implements BucketSource { +export class SyncStream implements BucketDataSource, BucketParameterSource { name: string; subscribedToByDefault: boolean; priority: BucketPriority; diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 9332418d5..265667cc8 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -17,7 +17,8 @@ describe('sync rules', () => { test('parse empty sync rules', () => { const rules = SqlSyncRules.fromYaml('bucket_definitions: {}', PARSE_OPTIONS); - expect(rules.bucketSources).toEqual([]); + expect(rules.bucketParameterSources).toEqual([]); + expect(rules.bucketDataSources).toEqual([]); }); test('parse global sync rules', () => { @@ -30,7 +31,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; + const bucket = rules.bucketDataSources[0] as SqlBucketDescriptor; expect(bucket.name).toEqual('mybucket'); expect(bucket.bucketParameters).toEqual([]); const dataQuery = bucket.dataQueries[0]; @@ -70,7 +71,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; + const bucket = rules.bucketParameterSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual([]); const param_query = bucket.globalParameterQueries[0]; @@ -102,7 +103,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; + const bucket = rules.bucketParameterSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual([]); const param_query = bucket.parameterQueries[0]; expect(param_query.bucketParameters).toEqual([]); @@ -126,9 +127,10 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual(['user_id', 'device_id']); - const param_query = bucket.globalParameterQueries[0]; + const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; + const bucketData = rules.bucketDataSources[0] as SqlBucketDescriptor; + expect(bucketParameters.bucketParameters).toEqual(['user_id', 'device_id']); + const param_query = bucketParameters.globalParameterQueries[0]; expect(param_query.bucketParameters).toEqual(['user_id', 'device_id']); expect( rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' })).querier @@ -137,7 +139,7 @@ bucket_definitions: { bucket: 'mybucket["user1","device1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 } ]); - const data_query = bucket.dataQueries[0]; + const data_query = bucketData.dataQueries[0]; expect(data_query.bucketParameters).toEqual(['user_id', 'device_id']); expect( rules.evaluateRow({ @@ -176,15 +178,16 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual(['user_id']); - const param_query = bucket.globalParameterQueries[0]; + const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; + const bucketData = rules.bucketDataSources[0] as SqlBucketDescriptor; + expect(bucketParameters.bucketParameters).toEqual(['user_id']); + const param_query = bucketParameters.globalParameterQueries[0]; expect(param_query.bucketParameters).toEqual(['user_id']); expect( rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier.staticBuckets ).toEqual([{ bucket: 'mybucket["user1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 }]); - const data_query = bucket.dataQueries[0]; + const data_query = bucketData.dataQueries[0]; expect(data_query.bucketParameters).toEqual(['user_id']); expect( rules.evaluateRow({ @@ -322,8 +325,8 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual(['user_id']); + const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; + expect(bucketParameters.bucketParameters).toEqual(['user_id']); expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false @@ -360,8 +363,8 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual(['user_id']); + const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; + expect(bucketParameters.bucketParameters).toEqual(['user_id']); expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false @@ -956,8 +959,8 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual(['user_id']); + const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; + expect(bucketParameters.bucketParameters).toEqual(['user_id']); expect(rules.hasDynamicBucketQueries()).toBe(true); expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ From 604bc6a82344774c0e2e4880789745ac57a8a931 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 9 Dec 2025 11:40:41 +0200 Subject: [PATCH 02/33] Split out sync rule definitions from compiled ones. --- .../src/routes/endpoints/sync-rules.ts | 2 +- .../sync-rules/src/BucketParameterQuerier.ts | 2 +- packages/sync-rules/src/BucketSource.ts | 107 ++++++++----- .../sync-rules/src/SqlBucketDescriptor.ts | 141 +++++++++------- packages/sync-rules/src/SqlSyncRules.ts | 131 +++------------ packages/sync-rules/src/SyncRules.ts | 114 +++++++++++++ packages/sync-rules/src/streams/stream.ts | 151 +++++++++++------- packages/sync-rules/src/types.ts | 10 +- .../sync-rules/test/src/sync_rules.test.ts | 109 +++++++------ 9 files changed, 434 insertions(+), 333 deletions(-) create mode 100644 packages/sync-rules/src/SyncRules.ts diff --git a/packages/service-core/src/routes/endpoints/sync-rules.ts b/packages/service-core/src/routes/endpoints/sync-rules.ts index ebac1c23e..e3ed68422 100644 --- a/packages/service-core/src/routes/endpoints/sync-rules.ts +++ b/packages/service-core/src/routes/endpoints/sync-rules.ts @@ -202,7 +202,7 @@ async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) { return { valid: true, - bucket_definitions: rules.bucketSources.map((source) => source.debugRepresentation()), + bucket_definitions: rules.debugRepresentation(), source_tables: resolved_tables, data_tables: rules.debugGetOutputTables() }; diff --git a/packages/sync-rules/src/BucketParameterQuerier.ts b/packages/sync-rules/src/BucketParameterQuerier.ts index 8842c6d7c..c3a947b43 100644 --- a/packages/sync-rules/src/BucketParameterQuerier.ts +++ b/packages/sync-rules/src/BucketParameterQuerier.ts @@ -1,4 +1,4 @@ -import { BucketDescription, ResolvedBucket } from './BucketDescription.js'; +import { ResolvedBucket } from './BucketDescription.js'; import { RequestedStream } from './SqlSyncRules.js'; import { RequestParameters, SqliteJsonRow, SqliteJsonValue } from './types.js'; import { normalizeParameterValue } from './utils.js'; diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index d81aec21e..5ae504add 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -3,41 +3,32 @@ import { ColumnDefinition } from './ExpressionType.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { GetQuerierOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; -import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, SourceSchema, SqliteRow } from './types.js'; +import { + BucketIdTransformer, + EvaluatedParametersResult, + EvaluateRowOptions, + EvaluationResult, + SourceSchema, + SqliteRow +} from './types.js'; + +export interface CreateSourceParams { + bucketIdTransformer: BucketIdTransformer; +} /** - * An interface declaring - * - * - which buckets the sync service should create when processing change streams from the database. - * - how data in source tables maps to data in buckets (e.g. when we're not selecting all columns). - * - which buckets a given connection has access to. - * - * There are two ways to define bucket sources: Via sync rules made up of parameter and data queries, and via stream - * definitions that only consist of a single query. + * Encodes a static definition of a bucket source, as parsed from sync rules or stream definitions. */ -export interface BucketDataSource { +export interface BucketDataSourceDefinition { readonly name: string; readonly type: BucketSourceType; - readonly subscribedToByDefault: boolean; - - /** - * Given a row as it appears in a table that affects sync data, return buckets, logical table names and transformed - * data for rows to add to buckets. - */ - evaluateRow(options: EvaluateRowOptions): EvaluationResult[]; - /** - * Reports {@link BucketParameterQuerier}s resolving buckets that a specific stream request should have access to. - * - * @param result The target array to insert queriers and errors into. - * @param options Options, including parameters that may affect the buckets loaded by this source. + * For debug use only. */ - pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void; - + readonly bucketParameters: string[]; getSourceTables(): Set; - - /** Whether the table possibly affects the contents of buckets resolved by this source. */ + createDataSource(params: CreateSourceParams): BucketDataSource; tableSyncsData(table: SourceTableInterface): boolean; /** @@ -53,6 +44,30 @@ export interface BucketDataSource { debugRepresentation(): any; } +export interface BucketParameterSourceDefinition { + readonly name: string; + readonly type: BucketSourceType; + readonly subscribedToByDefault: boolean; + + getSourceTables(): Set; + createParameterSource(params: CreateSourceParams): BucketParameterSource; + + /** + * Whether {@link pushBucketParameterQueriers} may include a querier where + * {@link BucketParameterQuerier.hasDynamicBuckets} is true. + * + * This is mostly used for testing. + */ + hasDynamicBucketQueries(): boolean; + + getSourceTables(): Set; + + /** Whether the table possibly affects the buckets resolved by this source. */ + tableSyncsParameters(table: SourceTableInterface): boolean; + + debugRepresentation(): any; +} + /** * An interface declaring * @@ -63,12 +78,28 @@ export interface BucketDataSource { * There are two ways to define bucket sources: Via sync rules made up of parameter and data queries, and via stream * definitions that only consist of a single query. */ -export interface BucketParameterSource { - readonly name: string; - readonly type: BucketSourceType; +export interface BucketDataSource { + readonly definition: BucketDataSourceDefinition; - readonly subscribedToByDefault: boolean; + /** + * Given a row as it appears in a table that affects sync data, return buckets, logical table names and transformed + * data for rows to add to buckets. + */ + evaluateRow(options: EvaluateRowOptions): EvaluationResult[]; +} +/** + * An interface declaring + * + * - which buckets the sync service should create when processing change streams from the database. + * - how data in source tables maps to data in buckets (e.g. when we're not selecting all columns). + * - which buckets a given connection has access to. + * + * There are two ways to define bucket sources: Via sync rules made up of parameter and data queries, and via stream + * definitions that only consist of a single query. + */ +export interface BucketParameterSource { + readonly definition: BucketParameterSourceDefinition; /** * Given a row in a source table that affects sync parameters, returns a structure to index which buckets rows should * be associated with. @@ -87,21 +118,9 @@ export interface BucketParameterSource { pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void; /** - * Whether {@link pushBucketParameterQueriers} may include a querier where - * {@link BucketParameterQuerier.hasDynamicBuckets} is true. - * - * This is mostly used for testing. + * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. */ - hasDynamicBucketQueries(): boolean; - - getSourceTables(): Set; - - /** Whether the table possibly affects the buckets resolved by this source. */ - tableSyncsParameters(table: SourceTableInterface): boolean; - - debugWriteOutputTables(result: Record): void; - - debugRepresentation(): any; + getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier; } export enum BucketSourceType { diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 3a1b734e2..78e80bf80 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -1,6 +1,14 @@ import { BucketInclusionReason, ResolvedBucket } from './BucketDescription.js'; import { BucketParameterQuerier, mergeBucketParameterQueriers, PendingQueriers } from './BucketParameterQuerier.js'; -import { BucketDataSource, BucketParameterSource, BucketSourceType, ResultSetDescription } from './BucketSource.js'; +import { + BucketDataSource, + BucketDataSourceDefinition, + BucketParameterSource, + BucketParameterSourceDefinition, + BucketSourceType, + CreateSourceParams, + ResultSetDescription +} from './BucketSource.js'; import { ColumnDefinition } from './ExpressionType.js'; import { IdSequence } from './IdSequence.js'; import { SourceTableInterface } from './SourceTableInterface.js'; @@ -32,9 +40,9 @@ export interface QueryParseResult { errors: SqlRuleError[]; } -export class SqlBucketDescriptor implements BucketDataSource, BucketParameterSource { +export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketParameterSourceDefinition { name: string; - bucketParameters?: string[]; + private bucketParametersInternal: string[] | null = null; constructor(name: string) { this.name = name; @@ -48,6 +56,10 @@ export class SqlBucketDescriptor implements BucketDataSource, BucketParameterSou return true; } + public get bucketParameters(): string[] { + return this.bucketParametersInternal ?? []; + } + /** * source table -> queries */ @@ -58,10 +70,10 @@ export class SqlBucketDescriptor implements BucketDataSource, BucketParameterSou parameterIdSequence = new IdSequence(); addDataQuery(sql: string, options: SyncRulesOptions, compatibility: CompatibilityContext): QueryParseResult { - if (this.bucketParameters == null) { + if (this.bucketParametersInternal == null) { throw new Error('Bucket parameters must be defined'); } - const dataRows = SqlDataQuery.fromSql(this.name, this.bucketParameters, sql, options, compatibility); + const dataRows = SqlDataQuery.fromSql(this.name, this.bucketParametersInternal, sql, options, compatibility); this.dataQueries.push(dataRows); @@ -73,11 +85,12 @@ export class SqlBucketDescriptor implements BucketDataSource, BucketParameterSou addParameterQuery(sql: string, options: QueryParseOptions): QueryParseResult { const parameterQuery = SqlParameterQuery.fromSql(this.name, sql, options, this.parameterIdSequence.nextId()); - if (this.bucketParameters == null) { - this.bucketParameters = parameterQuery.bucketParameters; + if (this.bucketParametersInternal == null) { + this.bucketParametersInternal = parameterQuery.bucketParameters; } else { if ( - new Set([...parameterQuery.bucketParameters!, ...this.bucketParameters]).size != this.bucketParameters.length + new Set([...parameterQuery.bucketParameters!, ...this.bucketParametersInternal]).size != + this.bucketParametersInternal.length ) { throw new Error('Bucket parameters must match for each parameter query within a bucket'); } @@ -94,61 +107,71 @@ export class SqlBucketDescriptor implements BucketDataSource, BucketParameterSou }; } - evaluateRow(options: EvaluateRowOptions): EvaluationResult[] { - let results: EvaluationResult[] = []; - for (let query of this.dataQueries) { - if (!query.applies(options.sourceTable)) { - continue; + createDataSource(params: CreateSourceParams): BucketDataSource { + return { + definition: this, + evaluateRow: (options) => { + let results: EvaluationResult[] = []; + for (let query of this.dataQueries) { + if (!query.applies(options.sourceTable)) { + continue; + } + + results.push(...query.evaluateRow(options.sourceTable, options.record, params.bucketIdTransformer)); + } + return results; } - - results.push(...query.evaluateRow(options.sourceTable, options.record, options.bucketIdTransformer)); - } - return results; + }; } - evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] { - let results: EvaluatedParametersResult[] = []; - for (let query of this.parameterQueries) { - if (query.applies(sourceTable)) { - results.push(...query.evaluateParameterRow(row)); + createParameterSource(params: CreateSourceParams): BucketParameterSource { + return { + definition: this, + + evaluateParameterRow: (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { + let results: EvaluatedParametersResult[] = []; + for (let query of this.parameterQueries) { + if (query.applies(sourceTable)) { + results.push(...query.evaluateParameterRow(row)); + } + } + return results; + }, + pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { + const reasons = [this.bucketInclusionReason()]; + const staticBuckets = this.getStaticBucketDescriptions( + options.globalParameters, + reasons, + params.bucketIdTransformer + ); + const staticQuerier = { + staticBuckets, + hasDynamicBuckets: false, + parameterQueryLookups: [], + queryDynamicBucketDescriptions: async () => [] + } satisfies BucketParameterQuerier; + result.queriers.push(staticQuerier); + + if (this.parameterQueries.length == 0) { + return; + } + + const dynamicQueriers = this.parameterQueries.map((query) => + query.getBucketParameterQuerier(options.globalParameters, reasons, params.bucketIdTransformer) + ); + result.queriers.push(...dynamicQueriers); + }, + + /** + * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. + */ + getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { + const queriers: BucketParameterQuerier[] = []; + this.pushBucketParameterQueriers({ queriers, errors: [] }, options); + + return mergeBucketParameterQueriers(queriers); } - } - return results; - } - - /** - * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. - */ - getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { - const queriers: BucketParameterQuerier[] = []; - this.pushBucketParameterQueriers({ queriers, errors: [] }, options); - - return mergeBucketParameterQueriers(queriers); - } - - pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions) { - const reasons = [this.bucketInclusionReason()]; - const staticBuckets = this.getStaticBucketDescriptions( - options.globalParameters, - reasons, - options.bucketIdTransformer - ); - const staticQuerier = { - staticBuckets, - hasDynamicBuckets: false, - parameterQueryLookups: [], - queryDynamicBucketDescriptions: async () => [] - } satisfies BucketParameterQuerier; - result.queriers.push(staticQuerier); - - if (this.parameterQueries.length == 0) { - return; - } - - const dynamicQueriers = this.parameterQueries.map((query) => - query.getBucketParameterQuerier(options.globalParameters, reasons, options.bucketIdTransformer) - ); - result.queriers.push(...dynamicQueriers); + }; } getStaticBucketDescriptions( diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index b69c3c8a8..684aae914 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -1,7 +1,7 @@ import { isScalar, LineCounter, parseDocument, Scalar, YAMLMap, YAMLSeq } from 'yaml'; import { isValidPriority } from './BucketDescription.js'; -import { BucketParameterQuerier, mergeBucketParameterQueriers, QuerierError } from './BucketParameterQuerier.js'; -import { BucketDataSource, BucketParameterSource } from './BucketSource.js'; +import { BucketParameterQuerier, QuerierError } from './BucketParameterQuerier.js'; +import { BucketDataSourceDefinition, BucketParameterSourceDefinition } from './BucketSource.js'; import { CompatibilityContext, CompatibilityEdition, CompatibilityOption } from './compatibility.js'; import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js'; import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; @@ -9,18 +9,10 @@ import { validateSyncRulesSchema } from './json_schema.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { QueryParseResult, SqlBucketDescriptor } from './SqlBucketDescriptor.js'; import { syncStreamFromSql } from './streams/from_sql.js'; +import { SyncRules } from './SyncRules.js'; import { TablePattern } from './TablePattern.js'; import { BucketIdTransformer, - EvaluatedParameters, - EvaluatedParametersResult, - EvaluatedRow, - EvaluateRowOptions, - EvaluationError, - EvaluationResult, - isEvaluatedParameters, - isEvaluatedRow, - isEvaluationError, QueryParseOptions, RequestParameters, SourceSchema, @@ -28,8 +20,7 @@ import { SqliteJsonRow, SqliteRow, SqliteValue, - StreamParseOptions, - SyncRules + StreamParseOptions } from './types.js'; import { applyRowContext } from './utils.js'; @@ -63,13 +54,6 @@ export interface RequestedStream { } export interface GetQuerierOptions { - /** - * A bucket id transformer, compatible to the one used when evaluating rows. - * - * Typically, this transformer only depends on the sync rule id (which is known to both the bucket storage - * implementation responsible for evaluating rows and the sync endpoint). - */ - bucketIdTransformer: BucketIdTransformer; globalParameters: RequestParameters; /** * Whether the client is subscribing to default query streams. @@ -93,9 +77,9 @@ export interface GetBucketParameterQuerierResult { errors: QuerierError[]; } -export class SqlSyncRules implements SyncRules { - bucketDataSources: BucketDataSource[] = []; - bucketParameterSources: BucketParameterSource[] = []; +export class SqlSyncRules { + bucketDataSources: BucketDataSourceDefinition[] = []; + bucketParameterSources: BucketParameterSourceDefinition[] = []; eventDescriptors: SqlEventDescriptor[] = []; compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; @@ -389,94 +373,24 @@ export class SqlSyncRules implements SyncRules { this.content = content; } + compile(params?: { bucketIdTransformer?: BucketIdTransformer }): SyncRules { + const bucketIdTransformer = this.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) + ? (params?.bucketIdTransformer ?? ((id: string) => id)) + : (id: string) => id; + return new SyncRules({ + bucketDataSources: this.bucketDataSources.map((d) => d.createDataSource({ bucketIdTransformer })), + bucketParameterSources: this.bucketParameterSources.map((d) => d.createParameterSource({ bucketIdTransformer })), + eventDescriptors: this.eventDescriptors, + compatibility: this.compatibility + }); + } + applyRowContext( source: SqliteRow ): SqliteRow { return applyRowContext(source, this.compatibility); } - /** - * Throws errors. - */ - evaluateRow(options: EvaluateRowOptions): EvaluatedRow[] { - const { results, errors } = this.evaluateRowWithErrors(options); - if (errors.length > 0) { - throw new Error(errors[0].error); - } - return results; - } - - evaluateRowWithErrors(options: EvaluateRowOptions): { results: EvaluatedRow[]; errors: EvaluationError[] } { - const resolvedOptions = this.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) - ? options - : { - ...options, - // Disable bucket id transformer when the option is unused. - bucketIdTransformer: (id: string) => id - }; - - let rawResults: EvaluationResult[] = []; - for (let source of this.bucketDataSources) { - rawResults.push(...source.evaluateRow(resolvedOptions)); - } - - const results = rawResults.filter(isEvaluatedRow) as EvaluatedRow[]; - const errors = rawResults.filter(isEvaluationError) as EvaluationError[]; - - return { results, errors }; - } - - /** - * Throws errors. - */ - evaluateParameterRow(table: SourceTableInterface, row: SqliteRow): EvaluatedParameters[] { - const { results, errors } = this.evaluateParameterRowWithErrors(table, row); - if (errors.length > 0) { - throw new Error(errors[0].error); - } - return results; - } - - evaluateParameterRowWithErrors( - table: SourceTableInterface, - row: SqliteRow - ): { results: EvaluatedParameters[]; errors: EvaluationError[] } { - let rawResults: EvaluatedParametersResult[] = []; - for (let source of this.bucketParameterSources) { - rawResults.push(...source.evaluateParameterRow(table, row)); - } - - const results = rawResults.filter(isEvaluatedParameters) as EvaluatedParameters[]; - const errors = rawResults.filter(isEvaluationError) as EvaluationError[]; - return { results, errors }; - } - - getBucketParameterQuerier(options: GetQuerierOptions): GetBucketParameterQuerierResult { - const resolvedOptions = this.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) - ? options - : { - ...options, - // Disable bucket id transformer when the option is unused. - bucketIdTransformer: (id: string) => id - }; - - const queriers: BucketParameterQuerier[] = []; - const errors: QuerierError[] = []; - const pending = { queriers, errors }; - - for (const source of this.bucketParameterSources) { - if ( - (source.subscribedToByDefault && resolvedOptions.hasDefaultStreams) || - source.name in resolvedOptions.streams - ) { - source.pushBucketParameterQueriers(pending, resolvedOptions); - } - } - - const querier = mergeBucketParameterQueriers(queriers); - return { querier, errors }; - } - hasDynamicBucketQueries() { return this.bucketParameterSources.some((s) => s.hasDynamicBucketQueries()); } @@ -535,12 +449,13 @@ export class SqlSyncRules implements SyncRules { for (let bucket of this.bucketDataSources) { bucket.debugWriteOutputTables(result); } - for (let bucket of this.bucketParameterSources) { - bucket.debugWriteOutputTables(result); - } return result; } + debugRepresentation() { + return this.bucketDataSources.map((rules) => rules.debugRepresentation()); + } + private parsePriority(value: YAMLMap) { if (value.has('priority')) { const priorityValue = value.get('priority', true)!; diff --git a/packages/sync-rules/src/SyncRules.ts b/packages/sync-rules/src/SyncRules.ts new file mode 100644 index 000000000..785cfa723 --- /dev/null +++ b/packages/sync-rules/src/SyncRules.ts @@ -0,0 +1,114 @@ +import { + BucketDataSource, + BucketDataSourceDefinition, + BucketParameterSource, + BucketParameterSourceDefinition +} from './BucketSource.js'; +import { + BucketParameterQuerier, + CompatibilityContext, + CompatibilityOption, + EvaluatedParameters, + EvaluatedRow, + EvaluationError, + GetBucketParameterQuerierResult, + GetQuerierOptions, + isEvaluatedParameters, + isEvaluatedRow, + isEvaluationError, + mergeBucketParameterQueriers, + QuerierError, + SqlEventDescriptor +} from './index.js'; +import { SourceTableInterface } from './SourceTableInterface.js'; +import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, SqliteRow } from './types.js'; + +export class SyncRules { + bucketDataSources: BucketDataSource[] = []; + bucketParameterSources: BucketParameterSource[] = []; + + eventDescriptors: SqlEventDescriptor[] = []; + compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; + + constructor(params: { + bucketDataSources: BucketDataSource[]; + bucketParameterSources: BucketParameterSource[]; + eventDescriptors?: SqlEventDescriptor[]; + compatibility?: CompatibilityContext; + }) { + this.bucketDataSources = params.bucketDataSources; + this.bucketParameterSources = params.bucketParameterSources; + if (params.eventDescriptors) { + this.eventDescriptors = params.eventDescriptors; + } + if (params.compatibility) { + this.compatibility = params.compatibility; + } + } + + /** + * Throws errors. + */ + evaluateRow(options: EvaluateRowOptions): EvaluatedRow[] { + const { results, errors } = this.evaluateRowWithErrors(options); + if (errors.length > 0) { + throw new Error(errors[0].error); + } + return results; + } + + evaluateRowWithErrors(options: EvaluateRowOptions): { results: EvaluatedRow[]; errors: EvaluationError[] } { + let rawResults: EvaluationResult[] = []; + for (let source of this.bucketDataSources) { + rawResults.push(...source.evaluateRow(options)); + } + + const results = rawResults.filter(isEvaluatedRow) as EvaluatedRow[]; + const errors = rawResults.filter(isEvaluationError) as EvaluationError[]; + + return { results, errors }; + } + + /** + * Throws errors. + */ + evaluateParameterRow(table: SourceTableInterface, row: SqliteRow): EvaluatedParameters[] { + const { results, errors } = this.evaluateParameterRowWithErrors(table, row); + if (errors.length > 0) { + throw new Error(errors[0].error); + } + return results; + } + + evaluateParameterRowWithErrors( + table: SourceTableInterface, + row: SqliteRow + ): { results: EvaluatedParameters[]; errors: EvaluationError[] } { + let rawResults: EvaluatedParametersResult[] = []; + for (let source of this.bucketParameterSources) { + rawResults.push(...source.evaluateParameterRow(table, row)); + } + + const results = rawResults.filter(isEvaluatedParameters) as EvaluatedParameters[]; + const errors = rawResults.filter(isEvaluationError) as EvaluationError[]; + return { results, errors }; + } + + getBucketParameterQuerier(options: GetQuerierOptions): GetBucketParameterQuerierResult { + const queriers: BucketParameterQuerier[] = []; + const errors: QuerierError[] = []; + const pending = { queriers, errors }; + + for (const source of this.bucketParameterSources) { + if ( + (source.definition.subscribedToByDefault && options.hasDefaultStreams) || + source.definition.name in options.streams + ) { + source.pushBucketParameterQueriers(pending, options); + } + } + + const querier = mergeBucketParameterQueriers(queriers); + return { querier, errors }; + } +} diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index b132dd676..ee1a62410 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -1,7 +1,14 @@ import { BaseSqlDataQuery } from '../BaseSqlDataQuery.js'; import { BucketInclusionReason, BucketPriority, DEFAULT_BUCKET_PRIORITY } from '../BucketDescription.js'; -import { BucketParameterQuerier, PendingQueriers } from '../BucketParameterQuerier.js'; -import { BucketDataSource, BucketParameterSource, BucketSourceType, ResultSetDescription } from '../BucketSource.js'; +import { BucketParameterQuerier, mergeBucketParameterQueriers, PendingQueriers } from '../BucketParameterQuerier.js'; +import { + BucketDataSource, + BucketDataSourceDefinition, + BucketParameterSource, + BucketParameterSourceDefinition, + BucketSourceType, + CreateSourceParams +} from '../BucketSource.js'; import { ColumnDefinition } from '../ExpressionType.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { GetQuerierOptions, RequestedStream } from '../SqlSyncRules.js'; @@ -13,12 +20,11 @@ import { EvaluationResult, RequestParameters, SourceSchema, - SqliteRow, TableRow } from '../types.js'; import { StreamVariant } from './variant.js'; -export class SyncStream implements BucketDataSource, BucketParameterSource { +export class SyncStream implements BucketDataSourceDefinition, BucketParameterSourceDefinition { name: string; subscribedToByDefault: boolean; priority: BucketPriority; @@ -37,31 +43,93 @@ export class SyncStream implements BucketDataSource, BucketParameterSource { return BucketSourceType.SYNC_STREAM; } - pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void { - const subscriptions = options.streams[this.name] ?? []; + public get bucketParameters(): string[] { + // FIXME: check whether this is correct. + // Could there be multiple variants with different bucket parameters? + return this.data.bucketParameters; + } - if (!this.subscribedToByDefault && !subscriptions.length) { - // The client is not subscribing to this stream, so don't query buckets related to it. - return; - } + createDataSource(params: CreateSourceParams): BucketDataSource { + return { + definition: this, + + evaluateRow: (options: EvaluateRowOptions): EvaluationResult[] => { + if (!this.data.applies(options.sourceTable)) { + return []; + } - let hasExplicitDefaultSubscription = false; - for (const subscription of subscriptions) { - let subscriptionParams = options.globalParameters; - if (subscription.parameters != null) { - subscriptionParams = subscriptionParams.withAddedStreamParameters(subscription.parameters); - } else { - hasExplicitDefaultSubscription = true; + const stream = this; + const row: TableRow = { + sourceTable: options.sourceTable, + record: options.record + }; + + return this.data.evaluateRowWithOptions({ + table: options.sourceTable, + row: options.record, + bucketIds() { + const bucketIds: string[] = []; + for (const variant of stream.variants) { + bucketIds.push(...variant.bucketIdsForRow(stream.name, row, params.bucketIdTransformer)); + } + + return bucketIds; + } + }); } + }; + } - this.queriersForSubscription(result, subscription, subscriptionParams, options.bucketIdTransformer); - } + createParameterSource(params: CreateSourceParams): BucketParameterSource { + return { + definition: this, - // If the stream is subscribed to by default and there is no explicit subscription that would match the default - // subscription, also include the default querier. - if (this.subscribedToByDefault && !hasExplicitDefaultSubscription) { - this.queriersForSubscription(result, null, options.globalParameters, options.bucketIdTransformer); - } + pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions): void => { + const subscriptions = options.streams[this.name] ?? []; + + if (!this.subscribedToByDefault && !subscriptions.length) { + // The client is not subscribing to this stream, so don't query buckets related to it. + return; + } + + let hasExplicitDefaultSubscription = false; + for (const subscription of subscriptions) { + let subscriptionParams = options.globalParameters; + if (subscription.parameters != null) { + subscriptionParams = subscriptionParams.withAddedStreamParameters(subscription.parameters); + } else { + hasExplicitDefaultSubscription = true; + } + + this.queriersForSubscription(result, subscription, subscriptionParams, params.bucketIdTransformer); + } + + // If the stream is subscribed to by default and there is no explicit subscription that would match the default + // subscription, also include the default querier. + if (this.subscribedToByDefault && !hasExplicitDefaultSubscription) { + this.queriersForSubscription(result, null, options.globalParameters, params.bucketIdTransformer); + } + }, + evaluateParameterRow: (sourceTable, row) => { + const result: EvaluatedParametersResult[] = []; + + for (const variant of this.variants) { + variant.pushParameterRowEvaluation(result, sourceTable, row); + } + + return result; + }, + + /** + * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. + */ + getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { + const queriers: BucketParameterQuerier[] = []; + this.pushBucketParameterQueriers({ queriers, errors: [] }, options); + + return mergeBucketParameterQueriers(queriers); + } + }; } private queriersForSubscription( @@ -149,39 +217,4 @@ export class SyncStream implements BucketDataSource, BucketParameterSource { result[this.data.table!.sqlName].push(r); } - - evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] { - const result: EvaluatedParametersResult[] = []; - - for (const variant of this.variants) { - variant.pushParameterRowEvaluation(result, sourceTable, row); - } - - return result; - } - - evaluateRow(options: EvaluateRowOptions): EvaluationResult[] { - if (!this.data.applies(options.sourceTable)) { - return []; - } - - const stream = this; - const row: TableRow = { - sourceTable: options.sourceTable, - record: options.record - }; - - return this.data.evaluateRowWithOptions({ - table: options.sourceTable, - row: options.record, - bucketIds() { - const bucketIds: string[] = []; - for (const variant of stream.variants) { - bucketIds.push(...variant.bucketIdsForRow(stream.name, row, options.bucketIdTransformer)); - } - - return bucketIds; - } - }); - } } diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index 8a607428c..1730ef803 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -10,12 +10,6 @@ import { CustomSqliteValue } from './types/custom_sqlite_value.js'; import { CompatibilityContext } from './compatibility.js'; import { RequestFunctionCall } from './request_functions.js'; -export interface SyncRules { - evaluateRow(options: EvaluateRowOptions): EvaluationResult[]; - - evaluateParameterRow(table: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[]; -} - export interface QueryParseOptions extends SyncRulesOptions { accept_potentially_dangerous_queries?: boolean; priority?: BucketPriority; @@ -305,9 +299,7 @@ export interface InputParameter { */ export type BucketIdTransformer = (regularId: string) => string; -export interface EvaluateRowOptions extends TableRow { - bucketIdTransformer: BucketIdTransformer; -} +export interface EvaluateRowOptions extends TableRow {} /** * A row associated with the table it's coming from. diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 265667cc8..dd61c9393 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -31,6 +31,7 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); const bucket = rules.bucketDataSources[0] as SqlBucketDescriptor; expect(bucket.name).toEqual('mybucket'); expect(bucket.bucketParameters).toEqual([]); @@ -38,9 +39,8 @@ bucket_definitions: expect(dataQuery.bucketParameters).toEqual([]); expect(dataQuery.columnOutputNames()).toEqual(['id', 'description']); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test' } }) ).toEqual([ @@ -55,7 +55,7 @@ bucket_definitions: } ]); expect(rules.hasDynamicBucketQueries()).toBe(false); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ + expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[]', priority: 3 }], hasDynamicBuckets: false }); @@ -71,6 +71,7 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); const bucket = rules.bucketParameterSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual([]); const param_query = bucket.globalParameterQueries[0]; @@ -79,15 +80,15 @@ bucket_definitions: expect(param_query.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 1n }))).toEqual(1n); expect(param_query.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 0n }))).toEqual(0n); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier).toMatchObject({ + expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[]', priority: 3 }], hasDynamicBuckets: false }); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: false })).querier).toMatchObject({ + expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: false })).querier).toMatchObject({ staticBuckets: [], hasDynamicBuckets: false }); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ + expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [], hasDynamicBuckets: false }); @@ -103,17 +104,18 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); const bucket = rules.bucketParameterSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual([]); const param_query = bucket.parameterQueries[0]; expect(param_query.bucketParameters).toEqual([]); - expect(rules.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ + expect(compiled.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ { bucketParameters: [{}], lookup: ParameterLookup.normalized('mybucket', '1', ['user1']) } ]); - expect(rules.evaluateParameterRow(USERS, { id: 'user1', is_admin: 0 })).toEqual([]); + expect(compiled.evaluateParameterRow(USERS, { id: 'user1', is_admin: 0 })).toEqual([]); }); test('parse bucket with parameters', () => { @@ -127,14 +129,15 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; const bucketData = rules.bucketDataSources[0] as SqlBucketDescriptor; expect(bucketParameters.bucketParameters).toEqual(['user_id', 'device_id']); const param_query = bucketParameters.globalParameterQueries[0]; expect(param_query.bucketParameters).toEqual(['user_id', 'device_id']); expect( - rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' })).querier - .staticBuckets + compiled.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' })) + .querier.staticBuckets ).toEqual([ { bucket: 'mybucket["user1","device1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 } ]); @@ -142,9 +145,8 @@ bucket_definitions: const data_query = bucketData.dataQueries[0]; expect(data_query.bucketParameters).toEqual(['user_id', 'device_id']); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', user_id: 'user1', device_id: 'device1' } }) ).toEqual([ @@ -159,9 +161,8 @@ bucket_definitions: } ]); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', user_id: 'user1', archived: 1, device_id: 'device1' } }) ).toEqual([]); @@ -178,21 +179,21 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; const bucketData = rules.bucketDataSources[0] as SqlBucketDescriptor; expect(bucketParameters.bucketParameters).toEqual(['user_id']); const param_query = bucketParameters.globalParameterQueries[0]; expect(param_query.bucketParameters).toEqual(['user_id']); expect( - rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier.staticBuckets + compiled.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier.staticBuckets ).toEqual([{ bucket: 'mybucket["user1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 }]); const data_query = bucketData.dataQueries[0]; expect(data_query.bucketParameters).toEqual(['user_id']); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', user_id: 'user1' } }) ).toEqual([ @@ -207,9 +208,8 @@ bucket_definitions: } ]); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', owner_id: 'user1' } }) ).toEqual([ @@ -325,17 +325,17 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; expect(bucketParameters.bucketParameters).toEqual(['user_id']); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ + expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false }); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', user_id: 'user1' } }) ).toEqual([ @@ -363,17 +363,17 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; expect(bucketParameters.bucketParameters).toEqual(['user_id']); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ + expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false }); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', user_id: 'user1' } }) ).toEqual([ @@ -399,10 +399,10 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', data: JSON.stringify({ count: 5, bool: true }) } }) ).toEqual([ @@ -433,11 +433,11 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', @@ -478,11 +478,11 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', role: 'admin' } }) ).toEqual([ @@ -500,9 +500,8 @@ bucket_definitions: ]); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset2', description: 'test', role: 'normal' } }) ).toEqual([ @@ -541,9 +540,9 @@ bucket_definitions: } ]); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier.staticBuckets).toEqual([ - { bucket: 'mybucket[1]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 } - ]); + expect( + compiled.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier.staticBuckets + ).toEqual([{ bucket: 'mybucket[1]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 }]); }); test('some math', () => { @@ -556,8 +555,9 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); - expect(rules.evaluateRow({ sourceTable: ASSETS, bucketIdTransformer, record: { id: 'asset1' } })).toEqual([ + expect(compiled.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1' } })).toEqual([ { bucket: 'mybucket[]', id: 'asset1', @@ -583,14 +583,14 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); expect( - rules.getBucketParameterQuerier(normalizeQuerierOptions({ int1: 314, float1: 3.14, float2: 314 })).querier + compiled.getBucketParameterQuerier(normalizeQuerierOptions({ int1: 314, float1: 3.14, float2: 314 })).querier ).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[314,3.14,314]', priority: 3 }] }); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', int1: 314n, float1: 3.14, float2: 314 } }) ).toEqual([ @@ -616,7 +616,8 @@ bucket_definitions: PARSE_OPTIONS ); expect(rules.errors).toEqual([]); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'test' })).querier).toMatchObject({ + const compiled = rules.compile({ bucketIdTransformer }); + expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'test' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["TEST"]', priority: 3 }], hasDynamicBuckets: false }); @@ -633,11 +634,11 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: new TestSourceTable('assets_123'), - bucketIdTransformer, record: { client_id: 'asset1', description: 'test', archived: 0n, other_id: 'other1' } }) ).toEqual([ @@ -674,11 +675,11 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: new TestSourceTable('assets_123'), - bucketIdTransformer, record: { client_id: 'asset1', description: 'test', archived: 0n, other_id: 'other1' } }) ).toEqual([ @@ -708,11 +709,11 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1', description: 'test', archived: 0n } }) ).toEqual([ @@ -744,11 +745,11 @@ bucket_definitions: `, PARSE_OPTIONS ); + const compiled = rules.compile({ bucketIdTransformer }); expect( - rules.evaluateRow({ + compiled.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer, record: { id: 'asset1' } }) ).toEqual([ @@ -873,7 +874,8 @@ bucket_definitions: expect(rules.errors).toEqual([]); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ + const compiled = rules.compile({ bucketIdTransformer }); + expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [ { bucket: 'highprio[]', priority: 0 }, { bucket: 'defaultprio[]', priority: 3 } @@ -898,7 +900,8 @@ bucket_definitions: expect(rules.errors).toEqual([]); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ + const compiled = rules.compile({ bucketIdTransformer }); + expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [ { bucket: 'highprio[]', priority: 0 }, { bucket: 'defaultprio[]', priority: 3 } @@ -963,7 +966,9 @@ bucket_definitions: expect(bucketParameters.bucketParameters).toEqual(['user_id']); expect(rules.hasDynamicBucketQueries()).toBe(true); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ + const compiled = rules.compile({ bucketIdTransformer }); + + expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ hasDynamicBuckets: true, parameterQueryLookups: [ ParameterLookup.normalized('mybucket', '2', ['user1']), From 223ae3e69aefcc10c39014cda8baf3735bfd64cb Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 9 Dec 2025 12:19:08 +0200 Subject: [PATCH 03/33] Update APIs to use hydrated sync rules. --- .../implementation/MongoBucketBatch.ts | 13 +-- .../implementation/MongoPersistedSyncRules.ts | 6 +- .../implementation/MongoSyncBucketStorage.ts | 10 +- .../src/replication/ChangeStream.ts | 11 +- .../module-mssql/src/replication/CDCStream.ts | 10 +- .../src/replication/BinLogStream.ts | 2 +- .../src/storage/PostgresSyncRulesStorage.ts | 10 +- .../src/storage/batch/PostgresBucketBatch.ts | 7 +- .../PostgresPersistedSyncRulesContent.ts | 7 +- .../src/replication/WalStream.ts | 3 +- .../src/test-utils/general-utils.ts | 8 +- .../register-data-storage-parameter-tests.ts | 11 +- .../src/tests/register-sync-tests.ts | 80 +++---------- .../src/routes/endpoints/admin.ts | 5 +- .../src/routes/endpoints/socket-route.ts | 5 +- .../src/routes/endpoints/sync-stream.ts | 5 +- .../src/storage/PersistedSyncRulesContent.ts | 4 +- .../src/storage/SyncRulesBucketStorage.ts | 6 +- .../src/sync/BucketChecksumState.ts | 40 +++---- packages/service-core/src/sync/sync.ts | 8 +- .../test/src/routes/stream.test.ts | 4 +- .../test/src/sync/BucketChecksumState.test.ts | 59 +++------- packages/sync-rules/src/SqlSyncRules.ts | 15 ++- packages/sync-rules/src/SyncRules.ts | 47 ++++++-- packages/sync-rules/src/index.ts | 1 + .../sync-rules/test/src/compatibility.test.ts | 51 +++------ packages/sync-rules/test/src/streams.test.ts | 77 ++++++++----- .../sync-rules/test/src/sync_rules.test.ts | 106 +++++++++--------- packages/sync-rules/test/src/util.ts | 6 +- 29 files changed, 298 insertions(+), 319 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 30952ff46..2961b2abb 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -1,14 +1,14 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { SqlEventDescriptor, SqliteRow, SqliteValue, SqlSyncRules } from '@powersync/service-sync-rules'; +import { SqlEventDescriptor, SqliteRow, SqliteValue, HydratedSyncRules } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { BaseObserver, container, + logger as defaultLogger, ErrorCode, errors, Logger, - logger as defaultLogger, ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework'; @@ -22,13 +22,13 @@ import { utils } from '@powersync/service-core'; import * as timers from 'node:timers/promises'; +import { idPrefixFilter } from '../../utils/util.js'; import { PowerSyncMongo } from './db.js'; import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; -import { idPrefixFilter } from '../../utils/util.js'; /** * 15MB @@ -44,7 +44,7 @@ const replicationMutex = new utils.Mutex(); export interface MongoBucketBatchOptions { db: PowerSyncMongo; - syncRules: SqlSyncRules; + syncRules: HydratedSyncRules; groupId: number; slotName: string; lastCheckpointLsn: string | null; @@ -71,7 +71,7 @@ export class MongoBucketBatch private readonly client: mongo.MongoClient; public readonly db: PowerSyncMongo; public readonly session: mongo.ClientSession; - private readonly sync_rules: SqlSyncRules; + private readonly sync_rules: HydratedSyncRules; private readonly group_id: number; @@ -474,8 +474,7 @@ export class MongoBucketBatch if (sourceTable.syncData) { const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({ record: after, - sourceTable, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${this.group_id}`) + sourceTable }); for (let error of syncErrors) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts index ce38cb683..131c35b9a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts @@ -1,4 +1,4 @@ -import { SqlSyncRules } from '@powersync/service-sync-rules'; +import { SqlSyncRules, HydratedSyncRules } from '@powersync/service-sync-rules'; import { storage } from '@powersync/service-core'; @@ -13,4 +13,8 @@ export class MongoPersistedSyncRules implements storage.PersistedSyncRules { ) { this.slot_name = slot_name ?? `powersync_${id}`; } + + hydratedSyncRules(): HydratedSyncRules { + return this.sync_rules.hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${this.id}`) }); + } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index c36a27322..8ab803726 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -25,7 +25,7 @@ import { WatchWriteCheckpointOptions } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; -import { ParameterLookup, SqliteJsonRow, SqlSyncRules } from '@powersync/service-sync-rules'; +import { ParameterLookup, SqliteJsonRow, SqlSyncRules, HydratedSyncRules } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { LRUCache } from 'lru-cache'; import * as timers from 'timers/promises'; @@ -61,7 +61,7 @@ export class MongoSyncBucketStorage private readonly db: PowerSyncMongo; readonly checksums: MongoChecksums; - private parsedSyncRulesCache: { parsed: SqlSyncRules; options: storage.ParseSyncRulesOptions } | undefined; + private parsedSyncRulesCache: { parsed: HydratedSyncRules; options: storage.ParseSyncRulesOptions } | undefined; private writeCheckpointAPI: MongoWriteCheckpointAPI; constructor( @@ -101,14 +101,14 @@ export class MongoSyncBucketStorage }); } - getParsedSyncRules(options: storage.ParseSyncRulesOptions): SqlSyncRules { + getParsedSyncRules(options: storage.ParseSyncRulesOptions): HydratedSyncRules { const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {}; /** * Check if the cached sync rules, if present, had the same options. * Parse sync rules if the options are different or if there is no cached value. */ if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) { - this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).sync_rules, options }; + this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncRules(), options }; } return this.parsedSyncRulesCache!.parsed; @@ -170,7 +170,7 @@ export class MongoSyncBucketStorage await using batch = new MongoBucketBatch({ logger: options.logger, db: this.db, - syncRules: this.sync_rules.parsed(options).sync_rules, + syncRules: this.sync_rules.parsed(options).hydratedSyncRules(), groupId: this.group_id, slotName: this.slot_name, lastCheckpointLsn: checkpoint_lsn, diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 0fa47f8a2..0347edf3c 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -10,7 +10,6 @@ import { ServiceError } from '@powersync/lib-services-framework'; import { - InternalOpId, MetricsEngine, RelationCache, SaveOperationTag, @@ -18,7 +17,13 @@ import { SourceTable, storage } from '@powersync/service-core'; -import { DatabaseInputRow, SqliteInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules'; +import { + DatabaseInputRow, + SqliteInputRow, + SqliteRow, + HydratedSyncRules, + TablePattern +} from '@powersync/service-sync-rules'; import { ReplicationMetric } from '@powersync/service-types'; import { MongoLSN } from '../common/MongoLSN.js'; import { PostImagesOption } from '../types/types.js'; @@ -75,7 +80,7 @@ export class ChangeStreamInvalidatedError extends DatabaseConnectionError { } export class ChangeStream { - sync_rules: SqlSyncRules; + sync_rules: HydratedSyncRules; group_id: number; connection_id = 1; diff --git a/modules/module-mssql/src/replication/CDCStream.ts b/modules/module-mssql/src/replication/CDCStream.ts index ca668aeff..d2b109f62 100644 --- a/modules/module-mssql/src/replication/CDCStream.ts +++ b/modules/module-mssql/src/replication/CDCStream.ts @@ -10,7 +10,13 @@ import { } from '@powersync/lib-services-framework'; import { getUuidReplicaIdentityBson, MetricsEngine, SourceEntityDescriptor, storage } from '@powersync/service-core'; -import { SqliteInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules'; +import { + SqliteInputRow, + SqliteRow, + SqlSyncRules, + HydratedSyncRules, + TablePattern +} from '@powersync/service-sync-rules'; import { ReplicationMetric } from '@powersync/service-types'; import { BatchedSnapshotQuery, MSSQLSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js'; @@ -82,7 +88,7 @@ export class CDCDataExpiredError extends DatabaseConnectionError { } export class CDCStream { - private readonly syncRules: SqlSyncRules; + private readonly syncRules: HydratedSyncRules; private readonly storage: storage.SyncRulesBucketStorage; private readonly connections: MSSQLConnectionManager; private readonly abortSignal: AbortSignal; diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index 98d9dc665..13c40062d 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -61,7 +61,7 @@ function createTableId(schema: string, tableName: string): string { } export class BinLogStream { - private readonly syncRules: sync_rules.SqlSyncRules; + private readonly syncRules: sync_rules.HydratedSyncRules; private readonly groupId: number; private readonly storage: storage.SyncRulesBucketStorage; diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 73600b0b8..faccb530a 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -60,7 +60,9 @@ export class PostgresSyncRulesStorage protected writeCheckpointAPI: PostgresWriteCheckpointAPI; // TODO we might be able to share this in an abstract class - private parsedSyncRulesCache: { parsed: sync_rules.SqlSyncRules; options: storage.ParseSyncRulesOptions } | undefined; + private parsedSyncRulesCache: + | { parsed: sync_rules.HydratedSyncRules; options: storage.ParseSyncRulesOptions } + | undefined; private _checksumCache: storage.ChecksumCache | undefined; constructor(protected options: PostgresSyncRulesStorageOptions) { @@ -96,14 +98,14 @@ export class PostgresSyncRulesStorage } // TODO we might be able to share this in an abstract class - getParsedSyncRules(options: storage.ParseSyncRulesOptions): sync_rules.SqlSyncRules { + getParsedSyncRules(options: storage.ParseSyncRulesOptions): sync_rules.HydratedSyncRules { const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {}; /** * Check if the cached sync rules, if present, had the same options. * Parse sync rules if the options are different or if there is no cached value. */ if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) { - this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).sync_rules, options }; + this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncRules(), options }; } return this.parsedSyncRulesCache!.parsed; @@ -349,7 +351,7 @@ export class PostgresSyncRulesStorage const batch = new PostgresBucketBatch({ logger: options.logger ?? framework.logger, db: this.db, - sync_rules: this.sync_rules.parsed(options).sync_rules, + sync_rules: this.sync_rules.parsed(options).hydratedSyncRules(), group_id: this.group_id, slot_name: this.slot_name, last_checkpoint_lsn: checkpoint_lsn, diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index 55ff1ccb6..27d4deb5a 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -24,7 +24,7 @@ import { PostgresPersistedBatch } from './PostgresPersistedBatch.js'; export interface PostgresBucketBatchOptions { logger: Logger; db: lib_postgres.DatabaseClient; - sync_rules: sync_rules.SqlSyncRules; + sync_rules: sync_rules.HydratedSyncRules; group_id: number; slot_name: string; last_checkpoint_lsn: string | null; @@ -72,7 +72,7 @@ export class PostgresBucketBatch protected persisted_op: InternalOpId | null; protected write_checkpoint_batch: storage.CustomWriteCheckpointOptions[]; - protected readonly sync_rules: sync_rules.SqlSyncRules; + protected readonly sync_rules: sync_rules.HydratedSyncRules; protected batch: OperationBatch | null; private lastWaitingLogThrottled = 0; private markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined; @@ -840,8 +840,7 @@ export class PostgresBucketBatch if (sourceTable.syncData) { const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({ record: after, - sourceTable, - bucketIdTransformer: sync_rules.SqlSyncRules.versionedBucketIdTransformer(`${this.group_id}`) + sourceTable }); for (const error of syncErrors) { diff --git a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts index c55cea108..e2dd67184 100644 --- a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +++ b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts @@ -35,7 +35,12 @@ export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncR return { id: this.id, slot_name: this.slot_name, - sync_rules: SqlSyncRules.fromYaml(this.sync_rules_content, options) + sync_rules: SqlSyncRules.fromYaml(this.sync_rules_content, options), + hydratedSyncRules() { + return this.sync_rules.hydrate({ + bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${this.id}`) + }); + } }; } diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index a68cf820d..a520a8b27 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -27,6 +27,7 @@ import { SqliteInputValue, SqliteRow, SqlSyncRules, + HydratedSyncRules, TablePattern, ToastableSqliteRow, toSyncRulesRow @@ -107,7 +108,7 @@ export class MissingReplicationSlotError extends Error { } export class WalStream { - sync_rules: SqlSyncRules; + sync_rules: HydratedSyncRules; group_id: number; connection_id = 1; diff --git a/packages/service-core-tests/src/test-utils/general-utils.ts b/packages/service-core-tests/src/test-utils/general-utils.ts index 6f93fffb9..5f48f38f4 100644 --- a/packages/service-core-tests/src/test-utils/general-utils.ts +++ b/packages/service-core-tests/src/test-utils/general-utils.ts @@ -25,7 +25,10 @@ export function testRules(content: string): storage.PersistedSyncRulesContent { return { id: 1, sync_rules: SqlSyncRules.fromYaml(content, options), - slot_name: 'test' + slot_name: 'test', + hydratedSyncRules() { + return this.sync_rules.hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); + } }; }, lock() { @@ -107,7 +110,6 @@ export function querierOptions(globalParameters: RequestParameters): GetQuerierO return { globalParameters, hasDefaultStreams: true, - streams: {}, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') + streams: {} }; } diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index d079aaa8c..bd10c6d82 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -333,7 +333,7 @@ bucket_definitions: const parameters = new RequestParameters({ sub: 'u1' }, {}); - const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0]; + const q1 = (sync_rules.bucketParameterSources[0] as SqlBucketDescriptor).parameterQueries[0]; const lookups = q1.getLookups(parameters); expect(lookups).toEqual([ParameterLookup.normalized('by_workspace', '1', ['u1'])]); @@ -342,6 +342,7 @@ bucket_definitions: expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]); const buckets = await sync_rules + .hydrate() .getBucketParameterQuerier(test_utils.querierOptions(parameters)) .querier.queryDynamicBucketDescriptions({ getParameterSets(lookups) { @@ -408,7 +409,7 @@ bucket_definitions: const parameters = new RequestParameters({ sub: 'unknown' }, {}); - const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0]; + const q1 = (sync_rules.bucketParameterSources[0] as SqlBucketDescriptor).parameterQueries[0]; const lookups = q1.getLookups(parameters); expect(lookups).toEqual([ParameterLookup.normalized('by_public_workspace', '1', [])]); @@ -418,6 +419,7 @@ bucket_definitions: expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]); const buckets = await sync_rules + .hydrate() .getBucketParameterQuerier(test_utils.querierOptions(parameters)) .querier.queryDynamicBucketDescriptions({ getParameterSets(lookups) { @@ -511,7 +513,7 @@ bucket_definitions: const parameters = new RequestParameters({ sub: 'u1' }, {}); // Test intermediate values - could be moved to sync_rules.test.ts - const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0]; + const q1 = (sync_rules.bucketParameterSources[0] as SqlBucketDescriptor).parameterQueries[0]; const lookups1 = q1.getLookups(parameters); expect(lookups1).toEqual([ParameterLookup.normalized('by_workspace', '1', [])]); @@ -519,7 +521,7 @@ bucket_definitions: parameter_sets1.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); expect(parameter_sets1).toEqual([{ workspace_id: 'workspace1' }]); - const q2 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[1]; + const q2 = (sync_rules.bucketParameterSources[0] as SqlBucketDescriptor).parameterQueries[1]; const lookups2 = q2.getLookups(parameters); expect(lookups2).toEqual([ParameterLookup.normalized('by_workspace', '2', ['u1'])]); @@ -530,6 +532,7 @@ bucket_definitions: // Test final values - the important part const buckets = ( await sync_rules + .hydrate() .getBucketParameterQuerier(test_utils.querierOptions(parameters)) .querier.queryDynamicBucketDescriptions({ getParameterSets(lookups) { diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index e721a2b31..25e323ace 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -82,10 +82,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { const stream = sync.streamResponse({ syncContext, bucketStorage: bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -146,10 +143,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -212,10 +206,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -325,10 +316,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -469,10 +457,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -588,10 +573,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -657,10 +639,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -689,10 +668,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -723,10 +699,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -800,10 +773,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -879,10 +849,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -949,10 +916,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -1020,10 +984,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -1086,10 +1047,7 @@ bucket_definitions: const stream = sync.streamResponse({ syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -1217,10 +1175,7 @@ bucket_definitions: const params: sync.SyncStreamParameters = { syncContext, bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, @@ -1293,10 +1248,7 @@ config: const stream = sync.streamResponse({ syncContext, bucketStorage: bucketStorage, - syncRules: { - syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), - version: bucketStorage.group_id - }, + syncRules: bucketStorage.getParsedSyncRules(test_utils.PARSE_OPTIONS), params: { buckets: [], include_checksum: true, diff --git a/packages/service-core/src/routes/endpoints/admin.ts b/packages/service-core/src/routes/endpoints/admin.ts index 8adab1795..6d162a072 100644 --- a/packages/service-core/src/routes/endpoints/admin.ts +++ b/packages/service-core/src/routes/endpoints/admin.ts @@ -177,7 +177,10 @@ export const validate = routeDefinition({ sync_rules: SqlSyncRules.fromYaml(content, { ...apiHandler.getParseSyncRulesOptions(), schema - }) + }), + hydratedSyncRules() { + return this.sync_rules.hydrate(); + } }; }, sync_rules_content: content, diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 81ac9d15b..e6367e09b 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -109,10 +109,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => for await (const data of sync.streamResponse({ syncContext: syncContext, bucketStorage: bucketStorage, - syncRules: { - syncRules, - version: bucketStorage.group_id - }, + syncRules, params: { ...params }, diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 46d02d1e4..fb1df9e96 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -92,10 +92,7 @@ export const syncStreamed = routeDefinition({ const syncLines = sync.streamResponse({ syncContext: syncContext, bucketStorage, - syncRules: { - syncRules, - version: bucketStorage.group_id - }, + syncRules, params: payload.params, token: payload.context.token_payload!, tracker, diff --git a/packages/service-core/src/storage/PersistedSyncRulesContent.ts b/packages/service-core/src/storage/PersistedSyncRulesContent.ts index 6c77ba03a..dd3922764 100644 --- a/packages/service-core/src/storage/PersistedSyncRulesContent.ts +++ b/packages/service-core/src/storage/PersistedSyncRulesContent.ts @@ -1,4 +1,4 @@ -import { SqlSyncRules } from '@powersync/service-sync-rules'; +import { SqlSyncRules, HydratedSyncRules } from '@powersync/service-sync-rules'; import { ReplicationLock } from './ReplicationLock.js'; export interface ParseSyncRulesOptions { @@ -30,4 +30,6 @@ export interface PersistedSyncRules { readonly id: number; readonly sync_rules: SqlSyncRules; readonly slot_name: string; + + hydratedSyncRules(): HydratedSyncRules; } diff --git a/packages/service-core/src/storage/SyncRulesBucketStorage.ts b/packages/service-core/src/storage/SyncRulesBucketStorage.ts index 37015766a..9621059c7 100644 --- a/packages/service-core/src/storage/SyncRulesBucketStorage.ts +++ b/packages/service-core/src/storage/SyncRulesBucketStorage.ts @@ -1,5 +1,5 @@ import { Logger, ObserverClient } from '@powersync/lib-services-framework'; -import { ParameterLookup, SqlSyncRules, SqliteJsonRow } from '@powersync/service-sync-rules'; +import { ParameterLookup, SqliteJsonRow, HydratedSyncRules } from '@powersync/service-sync-rules'; import * as util from '../util/util-index.js'; import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js'; import { BucketStorageFactory } from './BucketStorageFactory.js'; @@ -32,7 +32,7 @@ export interface SyncRulesBucketStorage callback: (batch: BucketStorageBatch) => Promise ): Promise; - getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules; + getParsedSyncRules(options: ParseSyncRulesOptions): HydratedSyncRules; /** * Terminate the sync rules. @@ -139,7 +139,7 @@ export interface ResolveTableOptions { connection_tag: string; entity_descriptor: SourceEntityDescriptor; - sync_rules: SqlSyncRules; + sync_rules: HydratedSyncRules; } export interface ResolveTableResult { diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index 4ba687500..f6b4d00dd 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -1,38 +1,33 @@ import { + BucketDataSourceDefinition, BucketDescription, BucketPriority, - BucketSource, + HydratedSyncRules, RequestedStream, RequestJwtPayload, RequestParameters, - ResolvedBucket, - SqlSyncRules + ResolvedBucket } from '@powersync/service-sync-rules'; import * as storage from '../storage/storage-index.js'; import * as util from '../util/util-index.js'; import { + logger as defaultLogger, ErrorCode, Logger, ServiceAssertionError, - ServiceError, - logger as defaultLogger + ServiceError } from '@powersync/lib-services-framework'; import { JSONBig } from '@powersync/service-jsonbig'; import { BucketParameterQuerier, QuerierError } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js'; import { SyncContext } from './SyncContext.js'; import { getIntersection, hasIntersection } from './util.js'; -export interface VersionedSyncRules { - syncRules: SqlSyncRules; - version: number; -} - export interface BucketChecksumStateOptions { syncContext: SyncContext; bucketStorage: BucketChecksumStateStorage; - syncRules: VersionedSyncRules; + syncRules: HydratedSyncRules; tokenPayload: RequestJwtPayload; syncRequest: util.StreamingSyncRequest; logger?: Logger; @@ -253,15 +248,15 @@ export class BucketChecksumState { const streamNameToIndex = new Map(); this.streamNameToIndex = streamNameToIndex; - for (const source of this.parameterState.syncRules.syncRules.bucketSources) { - if (this.parameterState.isSubscribedToStream(source)) { - streamNameToIndex.set(source.name, subscriptions.length); + for (const source of this.parameterState.syncRules.bucketDataSources) { + if (this.parameterState.isSubscribedToStream(source.definition)) { + streamNameToIndex.set(source.definition.name, subscriptions.length); subscriptions.push({ - name: source.name, - is_default: source.subscribedToByDefault, + name: source.definition.name, + is_default: source.definition.subscribedToByDefault, errors: - this.parameterState.streamErrors[source.name]?.map((e) => ({ + this.parameterState.streamErrors[source.definition.name]?.map((e) => ({ subscription: e.subscription?.opaque_id ?? 'default', message: e.message })) ?? [] @@ -381,7 +376,7 @@ export interface CheckpointUpdate { export class BucketParameterState { private readonly context: SyncContext; public readonly bucketStorage: BucketChecksumStateStorage; - public readonly syncRules: VersionedSyncRules; + public readonly syncRules: HydratedSyncRules; public readonly syncParams: RequestParameters; private readonly querier: BucketParameterQuerier; /** @@ -404,7 +399,7 @@ export class BucketParameterState { constructor( context: SyncContext, bucketStorage: BucketChecksumStateStorage, - syncRules: VersionedSyncRules, + syncRules: HydratedSyncRules, tokenPayload: RequestJwtPayload, request: util.StreamingSyncRequest, logger: Logger @@ -436,11 +431,10 @@ export class BucketParameterState { this.includeDefaultStreams = subscriptions?.include_defaults ?? true; this.explicitStreamSubscriptions = explicitStreamSubscriptions; - const { querier, errors } = syncRules.syncRules.getBucketParameterQuerier({ + const { querier, errors } = syncRules.getBucketParameterQuerier({ globalParameters: this.syncParams, hasDefaultStreams: this.includeDefaultStreams, - streams: streamsByName, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${syncRules.version}`) + streams: streamsByName }); this.querier = querier; this.streamErrors = Object.groupBy(errors, (e) => e.descriptor) as Record; @@ -490,7 +484,7 @@ export class BucketParameterState { }; } - isSubscribedToStream(desc: BucketSource): boolean { + isSubscribedToStream(desc: BucketDataSourceDefinition): boolean { return (desc.subscribedToByDefault && this.includeDefaultStreams) || this.subscribedStreamNames.has(desc.name); } diff --git a/packages/service-core/src/sync/sync.ts b/packages/service-core/src/sync/sync.ts index 01b848da0..ecd8e7b64 100644 --- a/packages/service-core/src/sync/sync.ts +++ b/packages/service-core/src/sync/sync.ts @@ -1,5 +1,5 @@ import { JSONBig, JsonContainer } from '@powersync/service-jsonbig'; -import { BucketDescription, BucketPriority, RequestJwtPayload } from '@powersync/service-sync-rules'; +import { BucketDescription, BucketPriority, RequestJwtPayload, HydratedSyncRules } from '@powersync/service-sync-rules'; import { AbortError } from 'ix/aborterror.js'; @@ -9,7 +9,7 @@ import * as util from '../util/util-index.js'; import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework'; import { mergeAsyncIterables } from '../streams/streams-index.js'; -import { BucketChecksumState, CheckpointLine, VersionedSyncRules } from './BucketChecksumState.js'; +import { BucketChecksumState, CheckpointLine } from './BucketChecksumState.js'; import { OperationsSentStats, RequestTracker, statsForBatch } from './RequestTracker.js'; import { SyncContext } from './SyncContext.js'; import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStream } from './util.js'; @@ -17,7 +17,7 @@ import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStr export interface SyncStreamParameters { syncContext: SyncContext; bucketStorage: storage.SyncRulesBucketStorage; - syncRules: VersionedSyncRules; + syncRules: HydratedSyncRules; params: util.StreamingSyncRequest; token: auth.JwtPayload; logger?: Logger; @@ -94,7 +94,7 @@ export async function* streamResponse( async function* streamResponseInner( syncContext: SyncContext, bucketStorage: storage.SyncRulesBucketStorage, - syncRules: VersionedSyncRules, + syncRules: HydratedSyncRules, params: util.StreamingSyncRequest, tokenPayload: RequestJwtPayload, tracker: RequestTracker, diff --git a/packages/service-core/test/src/routes/stream.test.ts b/packages/service-core/test/src/routes/stream.test.ts index baa74fc82..5e273d61c 100644 --- a/packages/service-core/test/src/routes/stream.test.ts +++ b/packages/service-core/test/src/routes/stream.test.ts @@ -45,7 +45,7 @@ describe('Stream Route', () => { const storage = { getParsedSyncRules() { - return new SqlSyncRules('bucket_definitions: {}'); + return new SqlSyncRules('bucket_definitions: {}').hydrate(); }, watchCheckpointChanges: async function* (options) { throw new Error('Simulated storage error'); @@ -83,7 +83,7 @@ describe('Stream Route', () => { it('logs the application metadata', async () => { const storage = { getParsedSyncRules() { - return new SqlSyncRules('bucket_definitions: {}'); + return new SqlSyncRules('bucket_definitions: {}').hydrate(); }, watchCheckpointChanges: async function* (options) { throw new Error('Simulated storage error'); diff --git a/packages/service-core/test/src/sync/BucketChecksumState.test.ts b/packages/service-core/test/src/sync/BucketChecksumState.test.ts index a27bf9a37..7ef36ac05 100644 --- a/packages/service-core/test/src/sync/BucketChecksumState.test.ts +++ b/packages/service-core/test/src/sync/BucketChecksumState.test.ts @@ -12,13 +12,7 @@ import { WatchFilterEvent } from '@/index.js'; import { JSONBig } from '@powersync/service-jsonbig'; -import { - SqliteJsonRow, - ParameterLookup, - SqlSyncRules, - RequestJwtPayload, - BucketSource -} from '@powersync/service-sync-rules'; +import { SqliteJsonRow, ParameterLookup, SqlSyncRules, RequestJwtPayload } from '@powersync/service-sync-rules'; import { describe, expect, test, beforeEach } from 'vitest'; describe('BucketChecksumState', () => { @@ -31,7 +25,7 @@ bucket_definitions: data: [] `, { defaultSchema: 'public' } - ); + ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); // global[1] and global[2] const SYNC_RULES_GLOBAL_TWO = SqlSyncRules.fromYaml( @@ -44,7 +38,7 @@ bucket_definitions: data: [] `, { defaultSchema: 'public' } - ); + ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('2') }); // by_project[n] const SYNC_RULES_DYNAMIC = SqlSyncRules.fromYaml( @@ -55,7 +49,7 @@ bucket_definitions: data: [] `, { defaultSchema: 'public' } - ); + ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('3') }); const syncContext = new SyncContext({ maxBuckets: 100, @@ -75,10 +69,7 @@ bucket_definitions: syncContext, syncRequest, tokenPayload, - syncRules: { - syncRules: SYNC_RULES_GLOBAL, - version: 1 - }, + syncRules: SYNC_RULES_GLOBAL, bucketStorage: storage }); @@ -148,10 +139,7 @@ bucket_definitions: tokenPayload, // Client sets the initial state here syncRequest: { buckets: [{ name: 'global[]', after: '1' }] }, - syncRules: { - syncRules: SYNC_RULES_GLOBAL, - version: 1 - }, + syncRules: SYNC_RULES_GLOBAL, bucketStorage: storage }); @@ -189,10 +177,7 @@ bucket_definitions: syncContext, tokenPayload, syncRequest, - syncRules: { - syncRules: SYNC_RULES_GLOBAL_TWO, - version: 2 - }, + syncRules: SYNC_RULES_GLOBAL_TWO, bucketStorage: storage }); @@ -260,10 +245,7 @@ bucket_definitions: tokenPayload, // Client sets the initial state here syncRequest: { buckets: [{ name: 'something_here[]', after: '1' }] }, - syncRules: { - syncRules: SYNC_RULES_GLOBAL, - version: 1 - }, + syncRules: SYNC_RULES_GLOBAL, bucketStorage: storage }); @@ -304,10 +286,7 @@ bucket_definitions: syncContext, tokenPayload, syncRequest, - syncRules: { - syncRules: SYNC_RULES_GLOBAL_TWO, - version: 1 - }, + syncRules: SYNC_RULES_GLOBAL_TWO, bucketStorage: storage }); @@ -360,10 +339,7 @@ bucket_definitions: syncContext, tokenPayload, syncRequest, - syncRules: { - syncRules: SYNC_RULES_GLOBAL_TWO, - version: 2 - }, + syncRules: SYNC_RULES_GLOBAL_TWO, bucketStorage: storage }); @@ -418,10 +394,7 @@ bucket_definitions: syncContext, tokenPayload, syncRequest, - syncRules: { - syncRules: SYNC_RULES_GLOBAL_TWO, - version: 2 - }, + syncRules: SYNC_RULES_GLOBAL_TWO, bucketStorage: storage }); @@ -525,10 +498,7 @@ bucket_definitions: syncContext, tokenPayload: { sub: 'u1' }, syncRequest, - syncRules: { - syncRules: SYNC_RULES_DYNAMIC, - version: 1 - }, + syncRules: SYNC_RULES_DYNAMIC, bucketStorage: storage }); @@ -627,7 +597,6 @@ bucket_definitions: }); describe('streams', () => { - let source: { -readonly [P in keyof BucketSource]: BucketSource[P] }; let storage: MockBucketChecksumStateStorage; function checksumState(source: string | boolean, options?: Partial) { @@ -645,13 +614,13 @@ config: const rules = SqlSyncRules.fromYaml(source, { defaultSchema: 'public' - }); + }).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); return new BucketChecksumState({ syncContext, syncRequest, tokenPayload, - syncRules: { syncRules: rules, version: 1 }, + syncRules: rules, bucketStorage: storage, ...options }); diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 684aae914..7e1ab7a19 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -9,7 +9,7 @@ import { validateSyncRulesSchema } from './json_schema.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { QueryParseResult, SqlBucketDescriptor } from './SqlBucketDescriptor.js'; import { syncStreamFromSql } from './streams/from_sql.js'; -import { SyncRules } from './SyncRules.js'; +import { HydratedSyncRules } from './SyncRules.js'; import { TablePattern } from './TablePattern.js'; import { BucketIdTransformer, @@ -373,11 +373,20 @@ export class SqlSyncRules { this.content = content; } - compile(params?: { bucketIdTransformer?: BucketIdTransformer }): SyncRules { + /** + * Hydrate the sync rule definitions with persisted state into runnable sync rules. + * + * Right now this is just the bucketIdTransformer, but this is expected to expand in the future to support + * incremental sync rule reprocessing. + * + * @param params.bucketIdTransformer A function that transforms bucket ids based on persisted state. May omit for tests. + */ + hydrate(params?: { bucketIdTransformer?: BucketIdTransformer }): HydratedSyncRules { const bucketIdTransformer = this.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) ? (params?.bucketIdTransformer ?? ((id: string) => id)) : (id: string) => id; - return new SyncRules({ + return new HydratedSyncRules({ + definition: this, bucketDataSources: this.bucketDataSources.map((d) => d.createDataSource({ bucketIdTransformer })), bucketParameterSources: this.bucketParameterSources.map((d) => d.createParameterSource({ bucketIdTransformer })), eventDescriptors: this.eventDescriptors, diff --git a/packages/sync-rules/src/SyncRules.ts b/packages/sync-rules/src/SyncRules.ts index 785cfa723..537e8efc9 100644 --- a/packages/sync-rules/src/SyncRules.ts +++ b/packages/sync-rules/src/SyncRules.ts @@ -1,13 +1,7 @@ -import { - BucketDataSource, - BucketDataSourceDefinition, - BucketParameterSource, - BucketParameterSourceDefinition -} from './BucketSource.js'; +import { BucketDataSource, BucketParameterSource } from './BucketSource.js'; import { BucketParameterQuerier, CompatibilityContext, - CompatibilityOption, EvaluatedParameters, EvaluatedRow, EvaluationError, @@ -18,19 +12,29 @@ import { isEvaluationError, mergeBucketParameterQueriers, QuerierError, - SqlEventDescriptor + SqlEventDescriptor, + SqliteInputValue, + SqliteValue, + SqlSyncRules } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, SqliteRow } from './types.js'; -export class SyncRules { +/** + * Hydrated sync rules is sync rule definitions along with persisted state. Currently, the persisted state + * specifically affects bucket names. + */ +export class HydratedSyncRules { bucketDataSources: BucketDataSource[] = []; bucketParameterSources: BucketParameterSource[] = []; eventDescriptors: SqlEventDescriptor[] = []; compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; + private definition: SqlSyncRules; + constructor(params: { + definition: SqlSyncRules; bucketDataSources: BucketDataSource[]; bucketParameterSources: BucketParameterSource[]; eventDescriptors?: SqlEventDescriptor[]; @@ -38,6 +42,7 @@ export class SyncRules { }) { this.bucketDataSources = params.bucketDataSources; this.bucketParameterSources = params.bucketParameterSources; + this.definition = params.definition; if (params.eventDescriptors) { this.eventDescriptors = params.eventDescriptors; } @@ -46,6 +51,30 @@ export class SyncRules { } } + // These methods do not depend on hydration, so we can just forward them to the definition. + + getSourceTables() { + return this.definition.getSourceTables(); + } + + tableTriggersEvent(table: SourceTableInterface): boolean { + return this.definition.tableTriggersEvent(table); + } + + tableSyncsData(table: SourceTableInterface): boolean { + return this.definition.tableSyncsData(table); + } + + tableSyncsParameters(table: SourceTableInterface): boolean { + return this.definition.tableSyncsParameters(table); + } + + applyRowContext( + source: SqliteRow + ): SqliteRow { + return this.definition.applyRowContext(source); + } + /** * Throws errors. */ diff --git a/packages/sync-rules/src/index.ts b/packages/sync-rules/src/index.ts index 8ebeb4115..08b82f11d 100644 --- a/packages/sync-rules/src/index.ts +++ b/packages/sync-rules/src/index.ts @@ -27,3 +27,4 @@ export * from './types.js'; export * from './types/custom_sqlite_value.js'; export * from './types/time.js'; export * from './utils.js'; +export * from './SyncRules.js'; diff --git a/packages/sync-rules/test/src/compatibility.test.ts b/packages/sync-rules/test/src/compatibility.test.ts index e1c4a7440..6d21f9dbc 100644 --- a/packages/sync-rules/test/src/compatibility.test.ts +++ b/packages/sync-rules/test/src/compatibility.test.ts @@ -16,12 +16,11 @@ bucket_definitions: - SELECT id, description FROM assets `, PARSE_OPTIONS - ); + ).hydrate(); expect( rules.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(''), record: rules.applyRowContext({ id: 'id', description: value @@ -44,12 +43,11 @@ config: timestamps_iso8601: true `, PARSE_OPTIONS - ); + ).hydrate(); expect( rules.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(''), record: rules.applyRowContext({ id: 'id', description: value @@ -72,11 +70,10 @@ config: edition: 2 `, PARSE_OPTIONS - ); + ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); expect( rules.evaluateRow({ - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1'), sourceTable: ASSETS, record: rules.applyRowContext({ id: 'id', @@ -87,11 +84,7 @@ config: { bucket: '1#stream|0[]', data: { description: '2025-08-19T09:21:00Z', id: 'id' }, id: 'id', table: 'assets' } ]); - expect( - rules.getBucketParameterQuerier( - normalizeQuerierOptions({}, {}, {}, SqlSyncRules.versionedBucketIdTransformer('1')) - ).querier.staticBuckets - ).toStrictEqual([ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}, {}, {})).querier.staticBuckets).toStrictEqual([ { bucket: '1#stream|0[]', definition: 'stream', @@ -115,11 +108,10 @@ config: versioned_bucket_ids: false `, PARSE_OPTIONS - ); + ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); expect( rules.evaluateRow({ - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1'), sourceTable: ASSETS, record: rules.applyRowContext({ id: 'id', @@ -129,11 +121,7 @@ config: ).toStrictEqual([ { bucket: 'stream|0[]', data: { description: '2025-08-19 09:21:00Z', id: 'id' }, id: 'id', table: 'assets' } ]); - expect( - rules.getBucketParameterQuerier( - normalizeQuerierOptions({}, {}, {}, SqlSyncRules.versionedBucketIdTransformer('1')) - ).querier.staticBuckets - ).toStrictEqual([ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}, {}, {})).querier.staticBuckets).toStrictEqual([ { bucket: 'stream|0[]', definition: 'stream', @@ -157,12 +145,11 @@ config: versioned_bucket_ids: true `, PARSE_OPTIONS - ); + ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); expect( rules.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1'), record: { id: 'id', description: 'desc' @@ -182,12 +169,11 @@ config: edition: 2 `, PARSE_OPTIONS - ); + ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); expect( rules.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1'), record: rules.applyRowContext({ id: 'id', description: new DateTimeValue('2025-08-19T09:21:00Z') @@ -210,7 +196,7 @@ bucket_definitions: - SELECT id, description ->> 'foo.bar' AS "desc" FROM assets `, PARSE_OPTIONS - ); + ).hydrate(); expect( rules.evaluateRow({ @@ -218,8 +204,7 @@ bucket_definitions: record: { id: 'id', description: description - }, - bucketIdTransformer: identityBucketTransformer + } }) ).toStrictEqual([{ bucket: 'a[]', data: { desc: 'baz', id: 'id' }, id: 'id', table: 'assets' }]); }); @@ -235,7 +220,7 @@ config: fixed_json_extract: true `, PARSE_OPTIONS - ); + ).hydrate(); expect( rules.evaluateRow({ @@ -243,8 +228,7 @@ config: record: { id: 'id', description: description - }, - bucketIdTransformer: identityBucketTransformer + } }) ).toStrictEqual([{ bucket: 'a[]', data: { desc: null, id: 'id' }, id: 'id', table: 'assets' }]); }); @@ -285,11 +269,12 @@ config: `; } - const rules = SqlSyncRules.fromYaml(syncRules, PARSE_OPTIONS); + const rules = SqlSyncRules.fromYaml(syncRules, PARSE_OPTIONS).hydrate({ + bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') + }); expect( rules.evaluateRow({ sourceTable: ASSETS, - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1'), record: rules.applyRowContext({ id: 'id', description: data @@ -309,11 +294,7 @@ config: } ]); - expect( - rules.getBucketParameterQuerier( - normalizeQuerierOptions({}, {}, {}, SqlSyncRules.versionedBucketIdTransformer('1')) - ).querier.staticBuckets - ).toStrictEqual([ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}, {}, {})).querier.staticBuckets).toStrictEqual([ { bucket: withFixedQuirk ? '1#mybucket[]' : 'mybucket[]', definition: 'mybucket', diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index 0be75a505..59310ef34 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -37,7 +37,9 @@ describe('streams', () => { expect(desc.variants).toHaveLength(1); expect(evaluateBucketIds(desc, COMMENTS, { id: 'foo' })).toStrictEqual(['1#stream|0[]']); - expect(desc.evaluateRow({ sourceTable: USERS, bucketIdTransformer, record: { id: 'foo' } })).toHaveLength(0); + expect( + desc.createDataSource({ bucketIdTransformer }).evaluateRow({ sourceTable: USERS, record: { id: 'foo' } }) + ).toHaveLength(0); }); test('row condition', () => { @@ -71,15 +73,12 @@ describe('streams', () => { const queriers: BucketParameterQuerier[] = []; const errors: QuerierError[] = []; const pending = { queriers, errors }; - desc.pushBucketParameterQueriers( - pending, - normalizeQuerierOptions( - { test: 'foo' }, - {}, - { stream: [{ opaque_id: 0, parameters: null }] }, - bucketIdTransformer - ) - ); + desc + .createParameterSource({ bucketIdTransformer }) + .pushBucketParameterQueriers( + pending, + normalizeQuerierOptions({ test: 'foo' }, {}, { stream: [{ opaque_id: 0, parameters: null }] }) + ); expect(mergeBucketParameterQueriers(queriers).staticBuckets).toEqual([ { @@ -220,7 +219,9 @@ describe('streams', () => { '1#stream|1[]' ]); - expect(desc.evaluateParameterRow(ISSUES, { id: 'i1', owner_id: 'u1' })).toStrictEqual([ + expect( + desc.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(ISSUES, { id: 'i1', owner_id: 'u1' }) + ).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', ['u1']), bucketParameters: [ @@ -253,7 +254,11 @@ describe('streams', () => { ); expect(desc.tableSyncsParameters(ISSUES)).toBe(true); - expect(desc.evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' })).toStrictEqual([ + expect( + desc + .createParameterSource({ bucketIdTransformer }) + .evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) + ).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', ['user1']), bucketParameters: [ @@ -285,7 +290,9 @@ describe('streams', () => { expect(desc.tableSyncsParameters(ISSUES)).toBe(false); expect(desc.tableSyncsParameters(USERS)).toBe(true); - expect(desc.evaluateParameterRow(USERS, { id: 'u', is_admin: 1n })).toStrictEqual([ + expect( + desc.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(USERS, { id: 'u', is_admin: 1n }) + ).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', ['u']), bucketParameters: [ @@ -295,7 +302,9 @@ describe('streams', () => { ] } ]); - expect(desc.evaluateParameterRow(USERS, { id: 'u', is_admin: 0n })).toStrictEqual([]); + expect( + desc.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(USERS, { id: 'u', is_admin: 0n }) + ).toStrictEqual([]); // Should return bucket id for admin users expect( @@ -331,7 +340,9 @@ describe('streams', () => { '1#stream|1["a"]' ]); - expect(desc.evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' })).toStrictEqual([ + expect( + desc.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' }) + ).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', ['b']), bucketParameters: [ @@ -437,7 +448,9 @@ describe('streams', () => { ); expect(desc.tableSyncsParameters(FRIENDS)).toBe(true); - expect(desc.evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' })).toStrictEqual([ + expect( + desc.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' }) + ).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', ['b']), bucketParameters: [ @@ -592,7 +605,11 @@ describe('streams', () => { 'select * from comments where NOT (issue_id not in (select id from issues where owner_id = auth.user_id()))' ); - expect(desc.evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' })).toStrictEqual([ + expect( + desc + .createParameterSource({ bucketIdTransformer }) + .evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) + ).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', ['user1']), bucketParameters: [ @@ -719,7 +736,9 @@ describe('streams', () => { expect(stream.tableSyncsParameters(accountMember)).toBeTruthy(); // Ensure lookup steps work. - expect(stream.evaluateParameterRow(accountMember, row)).toStrictEqual([ + expect( + stream.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(accountMember, row) + ).toStrictEqual([ { lookup: ParameterLookup.normalized('account_member', '0', ['id']), bucketParameters: [ @@ -800,7 +819,7 @@ WHERE expect(evaluateBucketIds(desc, scene, { _id: 'scene', project: 'foo' })).toStrictEqual(['1#stream|0["foo"]']); expect( - desc.evaluateParameterRow(projectInvitation, { + desc.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(projectInvitation, { project: 'foo', appliedTo: '[1,2]', status: 'CLAIMED' @@ -899,13 +918,16 @@ const options: StreamParseOptions = { const bucketIdTransformer = SqlSyncRules.versionedBucketIdTransformer('1'); function evaluateBucketIds(stream: SyncStream, sourceTable: SourceTableInterface, record: SqliteRow) { - return stream.evaluateRow({ sourceTable, record, bucketIdTransformer }).map((r) => { - if ('error' in r) { - throw new Error(`Unexpected error evaluating row: ${r.error}`); - } + return stream + .createDataSource({ bucketIdTransformer }) + .evaluateRow({ sourceTable, record }) + .map((r) => { + if ('error' in r) { + throw new Error(`Unexpected error evaluating row: ${r.error}`); + } - return r.bucket; - }); + return r.bucket; + }); } async function createQueriers( @@ -929,11 +951,10 @@ async function createQueriers( }, {} ), - streams: { [stream.name]: [{ opaque_id: 0, parameters: options?.parameters ?? null }] }, - bucketIdTransformer + streams: { [stream.name]: [{ opaque_id: 0, parameters: options?.parameters ?? null }] } }; - stream.pushBucketParameterQueriers(pending, querierOptions); + stream.createParameterSource({ bucketIdTransformer }).pushBucketParameterQueriers(pending, querierOptions); return { querier: mergeBucketParameterQueriers(queriers), errors }; } diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index dd61c9393..93bd6cc5c 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -31,7 +31,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); const bucket = rules.bucketDataSources[0] as SqlBucketDescriptor; expect(bucket.name).toEqual('mybucket'); expect(bucket.bucketParameters).toEqual([]); @@ -39,7 +39,7 @@ bucket_definitions: expect(dataQuery.bucketParameters).toEqual([]); expect(dataQuery.columnOutputNames()).toEqual(['id', 'description']); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1', description: 'test' } }) @@ -55,7 +55,7 @@ bucket_definitions: } ]); expect(rules.hasDynamicBucketQueries()).toBe(false); - expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[]', priority: 3 }], hasDynamicBuckets: false }); @@ -71,7 +71,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); const bucket = rules.bucketParameterSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual([]); const param_query = bucket.globalParameterQueries[0]; @@ -80,15 +80,15 @@ bucket_definitions: expect(param_query.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 1n }))).toEqual(1n); expect(param_query.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 0n }))).toEqual(0n); - expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier).toMatchObject({ + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[]', priority: 3 }], hasDynamicBuckets: false }); - expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: false })).querier).toMatchObject({ + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: false })).querier).toMatchObject({ staticBuckets: [], hasDynamicBuckets: false }); - expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [], hasDynamicBuckets: false }); @@ -104,18 +104,18 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); const bucket = rules.bucketParameterSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual([]); const param_query = bucket.parameterQueries[0]; expect(param_query.bucketParameters).toEqual([]); - expect(compiled.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ + expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ { bucketParameters: [{}], lookup: ParameterLookup.normalized('mybucket', '1', ['user1']) } ]); - expect(compiled.evaluateParameterRow(USERS, { id: 'user1', is_admin: 0 })).toEqual([]); + expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 0 })).toEqual([]); }); test('parse bucket with parameters', () => { @@ -129,14 +129,14 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; const bucketData = rules.bucketDataSources[0] as SqlBucketDescriptor; expect(bucketParameters.bucketParameters).toEqual(['user_id', 'device_id']); const param_query = bucketParameters.globalParameterQueries[0]; expect(param_query.bucketParameters).toEqual(['user_id', 'device_id']); expect( - compiled.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' })) + hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' })) .querier.staticBuckets ).toEqual([ { bucket: 'mybucket["user1","device1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 } @@ -145,7 +145,7 @@ bucket_definitions: const data_query = bucketData.dataQueries[0]; expect(data_query.bucketParameters).toEqual(['user_id', 'device_id']); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1', description: 'test', user_id: 'user1', device_id: 'device1' } }) @@ -161,7 +161,7 @@ bucket_definitions: } ]); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1', description: 'test', user_id: 'user1', archived: 1, device_id: 'device1' } }) @@ -179,20 +179,20 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; const bucketData = rules.bucketDataSources[0] as SqlBucketDescriptor; expect(bucketParameters.bucketParameters).toEqual(['user_id']); const param_query = bucketParameters.globalParameterQueries[0]; expect(param_query.bucketParameters).toEqual(['user_id']); expect( - compiled.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier.staticBuckets + hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier.staticBuckets ).toEqual([{ bucket: 'mybucket["user1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 }]); const data_query = bucketData.dataQueries[0]; expect(data_query.bucketParameters).toEqual(['user_id']); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1', description: 'test', user_id: 'user1' } }) @@ -208,7 +208,7 @@ bucket_definitions: } ]); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1', description: 'test', owner_id: 'user1' } }) @@ -325,16 +325,16 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; expect(bucketParameters.bucketParameters).toEqual(['user_id']); - expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false }); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1', description: 'test', user_id: 'user1' } }) @@ -363,16 +363,16 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; expect(bucketParameters.bucketParameters).toEqual(['user_id']); - expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false }); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1', description: 'test', user_id: 'user1' } }) @@ -399,9 +399,9 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1', data: JSON.stringify({ count: 5, bool: true }) } }) @@ -433,10 +433,10 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1', @@ -478,10 +478,10 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1', description: 'test', role: 'admin' } }) @@ -500,7 +500,7 @@ bucket_definitions: ]); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset2', description: 'test', role: 'normal' } }) @@ -541,7 +541,7 @@ bucket_definitions: ]); expect( - compiled.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier.staticBuckets + hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier.staticBuckets ).toEqual([{ bucket: 'mybucket[1]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 }]); }); @@ -555,9 +555,9 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); - expect(compiled.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1' } })).toEqual([ + expect(hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1' } })).toEqual([ { bucket: 'mybucket[]', id: 'asset1', @@ -583,13 +583,13 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); expect( - compiled.getBucketParameterQuerier(normalizeQuerierOptions({ int1: 314, float1: 3.14, float2: 314 })).querier + hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ int1: 314, float1: 3.14, float2: 314 })).querier ).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[314,3.14,314]', priority: 3 }] }); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1', int1: 314n, float1: 3.14, float2: 314 } }) @@ -616,8 +616,8 @@ bucket_definitions: PARSE_OPTIONS ); expect(rules.errors).toEqual([]); - const compiled = rules.compile({ bucketIdTransformer }); - expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'test' })).querier).toMatchObject({ + const hydrated = rules.hydrate({ bucketIdTransformer }); + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'test' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["TEST"]', priority: 3 }], hasDynamicBuckets: false }); @@ -634,10 +634,10 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: new TestSourceTable('assets_123'), record: { client_id: 'asset1', description: 'test', archived: 0n, other_id: 'other1' } }) @@ -675,10 +675,10 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: new TestSourceTable('assets_123'), record: { client_id: 'asset1', description: 'test', archived: 0n, other_id: 'other1' } }) @@ -709,10 +709,10 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1', description: 'test', archived: 0n } }) @@ -745,10 +745,10 @@ bucket_definitions: `, PARSE_OPTIONS ); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); expect( - compiled.evaluateRow({ + hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1' } }) @@ -874,8 +874,8 @@ bucket_definitions: expect(rules.errors).toEqual([]); - const compiled = rules.compile({ bucketIdTransformer }); - expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ + const hydrated = rules.hydrate({ bucketIdTransformer }); + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [ { bucket: 'highprio[]', priority: 0 }, { bucket: 'defaultprio[]', priority: 3 } @@ -900,8 +900,8 @@ bucket_definitions: expect(rules.errors).toEqual([]); - const compiled = rules.compile({ bucketIdTransformer }); - expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ + const hydrated = rules.hydrate({ bucketIdTransformer }); + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [ { bucket: 'highprio[]', priority: 0 }, { bucket: 'defaultprio[]', priority: 3 } @@ -966,9 +966,9 @@ bucket_definitions: expect(bucketParameters.bucketParameters).toEqual(['user_id']); expect(rules.hasDynamicBucketQueries()).toBe(true); - const compiled = rules.compile({ bucketIdTransformer }); + const hydrated = rules.hydrate({ bucketIdTransformer }); - expect(compiled.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ + expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ hasDynamicBuckets: true, parameterQueryLookups: [ ParameterLookup.normalized('mybucket', '2', ['user1']), diff --git a/packages/sync-rules/test/src/util.ts b/packages/sync-rules/test/src/util.ts index c86667e70..4c8946186 100644 --- a/packages/sync-rules/test/src/util.ts +++ b/packages/sync-rules/test/src/util.ts @@ -69,15 +69,13 @@ export function normalizeTokenParameters( export function normalizeQuerierOptions( token_parameters: Record, user_parameters?: Record, - streams?: Record, - bucketIdTransformer?: BucketIdTransformer + streams?: Record ): GetQuerierOptions { const globalParameters = normalizeTokenParameters(token_parameters, user_parameters); return { globalParameters, hasDefaultStreams: true, - streams: streams ?? {}, - bucketIdTransformer: bucketIdTransformer ?? identityBucketTransformer + streams: streams ?? {} }; } From 2b7787d948c87af7ca96b37e43dca4261b52e0a6 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 9 Dec 2025 14:55:20 +0200 Subject: [PATCH 04/33] Refactor parameter queries. --- packages/sync-rules/src/BucketSource.ts | 14 +-- .../sync-rules/src/SqlBucketDescriptor.ts | 108 ++---------------- packages/sync-rules/src/SqlParameterQuery.ts | 59 +++++++++- packages/sync-rules/src/SqlSyncRules.ts | 6 +- .../sync-rules/src/StaticSqlParameterQuery.ts | 77 ++++++++++++- .../TableValuedFunctionSqlParameterQuery.ts | 73 +++++++++++- packages/sync-rules/src/streams/stream.ts | 4 - packages/sync-rules/src/streams/variant.ts | 4 - .../sync-rules/test/src/sync_rules.test.ts | 39 +++---- 9 files changed, 230 insertions(+), 154 deletions(-) diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 5ae504add..e910c1742 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -48,24 +48,18 @@ export interface BucketParameterSourceDefinition { readonly name: string; readonly type: BucketSourceType; readonly subscribedToByDefault: boolean; + /** + * For debug use only. + */ + readonly bucketParameters: string[]; getSourceTables(): Set; createParameterSource(params: CreateSourceParams): BucketParameterSource; - /** - * Whether {@link pushBucketParameterQueriers} may include a querier where - * {@link BucketParameterQuerier.hasDynamicBuckets} is true. - * - * This is mostly used for testing. - */ - hasDynamicBucketQueries(): boolean; - getSourceTables(): Set; /** Whether the table possibly affects the buckets resolved by this source. */ tableSyncsParameters(table: SourceTableInterface): boolean; - - debugRepresentation(): any; } /** diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 78e80bf80..8ea906f82 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -1,35 +1,22 @@ -import { BucketInclusionReason, ResolvedBucket } from './BucketDescription.js'; -import { BucketParameterQuerier, mergeBucketParameterQueriers, PendingQueriers } from './BucketParameterQuerier.js'; import { BucketDataSource, BucketDataSourceDefinition, - BucketParameterSource, BucketParameterSourceDefinition, BucketSourceType, - CreateSourceParams, - ResultSetDescription + CreateSourceParams } from './BucketSource.js'; import { ColumnDefinition } from './ExpressionType.js'; import { IdSequence } from './IdSequence.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SqlDataQuery } from './SqlDataQuery.js'; import { SqlParameterQuery } from './SqlParameterQuery.js'; -import { GetQuerierOptions, SyncRulesOptions } from './SqlSyncRules.js'; +import { SyncRulesOptions } from './SqlSyncRules.js'; import { StaticSqlParameterQuery } from './StaticSqlParameterQuery.js'; import { TablePattern } from './TablePattern.js'; import { TableValuedFunctionSqlParameterQuery } from './TableValuedFunctionSqlParameterQuery.js'; -import { SqlRuleError } from './errors.js'; import { CompatibilityContext } from './compatibility.js'; -import { - BucketIdTransformer, - EvaluatedParametersResult, - EvaluateRowOptions, - EvaluationResult, - QueryParseOptions, - RequestParameters, - SourceSchema, - SqliteRow -} from './types.js'; +import { SqlRuleError } from './errors.js'; +import { EvaluationResult, QueryParseOptions, SourceSchema } from './types.js'; export interface QueryParseResult { /** @@ -40,7 +27,7 @@ export interface QueryParseResult { errors: SqlRuleError[]; } -export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketParameterSourceDefinition { +export class SqlBucketDescriptor implements BucketDataSourceDefinition { name: string; private bucketParametersInternal: string[] | null = null; @@ -124,76 +111,8 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketPa }; } - createParameterSource(params: CreateSourceParams): BucketParameterSource { - return { - definition: this, - - evaluateParameterRow: (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { - let results: EvaluatedParametersResult[] = []; - for (let query of this.parameterQueries) { - if (query.applies(sourceTable)) { - results.push(...query.evaluateParameterRow(row)); - } - } - return results; - }, - pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { - const reasons = [this.bucketInclusionReason()]; - const staticBuckets = this.getStaticBucketDescriptions( - options.globalParameters, - reasons, - params.bucketIdTransformer - ); - const staticQuerier = { - staticBuckets, - hasDynamicBuckets: false, - parameterQueryLookups: [], - queryDynamicBucketDescriptions: async () => [] - } satisfies BucketParameterQuerier; - result.queriers.push(staticQuerier); - - if (this.parameterQueries.length == 0) { - return; - } - - const dynamicQueriers = this.parameterQueries.map((query) => - query.getBucketParameterQuerier(options.globalParameters, reasons, params.bucketIdTransformer) - ); - result.queriers.push(...dynamicQueriers); - }, - - /** - * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. - */ - getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { - const queriers: BucketParameterQuerier[] = []; - this.pushBucketParameterQueriers({ queriers, errors: [] }, options); - - return mergeBucketParameterQueriers(queriers); - } - }; - } - - getStaticBucketDescriptions( - parameters: RequestParameters, - reasons: BucketInclusionReason[], - transformer: BucketIdTransformer - ): ResolvedBucket[] { - let results: ResolvedBucket[] = []; - for (let query of this.globalParameterQueries) { - for (const desc of query.getStaticBucketDescriptions(parameters, transformer)) { - results.push({ - ...desc, - definition: this.name, - inclusion_reasons: reasons - }); - } - } - return results; - } - - hasDynamicBucketQueries(): boolean { - return this.parameterQueries.length > 0; + getParameterSourceDefinitions(): BucketParameterSourceDefinition[] { + return [...this.parameterQueries, ...this.globalParameterQueries]; } getSourceTables(): Set { @@ -210,10 +129,6 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketPa return result; } - private bucketInclusionReason(): BucketInclusionReason { - return 'default'; - } - tableSyncsData(table: SourceTableInterface): boolean { for (let query of this.dataQueries) { if (query.applies(table)) { @@ -223,15 +138,6 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketPa return false; } - tableSyncsParameters(table: SourceTableInterface): boolean { - for (let query of this.parameterQueries) { - if (query.applies(table)) { - return true; - } - } - return false; - } - resolveResultSets(schema: SourceSchema, tables: Record>) { for (let query of this.dataQueries) { query.resolveResultSets(schema, tables); diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 76a83a5f0..d95d6228a 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -5,7 +5,13 @@ import { BucketPriority, DEFAULT_BUCKET_PRIORITY } from './BucketDescription.js'; -import { BucketParameterQuerier, ParameterLookup, ParameterLookupSource } from './BucketParameterQuerier.js'; +import { + BucketParameterQuerier, + mergeBucketParameterQueriers, + ParameterLookup, + ParameterLookupSource, + PendingQueriers +} from './BucketParameterQuerier.js'; import { SqlRuleError } from './errors.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; @@ -31,6 +37,13 @@ import { } from './types.js'; import { filterJsonRow, getBucketId, isJsonValue, isSelectStatement, normalizeParameterValue } from './utils.js'; import { DetectRequestParameters } from './validators.js'; +import { + BucketParameterSource, + BucketParameterSourceDefinition, + BucketSourceType, + CreateSourceParams +} from './BucketSource.js'; +import { GetQuerierOptions } from './index.js'; export interface SqlParameterQueryOptions { sourceTable: TablePattern; @@ -55,7 +68,7 @@ export interface SqlParameterQueryOptions { * SELECT id as user_id FROM users WHERE users.user_id = token_parameters.user_id * SELECT id as user_id, token_parameters.is_admin as is_admin FROM users WHERE users.user_id = token_parameters.user_id */ -export class SqlParameterQuery { +export class SqlParameterQuery implements BucketParameterSourceDefinition { static fromSql( descriptorName: string, sql: string, @@ -282,6 +295,10 @@ export class SqlParameterQuery { readonly queryId: string; readonly tools: SqlTools; + readonly type: BucketSourceType = BucketSourceType.SYNC_RULE; + + readonly subscribedToByDefault: boolean = true; + readonly errors: SqlRuleError[]; constructor(options: SqlParameterQueryOptions) { @@ -301,10 +318,46 @@ export class SqlParameterQuery { this.errors = options.errors ?? []; } - applies(table: SourceTableInterface) { + tableSyncsParameters(table: SourceTableInterface): boolean { return this.sourceTable.matches(table); } + get name(): string { + return this.descriptorName; + } + + getSourceTables(): Set { + return new Set([this.sourceTable]); + } + + createParameterSource(params: CreateSourceParams): BucketParameterSource { + return { + definition: this, + + evaluateParameterRow: (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { + if (this.tableSyncsParameters(sourceTable)) { + return this.evaluateParameterRow(row); + } else { + return []; + } + }, + pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { + const q = this.getBucketParameterQuerier(options.globalParameters, ['default'], params.bucketIdTransformer); + result.queriers.push(q); + }, + + /** + * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. + */ + getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { + const queriers: BucketParameterQuerier[] = []; + this.pushBucketParameterQueriers({ queriers, errors: [] }, options); + + return mergeBucketParameterQueriers(queriers); + } + }; + } + /** * Given a replicated row, results an array of bucket parameter rows to persist. */ diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 7e1ab7a19..e3e07ed9e 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -232,7 +232,7 @@ export class SqlSyncRules { }); } rules.bucketDataSources.push(descriptor); - rules.bucketParameterSources.push(descriptor); + rules.bucketParameterSources.push(...descriptor.getParameterSourceDefinitions()); } for (const entry of streamMap?.items ?? []) { @@ -400,10 +400,6 @@ export class SqlSyncRules { return applyRowContext(source, this.compatibility); } - hasDynamicBucketQueries() { - return this.bucketParameterSources.some((s) => s.hasDynamicBucketQueries()); - } - getSourceTables(): TablePattern[] { const sourceTables = new Map(); for (const bucket of this.bucketDataSources) { diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index 9a5d05f66..f41158102 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -1,17 +1,29 @@ import { SelectedColumn, SelectFromStatement } from 'pgsql-ast-parser'; -import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY } from './BucketDescription.js'; +import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY, ResolvedBucket } from './BucketDescription.js'; import { SqlRuleError } from './errors.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; import { checkUnsupportedFeatures, isClauseError, isParameterValueClause, sqliteBool } from './sql_support.js'; import { BucketIdTransformer, + EvaluatedParametersResult, ParameterValueClause, QueryParseOptions, RequestParameters, - SqliteJsonValue + SqliteJsonValue, + SqliteRow } from './types.js'; import { getBucketId, isJsonValue } from './utils.js'; import { DetectRequestParameters } from './validators.js'; +import { + BucketParameterSource, + BucketParameterSourceDefinition, + BucketSourceType, + CreateSourceParams +} from './BucketSource.js'; +import { SourceTableInterface } from './SourceTableInterface.js'; +import { TablePattern } from './TablePattern.js'; +import { BucketParameterQuerier, mergeBucketParameterQueriers, PendingQueriers } from './BucketParameterQuerier.js'; +import { GetQuerierOptions } from './index.js'; export interface StaticSqlParameterQueryOptions { sql: string; @@ -30,7 +42,7 @@ export interface StaticSqlParameterQueryOptions { * SELECT token_parameters.user_id * SELECT token_parameters.user_id as user_id WHERE token_parameters.is_admin */ -export class StaticSqlParameterQuery { +export class StaticSqlParameterQuery implements BucketParameterSourceDefinition { static fromSql( descriptorName: string, sql: string, @@ -148,6 +160,9 @@ export class StaticSqlParameterQuery { */ readonly filter: ParameterValueClause | undefined; + readonly subscribedToByDefault = true; + readonly type = BucketSourceType.SYNC_RULE; + readonly errors: SqlRuleError[]; constructor(options: StaticSqlParameterQueryOptions) { @@ -161,6 +176,62 @@ export class StaticSqlParameterQuery { this.errors = options.errors ?? []; } + get name() { + return this.descriptorName; + } + + getSourceTables() { + return new Set(); + } + + tableSyncsParameters(_table: SourceTableInterface): boolean { + return false; + } + + createParameterSource(params: CreateSourceParams): BucketParameterSource { + return { + definition: this, + + evaluateParameterRow: (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { + return []; + }, + + pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { + const staticBuckets = this.getStaticBucketDescriptions( + options.globalParameters, + params.bucketIdTransformer + ).map((desc) => { + return { + ...desc, + definition: this.name, + inclusion_reasons: ['default'] + } satisfies ResolvedBucket; + }); + + if (staticBuckets.length == 0) { + return; + } + const staticQuerier = { + staticBuckets, + hasDynamicBuckets: false, + parameterQueryLookups: [], + queryDynamicBucketDescriptions: async () => [] + } satisfies BucketParameterQuerier; + result.queriers.push(staticQuerier); + }, + + /** + * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. + */ + getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { + const queriers: BucketParameterQuerier[] = []; + this.pushBucketParameterQueriers({ queriers, errors: [] }, options); + + return mergeBucketParameterQueriers(queriers); + } + }; + } + getStaticBucketDescriptions(parameters: RequestParameters, transformer: BucketIdTransformer): BucketDescription[] { if (this.filter == null) { // Error in filter clause diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index 8b8da15bf..3c46120cf 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -5,6 +5,7 @@ import { checkUnsupportedFeatures, isClauseError, isParameterValueClause, sqlite import { generateTableValuedFunctions, TableValuedFunction } from './TableValuedFunctions.js'; import { BucketIdTransformer, + EvaluatedParametersResult, ParameterValueClause, ParameterValueSet, QueryParseOptions, @@ -13,8 +14,17 @@ import { SqliteRow } from './types.js'; import { getBucketId, isJsonValue } from './utils.js'; -import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY } from './BucketDescription.js'; +import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY, ResolvedBucket } from './BucketDescription.js'; import { DetectRequestParameters } from './validators.js'; +import { TablePattern } from './TablePattern.js'; +import { + BucketParameterSource, + BucketParameterSourceDefinition, + BucketSourceType, + CreateSourceParams +} from './BucketSource.js'; +import { SourceTableInterface } from './SourceTableInterface.js'; +import { BucketParameterQuerier, GetQuerierOptions, mergeBucketParameterQueriers, PendingQueriers } from './index.js'; export interface TableValuedFunctionSqlParameterQueryOptions { sql: string; @@ -41,7 +51,7 @@ export interface TableValuedFunctionSqlParameterQueryOptions { * * This can currently not be combined with parameter table queries or multiple table-valued functions. */ -export class TableValuedFunctionSqlParameterQuery { +export class TableValuedFunctionSqlParameterQuery implements BucketParameterSourceDefinition { static fromSql( descriptorName: string, sql: string, @@ -191,6 +201,9 @@ export class TableValuedFunctionSqlParameterQuery { readonly errors: SqlRuleError[]; + readonly subscribedToByDefault = true; + readonly type = BucketSourceType.SYNC_RULE; + constructor(options: TableValuedFunctionSqlParameterQueryOptions) { this.sql = options.sql; this.parameterExtractors = options.parameterExtractors; @@ -207,6 +220,62 @@ export class TableValuedFunctionSqlParameterQuery { this.errors = options.errors; } + get name() { + return this.descriptorName; + } + + getSourceTables() { + return new Set(); + } + + tableSyncsParameters(_table: SourceTableInterface): boolean { + return false; + } + + createParameterSource(params: CreateSourceParams): BucketParameterSource { + return { + definition: this, + + evaluateParameterRow: (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { + return []; + }, + + pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { + const staticBuckets = this.getStaticBucketDescriptions( + options.globalParameters, + params.bucketIdTransformer + ).map((desc) => { + return { + ...desc, + definition: this.name, + inclusion_reasons: ['default'] + } satisfies ResolvedBucket; + }); + + if (staticBuckets.length == 0) { + return; + } + const staticQuerier = { + staticBuckets, + hasDynamicBuckets: false, + parameterQueryLookups: [], + queryDynamicBucketDescriptions: async () => [] + } satisfies BucketParameterQuerier; + result.queriers.push(staticQuerier); + }, + + /** + * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. + */ + getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { + const queriers: BucketParameterQuerier[] = []; + this.pushBucketParameterQueriers({ queriers, errors: [] }, options); + + return mergeBucketParameterQueriers(queriers); + } + }; + } + getStaticBucketDescriptions(parameters: RequestParameters, transformer: BucketIdTransformer): BucketDescription[] { if (this.filter == null || this.callClause == null) { // Error in filter clause diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index ee1a62410..aee464b41 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -159,10 +159,6 @@ export class SyncStream implements BucketDataSourceDefinition, BucketParameterSo } } - hasDynamicBucketQueries(): boolean { - return this.variants.some((v) => v.hasDynamicBucketQueries); - } - tableSyncsData(table: SourceTableInterface): boolean { return this.data.applies(table); } diff --git a/packages/sync-rules/src/streams/variant.ts b/packages/sync-rules/src/streams/variant.ts index 1e9c7097e..c44d63397 100644 --- a/packages/sync-rules/src/streams/variant.ts +++ b/packages/sync-rules/src/streams/variant.ts @@ -114,10 +114,6 @@ export class StreamVariant { return [...cartesianProduct(...instantiations)]; } - get hasDynamicBucketQueries(): boolean { - return this.requestFilters.some((f) => f.type == 'dynamic'); - } - querier( stream: SyncStream, reason: BucketInclusionReason, diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 93bd6cc5c..ab308ce39 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { ParameterLookup, SqlSyncRules } from '../../src/index.js'; +import { ParameterLookup, SqlParameterQuery, SqlSyncRules } from '../../src/index.js'; import { ASSETS, @@ -11,6 +11,7 @@ import { normalizeTokenParameters } from './util.js'; import { SqlBucketDescriptor } from '../../src/SqlBucketDescriptor.js'; +import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; describe('sync rules', () => { const bucketIdTransformer = SqlSyncRules.versionedBucketIdTransformer(''); @@ -54,7 +55,6 @@ bucket_definitions: bucket: 'mybucket[]' } ]); - expect(rules.hasDynamicBucketQueries()).toBe(false); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[]', priority: 3 }], hasDynamicBuckets: false @@ -72,13 +72,13 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const bucket = rules.bucketParameterSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual([]); - const param_query = bucket.globalParameterQueries[0]; + const parameterSource = rules.bucketParameterSources[0]; + expect(parameterSource.bucketParameters).toEqual([]); // Internal API, subject to change - expect(param_query.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 1n }))).toEqual(1n); - expect(param_query.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 0n }))).toEqual(0n); + const parameterQuery = parameterSource as StaticSqlParameterQuery; + expect(parameterQuery.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 1n }))).toEqual(1n); + expect(parameterQuery.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 0n }))).toEqual(0n); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[]', priority: 3 }], @@ -105,10 +105,8 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const bucket = rules.bucketParameterSources[0] as SqlBucketDescriptor; - expect(bucket.bucketParameters).toEqual([]); - const param_query = bucket.parameterQueries[0]; - expect(param_query.bucketParameters).toEqual([]); + const parameterSource = rules.bucketParameterSources[0] as SqlParameterQuery; + expect(parameterSource.bucketParameters).toEqual([]); expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ { bucketParameters: [{}], @@ -130,11 +128,10 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; + const parameterSource = rules.bucketParameterSources[0] as StaticSqlParameterQuery; const bucketData = rules.bucketDataSources[0] as SqlBucketDescriptor; - expect(bucketParameters.bucketParameters).toEqual(['user_id', 'device_id']); - const param_query = bucketParameters.globalParameterQueries[0]; - expect(param_query.bucketParameters).toEqual(['user_id', 'device_id']); + expect(parameterSource.bucketParameters).toEqual(['user_id', 'device_id']); + expect(bucketData.bucketParameters).toEqual(['user_id', 'device_id']); expect( hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' })) .querier.staticBuckets @@ -180,11 +177,10 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; + const bucketParameters = rules.bucketParameterSources[0] as StaticSqlParameterQuery; const bucketData = rules.bucketDataSources[0] as SqlBucketDescriptor; expect(bucketParameters.bucketParameters).toEqual(['user_id']); - const param_query = bucketParameters.globalParameterQueries[0]; - expect(param_query.bucketParameters).toEqual(['user_id']); + expect(bucketData.bucketParameters).toEqual(['user_id']); expect( hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier.staticBuckets ).toEqual([{ bucket: 'mybucket["user1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 }]); @@ -326,7 +322,7 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; + const bucketParameters = rules.bucketParameterSources[0]; expect(bucketParameters.bucketParameters).toEqual(['user_id']); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], @@ -364,7 +360,7 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; + const bucketParameters = rules.bucketParameterSources[0]; expect(bucketParameters.bucketParameters).toEqual(['user_id']); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], @@ -962,9 +958,8 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucketParameters = rules.bucketParameterSources[0] as SqlBucketDescriptor; + const bucketParameters = rules.bucketParameterSources[0]; expect(bucketParameters.bucketParameters).toEqual(['user_id']); - expect(rules.hasDynamicBucketQueries()).toBe(true); const hydrated = rules.hydrate({ bucketIdTransformer }); From 308477a89e5c0d4fe344254fc11926c0af90b8e2 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 9 Dec 2025 15:48:45 +0200 Subject: [PATCH 05/33] Fix tests. --- .../register-data-storage-parameter-tests.ts | 82 +++++++++---------- 1 file changed, 37 insertions(+), 45 deletions(-) diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index bd10c6d82..d42672377 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -1,5 +1,10 @@ import { storage } from '@powersync/service-core'; -import { ParameterLookup, RequestParameters } from '@powersync/service-sync-rules'; +import { + mergeBucketParameterQueriers, + ParameterLookup, + PendingQueriers, + RequestParameters +} from '@powersync/service-sync-rules'; import { SqlBucketDescriptor } from '@powersync/service-sync-rules/src/SqlBucketDescriptor.js'; import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; @@ -314,7 +319,7 @@ bucket_definitions: data: [] ` }); - const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).sync_rules; + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); const bucketStorage = factory.getInstance(syncRules); await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -333,22 +338,19 @@ bucket_definitions: const parameters = new RequestParameters({ sub: 'u1' }, {}); - const q1 = (sync_rules.bucketParameterSources[0] as SqlBucketDescriptor).parameterQueries[0]; + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; - const lookups = q1.getLookups(parameters); + const lookups = querier.parameterQueryLookups; expect(lookups).toEqual([ParameterLookup.normalized('by_workspace', '1', ['u1'])]); const parameter_sets = await checkpoint.getParameterSets(lookups); expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]); - const buckets = await sync_rules - .hydrate() - .getBucketParameterQuerier(test_utils.querierOptions(parameters)) - .querier.queryDynamicBucketDescriptions({ - getParameterSets(lookups) { - return checkpoint.getParameterSets(lookups); - } - }); + const buckets = await querier.queryDynamicBucketDescriptions({ + getParameterSets(lookups) { + return checkpoint.getParameterSets(lookups); + } + }); expect(buckets).toEqual([ { bucket: 'by_workspace["workspace1"]', priority: 3, definition: 'by_workspace', inclusion_reasons: ['default'] } ]); @@ -368,7 +370,7 @@ bucket_definitions: data: [] ` }); - const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).sync_rules; + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); const bucketStorage = factory.getInstance(syncRules); await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -409,23 +411,20 @@ bucket_definitions: const parameters = new RequestParameters({ sub: 'unknown' }, {}); - const q1 = (sync_rules.bucketParameterSources[0] as SqlBucketDescriptor).parameterQueries[0]; + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; - const lookups = q1.getLookups(parameters); + const lookups = querier.parameterQueryLookups; expect(lookups).toEqual([ParameterLookup.normalized('by_public_workspace', '1', [])]); const parameter_sets = await checkpoint.getParameterSets(lookups); parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]); - const buckets = await sync_rules - .hydrate() - .getBucketParameterQuerier(test_utils.querierOptions(parameters)) - .querier.queryDynamicBucketDescriptions({ - getParameterSets(lookups) { - return checkpoint.getParameterSets(lookups); - } - }); + const buckets = await querier.queryDynamicBucketDescriptions({ + getParameterSets(lookups) { + return checkpoint.getParameterSets(lookups); + } + }); buckets.sort((a, b) => a.bucket.localeCompare(b.bucket)); expect(buckets).toEqual([ { @@ -459,7 +458,7 @@ bucket_definitions: data: [] ` }); - const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).sync_rules; + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); const bucketStorage = factory.getInstance(syncRules); await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -513,32 +512,25 @@ bucket_definitions: const parameters = new RequestParameters({ sub: 'u1' }, {}); // Test intermediate values - could be moved to sync_rules.test.ts - const q1 = (sync_rules.bucketParameterSources[0] as SqlBucketDescriptor).parameterQueries[0]; - const lookups1 = q1.getLookups(parameters); - expect(lookups1).toEqual([ParameterLookup.normalized('by_workspace', '1', [])]); - - const parameter_sets1 = await checkpoint.getParameterSets(lookups1); - parameter_sets1.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); - expect(parameter_sets1).toEqual([{ workspace_id: 'workspace1' }]); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; - const q2 = (sync_rules.bucketParameterSources[0] as SqlBucketDescriptor).parameterQueries[1]; - const lookups2 = q2.getLookups(parameters); - expect(lookups2).toEqual([ParameterLookup.normalized('by_workspace', '2', ['u1'])]); + const lookups = querier.parameterQueryLookups; + expect(lookups).toEqual([ + ParameterLookup.normalized('by_workspace', '1', []), + ParameterLookup.normalized('by_workspace', '2', ['u1']) + ]); - const parameter_sets2 = await checkpoint.getParameterSets(lookups2); - parameter_sets2.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); - expect(parameter_sets2).toEqual([{ workspace_id: 'workspace3' }]); + const parameter_sets = await checkpoint.getParameterSets(lookups); + parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); + expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]); // Test final values - the important part const buckets = ( - await sync_rules - .hydrate() - .getBucketParameterQuerier(test_utils.querierOptions(parameters)) - .querier.queryDynamicBucketDescriptions({ - getParameterSets(lookups) { - return checkpoint.getParameterSets(lookups); - } - }) + await querier.queryDynamicBucketDescriptions({ + getParameterSets(lookups) { + return checkpoint.getParameterSets(lookups); + } + }) ).map((e) => e.bucket); buckets.sort(); expect(buckets).toEqual(['by_workspace["workspace1"]', 'by_workspace["workspace3"]']); From 3f378db20c93227f691ceb9fcca88ee795ec0d9f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 9 Dec 2025 15:50:40 +0200 Subject: [PATCH 06/33] Remove redundant functions. --- .../src/tests/register-data-storage-parameter-tests.ts | 8 +------- packages/sync-rules/src/BucketSource.ts | 7 ------- packages/sync-rules/src/SqlParameterQuery.ts | 10 ---------- packages/sync-rules/src/StaticSqlParameterQuery.ts | 10 ---------- .../src/TableValuedFunctionSqlParameterQuery.ts | 10 ---------- packages/sync-rules/src/streams/stream.ts | 10 ---------- 6 files changed, 1 insertion(+), 54 deletions(-) diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index d42672377..679e316e7 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -1,11 +1,5 @@ import { storage } from '@powersync/service-core'; -import { - mergeBucketParameterQueriers, - ParameterLookup, - PendingQueriers, - RequestParameters -} from '@powersync/service-sync-rules'; -import { SqlBucketDescriptor } from '@powersync/service-sync-rules/src/SqlBucketDescriptor.js'; +import { ParameterLookup, RequestParameters } from '@powersync/service-sync-rules'; import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; import { TEST_TABLE } from './util.js'; diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index e910c1742..d5d1d2921 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -56,8 +56,6 @@ export interface BucketParameterSourceDefinition { getSourceTables(): Set; createParameterSource(params: CreateSourceParams): BucketParameterSource; - getSourceTables(): Set; - /** Whether the table possibly affects the buckets resolved by this source. */ tableSyncsParameters(table: SourceTableInterface): boolean; } @@ -110,11 +108,6 @@ export interface BucketParameterSource { * @param options Options, including parameters that may affect the buckets loaded by this source. */ pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void; - - /** - * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. - */ - getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier; } export enum BucketSourceType { diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index d95d6228a..c932a7afe 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -344,16 +344,6 @@ export class SqlParameterQuery implements BucketParameterSourceDefinition { pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { const q = this.getBucketParameterQuerier(options.globalParameters, ['default'], params.bucketIdTransformer); result.queriers.push(q); - }, - - /** - * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. - */ - getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { - const queriers: BucketParameterQuerier[] = []; - this.pushBucketParameterQueriers({ queriers, errors: [] }, options); - - return mergeBucketParameterQueriers(queriers); } }; } diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index f41158102..b79e7274e 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -218,16 +218,6 @@ export class StaticSqlParameterQuery implements BucketParameterSourceDefinition queryDynamicBucketDescriptions: async () => [] } satisfies BucketParameterQuerier; result.queriers.push(staticQuerier); - }, - - /** - * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. - */ - getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { - const queriers: BucketParameterQuerier[] = []; - this.pushBucketParameterQueriers({ queriers, errors: [] }, options); - - return mergeBucketParameterQueriers(queriers); } }; } diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index 3c46120cf..0262d552a 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -262,16 +262,6 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterSour queryDynamicBucketDescriptions: async () => [] } satisfies BucketParameterQuerier; result.queriers.push(staticQuerier); - }, - - /** - * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. - */ - getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { - const queriers: BucketParameterQuerier[] = []; - this.pushBucketParameterQueriers({ queriers, errors: [] }, options); - - return mergeBucketParameterQueriers(queriers); } }; } diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index aee464b41..1059f2e6a 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -118,16 +118,6 @@ export class SyncStream implements BucketDataSourceDefinition, BucketParameterSo } return result; - }, - - /** - * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. - */ - getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { - const queriers: BucketParameterQuerier[] = []; - this.pushBucketParameterQueriers({ queriers, errors: [] }, options); - - return mergeBucketParameterQueriers(queriers); } }; } From c1fd10dc433b1d657dd98c812f92365dde41542e Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 9 Dec 2025 16:53:08 +0200 Subject: [PATCH 07/33] Split out parameter lookup sources from parameter querier sources. --- packages/sync-rules/src/BucketSource.ts | 45 ++++++++++++------- .../sync-rules/src/SqlBucketDescriptor.ts | 9 +++- packages/sync-rules/src/SqlParameterQuery.ts | 27 +++++++---- packages/sync-rules/src/SqlSyncRules.ts | 26 ++++++++--- .../sync-rules/src/StaticSqlParameterQuery.ts | 13 +++--- packages/sync-rules/src/SyncRules.ts | 17 ++++--- .../TableValuedFunctionSqlParameterQuery.ts | 31 ++++++------- packages/sync-rules/src/streams/stream.ts | 37 +++++++++------ packages/sync-rules/test/src/streams.test.ts | 28 +++++++----- .../sync-rules/test/src/sync_rules.test.ts | 28 ++++++------ 10 files changed, 157 insertions(+), 104 deletions(-) diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index d5d1d2921..4a7246f2b 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -44,22 +44,43 @@ export interface BucketDataSourceDefinition { debugRepresentation(): any; } -export interface BucketParameterSourceDefinition { +/** + * A parameter lookup source defines how to extract parameter lookup values from parameter queries. + * + * This is only relevant for parameter queries that query tables. + */ +export interface BucketParameterLookupSourceDefinition { readonly name: string; readonly type: BucketSourceType; - readonly subscribedToByDefault: boolean; /** * For debug use only. */ readonly bucketParameters: string[]; getSourceTables(): Set; - createParameterSource(params: CreateSourceParams): BucketParameterSource; + createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource; /** Whether the table possibly affects the buckets resolved by this source. */ tableSyncsParameters(table: SourceTableInterface): boolean; } +/** + * Parameter querier source definitions define how to bucket parameter queries are evaluated. + * + * This may use request data only, or it may use parameter lookup data persisted by a BucketParameterLookupSourceDefinition. + */ +export interface BucketParameterQuerierSourceDefinition { + readonly name: string; + readonly type: BucketSourceType; + readonly subscribedToByDefault: boolean; + /** + * For debug use only. + */ + readonly bucketParameters: string[]; + + createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource; +} + /** * An interface declaring * @@ -80,18 +101,8 @@ export interface BucketDataSource { evaluateRow(options: EvaluateRowOptions): EvaluationResult[]; } -/** - * An interface declaring - * - * - which buckets the sync service should create when processing change streams from the database. - * - how data in source tables maps to data in buckets (e.g. when we're not selecting all columns). - * - which buckets a given connection has access to. - * - * There are two ways to define bucket sources: Via sync rules made up of parameter and data queries, and via stream - * definitions that only consist of a single query. - */ -export interface BucketParameterSource { - readonly definition: BucketParameterSourceDefinition; +export interface BucketParameterLookupSource { + readonly definition: BucketParameterLookupSourceDefinition; /** * Given a row in a source table that affects sync parameters, returns a structure to index which buckets rows should * be associated with. @@ -100,6 +111,10 @@ export interface BucketParameterSource { * system to find buckets. */ evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[]; +} + +export interface BucketParameterQuerierSource { + readonly definition: BucketParameterQuerierSourceDefinition; /** * Reports {@link BucketParameterQuerier}s resolving buckets that a specific stream request should have access to. diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 8ea906f82..7d10a3b5b 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -1,7 +1,8 @@ import { BucketDataSource, BucketDataSourceDefinition, - BucketParameterSourceDefinition, + BucketParameterLookupSourceDefinition, + BucketParameterQuerierSourceDefinition, BucketSourceType, CreateSourceParams } from './BucketSource.js'; @@ -111,10 +112,14 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition { }; } - getParameterSourceDefinitions(): BucketParameterSourceDefinition[] { + getParameterQuerierSourceDefinitions(): BucketParameterQuerierSourceDefinition[] { return [...this.parameterQueries, ...this.globalParameterQueries]; } + getParameterLookupSourceDefinitions(): BucketParameterLookupSourceDefinition[] { + return [...this.parameterQueries]; + } + getSourceTables(): Set { let result = new Set(); for (let query of this.parameterQueries) { diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index c932a7afe..a04e82523 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -38,8 +38,10 @@ import { import { filterJsonRow, getBucketId, isJsonValue, isSelectStatement, normalizeParameterValue } from './utils.js'; import { DetectRequestParameters } from './validators.js'; import { - BucketParameterSource, - BucketParameterSourceDefinition, + BucketParameterLookupSource, + BucketParameterLookupSourceDefinition, + BucketParameterQuerierSource, + BucketParameterQuerierSourceDefinition, BucketSourceType, CreateSourceParams } from './BucketSource.js'; @@ -68,7 +70,9 @@ export interface SqlParameterQueryOptions { * SELECT id as user_id FROM users WHERE users.user_id = token_parameters.user_id * SELECT id as user_id, token_parameters.is_admin as is_admin FROM users WHERE users.user_id = token_parameters.user_id */ -export class SqlParameterQuery implements BucketParameterSourceDefinition { +export class SqlParameterQuery + implements BucketParameterLookupSourceDefinition, BucketParameterQuerierSourceDefinition +{ static fromSql( descriptorName: string, sql: string, @@ -330,7 +334,18 @@ export class SqlParameterQuery implements BucketParameterSourceDefinition { return new Set([this.sourceTable]); } - createParameterSource(params: CreateSourceParams): BucketParameterSource { + createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { + return { + definition: this, + + pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { + const q = this.getBucketParameterQuerier(options.globalParameters, ['default'], params.bucketIdTransformer); + result.queriers.push(q); + } + }; + } + + createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { return { definition: this, @@ -340,10 +355,6 @@ export class SqlParameterQuery implements BucketParameterSourceDefinition { } else { return []; } - }, - pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { - const q = this.getBucketParameterQuerier(options.globalParameters, ['default'], params.bucketIdTransformer); - result.queriers.push(q); } }; } diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index e3e07ed9e..9b76f44a7 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -1,7 +1,11 @@ import { isScalar, LineCounter, parseDocument, Scalar, YAMLMap, YAMLSeq } from 'yaml'; import { isValidPriority } from './BucketDescription.js'; import { BucketParameterQuerier, QuerierError } from './BucketParameterQuerier.js'; -import { BucketDataSourceDefinition, BucketParameterSourceDefinition } from './BucketSource.js'; +import { + BucketDataSourceDefinition, + BucketParameterLookupSourceDefinition, + BucketParameterQuerierSourceDefinition +} from './BucketSource.js'; import { CompatibilityContext, CompatibilityEdition, CompatibilityOption } from './compatibility.js'; import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js'; import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; @@ -79,7 +83,8 @@ export interface GetBucketParameterQuerierResult { export class SqlSyncRules { bucketDataSources: BucketDataSourceDefinition[] = []; - bucketParameterSources: BucketParameterSourceDefinition[] = []; + bucketParameterLookupSources: BucketParameterLookupSourceDefinition[] = []; + bucketParameterQuerierSources: BucketParameterQuerierSourceDefinition[] = []; eventDescriptors: SqlEventDescriptor[] = []; compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; @@ -232,7 +237,8 @@ export class SqlSyncRules { }); } rules.bucketDataSources.push(descriptor); - rules.bucketParameterSources.push(...descriptor.getParameterSourceDefinitions()); + rules.bucketParameterLookupSources.push(...descriptor.getParameterLookupSourceDefinitions()); + rules.bucketParameterQuerierSources.push(...descriptor.getParameterQuerierSourceDefinitions()); } for (const entry of streamMap?.items ?? []) { @@ -258,7 +264,8 @@ export class SqlSyncRules { rules.withScalar(data, (q) => { const [parsed, errors] = syncStreamFromSql(key, q, queryOptions); rules.bucketDataSources.push(parsed); - rules.bucketParameterSources.push(parsed); + rules.bucketParameterLookupSources.push(parsed); + rules.bucketParameterQuerierSources.push(parsed); return { parsed: true, errors @@ -388,7 +395,12 @@ export class SqlSyncRules { return new HydratedSyncRules({ definition: this, bucketDataSources: this.bucketDataSources.map((d) => d.createDataSource({ bucketIdTransformer })), - bucketParameterSources: this.bucketParameterSources.map((d) => d.createParameterSource({ bucketIdTransformer })), + bucketParameterQuerierSources: this.bucketParameterQuerierSources.map((d) => + d.createParameterQuerierSource({ bucketIdTransformer }) + ), + bucketParameterLookupSources: this.bucketParameterLookupSources.map((d) => + d.createParameterLookupSource({ bucketIdTransformer }) + ), eventDescriptors: this.eventDescriptors, compatibility: this.compatibility }); @@ -408,7 +420,7 @@ export class SqlSyncRules { sourceTables.set(key, r); } } - for (const bucket of this.bucketParameterSources) { + for (const bucket of this.bucketParameterLookupSources) { for (const r of bucket.getSourceTables()) { const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`; sourceTables.set(key, r); @@ -446,7 +458,7 @@ export class SqlSyncRules { } tableSyncsParameters(table: SourceTableInterface): boolean { - return this.bucketParameterSources.some((b) => b.tableSyncsParameters(table)); + return this.bucketParameterLookupSources.some((b) => b.tableSyncsParameters(table)); } debugGetOutputTables() { diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index b79e7274e..da3d0b76a 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -15,8 +15,9 @@ import { import { getBucketId, isJsonValue } from './utils.js'; import { DetectRequestParameters } from './validators.js'; import { - BucketParameterSource, - BucketParameterSourceDefinition, + BucketParameterLookupSource, + BucketParameterQuerierSource, + BucketParameterQuerierSourceDefinition, BucketSourceType, CreateSourceParams } from './BucketSource.js'; @@ -42,7 +43,7 @@ export interface StaticSqlParameterQueryOptions { * SELECT token_parameters.user_id * SELECT token_parameters.user_id as user_id WHERE token_parameters.is_admin */ -export class StaticSqlParameterQuery implements BucketParameterSourceDefinition { +export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefinition { static fromSql( descriptorName: string, sql: string, @@ -188,14 +189,10 @@ export class StaticSqlParameterQuery implements BucketParameterSourceDefinition return false; } - createParameterSource(params: CreateSourceParams): BucketParameterSource { + createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { return { definition: this, - evaluateParameterRow: (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { - return []; - }, - pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { const staticBuckets = this.getStaticBucketDescriptions( options.globalParameters, diff --git a/packages/sync-rules/src/SyncRules.ts b/packages/sync-rules/src/SyncRules.ts index 537e8efc9..3ec461014 100644 --- a/packages/sync-rules/src/SyncRules.ts +++ b/packages/sync-rules/src/SyncRules.ts @@ -1,4 +1,4 @@ -import { BucketDataSource, BucketParameterSource } from './BucketSource.js'; +import { BucketDataSource, BucketParameterLookupSource, BucketParameterQuerierSource } from './BucketSource.js'; import { BucketParameterQuerier, CompatibilityContext, @@ -25,8 +25,9 @@ import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, Sqlite * specifically affects bucket names. */ export class HydratedSyncRules { - bucketDataSources: BucketDataSource[] = []; - bucketParameterSources: BucketParameterSource[] = []; + bucketDataSources: BucketDataSource[]; + bucketParameterQuerierSources: BucketParameterQuerierSource[]; + bucketParameterLookupSources: BucketParameterLookupSource[]; eventDescriptors: SqlEventDescriptor[] = []; compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; @@ -36,12 +37,14 @@ export class HydratedSyncRules { constructor(params: { definition: SqlSyncRules; bucketDataSources: BucketDataSource[]; - bucketParameterSources: BucketParameterSource[]; + bucketParameterQuerierSources: BucketParameterQuerierSource[]; + bucketParameterLookupSources: BucketParameterLookupSource[]; eventDescriptors?: SqlEventDescriptor[]; compatibility?: CompatibilityContext; }) { this.bucketDataSources = params.bucketDataSources; - this.bucketParameterSources = params.bucketParameterSources; + this.bucketParameterQuerierSources = params.bucketParameterQuerierSources; + this.bucketParameterLookupSources = params.bucketParameterLookupSources; this.definition = params.definition; if (params.eventDescriptors) { this.eventDescriptors = params.eventDescriptors; @@ -114,7 +117,7 @@ export class HydratedSyncRules { row: SqliteRow ): { results: EvaluatedParameters[]; errors: EvaluationError[] } { let rawResults: EvaluatedParametersResult[] = []; - for (let source of this.bucketParameterSources) { + for (let source of this.bucketParameterLookupSources) { rawResults.push(...source.evaluateParameterRow(table, row)); } @@ -128,7 +131,7 @@ export class HydratedSyncRules { const errors: QuerierError[] = []; const pending = { queriers, errors }; - for (const source of this.bucketParameterSources) { + for (const source of this.bucketParameterQuerierSources) { if ( (source.definition.subscribedToByDefault && options.hasDefaultStreams) || source.definition.name in options.streams diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index 0262d552a..878536422 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -1,11 +1,20 @@ import { FromCall, SelectFromStatement } from 'pgsql-ast-parser'; +import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY, ResolvedBucket } from './BucketDescription.js'; +import { + BucketParameterQuerierSource, + BucketParameterQuerierSourceDefinition, + BucketSourceType, + CreateSourceParams +} from './BucketSource.js'; import { SqlRuleError } from './errors.js'; +import { BucketParameterQuerier, GetQuerierOptions, PendingQueriers } from './index.js'; +import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; -import { checkUnsupportedFeatures, isClauseError, isParameterValueClause, sqliteBool } from './sql_support.js'; +import { checkUnsupportedFeatures, isClauseError, sqliteBool } from './sql_support.js'; +import { TablePattern } from './TablePattern.js'; import { generateTableValuedFunctions, TableValuedFunction } from './TableValuedFunctions.js'; import { BucketIdTransformer, - EvaluatedParametersResult, ParameterValueClause, ParameterValueSet, QueryParseOptions, @@ -14,17 +23,7 @@ import { SqliteRow } from './types.js'; import { getBucketId, isJsonValue } from './utils.js'; -import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY, ResolvedBucket } from './BucketDescription.js'; import { DetectRequestParameters } from './validators.js'; -import { TablePattern } from './TablePattern.js'; -import { - BucketParameterSource, - BucketParameterSourceDefinition, - BucketSourceType, - CreateSourceParams -} from './BucketSource.js'; -import { SourceTableInterface } from './SourceTableInterface.js'; -import { BucketParameterQuerier, GetQuerierOptions, mergeBucketParameterQueriers, PendingQueriers } from './index.js'; export interface TableValuedFunctionSqlParameterQueryOptions { sql: string; @@ -51,7 +50,7 @@ export interface TableValuedFunctionSqlParameterQueryOptions { * * This can currently not be combined with parameter table queries or multiple table-valued functions. */ -export class TableValuedFunctionSqlParameterQuery implements BucketParameterSourceDefinition { +export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuerierSourceDefinition { static fromSql( descriptorName: string, sql: string, @@ -232,14 +231,10 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterSour return false; } - createParameterSource(params: CreateSourceParams): BucketParameterSource { + createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { return { definition: this, - evaluateParameterRow: (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { - return []; - }, - pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { const staticBuckets = this.getStaticBucketDescriptions( options.globalParameters, diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index 1059f2e6a..e79db35fa 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -4,8 +4,10 @@ import { BucketParameterQuerier, mergeBucketParameterQueriers, PendingQueriers } import { BucketDataSource, BucketDataSourceDefinition, - BucketParameterSource, - BucketParameterSourceDefinition, + BucketParameterLookupSource, + BucketParameterLookupSourceDefinition, + BucketParameterQuerierSource, + BucketParameterQuerierSourceDefinition, BucketSourceType, CreateSourceParams } from '../BucketSource.js'; @@ -24,7 +26,9 @@ import { } from '../types.js'; import { StreamVariant } from './variant.js'; -export class SyncStream implements BucketDataSourceDefinition, BucketParameterSourceDefinition { +export class SyncStream + implements BucketDataSourceDefinition, BucketParameterLookupSourceDefinition, BucketParameterQuerierSourceDefinition +{ name: string; subscribedToByDefault: boolean; priority: BucketPriority; @@ -80,7 +84,23 @@ export class SyncStream implements BucketDataSourceDefinition, BucketParameterSo }; } - createParameterSource(params: CreateSourceParams): BucketParameterSource { + createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { + return { + definition: this, + + evaluateParameterRow: (sourceTable, row) => { + const result: EvaluatedParametersResult[] = []; + + for (const variant of this.variants) { + variant.pushParameterRowEvaluation(result, sourceTable, row); + } + + return result; + } + }; + } + + createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { return { definition: this, @@ -109,15 +129,6 @@ export class SyncStream implements BucketDataSourceDefinition, BucketParameterSo if (this.subscribedToByDefault && !hasExplicitDefaultSubscription) { this.queriersForSubscription(result, null, options.globalParameters, params.bucketIdTransformer); } - }, - evaluateParameterRow: (sourceTable, row) => { - const result: EvaluatedParametersResult[] = []; - - for (const variant of this.variants) { - variant.pushParameterRowEvaluation(result, sourceTable, row); - } - - return result; } }; } diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index 59310ef34..e0d071ab2 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -74,7 +74,7 @@ describe('streams', () => { const errors: QuerierError[] = []; const pending = { queriers, errors }; desc - .createParameterSource({ bucketIdTransformer }) + .createParameterQuerierSource({ bucketIdTransformer }) .pushBucketParameterQueriers( pending, normalizeQuerierOptions({ test: 'foo' }, {}, { stream: [{ opaque_id: 0, parameters: null }] }) @@ -220,7 +220,9 @@ describe('streams', () => { ]); expect( - desc.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(ISSUES, { id: 'i1', owner_id: 'u1' }) + desc + .createParameterLookupSource({ bucketIdTransformer }) + .evaluateParameterRow(ISSUES, { id: 'i1', owner_id: 'u1' }) ).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', ['u1']), @@ -256,7 +258,7 @@ describe('streams', () => { expect(desc.tableSyncsParameters(ISSUES)).toBe(true); expect( desc - .createParameterSource({ bucketIdTransformer }) + .createParameterLookupSource({ bucketIdTransformer }) .evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) ).toStrictEqual([ { @@ -291,7 +293,7 @@ describe('streams', () => { expect(desc.tableSyncsParameters(USERS)).toBe(true); expect( - desc.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(USERS, { id: 'u', is_admin: 1n }) + desc.createParameterLookupSource({ bucketIdTransformer }).evaluateParameterRow(USERS, { id: 'u', is_admin: 1n }) ).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', ['u']), @@ -303,7 +305,7 @@ describe('streams', () => { } ]); expect( - desc.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(USERS, { id: 'u', is_admin: 0n }) + desc.createParameterLookupSource({ bucketIdTransformer }).evaluateParameterRow(USERS, { id: 'u', is_admin: 0n }) ).toStrictEqual([]); // Should return bucket id for admin users @@ -341,7 +343,9 @@ describe('streams', () => { ]); expect( - desc.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' }) + desc + .createParameterLookupSource({ bucketIdTransformer }) + .evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' }) ).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', ['b']), @@ -449,7 +453,9 @@ describe('streams', () => { expect(desc.tableSyncsParameters(FRIENDS)).toBe(true); expect( - desc.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' }) + desc + .createParameterLookupSource({ bucketIdTransformer }) + .evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' }) ).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', ['b']), @@ -607,7 +613,7 @@ describe('streams', () => { expect( desc - .createParameterSource({ bucketIdTransformer }) + .createParameterLookupSource({ bucketIdTransformer }) .evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) ).toStrictEqual([ { @@ -737,7 +743,7 @@ describe('streams', () => { // Ensure lookup steps work. expect( - stream.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(accountMember, row) + stream.createParameterLookupSource({ bucketIdTransformer }).evaluateParameterRow(accountMember, row) ).toStrictEqual([ { lookup: ParameterLookup.normalized('account_member', '0', ['id']), @@ -819,7 +825,7 @@ WHERE expect(evaluateBucketIds(desc, scene, { _id: 'scene', project: 'foo' })).toStrictEqual(['1#stream|0["foo"]']); expect( - desc.createParameterSource({ bucketIdTransformer }).evaluateParameterRow(projectInvitation, { + desc.createParameterLookupSource({ bucketIdTransformer }).evaluateParameterRow(projectInvitation, { project: 'foo', appliedTo: '[1,2]', status: 'CLAIMED' @@ -954,7 +960,7 @@ async function createQueriers( streams: { [stream.name]: [{ opaque_id: 0, parameters: options?.parameters ?? null }] } }; - stream.createParameterSource({ bucketIdTransformer }).pushBucketParameterQueriers(pending, querierOptions); + stream.createParameterQuerierSource({ bucketIdTransformer }).pushBucketParameterQueriers(pending, querierOptions); return { querier: mergeBucketParameterQueriers(queriers), errors }; } diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index ab308ce39..1deeee6e5 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -18,7 +18,8 @@ describe('sync rules', () => { test('parse empty sync rules', () => { const rules = SqlSyncRules.fromYaml('bucket_definitions: {}', PARSE_OPTIONS); - expect(rules.bucketParameterSources).toEqual([]); + expect(rules.bucketParameterLookupSources).toEqual([]); + expect(rules.bucketParameterQuerierSources).toEqual([]); expect(rules.bucketDataSources).toEqual([]); }); @@ -72,7 +73,8 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const parameterSource = rules.bucketParameterSources[0]; + expect(rules.bucketParameterLookupSources).toEqual([]); + const parameterSource = rules.bucketParameterQuerierSources[0]; expect(parameterSource.bucketParameters).toEqual([]); // Internal API, subject to change @@ -105,8 +107,8 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const parameterSource = rules.bucketParameterSources[0] as SqlParameterQuery; - expect(parameterSource.bucketParameters).toEqual([]); + const parameterLookupSource = rules.bucketParameterLookupSources[0]; + expect(parameterLookupSource.bucketParameters).toEqual([]); expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ { bucketParameters: [{}], @@ -128,8 +130,8 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const parameterSource = rules.bucketParameterSources[0] as StaticSqlParameterQuery; - const bucketData = rules.bucketDataSources[0] as SqlBucketDescriptor; + const parameterSource = rules.bucketParameterQuerierSources[0]; + const bucketData = rules.bucketDataSources[0]; expect(parameterSource.bucketParameters).toEqual(['user_id', 'device_id']); expect(bucketData.bucketParameters).toEqual(['user_id', 'device_id']); expect( @@ -139,8 +141,6 @@ bucket_definitions: { bucket: 'mybucket["user1","device1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 } ]); - const data_query = bucketData.dataQueries[0]; - expect(data_query.bucketParameters).toEqual(['user_id', 'device_id']); expect( hydrated.evaluateRow({ sourceTable: ASSETS, @@ -177,16 +177,14 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const bucketParameters = rules.bucketParameterSources[0] as StaticSqlParameterQuery; - const bucketData = rules.bucketDataSources[0] as SqlBucketDescriptor; + const bucketParameters = rules.bucketParameterQuerierSources[0]; + const bucketData = rules.bucketDataSources[0]; expect(bucketParameters.bucketParameters).toEqual(['user_id']); expect(bucketData.bucketParameters).toEqual(['user_id']); expect( hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier.staticBuckets ).toEqual([{ bucket: 'mybucket["user1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 }]); - const data_query = bucketData.dataQueries[0]; - expect(data_query.bucketParameters).toEqual(['user_id']); expect( hydrated.evaluateRow({ sourceTable: ASSETS, @@ -322,7 +320,7 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const bucketParameters = rules.bucketParameterSources[0]; + const bucketParameters = rules.bucketParameterQuerierSources[0]; expect(bucketParameters.bucketParameters).toEqual(['user_id']); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], @@ -360,7 +358,7 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const bucketParameters = rules.bucketParameterSources[0]; + const bucketParameters = rules.bucketParameterQuerierSources[0]; expect(bucketParameters.bucketParameters).toEqual(['user_id']); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], @@ -958,7 +956,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucketParameters = rules.bucketParameterSources[0]; + const bucketParameters = rules.bucketParameterQuerierSources[0]; expect(bucketParameters.bucketParameters).toEqual(['user_id']); const hydrated = rules.hydrate({ bucketIdTransformer }); From 028c1d9405218fd4852d96d5a32f6f0be7594e28 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 10:08:07 +0200 Subject: [PATCH 08/33] Bring back BucketSource. --- .../src/sync/BucketChecksumState.ts | 15 ++++--- packages/sync-rules/src/BucketSource.ts | 41 +++++++++++++---- .../sync-rules/src/SqlBucketDescriptor.ts | 45 ++++++++++--------- packages/sync-rules/src/SqlSyncRules.ts | 19 +++++--- packages/sync-rules/src/SyncRules.ts | 34 ++++++++++++-- packages/sync-rules/src/streams/stream.ts | 19 +++++++- 6 files changed, 125 insertions(+), 48 deletions(-) diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index f6b4d00dd..a0b9130a6 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -2,6 +2,7 @@ import { BucketDataSourceDefinition, BucketDescription, BucketPriority, + BucketSource, HydratedSyncRules, RequestedStream, RequestJwtPayload, @@ -248,15 +249,15 @@ export class BucketChecksumState { const streamNameToIndex = new Map(); this.streamNameToIndex = streamNameToIndex; - for (const source of this.parameterState.syncRules.bucketDataSources) { - if (this.parameterState.isSubscribedToStream(source.definition)) { - streamNameToIndex.set(source.definition.name, subscriptions.length); + for (const source of this.parameterState.syncRules.definition.bucketSources) { + if (this.parameterState.isSubscribedToStream(source)) { + streamNameToIndex.set(source.name, subscriptions.length); subscriptions.push({ - name: source.definition.name, - is_default: source.definition.subscribedToByDefault, + name: source.name, + is_default: source.subscribedToByDefault, errors: - this.parameterState.streamErrors[source.definition.name]?.map((e) => ({ + this.parameterState.streamErrors[source.name]?.map((e) => ({ subscription: e.subscription?.opaque_id ?? 'default', message: e.message })) ?? [] @@ -484,7 +485,7 @@ export class BucketParameterState { }; } - isSubscribedToStream(desc: BucketDataSourceDefinition): boolean { + isSubscribedToStream(desc: BucketSource): boolean { return (desc.subscribedToByDefault && this.includeDefaultStreams) || this.subscribedStreamNames.has(desc.name); } diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 4a7246f2b..e8a406cba 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -16,13 +16,43 @@ export interface CreateSourceParams { bucketIdTransformer: BucketIdTransformer; } +export interface BucketSource { + readonly name: string; + readonly type: BucketSourceType; + readonly subscribedToByDefault: boolean; + + /** + * BucketDataSource describing the data in this bucket/stream definition. + * + * The same data source could in theory be present in multiple stream definitions. + */ + readonly dataSource: BucketDataSourceDefinition; + + /** + * BucketParameterQuerierSource describing the parameter queries / stream subqueries in this bucket/stream definition. + * + * The same source could in theory be present in multiple stream definitions. + */ + readonly parameterQuerierSources: BucketParameterQuerierSourceDefinition[]; + + /** + * BucketParameterLookupSource describing the parameter tables used in this bucket/stream definition. + * + * The same source could in theory be present in multiple stream definitions. + */ + readonly parameterLookupSources: BucketParameterLookupSourceDefinition[]; +} + +export interface HydratedBucketSource { + readonly definition: BucketSource; + + readonly parameterQuerierSources: BucketParameterQuerierSource[]; +} + /** * Encodes a static definition of a bucket source, as parsed from sync rules or stream definitions. */ export interface BucketDataSourceDefinition { - readonly name: string; - readonly type: BucketSourceType; - readonly subscribedToByDefault: boolean; /** * For debug use only. */ @@ -50,8 +80,6 @@ export interface BucketDataSourceDefinition { * This is only relevant for parameter queries that query tables. */ export interface BucketParameterLookupSourceDefinition { - readonly name: string; - readonly type: BucketSourceType; /** * For debug use only. */ @@ -70,9 +98,6 @@ export interface BucketParameterLookupSourceDefinition { * This may use request data only, or it may use parameter lookup data persisted by a BucketParameterLookupSourceDefinition. */ export interface BucketParameterQuerierSourceDefinition { - readonly name: string; - readonly type: BucketSourceType; - readonly subscribedToByDefault: boolean; /** * For debug use only. */ diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 7d10a3b5b..be18d83f5 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -4,7 +4,8 @@ import { BucketParameterLookupSourceDefinition, BucketParameterQuerierSourceDefinition, BucketSourceType, - CreateSourceParams + CreateSourceParams, + BucketSource } from './BucketSource.js'; import { ColumnDefinition } from './ExpressionType.js'; import { IdSequence } from './IdSequence.js'; @@ -28,10 +29,21 @@ export interface QueryParseResult { errors: SqlRuleError[]; } -export class SqlBucketDescriptor implements BucketDataSourceDefinition { +export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketSource { name: string; private bucketParametersInternal: string[] | null = null; + public readonly subscribedToByDefault: boolean = true; + + /** + * source table -> queries + */ + dataQueries: SqlDataQuery[] = []; + parameterQueries: SqlParameterQuery[] = []; + globalParameterQueries: (StaticSqlParameterQuery | TableValuedFunctionSqlParameterQuery)[] = []; + + parameterIdSequence = new IdSequence(); + constructor(name: string) { this.name = name; } @@ -40,22 +52,21 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition { return BucketSourceType.SYNC_RULE; } - public get subscribedToByDefault(): boolean { - return true; - } - public get bucketParameters(): string[] { return this.bucketParametersInternal ?? []; } - /** - * source table -> queries - */ - dataQueries: SqlDataQuery[] = []; - parameterQueries: SqlParameterQuery[] = []; - globalParameterQueries: (StaticSqlParameterQuery | TableValuedFunctionSqlParameterQuery)[] = []; + get dataSource() { + return this; + } - parameterIdSequence = new IdSequence(); + get parameterLookupSources() { + return this.parameterQueries; + } + + get parameterQuerierSources() { + return [...this.parameterQueries, ...this.globalParameterQueries]; + } addDataQuery(sql: string, options: SyncRulesOptions, compatibility: CompatibilityContext): QueryParseResult { if (this.bucketParametersInternal == null) { @@ -112,14 +123,6 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition { }; } - getParameterQuerierSourceDefinitions(): BucketParameterQuerierSourceDefinition[] { - return [...this.parameterQueries, ...this.globalParameterQueries]; - } - - getParameterLookupSourceDefinitions(): BucketParameterLookupSourceDefinition[] { - return [...this.parameterQueries]; - } - getSourceTables(): Set { let result = new Set(); for (let query of this.parameterQueries) { diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 9b76f44a7..460675907 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -4,7 +4,8 @@ import { BucketParameterQuerier, QuerierError } from './BucketParameterQuerier.j import { BucketDataSourceDefinition, BucketParameterLookupSourceDefinition, - BucketParameterQuerierSourceDefinition + BucketParameterQuerierSourceDefinition, + BucketSource } from './BucketSource.js'; import { CompatibilityContext, CompatibilityEdition, CompatibilityOption } from './compatibility.js'; import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js'; @@ -85,6 +86,7 @@ export class SqlSyncRules { bucketDataSources: BucketDataSourceDefinition[] = []; bucketParameterLookupSources: BucketParameterLookupSourceDefinition[] = []; bucketParameterQuerierSources: BucketParameterQuerierSourceDefinition[] = []; + bucketSources: BucketSource[] = []; eventDescriptors: SqlEventDescriptor[] = []; compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; @@ -236,9 +238,11 @@ export class SqlSyncRules { return descriptor.addDataQuery(q, queryOptions, compatibility); }); } - rules.bucketDataSources.push(descriptor); - rules.bucketParameterLookupSources.push(...descriptor.getParameterLookupSourceDefinitions()); - rules.bucketParameterQuerierSources.push(...descriptor.getParameterQuerierSourceDefinitions()); + + rules.bucketSources.push(descriptor); + rules.bucketDataSources.push(descriptor.dataSource); + rules.bucketParameterLookupSources.push(...descriptor.parameterLookupSources); + rules.bucketParameterQuerierSources.push(...descriptor.parameterQuerierSources); } for (const entry of streamMap?.items ?? []) { @@ -263,9 +267,10 @@ export class SqlSyncRules { if (data instanceof Scalar) { rules.withScalar(data, (q) => { const [parsed, errors] = syncStreamFromSql(key, q, queryOptions); - rules.bucketDataSources.push(parsed); - rules.bucketParameterLookupSources.push(parsed); - rules.bucketParameterQuerierSources.push(parsed); + rules.bucketSources.push(parsed); + rules.bucketDataSources.push(parsed.dataSource); + rules.bucketParameterLookupSources.push(...parsed.parameterLookupSources); + rules.bucketParameterQuerierSources.push(...parsed.parameterQuerierSources); return { parsed: true, errors diff --git a/packages/sync-rules/src/SyncRules.ts b/packages/sync-rules/src/SyncRules.ts index 3ec461014..bd55495c6 100644 --- a/packages/sync-rules/src/SyncRules.ts +++ b/packages/sync-rules/src/SyncRules.ts @@ -1,4 +1,10 @@ -import { BucketDataSource, BucketParameterLookupSource, BucketParameterQuerierSource } from './BucketSource.js'; +import { + BucketDataSource, + BucketParameterLookupSource, + BucketParameterQuerierSource, + BucketParameterQuerierSourceDefinition, + HydratedBucketSource +} from './BucketSource.js'; import { BucketParameterQuerier, CompatibilityContext, @@ -25,6 +31,7 @@ import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, Sqlite * specifically affects bucket names. */ export class HydratedSyncRules { + bucketSources: HydratedBucketSource[] = []; bucketDataSources: BucketDataSource[]; bucketParameterQuerierSources: BucketParameterQuerierSource[]; bucketParameterLookupSources: BucketParameterLookupSource[]; @@ -32,7 +39,7 @@ export class HydratedSyncRules { eventDescriptors: SqlEventDescriptor[] = []; compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; - private definition: SqlSyncRules; + readonly definition: SqlSyncRules; constructor(params: { definition: SqlSyncRules; @@ -46,12 +53,29 @@ export class HydratedSyncRules { this.bucketParameterQuerierSources = params.bucketParameterQuerierSources; this.bucketParameterLookupSources = params.bucketParameterLookupSources; this.definition = params.definition; + if (params.eventDescriptors) { this.eventDescriptors = params.eventDescriptors; } if (params.compatibility) { this.compatibility = params.compatibility; } + + let querierMap = new Map(); + for (let definition of this.definition.bucketSources) { + const hydratedBucketSource: HydratedBucketSource = { definition: definition, parameterQuerierSources: [] }; + this.bucketSources.push(hydratedBucketSource); + for (let querier of definition.parameterQuerierSources) { + querierMap.set(querier, hydratedBucketSource); + } + } + for (let querier of params.bucketParameterQuerierSources) { + const bucketSource = querierMap.get(querier.definition); + if (bucketSource == null) { + throw new Error('Cannot find BucketSource for BucketParameterQuerierSource'); + } + bucketSource.parameterQuerierSources.push(querier); + } } // These methods do not depend on hydration, so we can just forward them to the definition. @@ -131,12 +155,14 @@ export class HydratedSyncRules { const errors: QuerierError[] = []; const pending = { queriers, errors }; - for (const source of this.bucketParameterQuerierSources) { + for (const source of this.bucketSources) { if ( (source.definition.subscribedToByDefault && options.hasDefaultStreams) || source.definition.name in options.streams ) { - source.pushBucketParameterQueriers(pending, options); + for (let querier of source.parameterQuerierSources) { + querier.pushBucketParameterQueriers(pending, options); + } } } diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index e79db35fa..ab173993b 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -8,6 +8,7 @@ import { BucketParameterLookupSourceDefinition, BucketParameterQuerierSource, BucketParameterQuerierSourceDefinition, + BucketSource, BucketSourceType, CreateSourceParams } from '../BucketSource.js'; @@ -27,7 +28,11 @@ import { import { StreamVariant } from './variant.js'; export class SyncStream - implements BucketDataSourceDefinition, BucketParameterLookupSourceDefinition, BucketParameterQuerierSourceDefinition + implements + BucketDataSourceDefinition, + BucketParameterLookupSourceDefinition, + BucketParameterQuerierSourceDefinition, + BucketSource { name: string; subscribedToByDefault: boolean; @@ -53,6 +58,18 @@ export class SyncStream return this.data.bucketParameters; } + public get dataSource() { + return this; + } + + public get parameterLookupSources() { + return [this]; + } + + public get parameterQuerierSources() { + return [this]; + } + createDataSource(params: CreateSourceParams): BucketDataSource { return { definition: this, From eb0b860b4ebae64c26e1682f6f3221209c11dba5 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 10:10:38 +0200 Subject: [PATCH 09/33] Cleanup. --- packages/sync-rules/src/SqlParameterQuery.ts | 8 -------- packages/sync-rules/src/StaticSqlParameterQuery.ts | 9 +-------- .../src/TableValuedFunctionSqlParameterQuery.ts | 9 +-------- 3 files changed, 2 insertions(+), 24 deletions(-) diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index a04e82523..2ce837e8f 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -299,10 +299,6 @@ export class SqlParameterQuery readonly queryId: string; readonly tools: SqlTools; - readonly type: BucketSourceType = BucketSourceType.SYNC_RULE; - - readonly subscribedToByDefault: boolean = true; - readonly errors: SqlRuleError[]; constructor(options: SqlParameterQueryOptions) { @@ -326,10 +322,6 @@ export class SqlParameterQuery return this.sourceTable.matches(table); } - get name(): string { - return this.descriptorName; - } - getSourceTables(): Set { return new Set([this.sourceTable]); } diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index da3d0b76a..78d6df125 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -161,9 +161,6 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi */ readonly filter: ParameterValueClause | undefined; - readonly subscribedToByDefault = true; - readonly type = BucketSourceType.SYNC_RULE; - readonly errors: SqlRuleError[]; constructor(options: StaticSqlParameterQueryOptions) { @@ -177,10 +174,6 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi this.errors = options.errors ?? []; } - get name() { - return this.descriptorName; - } - getSourceTables() { return new Set(); } @@ -200,7 +193,7 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi ).map((desc) => { return { ...desc, - definition: this.name, + definition: this.descriptorName, inclusion_reasons: ['default'] } satisfies ResolvedBucket; }); diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index 878536422..0b3caacea 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -200,9 +200,6 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer readonly errors: SqlRuleError[]; - readonly subscribedToByDefault = true; - readonly type = BucketSourceType.SYNC_RULE; - constructor(options: TableValuedFunctionSqlParameterQueryOptions) { this.sql = options.sql; this.parameterExtractors = options.parameterExtractors; @@ -219,10 +216,6 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer this.errors = options.errors; } - get name() { - return this.descriptorName; - } - getSourceTables() { return new Set(); } @@ -242,7 +235,7 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer ).map((desc) => { return { ...desc, - definition: this.name, + definition: this.descriptorName, inclusion_reasons: ['default'] } satisfies ResolvedBucket; }); From 011ccb406decd7219d3f670f85c79f677e56e82f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 10:24:08 +0200 Subject: [PATCH 10/33] Cleanup imports. --- .../src/sync/BucketChecksumState.ts | 1 - .../sync-rules/src/SqlBucketDescriptor.ts | 6 ++-- packages/sync-rules/src/SqlParameterQuery.ts | 22 +++++++-------- .../sync-rules/src/StaticSqlParameterQuery.ts | 28 ++++++++----------- .../TableValuedFunctionSqlParameterQuery.ts | 1 - packages/sync-rules/src/request_functions.ts | 2 +- packages/sync-rules/src/streams/stream.ts | 2 +- 7 files changed, 26 insertions(+), 36 deletions(-) diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index a0b9130a6..d9afa0efb 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -1,5 +1,4 @@ import { - BucketDataSourceDefinition, BucketDescription, BucketPriority, BucketSource, diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index be18d83f5..d6fc829fc 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -1,11 +1,9 @@ import { BucketDataSource, BucketDataSourceDefinition, - BucketParameterLookupSourceDefinition, - BucketParameterQuerierSourceDefinition, + BucketSource, BucketSourceType, - CreateSourceParams, - BucketSource + CreateSourceParams } from './BucketSource.js'; import { ColumnDefinition } from './ExpressionType.js'; import { IdSequence } from './IdSequence.js'; diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 2ce837e8f..20af27837 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -1,4 +1,4 @@ -import { parse, SelectedColumn } from 'pgsql-ast-parser'; +import { parse } from 'pgsql-ast-parser'; import { BucketDescription, BucketInclusionReason, @@ -7,15 +7,22 @@ import { } from './BucketDescription.js'; import { BucketParameterQuerier, - mergeBucketParameterQueriers, ParameterLookup, ParameterLookupSource, PendingQueriers } from './BucketParameterQuerier.js'; +import { + BucketParameterLookupSource, + BucketParameterLookupSourceDefinition, + BucketParameterQuerierSource, + BucketParameterQuerierSourceDefinition, + CreateSourceParams +} from './BucketSource.js'; import { SqlRuleError } from './errors.js'; +import { GetQuerierOptions } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; -import { checkUnsupportedFeatures, isClauseError, isParameterValueClause } from './sql_support.js'; +import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; import { StaticSqlParameterQuery } from './StaticSqlParameterQuery.js'; import { TablePattern } from './TablePattern.js'; import { TableQuerySchema } from './TableQuerySchema.js'; @@ -37,15 +44,6 @@ import { } from './types.js'; import { filterJsonRow, getBucketId, isJsonValue, isSelectStatement, normalizeParameterValue } from './utils.js'; import { DetectRequestParameters } from './validators.js'; -import { - BucketParameterLookupSource, - BucketParameterLookupSourceDefinition, - BucketParameterQuerierSource, - BucketParameterQuerierSourceDefinition, - BucketSourceType, - CreateSourceParams -} from './BucketSource.js'; -import { GetQuerierOptions } from './index.js'; export interface SqlParameterQueryOptions { sourceTable: TablePattern; diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index 78d6df125..804de03b4 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -1,30 +1,26 @@ -import { SelectedColumn, SelectFromStatement } from 'pgsql-ast-parser'; +import { SelectFromStatement } from 'pgsql-ast-parser'; import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY, ResolvedBucket } from './BucketDescription.js'; +import { BucketParameterQuerier, PendingQueriers } from './BucketParameterQuerier.js'; +import { + BucketParameterQuerierSource, + BucketParameterQuerierSourceDefinition, + CreateSourceParams +} from './BucketSource.js'; import { SqlRuleError } from './errors.js'; +import { GetQuerierOptions } from './index.js'; +import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; -import { checkUnsupportedFeatures, isClauseError, isParameterValueClause, sqliteBool } from './sql_support.js'; +import { checkUnsupportedFeatures, isClauseError, sqliteBool } from './sql_support.js'; +import { TablePattern } from './TablePattern.js'; import { BucketIdTransformer, - EvaluatedParametersResult, ParameterValueClause, QueryParseOptions, RequestParameters, - SqliteJsonValue, - SqliteRow + SqliteJsonValue } from './types.js'; import { getBucketId, isJsonValue } from './utils.js'; import { DetectRequestParameters } from './validators.js'; -import { - BucketParameterLookupSource, - BucketParameterQuerierSource, - BucketParameterQuerierSourceDefinition, - BucketSourceType, - CreateSourceParams -} from './BucketSource.js'; -import { SourceTableInterface } from './SourceTableInterface.js'; -import { TablePattern } from './TablePattern.js'; -import { BucketParameterQuerier, mergeBucketParameterQueriers, PendingQueriers } from './BucketParameterQuerier.js'; -import { GetQuerierOptions } from './index.js'; export interface StaticSqlParameterQueryOptions { sql: string; diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index 0b3caacea..2ae485bb8 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -3,7 +3,6 @@ import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY, ResolvedBuc import { BucketParameterQuerierSource, BucketParameterQuerierSourceDefinition, - BucketSourceType, CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; diff --git a/packages/sync-rules/src/request_functions.ts b/packages/sync-rules/src/request_functions.ts index c6a79ae4f..dfc146618 100644 --- a/packages/sync-rules/src/request_functions.ts +++ b/packages/sync-rules/src/request_functions.ts @@ -1,5 +1,5 @@ import { ExpressionType } from './ExpressionType.js'; -import { CompatibilityContext, CompatibilityEdition, CompatibilityOption } from './compatibility.js'; +import { CompatibilityContext, CompatibilityEdition } from './compatibility.js'; import { generateSqlFunctions } from './sql_functions.js'; import { CompiledClause, ParameterValueClause, ParameterValueSet, SqliteValue } from './types.js'; diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index ab173993b..c5e2796cb 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -1,6 +1,6 @@ import { BaseSqlDataQuery } from '../BaseSqlDataQuery.js'; import { BucketInclusionReason, BucketPriority, DEFAULT_BUCKET_PRIORITY } from '../BucketDescription.js'; -import { BucketParameterQuerier, mergeBucketParameterQueriers, PendingQueriers } from '../BucketParameterQuerier.js'; +import { BucketParameterQuerier, PendingQueriers } from '../BucketParameterQuerier.js'; import { BucketDataSource, BucketDataSourceDefinition, From 0049d54be6ba85712ff39091f0babd99c7ac0865 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 10:53:01 +0200 Subject: [PATCH 11/33] Split out SyncStream into its components. --- packages/sync-rules/src/BaseSqlDataQuery.ts | 3 +- packages/sync-rules/src/BucketSource.ts | 5 - .../sync-rules/src/SqlBucketDescriptor.ts | 4 +- packages/sync-rules/src/streams/stream.ts | 153 ++++++++++-------- packages/sync-rules/test/src/streams.test.ts | 63 +++++--- 5 files changed, 127 insertions(+), 101 deletions(-) diff --git a/packages/sync-rules/src/BaseSqlDataQuery.ts b/packages/sync-rules/src/BaseSqlDataQuery.ts index b426cea03..3138544c8 100644 --- a/packages/sync-rules/src/BaseSqlDataQuery.ts +++ b/packages/sync-rules/src/BaseSqlDataQuery.ts @@ -26,7 +26,6 @@ export interface EvaluateRowOptions { table: SourceTableInterface; row: SqliteRow; bucketIds: (params: QueryParameters) => string[]; - bucketIdTransformer: BucketIdTransformer | null; } export interface BaseSqlDataQueryOptions { @@ -177,7 +176,7 @@ export class BaseSqlDataQuery { } } - evaluateRowWithOptions(options: Omit): EvaluationResult[] { + evaluateRowWithOptions(options: EvaluateRowOptions): EvaluationResult[] { try { const { table, row, bucketIds } = options; diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index e8a406cba..38b3011c2 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -80,11 +80,6 @@ export interface BucketDataSourceDefinition { * This is only relevant for parameter queries that query tables. */ export interface BucketParameterLookupSourceDefinition { - /** - * For debug use only. - */ - readonly bucketParameters: string[]; - getSourceTables(): Set; createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource; diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index d6fc829fc..6d9a82332 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -124,10 +124,10 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketSo getSourceTables(): Set { let result = new Set(); for (let query of this.parameterQueries) { - result.add(query.sourceTable!); + result.add(query.sourceTable); } for (let query of this.dataQueries) { - result.add(query.sourceTable!); + result.add(query.sourceTable); } // Note: No physical tables for global_parameter_queries diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index c5e2796cb..a848c83ed 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -27,47 +27,77 @@ import { } from '../types.js'; import { StreamVariant } from './variant.js'; -export class SyncStream - implements - BucketDataSourceDefinition, - BucketParameterLookupSourceDefinition, - BucketParameterQuerierSourceDefinition, - BucketSource -{ +export class SyncStream implements BucketSource { name: string; subscribedToByDefault: boolean; priority: BucketPriority; variants: StreamVariant[]; data: BaseSqlDataQuery; + public readonly dataSource: BucketDataSourceDefinition; + public readonly parameterLookupSources: BucketParameterLookupSourceDefinition[]; + public readonly parameterQuerierSources: BucketParameterQuerierSourceDefinition[]; + constructor(name: string, data: BaseSqlDataQuery) { this.name = name; this.subscribedToByDefault = false; this.priority = DEFAULT_BUCKET_PRIORITY; this.variants = []; this.data = data; + + this.dataSource = new SyncStreamDataSource(this, data); + this.parameterQuerierSources = [new SyncStreamParameterQuerierSource(this)]; + this.parameterLookupSources = [new SyncStreamParameterLookupSource(this)]; } public get type(): BucketSourceType { return BucketSourceType.SYNC_STREAM; } +} - public get bucketParameters(): string[] { +export class SyncStreamDataSource implements BucketDataSourceDefinition { + constructor( + private stream: SyncStream, + private data: BaseSqlDataQuery + ) {} + + get bucketParameters() { // FIXME: check whether this is correct. // Could there be multiple variants with different bucket parameters? return this.data.bucketParameters; } - public get dataSource() { - return this; + getSourceTables(): Set { + return new Set([this.data.sourceTable]); } - public get parameterLookupSources() { - return [this]; + tableSyncsData(table: SourceTableInterface): boolean { + return this.data.applies(table); + } + + resolveResultSets(schema: SourceSchema, tables: Record>): void { + return this.data.resolveResultSets(schema, tables); } - public get parameterQuerierSources() { - return [this]; + debugWriteOutputTables(result: Record): void { + result[this.data.table!.sqlName] ??= []; + const r = { + query: this.data.sql + }; + + result[this.data.table!.sqlName].push(r); + } + + debugRepresentation() { + return { + name: this.stream.name, + type: BucketSourceType[BucketSourceType.SYNC_STREAM], + variants: this.stream.variants.map((v) => v.debugRepresentation()), + data: { + table: this.data.sourceTable, + columns: this.data.columnOutputNames() + } + }; } createDataSource(params: CreateSourceParams): BucketDataSource { @@ -79,7 +109,7 @@ export class SyncStream return []; } - const stream = this; + const stream = this.stream; const row: TableRow = { sourceTable: options.sourceTable, record: options.record @@ -100,31 +130,28 @@ export class SyncStream } }; } +} - createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { - return { - definition: this, +export class SyncStreamParameterQuerierSource implements BucketParameterQuerierSourceDefinition { + // We could eventually split this into a separate source per variant. - evaluateParameterRow: (sourceTable, row) => { - const result: EvaluatedParametersResult[] = []; + constructor(private stream: SyncStream) {} - for (const variant of this.variants) { - variant.pushParameterRowEvaluation(result, sourceTable, row); - } - - return result; - } - }; + get bucketParameters(): string[] { + // FIXME: check whether this is correct. + // Could there be multiple variants with different bucket parameters? + return this.stream.data.bucketParameters; } createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { + const stream = this.stream; return { definition: this, pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions): void => { - const subscriptions = options.streams[this.name] ?? []; + const subscriptions = options.streams[stream.name] ?? []; - if (!this.subscribedToByDefault && !subscriptions.length) { + if (!stream.subscribedToByDefault && !subscriptions.length) { // The client is not subscribing to this stream, so don't query buckets related to it. return; } @@ -143,7 +170,7 @@ export class SyncStream // If the stream is subscribed to by default and there is no explicit subscription that would match the default // subscription, also include the default querier. - if (this.subscribedToByDefault && !hasExplicitDefaultSubscription) { + if (stream.subscribedToByDefault && !hasExplicitDefaultSubscription) { this.queriersForSubscription(result, null, options.globalParameters, params.bucketIdTransformer); } } @@ -160,8 +187,8 @@ export class SyncStream const queriers: BucketParameterQuerier[] = []; try { - for (const variant of this.variants) { - const querier = variant.querier(this, reason, params, bucketIdTransformer); + for (const variant of this.stream.variants) { + const querier = variant.querier(this.stream, reason, params, bucketIdTransformer); if (querier) { queriers.push(querier); } @@ -170,65 +197,55 @@ export class SyncStream result.queriers.push(...queriers); } catch (e) { result.errors.push({ - descriptor: this.name, + descriptor: this.stream.name, message: `Error evaluating bucket ids: ${e.message}`, subscription: subscription ?? undefined }); } } +} - tableSyncsData(table: SourceTableInterface): boolean { - return this.data.applies(table); - } - - tableSyncsParameters(table: SourceTableInterface): boolean { - for (const variant of this.variants) { - for (const subquery of variant.subqueries) { - if (subquery.parameterTable.matches(table)) { - return true; - } - } - } +export class SyncStreamParameterLookupSource implements BucketParameterLookupSourceDefinition { + // We could eventually split this into a separate source per variant. - return false; - } + constructor(private stream: SyncStream) {} getSourceTables(): Set { let result = new Set(); - result.add(this.data.sourceTable); - for (let variant of this.variants) { + for (let variant of this.stream.variants) { for (const subquery of variant.subqueries) { result.add(subquery.parameterTable); } } - // Note: No physical tables for global_parameter_queries - return result; } - resolveResultSets(schema: SourceSchema, tables: Record>) { - this.data.resolveResultSets(schema, tables); - } - - debugRepresentation() { + createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { return { - name: this.name, - type: BucketSourceType[this.type], - variants: this.variants.map((v) => v.debugRepresentation()), - data: { - table: this.data.sourceTable, - columns: this.data.columnOutputNames() + definition: this, + + evaluateParameterRow: (sourceTable, row) => { + const result: EvaluatedParametersResult[] = []; + + for (const variant of this.stream.variants) { + variant.pushParameterRowEvaluation(result, sourceTable, row); + } + + return result; } }; } - debugWriteOutputTables(result: Record): void { - result[this.data.table!.sqlName] ??= []; - const r = { - query: this.data.sql - }; + tableSyncsParameters(table: SourceTableInterface): boolean { + for (const variant of this.stream.variants) { + for (const subquery of variant.subqueries) { + if (subquery.parameterTable.matches(table)) { + return true; + } + } + } - result[this.data.table!.sqlName].push(r); + return false; } } diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index e0d071ab2..b80d524ea 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -38,7 +38,9 @@ describe('streams', () => { expect(desc.variants).toHaveLength(1); expect(evaluateBucketIds(desc, COMMENTS, { id: 'foo' })).toStrictEqual(['1#stream|0[]']); expect( - desc.createDataSource({ bucketIdTransformer }).evaluateRow({ sourceTable: USERS, record: { id: 'foo' } }) + desc.dataSource + .createDataSource({ bucketIdTransformer }) + .evaluateRow({ sourceTable: USERS, record: { id: 'foo' } }) ).toHaveLength(0); }); @@ -73,7 +75,7 @@ describe('streams', () => { const queriers: BucketParameterQuerier[] = []; const errors: QuerierError[] = []; const pending = { queriers, errors }; - desc + desc.parameterQuerierSources[0] .createParameterQuerierSource({ bucketIdTransformer }) .pushBucketParameterQueriers( pending, @@ -220,7 +222,7 @@ describe('streams', () => { ]); expect( - desc + desc.parameterLookupSources[0] .createParameterLookupSource({ bucketIdTransformer }) .evaluateParameterRow(ISSUES, { id: 'i1', owner_id: 'u1' }) ).toStrictEqual([ @@ -254,10 +256,11 @@ describe('streams', () => { const desc = parseStream( 'SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id())' ); + const lookup = desc.parameterLookupSources[0]; - expect(desc.tableSyncsParameters(ISSUES)).toBe(true); + expect(lookup.tableSyncsParameters(ISSUES)).toBe(true); expect( - desc + lookup .createParameterLookupSource({ bucketIdTransformer }) .evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) ).toStrictEqual([ @@ -288,12 +291,15 @@ describe('streams', () => { test('parameter value in subquery', async () => { const desc = parseStream('SELECT * FROM issues WHERE auth.user_id() IN (SELECT id FROM users WHERE is_admin)'); + const lookup = desc.parameterLookupSources[0]; - expect(desc.tableSyncsParameters(ISSUES)).toBe(false); - expect(desc.tableSyncsParameters(USERS)).toBe(true); + expect(lookup.tableSyncsParameters(ISSUES)).toBe(false); + expect(lookup.tableSyncsParameters(USERS)).toBe(true); expect( - desc.createParameterLookupSource({ bucketIdTransformer }).evaluateParameterRow(USERS, { id: 'u', is_admin: 1n }) + lookup + .createParameterLookupSource({ bucketIdTransformer }) + .evaluateParameterRow(USERS, { id: 'u', is_admin: 1n }) ).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', ['u']), @@ -305,7 +311,9 @@ describe('streams', () => { } ]); expect( - desc.createParameterLookupSource({ bucketIdTransformer }).evaluateParameterRow(USERS, { id: 'u', is_admin: 0n }) + lookup + .createParameterLookupSource({ bucketIdTransformer }) + .evaluateParameterRow(USERS, { id: 'u', is_admin: 0n }) ).toStrictEqual([]); // Should return bucket id for admin users @@ -343,7 +351,7 @@ describe('streams', () => { ]); expect( - desc + desc.parameterLookupSources[0] .createParameterLookupSource({ bucketIdTransformer }) .evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' }) ).toStrictEqual([ @@ -450,10 +458,11 @@ describe('streams', () => { const desc = parseStream( 'SELECT * FROM comments WHERE tagged_users && (SELECT user_a FROM friends WHERE user_b = auth.user_id())' ); + const lookup = desc.parameterLookupSources[0]; - expect(desc.tableSyncsParameters(FRIENDS)).toBe(true); + expect(lookup.tableSyncsParameters(FRIENDS)).toBe(true); expect( - desc + lookup .createParameterLookupSource({ bucketIdTransformer }) .evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' }) ).toStrictEqual([ @@ -612,7 +621,7 @@ describe('streams', () => { ); expect( - desc + desc.parameterLookupSources[0] .createParameterLookupSource({ bucketIdTransformer }) .evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) ).toStrictEqual([ @@ -738,12 +747,14 @@ describe('streams', () => { ); const row = { id: 'id', account_id: 'account_id' }; - expect(stream.tableSyncsData(accountMember)).toBeTruthy(); - expect(stream.tableSyncsParameters(accountMember)).toBeTruthy(); + expect(stream.dataSource.tableSyncsData(accountMember)).toBeTruthy(); + expect(stream.parameterLookupSources[0].tableSyncsParameters(accountMember)).toBeTruthy(); // Ensure lookup steps work. expect( - stream.createParameterLookupSource({ bucketIdTransformer }).evaluateParameterRow(accountMember, row) + stream.parameterLookupSources[0] + .createParameterLookupSource({ bucketIdTransformer }) + .evaluateParameterRow(accountMember, row) ).toStrictEqual([ { lookup: ParameterLookup.normalized('account_member', '0', ['id']), @@ -768,7 +779,7 @@ describe('streams', () => { // And that the data alias is respected for generated schemas. const outputSchema = {}; - stream.resolveResultSets(schema, outputSchema); + stream.dataSource.resolveResultSets(schema, outputSchema); expect(Object.keys(outputSchema)).toStrictEqual(['outer']); }); @@ -825,11 +836,13 @@ WHERE expect(evaluateBucketIds(desc, scene, { _id: 'scene', project: 'foo' })).toStrictEqual(['1#stream|0["foo"]']); expect( - desc.createParameterLookupSource({ bucketIdTransformer }).evaluateParameterRow(projectInvitation, { - project: 'foo', - appliedTo: '[1,2]', - status: 'CLAIMED' - }) + desc.parameterLookupSources[0] + .createParameterLookupSource({ bucketIdTransformer }) + .evaluateParameterRow(projectInvitation, { + project: 'foo', + appliedTo: '[1,2]', + status: 'CLAIMED' + }) ).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', [1n, 'foo']), @@ -924,7 +937,7 @@ const options: StreamParseOptions = { const bucketIdTransformer = SqlSyncRules.versionedBucketIdTransformer('1'); function evaluateBucketIds(stream: SyncStream, sourceTable: SourceTableInterface, record: SqliteRow) { - return stream + return stream.dataSource .createDataSource({ bucketIdTransformer }) .evaluateRow({ sourceTable, record }) .map((r) => { @@ -960,7 +973,9 @@ async function createQueriers( streams: { [stream.name]: [{ opaque_id: 0, parameters: options?.parameters ?? null }] } }; - stream.createParameterQuerierSource({ bucketIdTransformer }).pushBucketParameterQueriers(pending, querierOptions); + for (let querier of stream.parameterQuerierSources) { + querier.createParameterQuerierSource({ bucketIdTransformer }).pushBucketParameterQueriers(pending, querierOptions); + } return { querier: mergeBucketParameterQueriers(queriers), errors }; } From ec5851a46240d5ebfe2db06247232e5802e87909 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 13:40:41 +0200 Subject: [PATCH 12/33] Support multiple data sources. --- packages/sync-rules/src/BucketSource.ts | 6 +++- .../sync-rules/src/SqlBucketDescriptor.ts | 4 +-- packages/sync-rules/src/SqlSyncRules.ts | 4 +-- packages/sync-rules/src/streams/stream.ts | 4 +-- packages/sync-rules/test/src/streams.test.ts | 28 +++++++++++-------- .../sync-rules/test/src/sync_rules.test.ts | 1 - 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 38b3011c2..324203073 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -25,8 +25,12 @@ export interface BucketSource { * BucketDataSource describing the data in this bucket/stream definition. * * The same data source could in theory be present in multiple stream definitions. + * + * Sources must _only_ be split into multiple ones if they will result in different buckets being created. + * Specifically, bucket definitions would always have a single data source, while stream definitions may have + * one per variant. */ - readonly dataSource: BucketDataSourceDefinition; + readonly dataSources: BucketDataSourceDefinition[]; /** * BucketParameterQuerierSource describing the parameter queries / stream subqueries in this bucket/stream definition. diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 6d9a82332..55d8f4119 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -54,8 +54,8 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketSo return this.bucketParametersInternal ?? []; } - get dataSource() { - return this; + get dataSources() { + return [this]; } get parameterLookupSources() { diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 460675907..7c514f1de 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -240,7 +240,7 @@ export class SqlSyncRules { } rules.bucketSources.push(descriptor); - rules.bucketDataSources.push(descriptor.dataSource); + rules.bucketDataSources.push(...descriptor.dataSources); rules.bucketParameterLookupSources.push(...descriptor.parameterLookupSources); rules.bucketParameterQuerierSources.push(...descriptor.parameterQuerierSources); } @@ -268,7 +268,7 @@ export class SqlSyncRules { rules.withScalar(data, (q) => { const [parsed, errors] = syncStreamFromSql(key, q, queryOptions); rules.bucketSources.push(parsed); - rules.bucketDataSources.push(parsed.dataSource); + rules.bucketDataSources.push(...parsed.dataSources); rules.bucketParameterLookupSources.push(...parsed.parameterLookupSources); rules.bucketParameterQuerierSources.push(...parsed.parameterQuerierSources); return { diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index a848c83ed..4a9cb6d33 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -34,7 +34,7 @@ export class SyncStream implements BucketSource { variants: StreamVariant[]; data: BaseSqlDataQuery; - public readonly dataSource: BucketDataSourceDefinition; + public readonly dataSources: BucketDataSourceDefinition[]; public readonly parameterLookupSources: BucketParameterLookupSourceDefinition[]; public readonly parameterQuerierSources: BucketParameterQuerierSourceDefinition[]; @@ -45,7 +45,7 @@ export class SyncStream implements BucketSource { this.variants = []; this.data = data; - this.dataSource = new SyncStreamDataSource(this, data); + this.dataSources = [new SyncStreamDataSource(this, data)]; this.parameterQuerierSources = [new SyncStreamParameterQuerierSource(this)]; this.parameterLookupSources = [new SyncStreamParameterLookupSource(this)]; } diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index b80d524ea..b6801d7ed 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -38,7 +38,7 @@ describe('streams', () => { expect(desc.variants).toHaveLength(1); expect(evaluateBucketIds(desc, COMMENTS, { id: 'foo' })).toStrictEqual(['1#stream|0[]']); expect( - desc.dataSource + desc.dataSources[0] .createDataSource({ bucketIdTransformer }) .evaluateRow({ sourceTable: USERS, record: { id: 'foo' } }) ).toHaveLength(0); @@ -747,7 +747,7 @@ describe('streams', () => { ); const row = { id: 'id', account_id: 'account_id' }; - expect(stream.dataSource.tableSyncsData(accountMember)).toBeTruthy(); + expect(stream.dataSources[0].tableSyncsData(accountMember)).toBeTruthy(); expect(stream.parameterLookupSources[0].tableSyncsParameters(accountMember)).toBeTruthy(); // Ensure lookup steps work. @@ -779,7 +779,7 @@ describe('streams', () => { // And that the data alias is respected for generated schemas. const outputSchema = {}; - stream.dataSource.resolveResultSets(schema, outputSchema); + stream.dataSources[0].resolveResultSets(schema, outputSchema); expect(Object.keys(outputSchema)).toStrictEqual(['outer']); }); @@ -937,16 +937,20 @@ const options: StreamParseOptions = { const bucketIdTransformer = SqlSyncRules.versionedBucketIdTransformer('1'); function evaluateBucketIds(stream: SyncStream, sourceTable: SourceTableInterface, record: SqliteRow) { - return stream.dataSource - .createDataSource({ bucketIdTransformer }) - .evaluateRow({ sourceTable, record }) - .map((r) => { - if ('error' in r) { - throw new Error(`Unexpected error evaluating row: ${r.error}`); - } + return stream.dataSources + .map((s) => + s + .createDataSource({ bucketIdTransformer }) + .evaluateRow({ sourceTable, record }) + .map((r) => { + if ('error' in r) { + throw new Error(`Unexpected error evaluating row: ${r.error}`); + } - return r.bucket; - }); + return r.bucket; + }) + ) + .flat(); } async function createQueriers( diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 1deeee6e5..36a5e747a 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -108,7 +108,6 @@ bucket_definitions: ); const hydrated = rules.hydrate({ bucketIdTransformer }); const parameterLookupSource = rules.bucketParameterLookupSources[0]; - expect(parameterLookupSource.bucketParameters).toEqual([]); expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ { bucketParameters: [{}], From ddcb8a0d5290e260c3174d824b7c30a042f065e8 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 14:49:55 +0200 Subject: [PATCH 13/33] Use separate sources per sync stream variant. --- packages/sync-rules/src/BaseSqlDataQuery.ts | 4 + packages/sync-rules/src/BucketSource.ts | 72 ++++++++++++++- .../sync-rules/src/SqlBucketDescriptor.ts | 6 +- packages/sync-rules/src/SqlParameterQuery.ts | 4 - packages/sync-rules/src/SqlSyncRules.ts | 4 +- .../sync-rules/src/StaticSqlParameterQuery.ts | 2 - packages/sync-rules/src/SyncRules.ts | 15 +-- .../TableValuedFunctionSqlParameterQuery.ts | 2 - packages/sync-rules/src/streams/from_sql.ts | 7 +- packages/sync-rules/src/streams/stream.ts | 92 ++++++++----------- packages/sync-rules/test/src/streams.test.ts | 48 +++++----- .../sync-rules/test/src/sync_rules.test.ts | 1 - 12 files changed, 142 insertions(+), 115 deletions(-) diff --git a/packages/sync-rules/src/BaseSqlDataQuery.ts b/packages/sync-rules/src/BaseSqlDataQuery.ts index 3138544c8..915327135 100644 --- a/packages/sync-rules/src/BaseSqlDataQuery.ts +++ b/packages/sync-rules/src/BaseSqlDataQuery.ts @@ -182,6 +182,10 @@ export class BaseSqlDataQuery { const tables = { [this.table.nameInSchema]: this.addSpecialParameters(table, row) }; const resolvedBucketIds = bucketIds(tables); + if (resolvedBucketIds.length == 0) { + // Short-circuit: No need to transform the row if there are no matching buckets. + return []; + } const data = this.transformRow(tables); let id = data.id; diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 324203073..ebb73b167 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -16,6 +16,12 @@ export interface CreateSourceParams { bucketIdTransformer: BucketIdTransformer; } +/** + * A BucketSource is a _logical_ bucket or sync stream definition. It is primarily used to group together + * related BucketDataSource, BucketParameterLookupSource and BucketParameterQuerierSource definitions, + * for the purpose of subscribing to specific streams. It does not directly define the implementation + * or replication process. + */ export interface BucketSource { readonly name: string; readonly type: BucketSourceType; @@ -116,8 +122,6 @@ export interface BucketParameterQuerierSourceDefinition { * definitions that only consist of a single query. */ export interface BucketDataSource { - readonly definition: BucketDataSourceDefinition; - /** * Given a row as it appears in a table that affects sync data, return buckets, logical table names and transformed * data for rows to add to buckets. @@ -126,7 +130,6 @@ export interface BucketDataSource { } export interface BucketParameterLookupSource { - readonly definition: BucketParameterLookupSourceDefinition; /** * Given a row in a source table that affects sync parameters, returns a structure to index which buckets rows should * be associated with. @@ -138,8 +141,6 @@ export interface BucketParameterLookupSource { } export interface BucketParameterQuerierSource { - readonly definition: BucketParameterQuerierSourceDefinition; - /** * Reports {@link BucketParameterQuerier}s resolving buckets that a specific stream request should have access to. * @@ -149,9 +150,70 @@ export interface BucketParameterQuerierSource { pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void; } +export interface DebugMergedSource + extends BucketDataSource, + BucketParameterLookupSource, + BucketParameterQuerierSource {} + export enum BucketSourceType { SYNC_RULE, SYNC_STREAM } export type ResultSetDescription = { name: string; columns: ColumnDefinition[] }; + +export function mergeDataSources(sources: BucketDataSource[]): BucketDataSource { + return { + evaluateRow(options: EvaluateRowOptions): EvaluationResult[] { + let results: EvaluationResult[] = []; + for (let source of sources) { + results.push(...source.evaluateRow(options)); + } + return results; + } + }; +} + +export function mergeParameterLookupSources(sources: BucketParameterLookupSource[]): BucketParameterLookupSource { + return { + evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] { + let results: EvaluatedParametersResult[] = []; + for (let source of sources) { + results.push(...source.evaluateParameterRow(sourceTable, row)); + } + return results; + } + }; +} + +export function mergeParameterQuerierSources(sources: BucketParameterQuerierSource[]): BucketParameterQuerierSource { + return { + pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void { + for (let source of sources) { + source.pushBucketParameterQueriers(result, options); + } + } + }; +} + +/** + * For production purposes, we typically need to operate on the different sources separately. However, for debugging, + * it is useful to have a single merged source that can evaluate everything. + */ +export function debugHydratedMergedSource(bucketSource: BucketSource, params?: CreateSourceParams): DebugMergedSource { + const resolvedParams = params ?? { bucketIdTransformer: (id: string) => id }; + const dataSource = mergeDataSources( + bucketSource.dataSources.map((source) => source.createDataSource(resolvedParams)) + ); + const parameterLookupSource = mergeParameterLookupSources( + bucketSource.parameterLookupSources.map((source) => source.createParameterLookupSource(resolvedParams)) + ); + const parameterQuerierSource = mergeParameterQuerierSources( + bucketSource.parameterQuerierSources.map((source) => source.createParameterQuerierSource(resolvedParams)) + ); + return { + evaluateParameterRow: parameterLookupSource.evaluateParameterRow.bind(parameterLookupSource), + evaluateRow: dataSource.evaluateRow.bind(dataSource), + pushBucketParameterQueriers: parameterQuerierSource.pushBucketParameterQueriers.bind(parameterQuerierSource) + }; +} diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 55d8f4119..552681054 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -3,7 +3,10 @@ import { BucketDataSourceDefinition, BucketSource, BucketSourceType, - CreateSourceParams + CreateSourceParams, + DebugMergedSource, + mergeParameterLookupSources, + mergeParameterQuerierSources } from './BucketSource.js'; import { ColumnDefinition } from './ExpressionType.js'; import { IdSequence } from './IdSequence.js'; @@ -106,7 +109,6 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketSo createDataSource(params: CreateSourceParams): BucketDataSource { return { - definition: this, evaluateRow: (options) => { let results: EvaluationResult[] = []; for (let query of this.dataQueries) { diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 20af27837..356c13235 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -326,8 +326,6 @@ export class SqlParameterQuery createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { return { - definition: this, - pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { const q = this.getBucketParameterQuerier(options.globalParameters, ['default'], params.bucketIdTransformer); result.queriers.push(q); @@ -337,8 +335,6 @@ export class SqlParameterQuery createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { return { - definition: this, - evaluateParameterRow: (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { if (this.tableSyncsParameters(sourceTable)) { return this.evaluateParameterRow(row); diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 7c514f1de..5cb608faa 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -399,10 +399,8 @@ export class SqlSyncRules { : (id: string) => id; return new HydratedSyncRules({ definition: this, + createParams: { bucketIdTransformer }, bucketDataSources: this.bucketDataSources.map((d) => d.createDataSource({ bucketIdTransformer })), - bucketParameterQuerierSources: this.bucketParameterQuerierSources.map((d) => - d.createParameterQuerierSource({ bucketIdTransformer }) - ), bucketParameterLookupSources: this.bucketParameterLookupSources.map((d) => d.createParameterLookupSource({ bucketIdTransformer }) ), diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index 804de03b4..7b9d619bb 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -180,8 +180,6 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { return { - definition: this, - pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { const staticBuckets = this.getStaticBucketDescriptions( options.globalParameters, diff --git a/packages/sync-rules/src/SyncRules.ts b/packages/sync-rules/src/SyncRules.ts index bd55495c6..4e2491526 100644 --- a/packages/sync-rules/src/SyncRules.ts +++ b/packages/sync-rules/src/SyncRules.ts @@ -3,6 +3,7 @@ import { BucketParameterLookupSource, BucketParameterQuerierSource, BucketParameterQuerierSourceDefinition, + CreateSourceParams, HydratedBucketSource } from './BucketSource.js'; import { @@ -33,7 +34,6 @@ import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, Sqlite export class HydratedSyncRules { bucketSources: HydratedBucketSource[] = []; bucketDataSources: BucketDataSource[]; - bucketParameterQuerierSources: BucketParameterQuerierSource[]; bucketParameterLookupSources: BucketParameterLookupSource[]; eventDescriptors: SqlEventDescriptor[] = []; @@ -43,14 +43,13 @@ export class HydratedSyncRules { constructor(params: { definition: SqlSyncRules; + createParams: CreateSourceParams; bucketDataSources: BucketDataSource[]; - bucketParameterQuerierSources: BucketParameterQuerierSource[]; bucketParameterLookupSources: BucketParameterLookupSource[]; eventDescriptors?: SqlEventDescriptor[]; compatibility?: CompatibilityContext; }) { this.bucketDataSources = params.bucketDataSources; - this.bucketParameterQuerierSources = params.bucketParameterQuerierSources; this.bucketParameterLookupSources = params.bucketParameterLookupSources; this.definition = params.definition; @@ -61,21 +60,13 @@ export class HydratedSyncRules { this.compatibility = params.compatibility; } - let querierMap = new Map(); for (let definition of this.definition.bucketSources) { const hydratedBucketSource: HydratedBucketSource = { definition: definition, parameterQuerierSources: [] }; this.bucketSources.push(hydratedBucketSource); for (let querier of definition.parameterQuerierSources) { - querierMap.set(querier, hydratedBucketSource); + hydratedBucketSource.parameterQuerierSources.push(querier.createParameterQuerierSource(params.createParams)); } } - for (let querier of params.bucketParameterQuerierSources) { - const bucketSource = querierMap.get(querier.definition); - if (bucketSource == null) { - throw new Error('Cannot find BucketSource for BucketParameterQuerierSource'); - } - bucketSource.parameterQuerierSources.push(querier); - } } // These methods do not depend on hydration, so we can just forward them to the definition. diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index 2ae485bb8..5bafa20eb 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -225,8 +225,6 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { return { - definition: this, - pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { const staticBuckets = this.getStaticBucketDescriptions( options.globalParameters, diff --git a/packages/sync-rules/src/streams/from_sql.ts b/packages/sync-rules/src/streams/from_sql.ts index ba325167a..8f0b17681 100644 --- a/packages/sync-rules/src/streams/from_sql.ts +++ b/packages/sync-rules/src/streams/from_sql.ts @@ -100,14 +100,13 @@ class SyncStreamCompiler { let filter = this.whereClauseToFilters(tools, query.where); filter = filter.toDisjunctiveNormalForm(tools); + const variants = filter.isValid(tools) ? filter.compileVariants(this.descriptorName) : []; const stream = new SyncStream( this.descriptorName, - new BaseSqlDataQuery(this.compileDataQuery(tools, query, alias, sourceTable)) + new BaseSqlDataQuery(this.compileDataQuery(tools, query, alias, sourceTable)), + variants ); stream.subscribedToByDefault = this.options.auto_subscribe ?? false; - if (filter.isValid(tools)) { - stream.variants = filter.compileVariants(this.descriptorName); - } this.errors.push(...tools.errors); if (this.parameterDetector.usesStreamParameters && stream.subscribedToByDefault) { diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index 4a9cb6d33..33a979f5f 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -1,6 +1,6 @@ import { BaseSqlDataQuery } from '../BaseSqlDataQuery.js'; import { BucketInclusionReason, BucketPriority, DEFAULT_BUCKET_PRIORITY } from '../BucketDescription.js'; -import { BucketParameterQuerier, PendingQueriers } from '../BucketParameterQuerier.js'; +import { PendingQueriers } from '../BucketParameterQuerier.js'; import { BucketDataSource, BucketDataSourceDefinition, @@ -38,16 +38,16 @@ export class SyncStream implements BucketSource { public readonly parameterLookupSources: BucketParameterLookupSourceDefinition[]; public readonly parameterQuerierSources: BucketParameterQuerierSourceDefinition[]; - constructor(name: string, data: BaseSqlDataQuery) { + constructor(name: string, data: BaseSqlDataQuery, variants: StreamVariant[]) { this.name = name; this.subscribedToByDefault = false; this.priority = DEFAULT_BUCKET_PRIORITY; - this.variants = []; + this.variants = variants; this.data = data; - this.dataSources = [new SyncStreamDataSource(this, data)]; - this.parameterQuerierSources = [new SyncStreamParameterQuerierSource(this)]; - this.parameterLookupSources = [new SyncStreamParameterLookupSource(this)]; + this.dataSources = variants.map((variant) => new SyncStreamDataSource(this, data, variant)); + this.parameterQuerierSources = variants.map((variant) => new SyncStreamParameterQuerierSource(this, variant)); + this.parameterLookupSources = variants.map((variant) => new SyncStreamParameterLookupSource(this, variant)); } public get type(): BucketSourceType { @@ -58,13 +58,15 @@ export class SyncStream implements BucketSource { export class SyncStreamDataSource implements BucketDataSourceDefinition { constructor( private stream: SyncStream, - private data: BaseSqlDataQuery + private data: BaseSqlDataQuery, + private variant: StreamVariant ) {} + /** + * Not relevant for sync streams. + */ get bucketParameters() { - // FIXME: check whether this is correct. - // Could there be multiple variants with different bucket parameters? - return this.data.bucketParameters; + return []; } getSourceTables(): Set { @@ -102,8 +104,6 @@ export class SyncStreamDataSource implements BucketDataSourceDefinition { createDataSource(params: CreateSourceParams): BucketDataSource { return { - definition: this, - evaluateRow: (options: EvaluateRowOptions): EvaluationResult[] => { if (!this.data.applies(options.sourceTable)) { return []; @@ -115,16 +115,14 @@ export class SyncStreamDataSource implements BucketDataSourceDefinition { record: options.record }; + // There is some duplication in work here when there are multiple variants on a stream: + // Each variant does the same row transformation (only the filters / bucket ids differ). + // However, architecturally we do need to be able to evaluate each variant separately. return this.data.evaluateRowWithOptions({ table: options.sourceTable, row: options.record, - bucketIds() { - const bucketIds: string[] = []; - for (const variant of stream.variants) { - bucketIds.push(...variant.bucketIdsForRow(stream.name, row, params.bucketIdTransformer)); - } - - return bucketIds; + bucketIds: () => { + return this.variant.bucketIdsForRow(stream.name, row, params.bucketIdTransformer); } }); } @@ -135,19 +133,21 @@ export class SyncStreamDataSource implements BucketDataSourceDefinition { export class SyncStreamParameterQuerierSource implements BucketParameterQuerierSourceDefinition { // We could eventually split this into a separate source per variant. - constructor(private stream: SyncStream) {} + constructor( + private stream: SyncStream, + private variant: StreamVariant + ) {} - get bucketParameters(): string[] { - // FIXME: check whether this is correct. - // Could there be multiple variants with different bucket parameters? - return this.stream.data.bucketParameters; + /** + * Not relevant for sync streams. + */ + get bucketParameters() { + return []; } createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { const stream = this.stream; return { - definition: this, - pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions): void => { const subscriptions = options.streams[stream.name] ?? []; @@ -184,17 +184,12 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS bucketIdTransformer: BucketIdTransformer ) { const reason: BucketInclusionReason = subscription != null ? { subscription: subscription.opaque_id } : 'default'; - const queriers: BucketParameterQuerier[] = []; try { - for (const variant of this.stream.variants) { - const querier = variant.querier(this.stream, reason, params, bucketIdTransformer); - if (querier) { - queriers.push(querier); - } + const querier = this.variant.querier(this.stream, reason, params, bucketIdTransformer); + if (querier) { + result.queriers.push(querier); } - - result.queriers.push(...queriers); } catch (e) { result.errors.push({ descriptor: this.stream.name, @@ -206,16 +201,15 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS } export class SyncStreamParameterLookupSource implements BucketParameterLookupSourceDefinition { - // We could eventually split this into a separate source per variant. - - constructor(private stream: SyncStream) {} + constructor( + private stream: SyncStream, + private variant: StreamVariant + ) {} getSourceTables(): Set { let result = new Set(); - for (let variant of this.stream.variants) { - for (const subquery of variant.subqueries) { - result.add(subquery.parameterTable); - } + for (const subquery of this.variant.subqueries) { + result.add(subquery.parameterTable); } return result; @@ -223,26 +217,18 @@ export class SyncStreamParameterLookupSource implements BucketParameterLookupSou createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { return { - definition: this, - evaluateParameterRow: (sourceTable, row) => { const result: EvaluatedParametersResult[] = []; - - for (const variant of this.stream.variants) { - variant.pushParameterRowEvaluation(result, sourceTable, row); - } - + this.variant.pushParameterRowEvaluation(result, sourceTable, row); return result; } }; } tableSyncsParameters(table: SourceTableInterface): boolean { - for (const variant of this.stream.variants) { - for (const subquery of variant.subqueries) { - if (subquery.parameterTable.matches(table)) { - return true; - } + for (const subquery of this.variant.subqueries) { + if (subquery.parameterTable.matches(table)) { + return true; } } diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index b6801d7ed..d4cc22e64 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -4,6 +4,7 @@ import { BucketParameterQuerier, CompatibilityContext, CompatibilityEdition, + debugHydratedMergedSource, DEFAULT_TAG, GetBucketParameterQuerierResult, GetQuerierOptions, @@ -71,16 +72,15 @@ describe('streams', () => { test('legacy token parameter', async () => { const desc = parseStream(`SELECT * FROM issues WHERE owner_id = auth.parameter('$.parameters.test')`); + const source = debugHydratedMergedSource(desc, { bucketIdTransformer }); const queriers: BucketParameterQuerier[] = []; const errors: QuerierError[] = []; const pending = { queriers, errors }; - desc.parameterQuerierSources[0] - .createParameterQuerierSource({ bucketIdTransformer }) - .pushBucketParameterQueriers( - pending, - normalizeQuerierOptions({ test: 'foo' }, {}, { stream: [{ opaque_id: 0, parameters: null }] }) - ); + source.pushBucketParameterQueriers( + pending, + normalizeQuerierOptions({ test: 'foo' }, {}, { stream: [{ opaque_id: 0, parameters: null }] }) + ); expect(mergeBucketParameterQueriers(queriers).staticBuckets).toEqual([ { @@ -222,9 +222,10 @@ describe('streams', () => { ]); expect( - desc.parameterLookupSources[0] - .createParameterLookupSource({ bucketIdTransformer }) - .evaluateParameterRow(ISSUES, { id: 'i1', owner_id: 'u1' }) + debugHydratedMergedSource(desc, { bucketIdTransformer }).evaluateParameterRow(ISSUES, { + id: 'i1', + owner_id: 'u1' + }) ).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', ['u1']), @@ -350,11 +351,9 @@ describe('streams', () => { '1#stream|1["a"]' ]); - expect( - desc.parameterLookupSources[0] - .createParameterLookupSource({ bucketIdTransformer }) - .evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' }) - ).toStrictEqual([ + const source = debugHydratedMergedSource(desc, { bucketIdTransformer }); + + expect(source.evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' })).toStrictEqual([ { lookup: ParameterLookup.normalized('stream', '0', ['b']), bucketParameters: [ @@ -937,20 +936,15 @@ const options: StreamParseOptions = { const bucketIdTransformer = SqlSyncRules.versionedBucketIdTransformer('1'); function evaluateBucketIds(stream: SyncStream, sourceTable: SourceTableInterface, record: SqliteRow) { - return stream.dataSources - .map((s) => - s - .createDataSource({ bucketIdTransformer }) - .evaluateRow({ sourceTable, record }) - .map((r) => { - if ('error' in r) { - throw new Error(`Unexpected error evaluating row: ${r.error}`); - } + return debugHydratedMergedSource(stream, { bucketIdTransformer }) + .evaluateRow({ sourceTable, record }) + .map((r) => { + if ('error' in r) { + throw new Error(`Unexpected error evaluating row: ${r.error}`); + } - return r.bucket; - }) - ) - .flat(); + return r.bucket; + }); } async function createQueriers( diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 36a5e747a..3bb016ec2 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -107,7 +107,6 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const parameterLookupSource = rules.bucketParameterLookupSources[0]; expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ { bucketParameters: [{}], From 18bc3bf25af696982e2a7657a3bf244342d3e94e Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 14:54:39 +0200 Subject: [PATCH 14/33] Move debugRepresentation() back to BucketSource. --- packages/sync-rules/src/BucketSource.ts | 4 ++-- packages/sync-rules/src/SqlSyncRules.ts | 2 +- packages/sync-rules/src/streams/stream.ts | 24 +++++++++++------------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index ebb73b167..e63221894 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -51,6 +51,8 @@ export interface BucketSource { * The same source could in theory be present in multiple stream definitions. */ readonly parameterLookupSources: BucketParameterLookupSourceDefinition[]; + + debugRepresentation(): any; } export interface HydratedBucketSource { @@ -80,8 +82,6 @@ export interface BucketDataSourceDefinition { resolveResultSets(schema: SourceSchema, tables: Record>): void; debugWriteOutputTables(result: Record): void; - - debugRepresentation(): any; } /** diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 5cb608faa..10d7b9fb4 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -473,7 +473,7 @@ export class SqlSyncRules { } debugRepresentation() { - return this.bucketDataSources.map((rules) => rules.debugRepresentation()); + return this.bucketSources.map((rules) => rules.debugRepresentation()); } private parsePriority(value: YAMLMap) { diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index 33a979f5f..62c55b20b 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -53,6 +53,18 @@ export class SyncStream implements BucketSource { public get type(): BucketSourceType { return BucketSourceType.SYNC_STREAM; } + + debugRepresentation() { + return { + name: this.name, + type: BucketSourceType[BucketSourceType.SYNC_STREAM], + variants: this.variants.map((v) => v.debugRepresentation()), + data: { + table: this.data.sourceTable, + columns: this.data.columnOutputNames() + } + }; + } } export class SyncStreamDataSource implements BucketDataSourceDefinition { @@ -90,18 +102,6 @@ export class SyncStreamDataSource implements BucketDataSourceDefinition { result[this.data.table!.sqlName].push(r); } - debugRepresentation() { - return { - name: this.stream.name, - type: BucketSourceType[BucketSourceType.SYNC_STREAM], - variants: this.stream.variants.map((v) => v.debugRepresentation()), - data: { - table: this.data.sourceTable, - columns: this.data.columnOutputNames() - } - }; - } - createDataSource(params: CreateSourceParams): BucketDataSource { return { evaluateRow: (options: EvaluateRowOptions): EvaluationResult[] => { From 3bddef9350843d62bc29db440a4be0517e025374 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 15:06:49 +0200 Subject: [PATCH 15/33] Split out dataSource on SqlBucketDescriptor. --- .../sync-rules/src/SqlBucketDescriptor.ts | 96 ++++++++++--------- .../sync-rules/test/src/sync_rules.test.ts | 2 +- 2 files changed, 51 insertions(+), 47 deletions(-) diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 552681054..c2ac62ca1 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -3,10 +3,7 @@ import { BucketDataSourceDefinition, BucketSource, BucketSourceType, - CreateSourceParams, - DebugMergedSource, - mergeParameterLookupSources, - mergeParameterQuerierSources + CreateSourceParams } from './BucketSource.js'; import { ColumnDefinition } from './ExpressionType.js'; import { IdSequence } from './IdSequence.js'; @@ -30,12 +27,14 @@ export interface QueryParseResult { errors: SqlRuleError[]; } -export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketSource { +export class SqlBucketDescriptor implements BucketSource { name: string; private bucketParametersInternal: string[] | null = null; public readonly subscribedToByDefault: boolean = true; + private readonly dataSource = new BucketDefinitionDataSource(this); + /** * source table -> queries */ @@ -58,7 +57,7 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketSo } get dataSources() { - return [this]; + return [this.dataSource]; } get parameterLookupSources() { @@ -107,11 +106,51 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketSo }; } + debugRepresentation() { + let all_parameter_queries = [...this.parameterQueries.values()].flat(); + let all_data_queries = [...this.dataQueries.values()].flat(); + return { + name: this.name, + type: BucketSourceType[this.type], + bucket_parameters: this.bucketParameters, + global_parameter_queries: this.globalParameterQueries.map((q) => { + return { + sql: q.sql + }; + }), + parameter_queries: all_parameter_queries.map((q) => { + return { + sql: q.sql, + table: q.sourceTable, + input_parameters: q.inputParameters + }; + }), + data_queries: all_data_queries.map((q) => { + return { + sql: q.sql, + table: q.sourceTable, + columns: q.columnOutputNames() + }; + }) + }; + } +} + +export class BucketDefinitionDataSource implements BucketDataSourceDefinition { + constructor(private descriptor: SqlBucketDescriptor) {} + + /** + * For debug use only. + */ + get bucketParameters() { + return this.descriptor.bucketParameters; + } + createDataSource(params: CreateSourceParams): BucketDataSource { return { evaluateRow: (options) => { let results: EvaluationResult[] = []; - for (let query of this.dataQueries) { + for (let query of this.descriptor.dataQueries) { if (!query.applies(options.sourceTable)) { continue; } @@ -125,20 +164,14 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketSo getSourceTables(): Set { let result = new Set(); - for (let query of this.parameterQueries) { - result.add(query.sourceTable); - } - for (let query of this.dataQueries) { + for (let query of this.descriptor.dataQueries) { result.add(query.sourceTable); } - - // Note: No physical tables for global_parameter_queries - return result; } tableSyncsData(table: SourceTableInterface): boolean { - for (let query of this.dataQueries) { + for (let query of this.descriptor.dataQueries) { if (query.applies(table)) { return true; } @@ -147,13 +180,13 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketSo } resolveResultSets(schema: SourceSchema, tables: Record>) { - for (let query of this.dataQueries) { + for (let query of this.descriptor.dataQueries) { query.resolveResultSets(schema, tables); } } debugWriteOutputTables(result: Record): void { - for (let q of this.dataQueries) { + for (let q of this.descriptor.dataQueries) { result[q.table!.sqlName] ??= []; const r = { query: q.sql @@ -162,33 +195,4 @@ export class SqlBucketDescriptor implements BucketDataSourceDefinition, BucketSo result[q.table!.sqlName].push(r); } } - - debugRepresentation() { - let all_parameter_queries = [...this.parameterQueries.values()].flat(); - let all_data_queries = [...this.dataQueries.values()].flat(); - return { - name: this.name, - type: BucketSourceType[this.type], - bucket_parameters: this.bucketParameters, - global_parameter_queries: this.globalParameterQueries.map((q) => { - return { - sql: q.sql - }; - }), - parameter_queries: all_parameter_queries.map((q) => { - return { - sql: q.sql, - table: q.sourceTable, - input_parameters: q.inputParameters - }; - }), - data_queries: all_data_queries.map((q) => { - return { - sql: q.sql, - table: q.sourceTable, - columns: q.columnOutputNames() - }; - }) - }; - } } diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 3bb016ec2..21d3839ce 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -34,7 +34,7 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate({ bucketIdTransformer }); - const bucket = rules.bucketDataSources[0] as SqlBucketDescriptor; + const bucket = rules.bucketSources[0] as SqlBucketDescriptor; expect(bucket.name).toEqual('mybucket'); expect(bucket.bucketParameters).toEqual([]); const dataQuery = bucket.dataQueries[0]; From 60290ae7b6fe83f12f02cdb0adffde111ef8d660 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 17:30:28 +0200 Subject: [PATCH 16/33] Move SubqueryParameterLookupSource deeper to be specific to variants. --- packages/sync-rules/src/BucketSource.ts | 15 +++ .../sync-rules/src/SqlBucketDescriptor.ts | 4 + packages/sync-rules/src/SqlParameterQuery.ts | 8 ++ packages/sync-rules/src/streams/filter.ts | 104 +++++++++++++++--- packages/sync-rules/src/streams/parameter.ts | 4 +- packages/sync-rules/src/streams/stream.ts | 42 +------ packages/sync-rules/src/streams/variant.ts | 49 +++------ 7 files changed, 139 insertions(+), 87 deletions(-) diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index e63221894..e14573d2a 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -65,6 +65,13 @@ export interface HydratedBucketSource { * Encodes a static definition of a bucket source, as parsed from sync rules or stream definitions. */ export interface BucketDataSourceDefinition { + /** + * Bucket prefix if no transformations are defined. + * + * Transformations may use this as a base, or may generate an entirely different prefix. + */ + readonly defaultBucketPrefix: string; + /** * For debug use only. */ @@ -90,6 +97,14 @@ export interface BucketDataSourceDefinition { * This is only relevant for parameter queries that query tables. */ export interface BucketParameterLookupSourceDefinition { + /** + * lookupName + queryId is used to uniquely identify parameter queries for parameter storage. + * + * This defines the default values if no transformations are applied. + */ + defaultLookupName: string; + defaultQueryId: string; + getSourceTables(): Set; createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource; diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index c2ac62ca1..1a70f1129 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -146,6 +146,10 @@ export class BucketDefinitionDataSource implements BucketDataSourceDefinition { return this.descriptor.bucketParameters; } + public get defaultBucketPrefix(): string { + return this.descriptor.name; + } + createDataSource(params: CreateSourceParams): BucketDataSource { return { evaluateRow: (options) => { diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 356c13235..9bb0427a9 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -316,6 +316,14 @@ export class SqlParameterQuery this.errors = options.errors ?? []; } + public get defaultLookupName(): string { + return this.descriptorName; + } + + public get defaultQueryId(): string { + return this.queryId; + } + tableSyncsParameters(table: SourceTableInterface): boolean { return this.sourceTable.matches(table); } diff --git a/packages/sync-rules/src/streams/filter.ts b/packages/sync-rules/src/streams/filter.ts index 05c6e34f1..ca149f3b2 100644 --- a/packages/sync-rules/src/streams/filter.ts +++ b/packages/sync-rules/src/streams/filter.ts @@ -1,6 +1,13 @@ import { isParameterValueClause, isRowValueClause, SQLITE_TRUE, sqliteBool } from '../sql_support.js'; import { TablePattern } from '../TablePattern.js'; -import { ParameterMatchClause, ParameterValueClause, RowValueClause, SqliteJsonValue } from '../types.js'; +import { + EvaluatedParametersResult, + ParameterMatchClause, + ParameterValueClause, + RowValueClause, + SqliteJsonValue, + SqliteRow +} from '../types.js'; import { isJsonValue, normalizeParameterValue } from '../utils.js'; import { SqlTools } from '../sql_filters.js'; import { checkJsonArray, OPERATOR_NOT } from '../sql_functions.js'; @@ -10,6 +17,12 @@ import { StreamVariant } from './variant.js'; import { SubqueryEvaluator } from './parameter.js'; import { cartesianProduct } from './utils.js'; import { NodeLocation } from 'pgsql-ast-parser'; +import { + BucketParameterLookupSource, + BucketParameterLookupSourceDefinition, + CreateSourceParams +} from '../BucketSource.js'; +import { SourceTableInterface } from '../SourceTableInterface.js'; /** * An intermediate representation of a `WHERE` clause for stream queries. @@ -253,19 +266,10 @@ export class Subquery { const evaluator: SubqueryEvaluator = { parameterTable: this.table, - lookupsForParameterRow(sourceTable, row) { - const value = column.evaluate({ [sourceTable.name]: row }); - if (!isJsonValue(value)) { - return null; - } - - const lookups: ParameterLookup[] = []; - for (const [variant, id] of innerVariants) { - for (const instantiation of variant.instantiationsForRow({ sourceTable, record: row })) { - lookups.push(ParameterLookup.normalized(context.streamName, id, instantiation)); - } - } - return { value, lookups }; + lookupSources(streamName) { + return innerVariants.map(([variant, id]) => { + return new SubqueryParameterLookupSource(evaluator, column, variant, id, streamName); + }); }, lookupsForRequest(parameters) { const lookups: ParameterLookup[] = []; @@ -510,3 +514,75 @@ export class EvaluateSimpleCondition extends FilterOperator { ); } } + +export class SubqueryParameterLookupSource implements BucketParameterLookupSourceDefinition { + constructor( + private subquery: SubqueryEvaluator, + private column: RowValueClause, + private innerVariant: StreamVariant, + public readonly defaultQueryId: string, + private streamName: string + ) {} + + get defaultLookupName() { + return this.streamName; + } + + getSourceTables(): Set { + let result = new Set(); + result.add(this.subquery.parameterTable); + return result; + } + + /** + * Creates lookup indices for dynamically-resolved parameters. + * + * Resolving dynamic parameters is a two-step process: First, for tables referenced in subqueries, we create an index + * to resolve which request parameters would match rows in subqueries. Then, when resolving bucket ids for a request, + * we compute subquery results by looking up results in that index. + * + * This implements the first step of that process. + * + * @param result The array into which evaluation results should be written to. + * @param sourceTable A table we depend on in a subquery. + * @param row Row data to index. + */ + evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] { + if (this.subquery.parameterTable.matches(sourceTable)) { + // Theoretically we're doing duplicate work by doing this for each innerVariant in a subquery. + // In practice, we don't have more than one innerVariant per subquery right now, so this is fine. + const value = this.column.evaluate({ [sourceTable.name]: row }); + if (!isJsonValue(value)) { + return []; + } + + const lookups: ParameterLookup[] = []; + for (const instantiation of this.innerVariant.instantiationsForRow({ sourceTable, record: row })) { + // TODO: dynamic lookup name and query id + lookups.push(ParameterLookup.normalized(this.defaultLookupName, this.defaultQueryId, instantiation)); + } + + // The row of the subquery. Since we only support subqueries with a single column, we unconditionally name the + // column `result` for simplicity. + const resultRow = { result: value }; + + return lookups.map((l) => ({ + lookup: l, + bucketParameters: [resultRow] + })); + } + return []; + } + + createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { + return { + evaluateParameterRow: (sourceTable, row) => { + return this.evaluateParameterRow(sourceTable, row); + } + }; + } + + tableSyncsParameters(table: SourceTableInterface): boolean { + return this.subquery.parameterTable.matches(table); + } +} diff --git a/packages/sync-rules/src/streams/parameter.ts b/packages/sync-rules/src/streams/parameter.ts index 873216385..0b23b58ab 100644 --- a/packages/sync-rules/src/streams/parameter.ts +++ b/packages/sync-rules/src/streams/parameter.ts @@ -1,4 +1,5 @@ import { ParameterLookup } from '../BucketParameterQuerier.js'; +import { BucketParameterLookupSource, BucketParameterLookupSourceDefinition } from '../BucketSource.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { TablePattern } from '../TablePattern.js'; import { @@ -38,8 +39,9 @@ export interface BucketParameter { export interface SubqueryEvaluator { parameterTable: TablePattern; - lookupsForParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): SubqueryLookups | null; lookupsForRequest(params: RequestParameters): ParameterLookup[]; + + lookupSources(streamName: string): BucketParameterLookupSourceDefinition[]; } export interface SubqueryLookups { diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index 62c55b20b..d0c8386b4 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -47,7 +47,7 @@ export class SyncStream implements BucketSource { this.dataSources = variants.map((variant) => new SyncStreamDataSource(this, data, variant)); this.parameterQuerierSources = variants.map((variant) => new SyncStreamParameterQuerierSource(this, variant)); - this.parameterLookupSources = variants.map((variant) => new SyncStreamParameterLookupSource(this, variant)); + this.parameterLookupSources = variants.flatMap((variant) => variant.lookupSources(name)); } public get type(): BucketSourceType { @@ -81,6 +81,10 @@ export class SyncStreamDataSource implements BucketDataSourceDefinition { return []; } + public get defaultBucketPrefix(): string { + return this.variant.defaultBucketPrefix(this.stream.name); + } + getSourceTables(): Set { return new Set([this.data.sourceTable]); } @@ -199,39 +203,3 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS } } } - -export class SyncStreamParameterLookupSource implements BucketParameterLookupSourceDefinition { - constructor( - private stream: SyncStream, - private variant: StreamVariant - ) {} - - getSourceTables(): Set { - let result = new Set(); - for (const subquery of this.variant.subqueries) { - result.add(subquery.parameterTable); - } - - return result; - } - - createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { - return { - evaluateParameterRow: (sourceTable, row) => { - const result: EvaluatedParametersResult[] = []; - this.variant.pushParameterRowEvaluation(result, sourceTable, row); - return result; - } - }; - } - - tableSyncsParameters(table: SourceTableInterface): boolean { - for (const subquery of this.variant.subqueries) { - if (subquery.parameterTable.matches(table)) { - return true; - } - } - - return false; - } -} diff --git a/packages/sync-rules/src/streams/variant.ts b/packages/sync-rules/src/streams/variant.ts index c44d63397..98b8ef9c0 100644 --- a/packages/sync-rules/src/streams/variant.ts +++ b/packages/sync-rules/src/streams/variant.ts @@ -1,6 +1,12 @@ import { BucketInclusionReason, ResolvedBucket } from '../BucketDescription.js'; import { BucketParameterQuerier, ParameterLookup } from '../BucketParameterQuerier.js'; +import { + BucketParameterLookupSource, + BucketParameterLookupSourceDefinition, + CreateSourceParams +} from '../BucketSource.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; +import { TablePattern } from '../TablePattern.js'; import { BucketIdTransformer, EvaluatedParametersResult, @@ -65,6 +71,14 @@ export class StreamVariant { this.requestFilters = []; } + defaultBucketPrefix(streamName: string): string { + return `${streamName}|${this.id}`; + } + + lookupSources(streamName: string): BucketParameterLookupSourceDefinition[] { + return this.subqueries.flatMap((subquery) => subquery.lookupSources(streamName)); + } + /** * Given a row in the table this stream selects from, returns all ids of buckets to which that row belongs to. */ @@ -216,41 +230,6 @@ export class StreamVariant { ); } - /** - * Creates lookup indices for dynamically-resolved parameters. - * - * Resolving dynamic parameters is a two-step process: First, for tables referenced in subqueries, we create an index - * to resolve which request parameters would match rows in subqueries. Then, when resolving bucket ids for a request, - * we compute subquery results by looking up results in that index. - * - * This implements the first step of that process. - * - * @param result The array into which evaluation results should be written to. - * @param sourceTable A table we depend on in a subquery. - * @param row Row data to index. - */ - pushParameterRowEvaluation(result: EvaluatedParametersResult[], sourceTable: SourceTableInterface, row: SqliteRow) { - for (const subquery of this.subqueries) { - if (subquery.parameterTable.matches(sourceTable)) { - const lookups = subquery.lookupsForParameterRow(sourceTable, row); - if (lookups == null) { - continue; - } - - // The row of the subquery. Since we only support subqueries with a single column, we unconditionally name the - // column `result` for simplicity. - const resultRow = { result: lookups.value }; - - result.push( - ...lookups.lookups.map((l) => ({ - lookup: l, - bucketParameters: [resultRow] - })) - ); - } - } - } - debugRepresentation(): any { return { id: this.id, From 9d8f7fba94002d1aff84b42676b3573a6ed65b03 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 18:12:18 +0200 Subject: [PATCH 17/33] Use HydrationState for bucket names. --- packages/sync-rules/src/BucketSource.ts | 14 +- packages/sync-rules/src/HydrationState.ts | 105 +++++ .../sync-rules/src/SqlBucketDescriptor.ts | 13 +- packages/sync-rules/src/SqlDataQuery.ts | 10 +- packages/sync-rules/src/SqlParameterQuery.ts | 36 +- .../sync-rules/src/StaticSqlParameterQuery.ts | 22 +- .../TableValuedFunctionSqlParameterQuery.ts | 26 +- packages/sync-rules/src/streams/stream.ts | 28 +- packages/sync-rules/src/streams/variant.ts | 20 +- packages/sync-rules/src/utils.ts | 11 +- .../sync-rules/test/src/compatibility.test.ts | 4 +- .../sync-rules/test/src/data_queries.test.ts | 34 +- .../test/src/parameter_queries.test.ts | 402 +++++++++++++++--- .../test/src/static_parameter_queries.test.ts | 278 ++++++++---- .../src/table_valued_function_queries.test.ts | 113 ++--- packages/sync-rules/test/src/util.ts | 32 +- 16 files changed, 861 insertions(+), 287 deletions(-) create mode 100644 packages/sync-rules/src/HydrationState.ts diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index e14573d2a..b53ccfe09 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -1,5 +1,6 @@ import { BucketParameterQuerier, ParameterLookup, PendingQueriers } from './BucketParameterQuerier.js'; import { ColumnDefinition } from './ExpressionType.js'; +import { HydrationState } from './HydrationState.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { GetQuerierOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; @@ -13,7 +14,11 @@ import { } from './types.js'; export interface CreateSourceParams { - bucketIdTransformer: BucketIdTransformer; + hydrationState?: HydrationState; + /** + * @deprecated Use hydrationState instead. + */ + bucketIdTransformer?: BucketIdTransformer; } /** @@ -123,6 +128,13 @@ export interface BucketParameterQuerierSourceDefinition { */ readonly bucketParameters: string[]; + /** + * The data source linked to this querier. This determines the bucket names that the querier generates. + * + * Note that queriers do not persist data themselves; they only resolve which buckets to load based on request parameters. + */ + readonly querierDataSource: BucketDataSourceDefinition; + createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource; } diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts new file mode 100644 index 000000000..f3723b5cf --- /dev/null +++ b/packages/sync-rules/src/HydrationState.ts @@ -0,0 +1,105 @@ +import { BucketDataSource, BucketDataSourceDefinition, BucketParameterLookupSourceDefinition } from './BucketSource.js'; +import { BucketIdTransformer, CompatibilityContext, CompatibilityOption, CreateSourceParams } from './index.js'; + +export interface BucketSourceState { + /** The prefix is the bucket name before the parameters. */ + bucketPrefix: string; +} + +export interface BucketParameterLookupSourceState { + /** The lookup name + queryid is used to reference the parameter lookup record. */ + lookupName: string; + queryId: string; +} + +/** + * Hydration state information for a source. + * + * This is what keeps track of bucket name and parameter lookup mappings for hydration. This can be used + * both to re-use mappings across hydrations of different sync rule versions, or to generate new mappings. + */ +export interface HydrationState< + T extends BucketSourceState = BucketSourceState, + U extends BucketParameterLookupSourceState = BucketParameterLookupSourceState +> { + /** + * Given a bucket data source definition, get the bucket prefix to use for it. + */ + getBucketSourceState(source: BucketDataSourceDefinition): T; + + /** + * Given a bucket parameter lookup definition, get the persistence name to use. + */ + getParameterLookupState(source: BucketParameterLookupSourceDefinition): U; +} + +/** + * This represents hydration state that performs no transformations. + * + * This is the legacy default behavior with no bucket versioning. + */ +export const DEFAULT_HYDRATION_STATE: HydrationState = { + getBucketSourceState(source: BucketDataSourceDefinition) { + return { + bucketPrefix: source.defaultBucketPrefix + }; + }, + getParameterLookupState(source) { + return { + lookupName: source.defaultLookupName, + queryId: source.defaultQueryId + }; + } +}; + +export function versionedHydrationState(version: number) { + return new VersionedHydrationState((bucketId: string) => { + return `${version}#${bucketId}`; + }); +} + +export class VersionedHydrationState implements HydrationState { + constructor(private transformer: BucketIdTransformer) {} + + getBucketSourceState(source: BucketDataSourceDefinition): BucketSourceState { + return { + bucketPrefix: this.transformer(source.defaultBucketPrefix) + }; + } + + getParameterLookupState(source: BucketParameterLookupSourceDefinition): BucketParameterLookupSourceState { + // No transformations applied here + return { + lookupName: source.defaultLookupName, + queryId: source.defaultQueryId + }; + } +} + +export class BucketIdTransformerHydrationState implements HydrationState { + constructor(private transformer: BucketIdTransformer) {} + + getBucketSourceState(source: BucketDataSourceDefinition): BucketSourceState { + return { + bucketPrefix: this.transformer(source.defaultBucketPrefix) + }; + } + + getParameterLookupState(source: BucketParameterLookupSourceDefinition): BucketParameterLookupSourceState { + // No transformations applied here + return { + lookupName: source.defaultLookupName, + queryId: source.defaultQueryId + }; + } +} + +export function resolveHydrationState(params: CreateSourceParams): HydrationState { + if (params.hydrationState) { + return params.hydrationState; + } else if (params.bucketIdTransformer) { + return new BucketIdTransformerHydrationState(params.bucketIdTransformer); + } else { + return DEFAULT_HYDRATION_STATE; + } +} diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 1a70f1129..68eb492f8 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -6,6 +6,7 @@ import { CreateSourceParams } from './BucketSource.js'; import { ColumnDefinition } from './ExpressionType.js'; +import { resolveHydrationState } from './HydrationState.js'; import { IdSequence } from './IdSequence.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SqlDataQuery } from './SqlDataQuery.js'; @@ -83,7 +84,13 @@ export class SqlBucketDescriptor implements BucketSource { } addParameterQuery(sql: string, options: QueryParseOptions): QueryParseResult { - const parameterQuery = SqlParameterQuery.fromSql(this.name, sql, options, this.parameterIdSequence.nextId()); + const parameterQuery = SqlParameterQuery.fromSql( + this.name, + sql, + options, + this.parameterIdSequence.nextId(), + this.dataSource + ); if (this.bucketParametersInternal == null) { this.bucketParametersInternal = parameterQuery.bucketParameters; } else { @@ -151,6 +158,8 @@ export class BucketDefinitionDataSource implements BucketDataSourceDefinition { } createDataSource(params: CreateSourceParams): BucketDataSource { + const hydrationState = resolveHydrationState(params); + const bucketPrefix = hydrationState.getBucketSourceState(this).bucketPrefix; return { evaluateRow: (options) => { let results: EvaluationResult[] = []; @@ -159,7 +168,7 @@ export class BucketDefinitionDataSource implements BucketDataSourceDefinition { continue; } - results.push(...query.evaluateRow(options.sourceTable, options.record, params.bucketIdTransformer)); + results.push(...query.evaluateRow(options.sourceTable, options.record, bucketPrefix)); } return results; } diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index 30f0a2802..b8803dcaa 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -192,19 +192,13 @@ export class SqlDataQuery extends BaseSqlDataQuery { this.filter = options.filter; } - evaluateRow( - table: SourceTableInterface, - row: SqliteRow, - bucketIdTransformer: BucketIdTransformer - ): EvaluationResult[] { + evaluateRow(table: SourceTableInterface, row: SqliteRow, bucketPrefix: string): EvaluationResult[] { return this.evaluateRowWithOptions({ table, row, bucketIds: (tables) => { const bucketParameters = this.filter.filterRow(tables); - return bucketParameters.map((params) => - getBucketId(this.descriptorName, this.bucketParameters, params, bucketIdTransformer) - ); + return bucketParameters.map((params) => getBucketId(bucketPrefix, this.bucketParameters, params)); } }); } diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 9bb0427a9..9e41dd228 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -19,7 +19,7 @@ import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; -import { GetQuerierOptions } from './index.js'; +import { BucketDataSourceDefinition, GetQuerierOptions } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; @@ -44,6 +44,7 @@ import { } from './types.js'; import { filterJsonRow, getBucketId, isJsonValue, isSelectStatement, normalizeParameterValue } from './utils.js'; import { DetectRequestParameters } from './validators.js'; +import { HydrationState, resolveHydrationState } from './HydrationState.js'; export interface SqlParameterQueryOptions { sourceTable: TablePattern; @@ -59,6 +60,7 @@ export interface SqlParameterQueryOptions { bucketParameters: string[]; queryId: string; tools: SqlTools; + querierDataSource: BucketDataSourceDefinition; errors?: SqlRuleError[]; } @@ -75,7 +77,8 @@ export class SqlParameterQuery descriptorName: string, sql: string, options: QueryParseOptions, - queryId: string + queryId: string, + querierDataSource: BucketDataSourceDefinition ): SqlParameterQuery | StaticSqlParameterQuery | TableValuedFunctionSqlParameterQuery { const parsed = parse(sql, { locationTracking: true }); const schema = options?.schema; @@ -91,7 +94,7 @@ export class SqlParameterQuery if (q.from == null) { // E.g. SELECT token_parameters.user_id as user_id WHERE token_parameters.is_admin - return StaticSqlParameterQuery.fromSql(descriptorName, sql, q, options, queryId); + return StaticSqlParameterQuery.fromSql(descriptorName, sql, q, options, queryId, querierDataSource); } let errors: SqlRuleError[] = []; @@ -102,7 +105,15 @@ export class SqlParameterQuery throw new SqlRuleError('Must SELECT from a single table', sql, q.from?.[0]._location); } else if (q.from[0].type == 'call') { const from = q.from[0]; - return TableValuedFunctionSqlParameterQuery.fromSql(descriptorName, sql, from, q, options, queryId); + return TableValuedFunctionSqlParameterQuery.fromSql( + descriptorName, + sql, + from, + q, + options, + queryId, + querierDataSource + ); } else if (q.from[0].type == 'statement') { throw new SqlRuleError('Subqueries are not supported yet', sql, q.from?.[0]._location); } @@ -203,6 +214,7 @@ export class SqlParameterQuery bucketParameters, queryId, tools, + querierDataSource, errors }); @@ -297,6 +309,8 @@ export class SqlParameterQuery readonly queryId: string; readonly tools: SqlTools; + readonly querierDataSource: BucketDataSourceDefinition; + readonly errors: SqlRuleError[]; constructor(options: SqlParameterQueryOptions) { @@ -314,6 +328,7 @@ export class SqlParameterQuery this.queryId = options.queryId; this.tools = options.tools; this.errors = options.errors ?? []; + this.querierDataSource = options.querierDataSource; } public get defaultLookupName(): string { @@ -333,15 +348,18 @@ export class SqlParameterQuery } createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { + const hydrationState = resolveHydrationState(params); + const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; return { pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { - const q = this.getBucketParameterQuerier(options.globalParameters, ['default'], params.bucketIdTransformer); + const q = this.getBucketParameterQuerier(options.globalParameters, ['default'], bucketPrefix); result.queriers.push(q); } }; } createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { + // FIXME: Use HydrationState for lookups. return { evaluateParameterRow: (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { if (this.tableSyncsParameters(sourceTable)) { @@ -403,7 +421,7 @@ export class SqlParameterQuery resolveBucketDescriptions( bucketParameters: SqliteJsonRow[], parameters: RequestParameters, - transformer: BucketIdTransformer + bucketPrefix: string ): BucketDescription[] { // Filters have already been applied and gotten us the set of bucketParameters - don't attempt to filter again. // We _do_ need to evaluate the output columns here, using a combination of precomputed bucketParameters, @@ -428,7 +446,7 @@ export class SqlParameterQuery } return { - bucket: getBucketId(this.descriptorName, this.bucketParameters, result, transformer), + bucket: getBucketId(bucketPrefix, this.bucketParameters, result), priority: this.priority }; }) @@ -514,7 +532,7 @@ export class SqlParameterQuery getBucketParameterQuerier( requestParameters: RequestParameters, reasons: BucketInclusionReason[], - transformer: BucketIdTransformer + bucketPrefix: string ): BucketParameterQuerier { const lookups = this.getLookups(requestParameters); if (lookups.length == 0) { @@ -534,7 +552,7 @@ export class SqlParameterQuery parameterQueryLookups: lookups, queryDynamicBucketDescriptions: async (source: ParameterLookupSource) => { const bucketParameters = await source.getParameterSets(lookups); - return this.resolveBucketDescriptions(bucketParameters, requestParameters, transformer).map((bucket) => ({ + return this.resolveBucketDescriptions(bucketParameters, requestParameters, bucketPrefix).map((bucket) => ({ ...bucket, definition: this.descriptorName, inclusion_reasons: reasons diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index 7b9d619bb..9643cd606 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -7,7 +7,7 @@ import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; -import { GetQuerierOptions } from './index.js'; +import { BucketDataSourceDefinition, GetQuerierOptions } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; import { checkUnsupportedFeatures, isClauseError, sqliteBool } from './sql_support.js'; @@ -21,6 +21,7 @@ import { } from './types.js'; import { getBucketId, isJsonValue } from './utils.js'; import { DetectRequestParameters } from './validators.js'; +import { resolveHydrationState } from './HydrationState.js'; export interface StaticSqlParameterQueryOptions { sql: string; @@ -30,6 +31,7 @@ export interface StaticSqlParameterQueryOptions { bucketParameters: string[]; queryId: string; filter: ParameterValueClause | undefined; + querierDataSource: BucketDataSourceDefinition; errors?: SqlRuleError[]; } @@ -45,7 +47,8 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi sql: string, q: SelectFromStatement, options: QueryParseOptions, - queryId: string + queryId: string, + querierDataSource: BucketDataSourceDefinition ) { let errors: SqlRuleError[] = []; @@ -102,6 +105,7 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi priority: priority ?? DEFAULT_BUCKET_PRIORITY, filter: isClauseError(filter) ? undefined : filter, queryId, + querierDataSource, errors }); if (query.usesDangerousRequestParameters && !options?.accept_potentially_dangerous_queries) { @@ -157,6 +161,8 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi */ readonly filter: ParameterValueClause | undefined; + public readonly querierDataSource: BucketDataSourceDefinition; + readonly errors: SqlRuleError[]; constructor(options: StaticSqlParameterQueryOptions) { @@ -167,6 +173,7 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi this.bucketParameters = options.bucketParameters; this.queryId = options.queryId; this.filter = options.filter; + this.querierDataSource = options.querierDataSource; this.errors = options.errors ?? []; } @@ -179,12 +186,11 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi } createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { + const hydrationState = resolveHydrationState(params); + const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; return { pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { - const staticBuckets = this.getStaticBucketDescriptions( - options.globalParameters, - params.bucketIdTransformer - ).map((desc) => { + const staticBuckets = this.getStaticBucketDescriptions(options.globalParameters, bucketPrefix).map((desc) => { return { ...desc, definition: this.descriptorName, @@ -206,7 +212,7 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi }; } - getStaticBucketDescriptions(parameters: RequestParameters, transformer: BucketIdTransformer): BucketDescription[] { + getStaticBucketDescriptions(parameters: RequestParameters, bucketPrefix: string): BucketDescription[] { if (this.filter == null) { // Error in filter clause return []; @@ -230,7 +236,7 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi return [ { - bucket: getBucketId(this.descriptorName, this.bucketParameters, result, transformer), + bucket: getBucketId(bucketPrefix, this.bucketParameters, result), priority: this.priority } ]; diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index 5bafa20eb..767d62242 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -6,7 +6,7 @@ import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; -import { BucketParameterQuerier, GetQuerierOptions, PendingQueriers } from './index.js'; +import { BucketDataSourceDefinition, BucketParameterQuerier, GetQuerierOptions, PendingQueriers } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; import { checkUnsupportedFeatures, isClauseError, sqliteBool } from './sql_support.js'; @@ -23,6 +23,7 @@ import { } from './types.js'; import { getBucketId, isJsonValue } from './utils.js'; import { DetectRequestParameters } from './validators.js'; +import { resolveHydrationState } from './HydrationState.js'; export interface TableValuedFunctionSqlParameterQueryOptions { sql: string; @@ -36,6 +37,7 @@ export interface TableValuedFunctionSqlParameterQueryOptions { callClause: ParameterValueClause | undefined; function: TableValuedFunction; callTable: AvailableTable; + querierDataSource: BucketDataSourceDefinition; errors: SqlRuleError[]; } @@ -56,7 +58,8 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer call: FromCall, q: SelectFromStatement, options: QueryParseOptions, - queryId: string + queryId: string, + querierDataSource: BucketDataSourceDefinition ): TableValuedFunctionSqlParameterQuery { const compatibility = options.compatibility; let errors: SqlRuleError[] = []; @@ -120,6 +123,7 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer callTable, priority: priority ?? DEFAULT_BUCKET_PRIORITY, queryId, + querierDataSource, errors }); @@ -197,6 +201,8 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer */ readonly callTable: AvailableTable; + public readonly querierDataSource: BucketDataSourceDefinition; + readonly errors: SqlRuleError[]; constructor(options: TableValuedFunctionSqlParameterQueryOptions) { @@ -206,6 +212,7 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer this.descriptorName = options.descriptorName; this.bucketParameters = options.bucketParameters; this.queryId = options.queryId; + this.querierDataSource = options.querierDataSource; this.filter = options.filter; this.callClause = options.callClause; @@ -224,12 +231,11 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer } createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { + const hydrationState = resolveHydrationState(params); + const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; return { pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { - const staticBuckets = this.getStaticBucketDescriptions( - options.globalParameters, - params.bucketIdTransformer - ).map((desc) => { + const staticBuckets = this.getStaticBucketDescriptions(options.globalParameters, bucketPrefix).map((desc) => { return { ...desc, definition: this.descriptorName, @@ -251,7 +257,7 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer }; } - getStaticBucketDescriptions(parameters: RequestParameters, transformer: BucketIdTransformer): BucketDescription[] { + getStaticBucketDescriptions(parameters: RequestParameters, bucketPrefix: string): BucketDescription[] { if (this.filter == null || this.callClause == null) { // Error in filter clause return []; @@ -261,7 +267,7 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer const rows = this.function.call([valueString]); let total: BucketDescription[] = []; for (let row of rows) { - const description = this.getIndividualBucketDescription(row, parameters, transformer); + const description = this.getIndividualBucketDescription(row, parameters, bucketPrefix); if (description !== null) { total.push(description); } @@ -272,7 +278,7 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer private getIndividualBucketDescription( row: SqliteRow, parameters: RequestParameters, - transformer: BucketIdTransformer + bucketPrefix: string ): BucketDescription | null { const mergedParams: ParameterValueSet = { ...parameters, @@ -300,7 +306,7 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer } return { - bucket: getBucketId(this.descriptorName, this.bucketParameters, result, transformer), + bucket: getBucketId(bucketPrefix, this.bucketParameters, result), priority: this.priority }; } diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index d0c8386b4..d87166d28 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -13,6 +13,7 @@ import { CreateSourceParams } from '../BucketSource.js'; import { ColumnDefinition } from '../ExpressionType.js'; +import { resolveHydrationState } from '../HydrationState.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { GetQuerierOptions, RequestedStream } from '../SqlSyncRules.js'; import { TablePattern } from '../TablePattern.js'; @@ -45,8 +46,14 @@ export class SyncStream implements BucketSource { this.variants = variants; this.data = data; - this.dataSources = variants.map((variant) => new SyncStreamDataSource(this, data, variant)); - this.parameterQuerierSources = variants.map((variant) => new SyncStreamParameterQuerierSource(this, variant)); + this.dataSources = []; + this.parameterQuerierSources = []; + + for (let variant of variants) { + const dataSource = new SyncStreamDataSource(this, data, variant); + this.dataSources.push(dataSource); + this.parameterQuerierSources.push(new SyncStreamParameterQuerierSource(this, variant, dataSource)); + } this.parameterLookupSources = variants.flatMap((variant) => variant.lookupSources(name)); } @@ -107,6 +114,8 @@ export class SyncStreamDataSource implements BucketDataSourceDefinition { } createDataSource(params: CreateSourceParams): BucketDataSource { + const hydrationState = resolveHydrationState(params); + const bucketPrefix = hydrationState.getBucketSourceState(this).bucketPrefix; return { evaluateRow: (options: EvaluateRowOptions): EvaluationResult[] => { if (!this.data.applies(options.sourceTable)) { @@ -126,7 +135,7 @@ export class SyncStreamDataSource implements BucketDataSourceDefinition { table: options.sourceTable, row: options.record, bucketIds: () => { - return this.variant.bucketIdsForRow(stream.name, row, params.bucketIdTransformer); + return this.variant.bucketIdsForRow(bucketPrefix, row); } }); } @@ -139,7 +148,8 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS constructor( private stream: SyncStream, - private variant: StreamVariant + private variant: StreamVariant, + public readonly querierDataSource: BucketDataSourceDefinition ) {} /** @@ -150,6 +160,8 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS } createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { + const hydrationState = resolveHydrationState(params); + const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; const stream = this.stream; return { pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions): void => { @@ -169,13 +181,13 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS hasExplicitDefaultSubscription = true; } - this.queriersForSubscription(result, subscription, subscriptionParams, params.bucketIdTransformer); + this.queriersForSubscription(result, subscription, subscriptionParams, bucketPrefix); } // If the stream is subscribed to by default and there is no explicit subscription that would match the default // subscription, also include the default querier. if (stream.subscribedToByDefault && !hasExplicitDefaultSubscription) { - this.queriersForSubscription(result, null, options.globalParameters, params.bucketIdTransformer); + this.queriersForSubscription(result, null, options.globalParameters, bucketPrefix); } } }; @@ -185,12 +197,12 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS result: PendingQueriers, subscription: RequestedStream | null, params: RequestParameters, - bucketIdTransformer: BucketIdTransformer + bucketPrefix: string ) { const reason: BucketInclusionReason = subscription != null ? { subscription: subscription.opaque_id } : 'default'; try { - const querier = this.variant.querier(this.stream, reason, params, bucketIdTransformer); + const querier = this.variant.querier(this.stream, reason, params, bucketPrefix); if (querier) { result.queriers.push(querier); } diff --git a/packages/sync-rules/src/streams/variant.ts b/packages/sync-rules/src/streams/variant.ts index 98b8ef9c0..35e016330 100644 --- a/packages/sync-rules/src/streams/variant.ts +++ b/packages/sync-rules/src/streams/variant.ts @@ -82,8 +82,8 @@ export class StreamVariant { /** * Given a row in the table this stream selects from, returns all ids of buckets to which that row belongs to. */ - bucketIdsForRow(streamName: string, options: TableRow, transformer: BucketIdTransformer): string[] { - return this.instantiationsForRow(options).map((values) => this.buildBucketId(streamName, values, transformer)); + bucketIdsForRow(bucketPrefix: string, options: TableRow): string[] { + return this.instantiationsForRow(options).map((values) => this.buildBucketId(bucketPrefix, values)); } /** @@ -132,7 +132,7 @@ export class StreamVariant { stream: SyncStream, reason: BucketInclusionReason, params: RequestParameters, - bucketIdTransformer: BucketIdTransformer + bucketPrefix: string ): BucketParameterQuerier | null { const instantiation = this.partiallyEvaluateParameters(params); if (instantiation == null) { @@ -169,7 +169,7 @@ export class StreamVariant { // When we have no dynamic parameters, the partial evaluation is a full instantiation. const instantiations = this.cartesianProductOfParameterInstantiations(instantiation as SqliteJsonValue[][]); for (const instantiation of instantiations) { - staticBuckets.push(this.resolveBucket(stream, instantiation, reason, bucketIdTransformer)); + staticBuckets.push(this.resolveBucket(stream, instantiation, reason, bucketPrefix)); } } @@ -214,7 +214,7 @@ export class StreamVariant { perParameterInstantiation as SqliteJsonValue[][] ); - return Promise.resolve(product.map((e) => variant.resolveBucket(stream, e, reason, bucketIdTransformer))); + return Promise.resolve(product.map((e) => variant.resolveBucket(stream, e, reason, bucketPrefix))); } }; } @@ -280,29 +280,29 @@ export class StreamVariant { /** * Builds a bucket id for an instantiation, like `stream|0[1,2,"foo"]`. * - * @param streamName The name of the stream, included in the bucket id + * @param bucketPrefix The name of the the bucket, excluding parameters * @param instantiation An instantiation for all parameters in this variant. * @param transformer A transformer adding version information to the inner id. * @returns The generated bucket id */ - private buildBucketId(streamName: string, instantiation: SqliteJsonValue[], transformer: BucketIdTransformer) { + private buildBucketId(bucketPrefix: string, instantiation: SqliteJsonValue[]) { if (instantiation.length != this.parameters.length) { throw Error('Internal error, instantiation length mismatch'); } - return transformer(`${streamName}|${this.id}${JSONBucketNameSerialize.stringify(instantiation)}`); + return `${bucketPrefix}${JSONBucketNameSerialize.stringify(instantiation)}`; } private resolveBucket( stream: SyncStream, instantiation: SqliteJsonValue[], reason: BucketInclusionReason, - bucketIdTransformer: BucketIdTransformer + bucketPrefix: string ): ResolvedBucket { return { definition: stream.name, inclusion_reasons: [reason], - bucket: this.buildBucketId(stream.name, instantiation, bucketIdTransformer), + bucket: this.buildBucketId(bucketPrefix, instantiation), priority: stream.priority }; } diff --git a/packages/sync-rules/src/utils.ts b/packages/sync-rules/src/utils.ts index 04f849e08..e66d8367b 100644 --- a/packages/sync-rules/src/utils.ts +++ b/packages/sync-rules/src/utils.ts @@ -21,14 +21,13 @@ export function isSelectStatement(q: Statement): q is SelectFromStatement { } export function getBucketId( - descriptor_id: string, - bucket_parameters: string[], - params: Record, - transformer: BucketIdTransformer + bucketPrefix: string, + bucketParameters: string[], + params: Record ): string { // Important: REAL and INTEGER values matching the same number needs the same representation in the bucket name. - const paramArray = bucket_parameters.map((name) => params[`bucket.${name}`]); - return transformer(`${descriptor_id}${JSONBucketNameSerialize.stringify(paramArray)}`); + const paramArray = bucketParameters.map((name) => params[`bucket.${name}`]); + return `${bucketPrefix}${JSONBucketNameSerialize.stringify(paramArray)}`; } const DEPTH_LIMIT = 10; diff --git a/packages/sync-rules/test/src/compatibility.test.ts b/packages/sync-rules/test/src/compatibility.test.ts index 6d21f9dbc..624468a58 100644 --- a/packages/sync-rules/test/src/compatibility.test.ts +++ b/packages/sync-rules/test/src/compatibility.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'vitest'; -import { SqlSyncRules, DateTimeValue, toSyncRulesValue, SqliteInputRow } from '../../src/index.js'; +import { DateTimeValue, SqlSyncRules, toSyncRulesValue } from '../../src/index.js'; -import { ASSETS, identityBucketTransformer, normalizeQuerierOptions, PARSE_OPTIONS } from './util.js'; +import { ASSETS, normalizeQuerierOptions, PARSE_OPTIONS } from './util.js'; describe('compatibility options', () => { describe('timestamps', () => { diff --git a/packages/sync-rules/test/src/data_queries.test.ts b/packages/sync-rules/test/src/data_queries.test.ts index 34fba0db3..6959f7dc3 100644 --- a/packages/sync-rules/test/src/data_queries.test.ts +++ b/packages/sync-rules/test/src/data_queries.test.ts @@ -8,9 +8,7 @@ describe('data queries', () => { const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect( - query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, SqlSyncRules.versionedBucketIdTransformer('1')) - ).toEqual([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, '1#mybucket')).toEqual([ { bucket: '1#mybucket["org1"]', table: 'assets', @@ -25,7 +23,7 @@ describe('data queries', () => { const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, identityBucketTransformer)).toEqual([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, 'mybucket')).toEqual([ { bucket: 'mybucket["org1"]', table: 'assets', @@ -34,7 +32,7 @@ describe('data queries', () => { } ]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null }, identityBucketTransformer)).toEqual([]); + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null }, 'mybucket')).toEqual([]); }); test('bucket parameters IN query', function () { @@ -43,11 +41,7 @@ describe('data queries', () => { expect(query.errors).toEqual([]); expect( - query.evaluateRow( - ASSETS, - { id: 'asset1', categories: JSON.stringify(['red', 'green']) }, - identityBucketTransformer - ) + query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'green']) }, 'mybucket') ).toMatchObject([ { bucket: 'mybucket["red"]', @@ -61,7 +55,7 @@ describe('data queries', () => { } ]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null }, identityBucketTransformer)).toEqual([]); + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null }, 'mybucket')).toEqual([]); }); test('static IN data query', function () { @@ -70,11 +64,7 @@ describe('data queries', () => { expect(query.errors).toEqual([]); expect( - query.evaluateRow( - ASSETS, - { id: 'asset1', categories: JSON.stringify(['red', 'green']) }, - identityBucketTransformer - ) + query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'green']) }, 'mybucket') ).toMatchObject([ { bucket: 'mybucket[]', @@ -84,11 +74,7 @@ describe('data queries', () => { ]); expect( - query.evaluateRow( - ASSETS, - { id: 'asset1', categories: JSON.stringify(['red', 'blue']) }, - identityBucketTransformer - ) + query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'blue']) }, 'mybucket') ).toEqual([]); }); @@ -97,7 +83,7 @@ describe('data queries', () => { const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'good' }, identityBucketTransformer)).toMatchObject([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'good' }, 'mybucket')).toMatchObject([ { bucket: 'mybucket[]', table: 'assets', @@ -105,7 +91,7 @@ describe('data queries', () => { } ]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'bad' }, identityBucketTransformer)).toEqual([]); + expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'bad' }, 'mybucket')).toEqual([]); }); test('table alias', function () { @@ -113,7 +99,7 @@ describe('data queries', () => { const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, identityBucketTransformer)).toEqual([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, 'mybucket')).toEqual([ { bucket: 'mybucket["org1"]', table: 'others', diff --git a/packages/sync-rules/test/src/parameter_queries.test.ts b/packages/sync-rules/test/src/parameter_queries.test.ts index acabec993..653eb56ed 100644 --- a/packages/sync-rules/test/src/parameter_queries.test.ts +++ b/packages/sync-rules/test/src/parameter_queries.test.ts @@ -1,12 +1,18 @@ import { describe, expect, test } from 'vitest'; import { ParameterLookup, SqlParameterQuery } from '../../src/index.js'; -import { BASIC_SCHEMA, identityBucketTransformer, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; +import { BASIC_SCHEMA, EMPTY_DATA_SOURCE, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; describe('parameter queries', () => { test('token_parameters IN query', function () { const sql = 'SELECT id as group_id FROM groups WHERE token_parameters.user_id IN groups.user_ids'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'group1', user_ids: JSON.stringify(['user1', 'user2']) })).toEqual([ { @@ -37,7 +43,13 @@ describe('parameter queries', () => { test('IN token_parameters query', function () { const sql = 'SELECT id as region_id FROM regions WHERE name IN token_parameters.region_names'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'region1', name: 'colorado' })).toEqual([ { @@ -64,7 +76,13 @@ describe('parameter queries', () => { test('queried numeric parameters', () => { const sql = 'SELECT users.int1, users.float1, users.float2 FROM users WHERE users.int1 = token_parameters.int1 AND users.float1 = token_parameters.float1 AND users.float2 = token_parameters.float2'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); // Note: We don't need to worry about numeric vs decimal types in the lookup - JSONB handles normalization for us. expect(query.evaluateParameterRow({ int1: 314n, float1: 3.14, float2: 314 })).toEqual([ @@ -86,7 +104,7 @@ describe('parameter queries', () => { query.resolveBucketDescriptions( [{ int1: 314, float1: 3.14, float2: 314 }], normalizeTokenParameters({}), - identityBucketTransformer + 'mybucket' ) ).toEqual([{ bucket: 'mybucket[314,3.14,314]', priority: 3 }]); @@ -94,14 +112,20 @@ describe('parameter queries', () => { query.resolveBucketDescriptions( [{ int1: 314n, float1: 3.14, float2: 314 }], normalizeTokenParameters({}), - identityBucketTransformer + 'mybucket' ) ).toEqual([{ bucket: 'mybucket[314,3.14,314]', priority: 3 }]); }); test('plain token_parameter (baseline)', () => { const sql = 'SELECT id from users WHERE filter_param = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ @@ -119,7 +143,13 @@ describe('parameter queries', () => { test('function on token_parameter', () => { const sql = 'SELECT id from users WHERE filter_param = upper(token_parameters.user_id)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ @@ -137,7 +167,13 @@ describe('parameter queries', () => { test('token parameter member operator', () => { const sql = "SELECT id from users WHERE filter_param = token_parameters.some_param ->> 'description'"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ @@ -155,7 +191,13 @@ describe('parameter queries', () => { test('token parameter and binary operator', () => { const sql = 'SELECT id from users WHERE filter_param = token_parameters.some_param + 2'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getLookups(normalizeTokenParameters({ some_param: 3 }))).toEqual([ @@ -165,7 +207,13 @@ describe('parameter queries', () => { test('token parameter IS NULL as filter', () => { const sql = 'SELECT id from users WHERE filter_param = (token_parameters.some_param IS NULL)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getLookups(normalizeTokenParameters({ some_param: null }))).toEqual([ @@ -178,7 +226,13 @@ describe('parameter queries', () => { test('direct token parameter', () => { const sql = 'SELECT FROM users WHERE token_parameters.some_param'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ @@ -198,7 +252,13 @@ describe('parameter queries', () => { test('token parameter IS NULL', () => { const sql = 'SELECT FROM users WHERE token_parameters.some_param IS NULL'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ @@ -218,7 +278,13 @@ describe('parameter queries', () => { test('token parameter IS NOT NULL', () => { const sql = 'SELECT FROM users WHERE token_parameters.some_param IS NOT NULL'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ @@ -238,7 +304,13 @@ describe('parameter queries', () => { test('token parameter NOT', () => { const sql = 'SELECT FROM users WHERE NOT token_parameters.is_admin'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ @@ -258,7 +330,13 @@ describe('parameter queries', () => { test('row filter and token parameter IS NULL', () => { const sql = 'SELECT FROM users WHERE users.id = token_parameters.user_id AND token_parameters.some_param IS NULL'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ @@ -278,7 +356,13 @@ describe('parameter queries', () => { test('row filter and direct token parameter', () => { const sql = 'SELECT FROM users WHERE users.id = token_parameters.user_id AND token_parameters.some_param'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ @@ -298,7 +382,13 @@ describe('parameter queries', () => { test('cast', () => { const sql = 'SELECT FROM users WHERE users.id = cast(token_parameters.user_id as text)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ @@ -311,7 +401,13 @@ describe('parameter queries', () => { test('IS NULL row filter', () => { const sql = 'SELECT id FROM users WHERE role IS NULL'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'user1', role: null })).toEqual([ @@ -331,7 +427,13 @@ describe('parameter queries', () => { // Not supported: token_parameters.is_admin != false // Support could be added later. const sql = 'SELECT FROM users WHERE users.id = token_parameters.user_id AND token_parameters.is_admin'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ @@ -353,7 +455,13 @@ describe('parameter queries', () => { test('token filter (2)', () => { const sql = 'SELECT users.id AS user_id, token_parameters.is_admin as is_admin FROM users WHERE users.id = token_parameters.user_id AND token_parameters.is_admin'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ @@ -372,14 +480,20 @@ describe('parameter queries', () => { query.resolveBucketDescriptions( [{ user_id: 'user1' }], normalizeTokenParameters({ user_id: 'user1', is_admin: true }), - identityBucketTransformer + 'mybucket' ) ).toEqual([{ bucket: 'mybucket["user1",1]', priority: 3 }]); }); test('case-sensitive parameter queries (1)', () => { const sql = 'SELECT users."userId" AS user_id FROM users WHERE users."userId" = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ userId: 'user1' })).toEqual([ @@ -396,7 +510,13 @@ describe('parameter queries', () => { // This may change in the future - we should check against expected behavior for // Postgres and/or SQLite. const sql = 'SELECT users.userId AS user_id FROM users WHERE users.userId = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "userId" instead.` }, { message: `Unquoted identifiers are converted to lower-case. Use "userId" instead.` } @@ -414,7 +534,13 @@ describe('parameter queries', () => { test('case-sensitive parameter queries (3)', () => { const sql = 'SELECT user_id FROM users WHERE Users.user_id = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "Users" instead.` } ]); @@ -422,7 +548,13 @@ describe('parameter queries', () => { test('case-sensitive parameter queries (4)', () => { const sql = 'SELECT Users.user_id FROM users WHERE user_id = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "Users" instead.` } ]); @@ -430,7 +562,13 @@ describe('parameter queries', () => { test('case-sensitive parameter queries (5)', () => { const sql = 'SELECT user_id FROM Users WHERE user_id = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "Users" instead.` } ]); @@ -438,7 +576,13 @@ describe('parameter queries', () => { test('case-sensitive parameter queries (6)', () => { const sql = 'SELECT userId FROM users'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "userId" instead.` } ]); @@ -446,7 +590,13 @@ describe('parameter queries', () => { test('case-sensitive parameter queries (7)', () => { const sql = 'SELECT user_id as userId FROM users'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "userId" instead.` } ]); @@ -454,7 +604,13 @@ describe('parameter queries', () => { test('dynamic global parameter query', () => { const sql = "SELECT workspaces.id AS workspace_id FROM workspaces WHERE visibility = 'public'"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'workspace1', visibility: 'public' })).toEqual([ @@ -472,7 +628,13 @@ describe('parameter queries', () => { // This is treated as two separate lookup index values const sql = 'SELECT id from users WHERE filter_param = upper(token_parameters.user_id) AND filter_param = lower(token_parameters.user_id)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ @@ -492,7 +654,13 @@ describe('parameter queries', () => { // This is treated as the same index lookup value, can use OR with the two clauses const sql = 'SELECT id from users WHERE filter_param1 = upper(token_parameters.user_id) OR filter_param2 = upper(token_parameters.user_id)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'test_id', filter_param1: 'test1', filter_param2: 'test2' })).toEqual([ @@ -520,7 +688,8 @@ describe('parameter queries', () => { accept_potentially_dangerous_queries: true, ...PARSE_OPTIONS }, - '1' + '1', + EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'group1', category: 'red' })).toEqual([ @@ -543,7 +712,8 @@ describe('parameter queries', () => { accept_potentially_dangerous_queries: true, ...PARSE_OPTIONS }, - '1' + '1', + EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getLookups(normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ @@ -560,7 +730,8 @@ describe('parameter queries', () => { accept_potentially_dangerous_queries: true, ...PARSE_OPTIONS }, - '1' + '1', + EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getLookups(normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ @@ -578,7 +749,8 @@ describe('parameter queries', () => { accept_potentially_dangerous_queries: true, ...PARSE_OPTIONS }, - '1' + '1', + EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'region1', name: 'colorado' })).toEqual([ @@ -608,7 +780,13 @@ describe('parameter queries', () => { test('user_parameters in SELECT', function () { const sql = 'SELECT id, user_parameters.other_id as other_id FROM users WHERE id = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ { @@ -623,7 +801,13 @@ describe('parameter queries', () => { test('request.parameters() in SELECT', function () { const sql = "SELECT id, request.parameters() ->> 'other_id' as other_id FROM users WHERE id = token_parameters.user_id"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ { @@ -637,7 +821,13 @@ describe('parameter queries', () => { test('request.jwt()', function () { const sql = "SELECT FROM users WHERE id = request.jwt() ->> 'sub'"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }); @@ -646,7 +836,13 @@ describe('parameter queries', () => { test('request.user_id()', function () { const sql = 'SELECT FROM users WHERE id = request.user_id()'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }); @@ -658,51 +854,99 @@ describe('parameter queries', () => { // into separate queries, but it's a significant change. For now, developers should do that manually. const sql = "SELECT workspaces.id AS workspace_id FROM workspaces WHERE workspaces.user_id = token_parameters.user_id OR visibility = 'public'"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/must use the same parameters/); }); test('invalid OR in parameter queries (2)', () => { const sql = 'SELECT id from users WHERE filter_param = upper(token_parameters.user_id) OR filter_param = lower(token_parameters.user_id)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/must use the same parameters/); }); test('invalid parameter match clause (1)', () => { const sql = 'SELECT FROM users WHERE (id = token_parameters.user_id) = false'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/Parameter match clauses cannot be used here/); }); test('invalid parameter match clause (2)', () => { const sql = 'SELECT FROM users WHERE NOT (id = token_parameters.user_id)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/Parameter match clauses cannot be used here/); }); test('invalid parameter match clause (3)', () => { // May be supported in the future const sql = 'SELECT FROM users WHERE token_parameters.start_at < users.created_at'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/Cannot use table values and parameters in the same clauses/); }); test('invalid parameter match clause (4)', () => { const sql = 'SELECT FROM users WHERE json_extract(users.description, token_parameters.path)'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/Cannot use table values and parameters in the same clauses/); }); test('invalid parameter match clause (5)', () => { const sql = 'SELECT (user_parameters.role = posts.roles) as r FROM posts'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/Parameter match expression is not allowed here/); }); test('invalid function schema', () => { const sql = 'SELECT FROM users WHERE something.length(users.id) = 0'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors[0].message).toMatch(/Function 'something.length' is not defined/); }); @@ -716,7 +960,8 @@ describe('parameter queries', () => { ...PARSE_OPTIONS, schema }, - '1' + '1', + EMPTY_DATA_SOURCE ); expect(q1.errors).toMatchObject([]); @@ -724,7 +969,8 @@ describe('parameter queries', () => { 'q5', 'SELECT id as asset_id FROM assets WHERE other_id = token_parameters.user_id', { ...PARSE_OPTIONS, schema }, - '1' + '1', + EMPTY_DATA_SOURCE ); expect(q2.errors).toMatchObject([ @@ -738,7 +984,13 @@ describe('parameter queries', () => { describe('dangerous queries', function () { function testDangerousQuery(sql: string) { test(sql, function () { - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: @@ -750,7 +1002,13 @@ describe('parameter queries', () => { } function testSafeQuery(sql: string) { test(sql, function () { - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.usesDangerousRequestParameters).toEqual(false); }); @@ -779,7 +1037,13 @@ describe('parameter queries', () => { describe('bucket priorities', () => { test('valid definition', function () { const sql = 'SELECT id as group_id, 1 AS _priority FROM groups WHERE token_parameters.user_id IN groups.user_ids'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(Object.entries(query.lookupExtractors)).toHaveLength(1); expect(Object.entries(query.parameterExtractors)).toHaveLength(0); @@ -789,7 +1053,13 @@ describe('parameter queries', () => { test('valid definition, static query', function () { const sql = 'SELECT token_parameters.user_id, 0 AS _priority'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(Object.entries(query.parameterExtractors)).toHaveLength(1); expect(query.bucketParameters).toEqual(['user_id']); @@ -798,21 +1068,39 @@ describe('parameter queries', () => { test('invalid dynamic query', function () { const sql = 'SELECT LENGTH(assets.name) AS _priority FROM assets'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([{ message: 'Priority must be a simple integer literal' }]); }); test('invalid literal type', function () { const sql = "SELECT 'very fast please' AS _priority"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toMatchObject([{ message: 'Priority must be a simple integer literal' }]); }); test('invalid literal value', function () { const sql = 'SELECT 15 AS _priority'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toMatchObject([ { message: 'Invalid value for priority, must be between 0 and 3 (inclusive).' } diff --git a/packages/sync-rules/test/src/static_parameter_queries.test.ts b/packages/sync-rules/test/src/static_parameter_queries.test.ts index 3e2861307..bb78ca4a2 100644 --- a/packages/sync-rules/test/src/static_parameter_queries.test.ts +++ b/packages/sync-rules/test/src/static_parameter_queries.test.ts @@ -1,92 +1,123 @@ import { describe, expect, test } from 'vitest'; -import { RequestParameters, SqlParameterQuery, SqlSyncRules } from '../../src/index.js'; +import { RequestParameters, SqlParameterQuery } from '../../src/index.js'; import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; -import { identityBucketTransformer, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; +import { EMPTY_DATA_SOURCE, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; describe('static parameter queries', () => { test('basic query', function () { const sql = 'SELECT token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters!).toEqual(['user_id']); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket["user1"]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), 'mybucket')).toEqual([ + { bucket: 'mybucket["user1"]', priority: 3 } + ]); }); - test('uses bucket id transformer', function () { + test('uses bucketPrefix', function () { const sql = 'SELECT token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters!).toEqual(['user_id']); - expect( - query.getStaticBucketDescriptions( - normalizeTokenParameters({ user_id: 'user1' }), - SqlSyncRules.versionedBucketIdTransformer('1') - ) - ).toEqual([{ bucket: '1#mybucket["user1"]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), '1#mybucket')).toEqual([ + { bucket: '1#mybucket["user1"]', priority: 3 } + ]); }); test('global query', function () { const sql = 'SELECT'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters!).toEqual([]); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), 'mybucket')).toEqual([ + { bucket: 'mybucket[]', priority: 3 } + ]); }); test('query with filter', function () { const sql = 'SELECT token_parameters.user_id WHERE token_parameters.is_admin'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect( - query.getStaticBucketDescriptions( - normalizeTokenParameters({ user_id: 'user1', is_admin: true }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1', is_admin: true }), 'mybucket') ).toEqual([{ bucket: 'mybucket["user1"]', priority: 3 }]); expect( - query.getStaticBucketDescriptions( - normalizeTokenParameters({ user_id: 'user1', is_admin: false }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1', is_admin: false }), 'mybucket') ).toEqual([]); }); test('function in select clause', function () { const sql = 'SELECT upper(token_parameters.user_id) as upper_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket["USER1"]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), 'mybucket')).toEqual([ + { bucket: 'mybucket["USER1"]', priority: 3 } + ]); expect(query.bucketParameters!).toEqual(['upper_id']); }); test('function in filter clause', function () { const sql = "SELECT WHERE upper(token_parameters.role) = 'ADMIN'"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'admin' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'user' }), identityBucketTransformer) - ).toEqual([]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'admin' }), 'mybucket')).toEqual([ + { bucket: 'mybucket[]', priority: 3 } + ]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'user' }), 'mybucket')).toEqual([]); }); test('comparison in filter clause', function () { const sql = 'SELECT WHERE token_parameters.id1 = token_parameters.id2'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't1' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't2' }), identityBucketTransformer) - ).toEqual([]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't1' }), 'mybucket')).toEqual([ + { bucket: 'mybucket[]', priority: 3 } + ]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't2' }), 'mybucket')).toEqual( + [] + ); }); test('request.parameters()', function () { @@ -98,88 +129,129 @@ describe('static parameter queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({}, { org_id: 'test' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket["test"]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({}, { org_id: 'test' }), 'mybucket')).toEqual([ + { bucket: 'mybucket["test"]', priority: 3 } + ]); }); test('request.jwt()', function () { const sql = "SELECT request.jwt() ->> 'sub' as user_id"; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['user_id']); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket["user1"]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), 'mybucket')).toEqual([ + { bucket: 'mybucket["user1"]', priority: 3 } + ]); }); test('request.user_id()', function () { const sql = 'SELECT request.user_id() as user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['user_id']); - expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket["user1"]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), 'mybucket')).toEqual([ + { bucket: 'mybucket["user1"]', priority: 3 } + ]); }); test('static value', function () { const sql = `SELECT WHERE 1`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([ + { bucket: 'mybucket[]', priority: 3 } + ]); }); test('static expression (1)', function () { const sql = `SELECT WHERE 1 = 1`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([ + { bucket: 'mybucket[]', priority: 3 } + ]); }); test('static expression (2)', function () { const sql = `SELECT WHERE 1 != 1`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) - ).toEqual([]); + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([]); }); test('static IN expression', function () { const sql = `SELECT WHERE 'admin' IN '["admin", "superuser"]'`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) - ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([ + { bucket: 'mybucket[]', priority: 3 } + ]); }); test('IN for permissions in request.jwt() (1)', function () { // Can use -> or ->> here const sql = `SELECT 'read:users' IN (request.jwt() ->> 'permissions') as access_granted`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect( query.getStaticBucketDescriptions( new RequestParameters({ sub: '', permissions: ['write', 'read:users'] }, {}), - identityBucketTransformer + 'mybucket' ) ).toEqual([{ bucket: 'mybucket[1]', priority: 3 }]); expect( query.getStaticBucketDescriptions( new RequestParameters({ sub: '', permissions: ['write', 'write:users'] }, {}), - identityBucketTransformer + 'mybucket' ) ).toEqual([{ bucket: 'mybucket[0]', priority: 3 }]); }); @@ -187,43 +259,55 @@ describe('static parameter queries', () => { test('IN for permissions in request.jwt() (2)', function () { // Can use -> or ->> here const sql = `SELECT WHERE 'read:users' IN (request.jwt() ->> 'permissions')`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect( query.getStaticBucketDescriptions( new RequestParameters({ sub: '', permissions: ['write', 'read:users'] }, {}), - identityBucketTransformer + 'mybucket' ) ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); expect( query.getStaticBucketDescriptions( new RequestParameters({ sub: '', permissions: ['write', 'write:users'] }, {}), - identityBucketTransformer + 'mybucket' ) ).toEqual([]); }); test('IN for permissions in request.jwt() (3)', function () { const sql = `SELECT WHERE request.jwt() ->> 'role' IN '["admin", "superuser"]'`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '', role: 'superuser' }, {}), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '', role: 'superuser' }, {}), 'mybucket') ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '', role: 'superadmin' }, {}), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '', role: 'superadmin' }, {}), 'mybucket') ).toEqual([]); }); test('case-sensitive queries (1)', () => { const sql = 'SELECT request.user_id() as USER_ID'; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "USER_ID" instead.` } ]); @@ -232,7 +316,13 @@ describe('static parameter queries', () => { describe('dangerous queries', function () { function testDangerousQuery(sql: string) { test(sql, function () { - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: @@ -244,7 +334,13 @@ describe('static parameter queries', () => { } function testSafeQuery(sql: string) { test(sql, function () { - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.usesDangerousRequestParameters).toEqual(false); }); diff --git a/packages/sync-rules/test/src/table_valued_function_queries.test.ts b/packages/sync-rules/test/src/table_valued_function_queries.test.ts index 997270381..5b6d7c2a9 100644 --- a/packages/sync-rules/test/src/table_valued_function_queries.test.ts +++ b/packages/sync-rules/test/src/table_valued_function_queries.test.ts @@ -7,7 +7,7 @@ import { SqlParameterQuery } from '../../src/index.js'; import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; -import { identityBucketTransformer, PARSE_OPTIONS } from './util.js'; +import { EMPTY_DATA_SOURCE, PARSE_OPTIONS } from './util.js'; describe('table-valued function queries', () => { test('json_each(array param)', function () { @@ -19,16 +19,14 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '' }, { array: [1, 2, 3, null] }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3, null] }), 'mybucket') ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, @@ -50,16 +48,14 @@ describe('table-valued function queries', () => { new Map([[CompatibilityOption.fixedJsonExtract, true]]) ) }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '' }, { array: [1, 2, 3, null] }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3, null] }), 'mybucket') ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, @@ -70,13 +66,17 @@ describe('table-valued function queries', () => { test('json_each(static string)', function () { const sql = `SELECT json_each.value as v FROM json_each('[1,2,3]')`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); - expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) - ).toEqual([ + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, { bucket: 'mybucket[3]', priority: 3 } @@ -85,13 +85,17 @@ describe('table-valued function queries', () => { test('json_each(null)', function () { const sql = `SELECT json_each.value as v FROM json_each(null)`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); - expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) - ).toEqual([]); + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([]); }); test('json_each(array param not present)', function () { @@ -103,14 +107,13 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); - expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) - ).toEqual([]); + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([]); }); test('json_each(array param not present, ifnull)', function () { @@ -122,25 +125,28 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); - expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) - ).toEqual([]); + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([]); }); test('json_each on json_keys', function () { const sql = `SELECT value FROM json_each(json_keys('{"a": [], "b": 2, "c": null}'))`; - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['value']); - expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), identityBucketTransformer) - ).toEqual([ + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([ { bucket: 'mybucket["a"]', priority: 3 }, { bucket: 'mybucket["b"]', priority: 3 }, { bucket: 'mybucket["c"]', priority: 3 } @@ -156,16 +162,14 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['value']); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), 'mybucket') ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, @@ -182,16 +186,14 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['value']); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), 'mybucket') ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, @@ -208,16 +210,14 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); expect( - query.getStaticBucketDescriptions( - new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), - identityBucketTransformer - ) + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), 'mybucket') ).toEqual([ { bucket: 'mybucket[2]', priority: 3 }, { bucket: 'mybucket[3]', priority: 3 } @@ -234,7 +234,8 @@ describe('table-valued function queries', () => { ...PARSE_OPTIONS, accept_potentially_dangerous_queries: true }, - '1' + '1', + EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['project_id']); @@ -251,7 +252,7 @@ describe('table-valued function queries', () => { }, {} ), - identityBucketTransformer + 'mybucket' ) ).toEqual([{ bucket: 'mybucket[1]', priority: 3 }]); }); @@ -259,7 +260,13 @@ describe('table-valued function queries', () => { describe('dangerous queries', function () { function testDangerousQuery(sql: string) { test(sql, function () { - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toMatchObject([ { message: @@ -271,7 +278,13 @@ describe('table-valued function queries', () => { } function testSafeQuery(sql: string) { test(sql, function () { - const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as SqlParameterQuery; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect(query.usesDangerousRequestParameters).toEqual(false); }); diff --git a/packages/sync-rules/test/src/util.ts b/packages/sync-rules/test/src/util.ts index 4c8946186..7e714815f 100644 --- a/packages/sync-rules/test/src/util.ts +++ b/packages/sync-rules/test/src/util.ts @@ -7,7 +7,13 @@ import { RequestJwtPayload, RequestParameters, SourceTableInterface, - StaticSchema + StaticSchema, + BucketDataSourceDefinition, + TablePattern, + CreateSourceParams, + BucketDataSource, + SourceSchema, + ColumnDefinition } from '../../src/index.js'; export class TestSourceTable implements SourceTableInterface { @@ -82,3 +88,27 @@ export function normalizeQuerierOptions( export function identityBucketTransformer(id: string) { return id; } + +/** + * Empty data source that can be used for testing parameter queries, where most of the functionality here is not used. + */ +export const EMPTY_DATA_SOURCE: BucketDataSourceDefinition = { + defaultBucketPrefix: 'mybucket', + bucketParameters: [], + // These are not used in the tests. + getSourceTables: function (): Set { + return new Set(); + }, + createDataSource: function (params: CreateSourceParams): BucketDataSource { + throw new Error('Function not implemented.'); + }, + tableSyncsData: function (table: SourceTableInterface): boolean { + throw new Error('Function not implemented.'); + }, + resolveResultSets: function (schema: SourceSchema, tables: Record>): void { + throw new Error('Function not implemented.'); + }, + debugWriteOutputTables: function (result: Record): void { + throw new Error('Function not implemented.'); + } +}; From 1075d042ee25dbce5df1d5d0e22936c350b58047 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 20:01:55 +0200 Subject: [PATCH 18/33] Remove some unused descriptorName references. --- packages/sync-rules/src/BaseSqlDataQuery.ts | 6 --- .../sync-rules/src/SqlBucketDescriptor.ts | 2 +- packages/sync-rules/src/SqlDataQuery.ts | 2 - .../src/events/SqlEventDescriptor.ts | 2 +- .../src/events/SqlEventSourceQuery.ts | 3 +- packages/sync-rules/src/streams/from_sql.ts | 1 - .../sync-rules/test/src/data_queries.test.ts | 46 ++++++++----------- 7 files changed, 23 insertions(+), 39 deletions(-) diff --git a/packages/sync-rules/src/BaseSqlDataQuery.ts b/packages/sync-rules/src/BaseSqlDataQuery.ts index 915327135..7bec16db8 100644 --- a/packages/sync-rules/src/BaseSqlDataQuery.ts +++ b/packages/sync-rules/src/BaseSqlDataQuery.ts @@ -34,7 +34,6 @@ export interface BaseSqlDataQueryOptions { sql: string; columns: SelectedColumn[]; extractors: RowValueExtractor[]; - descriptorName: string; bucketParameters: string[]; tools: SqlTools; errors?: SqlRuleError[]; @@ -70,10 +69,6 @@ export class BaseSqlDataQuery { */ readonly extractors: RowValueExtractor[]; - /** - * Bucket definition name. - */ - readonly descriptorName: string; /** * Bucket parameter names, without the `bucket.` prefix. * @@ -94,7 +89,6 @@ export class BaseSqlDataQuery { this.sql = options.sql; this.columns = options.columns; this.extractors = options.extractors; - this.descriptorName = options.descriptorName; this.bucketParameters = options.bucketParameters; this.tools = options.tools; this.errors = options.errors ?? []; diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 68eb492f8..d3e23e996 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -73,7 +73,7 @@ export class SqlBucketDescriptor implements BucketSource { if (this.bucketParametersInternal == null) { throw new Error('Bucket parameters must be defined'); } - const dataRows = SqlDataQuery.fromSql(this.name, this.bucketParametersInternal, sql, options, compatibility); + const dataRows = SqlDataQuery.fromSql(this.bucketParametersInternal, sql, options, compatibility); this.dataQueries.push(dataRows); diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index b8803dcaa..bec1daad2 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -19,7 +19,6 @@ export interface SqlDataQueryOptions extends BaseSqlDataQueryOptions { export class SqlDataQuery extends BaseSqlDataQuery { static fromSql( - descriptorName: string, bucketParameters: string[], sql: string, options: SyncRulesOptions, @@ -170,7 +169,6 @@ export class SqlDataQuery extends BaseSqlDataQuery { sql, filter, columns: q.columns ?? [], - descriptorName, bucketParameters, tools, errors, diff --git a/packages/sync-rules/src/events/SqlEventDescriptor.ts b/packages/sync-rules/src/events/SqlEventDescriptor.ts index 8ea79f67e..33c34b98e 100644 --- a/packages/sync-rules/src/events/SqlEventDescriptor.ts +++ b/packages/sync-rules/src/events/SqlEventDescriptor.ts @@ -22,7 +22,7 @@ export class SqlEventDescriptor { } addSourceQuery(sql: string, options: SyncRulesOptions): QueryParseResult { - const source = SqlEventSourceQuery.fromSql(this.name, sql, options, this.compatibility); + const source = SqlEventSourceQuery.fromSql(sql, options, this.compatibility); // Each source query should be for a unique table const existingSourceQuery = this.sourceQueries.find((q) => q.table == source.table); diff --git a/packages/sync-rules/src/events/SqlEventSourceQuery.ts b/packages/sync-rules/src/events/SqlEventSourceQuery.ts index 3558eb6d3..17d263e8c 100644 --- a/packages/sync-rules/src/events/SqlEventSourceQuery.ts +++ b/packages/sync-rules/src/events/SqlEventSourceQuery.ts @@ -25,7 +25,7 @@ export type EvaluatedEventRowWithErrors = { * Defines how a Replicated Row is mapped to source parameters for events. */ export class SqlEventSourceQuery extends BaseSqlDataQuery { - static fromSql(descriptor_name: string, sql: string, options: SyncRulesOptions, compatibility: CompatibilityContext) { + static fromSql(sql: string, options: SyncRulesOptions, compatibility: CompatibilityContext) { const parsed = parse(sql, { locationTracking: true }); const schema = options.schema; @@ -121,7 +121,6 @@ export class SqlEventSourceQuery extends BaseSqlDataQuery { sourceTable, table: alias, sql, - descriptorName: descriptor_name, columns: q.columns ?? [], extractors: extractors, tools, diff --git a/packages/sync-rules/src/streams/from_sql.ts b/packages/sync-rules/src/streams/from_sql.ts index 8f0b17681..e11c594b2 100644 --- a/packages/sync-rules/src/streams/from_sql.ts +++ b/packages/sync-rules/src/streams/from_sql.ts @@ -201,7 +201,6 @@ class SyncStreamCompiler { table: alias, sql: this.sql, columns: query.columns ?? [], - descriptorName: this.descriptorName, tools, extractors, // Streams don't have traditional parameters, and parameters aren't used in the rest of the stream implementation. diff --git a/packages/sync-rules/test/src/data_queries.test.ts b/packages/sync-rules/test/src/data_queries.test.ts index 6959f7dc3..b029e7304 100644 --- a/packages/sync-rules/test/src/data_queries.test.ts +++ b/packages/sync-rules/test/src/data_queries.test.ts @@ -5,7 +5,7 @@ import { ASSETS, BASIC_SCHEMA, identityBucketTransformer, PARSE_OPTIONS } from ' describe('data queries', () => { test('uses bucket id transformer', function () { const sql = 'SELECT * FROM assets WHERE assets.org_id = bucket.org_id'; - const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, '1#mybucket')).toEqual([ @@ -20,7 +20,7 @@ describe('data queries', () => { test('bucket parameters = query', function () { const sql = 'SELECT * FROM assets WHERE assets.org_id = bucket.org_id'; - const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, 'mybucket')).toEqual([ @@ -37,7 +37,7 @@ describe('data queries', () => { test('bucket parameters IN query', function () { const sql = 'SELECT * FROM assets WHERE bucket.category IN assets.categories'; - const query = SqlDataQuery.fromSql('mybucket', ['category'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['category'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); expect( @@ -60,7 +60,7 @@ describe('data queries', () => { test('static IN data query', function () { const sql = `SELECT * FROM assets WHERE 'green' IN assets.categories`; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); expect( @@ -80,7 +80,7 @@ describe('data queries', () => { test('data IN static query', function () { const sql = `SELECT * FROM assets WHERE assets.condition IN '["good","great"]'`; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'good' }, 'mybucket')).toMatchObject([ @@ -96,7 +96,7 @@ describe('data queries', () => { test('table alias', function () { const sql = 'SELECT * FROM assets as others WHERE others.org_id = bucket.org_id'; - const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, 'mybucket')).toEqual([ @@ -113,7 +113,6 @@ describe('data queries', () => { const schema = BASIC_SCHEMA; const q1 = SqlDataQuery.fromSql( - 'q1', ['user_id'], `SELECT * FROM assets WHERE owner_id = bucket.user_id`, PARSE_OPTIONS, @@ -132,7 +131,6 @@ describe('data queries', () => { ]); const q2 = SqlDataQuery.fromSql( - 'q1', ['user_id'], ` SELECT id :: integer as id, @@ -167,7 +165,6 @@ describe('data queries', () => { test('validate columns', () => { const schema = BASIC_SCHEMA; const q1 = SqlDataQuery.fromSql( - 'q1', ['user_id'], 'SELECT id, name, count FROM assets WHERE owner_id = bucket.user_id', { ...PARSE_OPTIONS, schema }, @@ -176,7 +173,6 @@ describe('data queries', () => { expect(q1.errors).toEqual([]); const q2 = SqlDataQuery.fromSql( - 'q2', ['user_id'], 'SELECT id, upper(description) as d FROM assets WHERE other_id = bucket.user_id', { ...PARSE_OPTIONS, schema }, @@ -194,7 +190,6 @@ describe('data queries', () => { ]); const q3 = SqlDataQuery.fromSql( - 'q3', ['user_id'], 'SELECT id, description, * FROM nope WHERE other_id = bucket.user_id', { ...PARSE_OPTIONS, schema }, @@ -207,7 +202,7 @@ describe('data queries', () => { } ]); - const q4 = SqlDataQuery.fromSql('q4', [], 'SELECT * FROM other', { ...PARSE_OPTIONS, schema }, compatibility); + const q4 = SqlDataQuery.fromSql([], 'SELECT * FROM other', { ...PARSE_OPTIONS, schema }, compatibility); expect(q4.errors).toMatchObject([ { message: `Query must return an "id" column`, @@ -216,7 +211,6 @@ describe('data queries', () => { ]); const q5 = SqlDataQuery.fromSql( - 'q5', [], 'SELECT other_id as id, * FROM other', { ...PARSE_OPTIONS, schema }, @@ -227,7 +221,7 @@ describe('data queries', () => { test('invalid query - invalid IN', function () { const sql = 'SELECT * FROM assets WHERE assets.category IN bucket.categories'; - const query = SqlDataQuery.fromSql('mybucket', ['categories'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['categories'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { type: 'fatal', message: 'Cannot use bucket parameters on the right side of IN operators' } ]); @@ -235,7 +229,7 @@ describe('data queries', () => { test('invalid query - not all parameters used', function () { const sql = 'SELECT * FROM assets WHERE 1'; - const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { type: 'fatal', message: 'Query must cover all bucket parameters. Expected: ["bucket.org_id"] Got: []' } ]); @@ -243,7 +237,7 @@ describe('data queries', () => { test('invalid query - parameter not defined', function () { const sql = 'SELECT * FROM assets WHERE assets.org_id = bucket.org_id'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { type: 'fatal', message: 'Query must cover all bucket parameters. Expected: [] Got: ["bucket.org_id"]' } ]); @@ -251,25 +245,25 @@ describe('data queries', () => { test('invalid query - function on parameter (1)', function () { const sql = 'SELECT * FROM assets WHERE assets.org_id = upper(bucket.org_id)'; - const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([{ type: 'fatal', message: 'Cannot use bucket parameters in expressions' }]); }); test('invalid query - function on parameter (2)', function () { const sql = 'SELECT * FROM assets WHERE assets.org_id = upper(bucket.org_id)'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([{ type: 'fatal', message: 'Cannot use bucket parameters in expressions' }]); }); test('invalid query - match clause in select', () => { const sql = 'SELECT id, (bucket.org_id = assets.org_id) as org_matches FROM assets where org_id = bucket.org_id'; - const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors[0].message).toMatch(/Parameter match expression is not allowed here/); }); test('case-sensitive queries (1)', () => { const sql = 'SELECT * FROM Assets'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "Assets" instead.` } ]); @@ -277,7 +271,7 @@ describe('data queries', () => { test('case-sensitive queries (2)', () => { const sql = 'SELECT *, Name FROM assets'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "Name" instead.` } ]); @@ -285,7 +279,7 @@ describe('data queries', () => { test('case-sensitive queries (3)', () => { const sql = 'SELECT * FROM assets WHERE Archived = False'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "Archived" instead.` } ]); @@ -294,7 +288,7 @@ describe('data queries', () => { test.skip('case-sensitive queries (4)', () => { // Cannot validate table alias yet const sql = 'SELECT * FROM assets as myAssets'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "myAssets" instead.` } ]); @@ -303,7 +297,7 @@ describe('data queries', () => { test.skip('case-sensitive queries (5)', () => { // Cannot validate table alias yet const sql = 'SELECT * FROM assets myAssets'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "myAssets" instead.` } ]); @@ -312,7 +306,7 @@ describe('data queries', () => { test.skip('case-sensitive queries (6)', () => { // Cannot validate anything with a schema yet const sql = 'SELECT * FROM public.ASSETS'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "ASSETS" instead.` } ]); @@ -321,7 +315,7 @@ describe('data queries', () => { test.skip('case-sensitive queries (7)', () => { // Cannot validate schema yet const sql = 'SELECT * FROM PUBLIC.assets'; - const query = SqlDataQuery.fromSql('mybucket', [], sql, PARSE_OPTIONS, compatibility); + const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toMatchObject([ { message: `Unquoted identifiers are converted to lower-case. Use "PUBLIC" instead.` } ]); From fb6f76a33dc30736440476ce7e5202c23fb21962 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 21:03:15 +0200 Subject: [PATCH 19/33] Refactor parameter lookups to make the names configurable. --- .../register-data-storage-parameter-tests.ts | 37 ++- .../register-parameter-compacting-tests.ts | 4 +- .../test/src/sync/BucketChecksumState.test.ts | 4 +- .../sync-rules/src/BucketParameterQuerier.ts | 7 +- packages/sync-rules/src/HydrationState.ts | 12 +- packages/sync-rules/src/SqlParameterQuery.ts | 37 +-- packages/sync-rules/src/streams/filter.ts | 68 +++-- packages/sync-rules/src/streams/parameter.ts | 6 +- packages/sync-rules/src/streams/stream.ts | 79 +----- packages/sync-rules/src/streams/variant.ts | 105 ++++++- .../test/src/parameter_queries.test.ts | 257 +++++++++--------- packages/sync-rules/test/src/streams.test.ts | 58 ++-- .../sync-rules/test/src/sync_rules.test.ts | 8 +- 13 files changed, 378 insertions(+), 304 deletions(-) diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index 679e316e7..ff39db906 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -3,6 +3,7 @@ import { ParameterLookup, RequestParameters } from '@powersync/service-sync-rule import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; import { TEST_TABLE } from './util.js'; +import { ParameterLookupScope } from '@powersync/service-sync-rules/src/HydrationState.js'; /** * @example @@ -15,6 +16,8 @@ import { TEST_TABLE } from './util.js'; * ``` */ export function registerDataStorageParameterTests(generateStorageFactory: storage.TestStorageFactory) { + const MYBUCKET_1: ParameterLookupScope = { lookupName: 'mybucket', queryId: '1' }; + test('save and load parameters', async () => { await using factory = await generateStorageFactory(); const syncRules = await factory.updateSyncRules({ @@ -57,7 +60,7 @@ bucket_definitions: }); const checkpoint = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint.getParameterSets([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + const parameters = await checkpoint.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); expect(parameters).toEqual([ { group_id: 'group1a' @@ -105,7 +108,7 @@ bucket_definitions: }); const checkpoint2 = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint2.getParameterSets([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + const parameters = await checkpoint2.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); expect(parameters).toEqual([ { group_id: 'group2' @@ -113,7 +116,7 @@ bucket_definitions: ]); // Use the checkpoint to get older data if relevant - const parameters2 = await checkpoint1.getParameterSets([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + const parameters2 = await checkpoint1.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); expect(parameters2).toEqual([ { group_id: 'group1' @@ -182,8 +185,8 @@ bucket_definitions: // association of `list1`::`todo2` const checkpoint = await bucketStorage.getCheckpoint(); const parameters = await checkpoint.getParameterSets([ - ParameterLookup.normalized('mybucket', '1', ['list1']), - ParameterLookup.normalized('mybucket', '1', ['list2']) + ParameterLookup.normalized(MYBUCKET_1, ['list1']), + ParameterLookup.normalized(MYBUCKET_1, ['list2']) ]); expect(parameters.sort((a, b) => (a.todo_id as string).localeCompare(b.todo_id as string))).toEqual([ @@ -230,17 +233,11 @@ bucket_definitions: const checkpoint = await bucketStorage.getCheckpoint(); - const parameters1 = await checkpoint.getParameterSets([ - ParameterLookup.normalized('mybucket', '1', [314n, 314, 3.14]) - ]); + const parameters1 = await checkpoint.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, [314n, 314, 3.14])]); expect(parameters1).toEqual([TEST_PARAMS]); - const parameters2 = await checkpoint.getParameterSets([ - ParameterLookup.normalized('mybucket', '1', [314, 314n, 3.14]) - ]); + const parameters2 = await checkpoint.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, [314, 314n, 3.14])]); expect(parameters2).toEqual([TEST_PARAMS]); - const parameters3 = await checkpoint.getParameterSets([ - ParameterLookup.normalized('mybucket', '1', [314n, 314, 3]) - ]); + const parameters3 = await checkpoint.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, [314n, 314, 3])]); expect(parameters3).toEqual([]); }); @@ -294,7 +291,7 @@ bucket_definitions: const checkpoint = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint.getParameterSets([ - ParameterLookup.normalized('mybucket', '1', [1152921504606846976n]) + ParameterLookup.normalized(MYBUCKET_1, [1152921504606846976n]) ]); expect(parameters1).toEqual([TEST_PARAMS]); }); @@ -335,7 +332,7 @@ bucket_definitions: const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; const lookups = querier.parameterQueryLookups; - expect(lookups).toEqual([ParameterLookup.normalized('by_workspace', '1', ['u1'])]); + expect(lookups).toEqual([ParameterLookup.normalized({ lookupName: 'by_workspace', queryId: '1' }, ['u1'])]); const parameter_sets = await checkpoint.getParameterSets(lookups); expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]); @@ -408,7 +405,7 @@ bucket_definitions: const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; const lookups = querier.parameterQueryLookups; - expect(lookups).toEqual([ParameterLookup.normalized('by_public_workspace', '1', [])]); + expect(lookups).toEqual([ParameterLookup.normalized({ lookupName: 'by_public_workspace', queryId: '1' }, [])]); const parameter_sets = await checkpoint.getParameterSets(lookups); parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); @@ -510,8 +507,8 @@ bucket_definitions: const lookups = querier.parameterQueryLookups; expect(lookups).toEqual([ - ParameterLookup.normalized('by_workspace', '1', []), - ParameterLookup.normalized('by_workspace', '2', ['u1']) + ParameterLookup.normalized({ lookupName: 'by_workspace', queryId: '1' }, []), + ParameterLookup.normalized({ lookupName: 'by_workspace', queryId: '2' }, ['u1']) ]); const parameter_sets = await checkpoint.getParameterSets(lookups); @@ -561,7 +558,7 @@ bucket_definitions: const checkpoint = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint.getParameterSets([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + const parameters = await checkpoint.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); expect(parameters).toEqual([]); }); diff --git a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts index 59499fa02..0b63b62ba 100644 --- a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts @@ -40,7 +40,7 @@ bucket_definitions: await batch.commit('1/1'); }); - const lookup = ParameterLookup.normalized('test', '1', ['t1']); + const lookup = ParameterLookup.normalized({ lookupName: 'test', queryId: '1' }, ['t1']); const checkpoint1 = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint1.getParameterSets([lookup]); @@ -151,7 +151,7 @@ bucket_definitions: await batch.commit('3/1'); }); - const lookup = ParameterLookup.normalized('test', '1', ['u1']); + const lookup = ParameterLookup.normalized({ lookupName: 'test', queryId: '1' }, ['u1']); const checkpoint1 = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint1.getParameterSets([lookup]); diff --git a/packages/service-core/test/src/sync/BucketChecksumState.test.ts b/packages/service-core/test/src/sync/BucketChecksumState.test.ts index 7ef36ac05..2d3fbde45 100644 --- a/packages/service-core/test/src/sync/BucketChecksumState.test.ts +++ b/packages/service-core/test/src/sync/BucketChecksumState.test.ts @@ -504,7 +504,7 @@ bucket_definitions: const line = (await state.buildNextCheckpointLine({ base: storage.makeCheckpoint(1n, (lookups) => { - expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]); + expect(lookups).toEqual([ParameterLookup.normalized({ lookupName: 'by_project', queryId: '1' }, ['u1'])]); return [{ id: 1 }, { id: 2 }]; }), writeCheckpoint: null, @@ -565,7 +565,7 @@ bucket_definitions: // Now we get a new line const line2 = (await state.buildNextCheckpointLine({ base: storage.makeCheckpoint(2n, (lookups) => { - expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]); + expect(lookups).toEqual([ParameterLookup.normalized({ lookupName: 'by_project', queryId: '1' }, ['u1'])]); return [{ id: 1 }, { id: 2 }, { id: 3 }]; }), writeCheckpoint: null, diff --git a/packages/sync-rules/src/BucketParameterQuerier.ts b/packages/sync-rules/src/BucketParameterQuerier.ts index c3a947b43..025744beb 100644 --- a/packages/sync-rules/src/BucketParameterQuerier.ts +++ b/packages/sync-rules/src/BucketParameterQuerier.ts @@ -1,4 +1,5 @@ import { ResolvedBucket } from './BucketDescription.js'; +import { ParameterLookupScope } from './HydrationState.js'; import { RequestedStream } from './SqlSyncRules.js'; import { RequestParameters, SqliteJsonRow, SqliteJsonValue } from './types.js'; import { normalizeParameterValue } from './utils.js'; @@ -92,15 +93,15 @@ export class ParameterLookup { // bucket definition name, parameter query index, ...lookup values readonly values: SqliteJsonValue[]; - static normalized(bucketDefinitionName: string, queryIndex: string, values: SqliteJsonValue[]): ParameterLookup { - return new ParameterLookup([bucketDefinitionName, queryIndex, ...values.map(normalizeParameterValue)]); + static normalized(scope: ParameterLookupScope, values: SqliteJsonValue[]): ParameterLookup { + return new ParameterLookup([scope.lookupName, scope.queryId, ...values.map(normalizeParameterValue)]); } /** * * @param values must be pre-normalized (any integer converted into bigint) */ - constructor(values: SqliteJsonValue[]) { + private constructor(values: SqliteJsonValue[]) { this.values = values; } } diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts index f3723b5cf..ed31e0aaf 100644 --- a/packages/sync-rules/src/HydrationState.ts +++ b/packages/sync-rules/src/HydrationState.ts @@ -6,7 +6,7 @@ export interface BucketSourceState { bucketPrefix: string; } -export interface BucketParameterLookupSourceState { +export interface ParameterLookupScope { /** The lookup name + queryid is used to reference the parameter lookup record. */ lookupName: string; queryId: string; @@ -20,7 +20,7 @@ export interface BucketParameterLookupSourceState { */ export interface HydrationState< T extends BucketSourceState = BucketSourceState, - U extends BucketParameterLookupSourceState = BucketParameterLookupSourceState + U extends ParameterLookupScope = ParameterLookupScope > { /** * Given a bucket data source definition, get the bucket prefix to use for it. @@ -30,7 +30,7 @@ export interface HydrationState< /** * Given a bucket parameter lookup definition, get the persistence name to use. */ - getParameterLookupState(source: BucketParameterLookupSourceDefinition): U; + getParameterLookupScope(source: BucketParameterLookupSourceDefinition): U; } /** @@ -44,7 +44,7 @@ export const DEFAULT_HYDRATION_STATE: HydrationState = { bucketPrefix: source.defaultBucketPrefix }; }, - getParameterLookupState(source) { + getParameterLookupScope(source) { return { lookupName: source.defaultLookupName, queryId: source.defaultQueryId @@ -67,7 +67,7 @@ export class VersionedHydrationState implements HydrationState { }; } - getParameterLookupState(source: BucketParameterLookupSourceDefinition): BucketParameterLookupSourceState { + getParameterLookupScope(source: BucketParameterLookupSourceDefinition): ParameterLookupScope { // No transformations applied here return { lookupName: source.defaultLookupName, @@ -85,7 +85,7 @@ export class BucketIdTransformerHydrationState implements HydrationState { }; } - getParameterLookupState(source: BucketParameterLookupSourceDefinition): BucketParameterLookupSourceState { + getParameterLookupScope(source: BucketParameterLookupSourceDefinition): ParameterLookupScope { // No transformations applied here return { lookupName: source.defaultLookupName, diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 9e41dd228..9bd4303b4 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -44,7 +44,7 @@ import { } from './types.js'; import { filterJsonRow, getBucketId, isJsonValue, isSelectStatement, normalizeParameterValue } from './utils.js'; import { DetectRequestParameters } from './validators.js'; -import { HydrationState, resolveHydrationState } from './HydrationState.js'; +import { ParameterLookupScope, HydrationState, resolveHydrationState } from './HydrationState.js'; export interface SqlParameterQueryOptions { sourceTable: TablePattern; @@ -350,9 +350,11 @@ export class SqlParameterQuery createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { const hydrationState = resolveHydrationState(params); const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; + const lookupState = hydrationState.getParameterLookupScope(this); + return { pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { - const q = this.getBucketParameterQuerier(options.globalParameters, ['default'], bucketPrefix); + const q = this.getBucketParameterQuerier(options.globalParameters, ['default'], bucketPrefix, lookupState); result.queriers.push(q); } }; @@ -360,10 +362,12 @@ export class SqlParameterQuery createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { // FIXME: Use HydrationState for lookups. + const hydrationState = resolveHydrationState(params); + const lookupState = hydrationState.getParameterLookupScope(this); return { evaluateParameterRow: (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { if (this.tableSyncsParameters(sourceTable)) { - return this.evaluateParameterRow(row); + return this.evaluateParameterRow(lookupState, row); } else { return []; } @@ -374,7 +378,7 @@ export class SqlParameterQuery /** * Given a replicated row, results an array of bucket parameter rows to persist. */ - evaluateParameterRow(row: SqliteRow): EvaluatedParametersResult[] { + evaluateParameterRow(scope: ParameterLookupScope, row: SqliteRow): EvaluatedParametersResult[] { const tables = { [this.table.nameInSchema]: row }; @@ -382,8 +386,8 @@ export class SqlParameterQuery const filterParameters = this.filter.filterRow(tables); let result: EvaluatedParametersResult[] = []; for (let filterParamSet of filterParameters) { - let lookup: SqliteJsonValue[] = [this.descriptorName, this.queryId]; - lookup.push( + let lookupValues: SqliteJsonValue[] = []; + lookupValues.push( ...this.inputParameters.map((param) => { return normalizeParameterValue(param.filteredRowToLookupValue(filterParamSet)); }) @@ -393,7 +397,7 @@ export class SqlParameterQuery const role: EvaluatedParameters = { bucketParameters: data.map((row) => filterJsonRow(row)), - lookup: new ParameterLookup(lookup) + lookup: ParameterLookup.normalized(scope, lookupValues) }; result.push(role); } @@ -458,12 +462,12 @@ export class SqlParameterQuery * * Each lookup is [bucket definition name, parameter query index, ...lookup values] */ - getLookups(parameters: RequestParameters): ParameterLookup[] { + getLookups(scope: ParameterLookupScope, parameters: RequestParameters): ParameterLookup[] { if (!this.expandedInputParameter) { - let lookup: SqliteJsonValue[] = [this.descriptorName, this.queryId]; + let lookupValues: SqliteJsonValue[] = []; let valid = true; - lookup.push( + lookupValues.push( ...this.inputParameters.map((param): SqliteJsonValue => { // Scalar value const value = param.parametersToLookupValue(parameters); @@ -479,7 +483,7 @@ export class SqlParameterQuery if (!valid) { return []; } - return [new ParameterLookup(lookup)]; + return [ParameterLookup.normalized(scope, lookupValues)]; } else { const arrayString = this.expandedInputParameter.parametersToLookupValue(parameters); @@ -498,10 +502,10 @@ export class SqlParameterQuery return values .map((expandedValue) => { - let lookup: SqliteJsonValue[] = [this.descriptorName, this.queryId]; + let lookupValues: SqliteJsonValue[] = []; let valid = true; const normalizedExpandedValue = normalizeParameterValue(expandedValue); - lookup.push( + lookupValues.push( ...this.inputParameters.map((param): SqliteJsonValue => { if (param == this.expandedInputParameter) { // Expand array value @@ -523,7 +527,7 @@ export class SqlParameterQuery return null; } - return new ParameterLookup(lookup); + return ParameterLookup.normalized(scope, lookupValues); }) .filter((lookup) => lookup != null) as ParameterLookup[]; } @@ -532,9 +536,10 @@ export class SqlParameterQuery getBucketParameterQuerier( requestParameters: RequestParameters, reasons: BucketInclusionReason[], - bucketPrefix: string + bucketPrefix: string, + scope: ParameterLookupScope ): BucketParameterQuerier { - const lookups = this.getLookups(requestParameters); + const lookups = this.getLookups(scope, requestParameters); if (lookups.length == 0) { // This typically happens when the query is pre-filtered using a where clause // on the parameters, and does not depend on the database state. diff --git a/packages/sync-rules/src/streams/filter.ts b/packages/sync-rules/src/streams/filter.ts index ca149f3b2..38f188ca9 100644 --- a/packages/sync-rules/src/streams/filter.ts +++ b/packages/sync-rules/src/streams/filter.ts @@ -4,6 +4,7 @@ import { EvaluatedParametersResult, ParameterMatchClause, ParameterValueClause, + RequestParameters, RowValueClause, SqliteJsonValue, SqliteRow @@ -23,6 +24,7 @@ import { CreateSourceParams } from '../BucketSource.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; +import { HydrationState, ParameterLookupScope, resolveHydrationState } from '../HydrationState.js'; /** * An intermediate representation of a `WHERE` clause for stream queries. @@ -264,24 +266,41 @@ export class Subquery { const column = this.column; - const evaluator: SubqueryEvaluator = { - parameterTable: this.table, - lookupSources(streamName) { - return innerVariants.map(([variant, id]) => { - return new SubqueryParameterLookupSource(evaluator, column, variant, id, streamName); - }); - }, - lookupsForRequest(parameters) { - const lookups: ParameterLookup[] = []; - - for (const [variant, id] of innerVariants) { + let lookupSources: BucketParameterLookupSourceDefinition[] = []; + let lookupsForRequest: (( + hydrationState: HydrationState + ) => (parameters: RequestParameters) => ParameterLookup[])[] = []; + + for (let [variant, id] of innerVariants) { + const source = new SubqueryParameterLookupSource(this.table, column, variant, id, context.streamName); + lookupSources.push(source); + lookupsForRequest.push((hydrationState: HydrationState) => { + const scope = hydrationState.getParameterLookupScope(source); + return (parameters: RequestParameters) => { + const lookups: ParameterLookup[] = []; const instantiations = variant.findStaticInstantiations(parameters); for (const instantiation of instantiations) { - lookups.push(ParameterLookup.normalized(context.streamName, id, instantiation)); + lookups.push(ParameterLookup.normalized(scope, instantiation)); } - } + return lookups; + }; + }); + } - return lookups; + const evaluator: SubqueryEvaluator = { + parameterTable: this.table, + lookupSources() { + return lookupSources; + }, + hydrateLookupsForRequest(hydrationState: HydrationState) { + const hydrated = lookupsForRequest.map((fn) => fn(hydrationState)); + return (parameters: RequestParameters) => { + const lookups: ParameterLookup[] = []; + for (const getLookups of hydrated) { + lookups.push(...getLookups(parameters)); + } + return lookups; + }; } }; @@ -517,7 +536,7 @@ export class EvaluateSimpleCondition extends FilterOperator { export class SubqueryParameterLookupSource implements BucketParameterLookupSourceDefinition { constructor( - private subquery: SubqueryEvaluator, + private parameterTable: TablePattern, private column: RowValueClause, private innerVariant: StreamVariant, public readonly defaultQueryId: string, @@ -530,7 +549,7 @@ export class SubqueryParameterLookupSource implements BucketParameterLookupSourc getSourceTables(): Set { let result = new Set(); - result.add(this.subquery.parameterTable); + result.add(this.parameterTable); return result; } @@ -547,8 +566,12 @@ export class SubqueryParameterLookupSource implements BucketParameterLookupSourc * @param sourceTable A table we depend on in a subquery. * @param row Row data to index. */ - evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] { - if (this.subquery.parameterTable.matches(sourceTable)) { + evaluateParameterRow( + lookupScope: ParameterLookupScope, + sourceTable: SourceTableInterface, + row: SqliteRow + ): EvaluatedParametersResult[] { + if (this.parameterTable.matches(sourceTable)) { // Theoretically we're doing duplicate work by doing this for each innerVariant in a subquery. // In practice, we don't have more than one innerVariant per subquery right now, so this is fine. const value = this.column.evaluate({ [sourceTable.name]: row }); @@ -558,8 +581,7 @@ export class SubqueryParameterLookupSource implements BucketParameterLookupSourc const lookups: ParameterLookup[] = []; for (const instantiation of this.innerVariant.instantiationsForRow({ sourceTable, record: row })) { - // TODO: dynamic lookup name and query id - lookups.push(ParameterLookup.normalized(this.defaultLookupName, this.defaultQueryId, instantiation)); + lookups.push(ParameterLookup.normalized(lookupScope, instantiation)); } // The row of the subquery. Since we only support subqueries with a single column, we unconditionally name the @@ -575,14 +597,16 @@ export class SubqueryParameterLookupSource implements BucketParameterLookupSourc } createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { + const hydrationState = resolveHydrationState(params); + const lookupScope = hydrationState.getParameterLookupScope(this); return { evaluateParameterRow: (sourceTable, row) => { - return this.evaluateParameterRow(sourceTable, row); + return this.evaluateParameterRow(lookupScope, sourceTable, row); } }; } tableSyncsParameters(table: SourceTableInterface): boolean { - return this.subquery.parameterTable.matches(table); + return this.parameterTable.matches(table); } } diff --git a/packages/sync-rules/src/streams/parameter.ts b/packages/sync-rules/src/streams/parameter.ts index 0b23b58ab..75c6b44e4 100644 --- a/packages/sync-rules/src/streams/parameter.ts +++ b/packages/sync-rules/src/streams/parameter.ts @@ -1,5 +1,6 @@ import { ParameterLookup } from '../BucketParameterQuerier.js'; import { BucketParameterLookupSource, BucketParameterLookupSourceDefinition } from '../BucketSource.js'; +import { HydrationState } from '../HydrationState.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { TablePattern } from '../TablePattern.js'; import { @@ -39,9 +40,8 @@ export interface BucketParameter { export interface SubqueryEvaluator { parameterTable: TablePattern; - lookupsForRequest(params: RequestParameters): ParameterLookup[]; - - lookupSources(streamName: string): BucketParameterLookupSourceDefinition[]; + lookupSources(): BucketParameterLookupSourceDefinition[]; + hydrateLookupsForRequest(hydrationState: HydrationState): (params: RequestParameters) => ParameterLookup[]; } export interface SubqueryLookups { diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index d87166d28..89496c75f 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -47,14 +47,16 @@ export class SyncStream implements BucketSource { this.data = data; this.dataSources = []; + this.parameterLookupSources = []; this.parameterQuerierSources = []; for (let variant of variants) { const dataSource = new SyncStreamDataSource(this, data, variant); this.dataSources.push(dataSource); - this.parameterQuerierSources.push(new SyncStreamParameterQuerierSource(this, variant, dataSource)); + const lookupSources = variant.lookupSources(); + this.parameterQuerierSources.push(variant.querierSource(this, dataSource)); + this.parameterLookupSources.push(...lookupSources); } - this.parameterLookupSources = variants.flatMap((variant) => variant.lookupSources(name)); } public get type(): BucketSourceType { @@ -142,76 +144,3 @@ export class SyncStreamDataSource implements BucketDataSourceDefinition { }; } } - -export class SyncStreamParameterQuerierSource implements BucketParameterQuerierSourceDefinition { - // We could eventually split this into a separate source per variant. - - constructor( - private stream: SyncStream, - private variant: StreamVariant, - public readonly querierDataSource: BucketDataSourceDefinition - ) {} - - /** - * Not relevant for sync streams. - */ - get bucketParameters() { - return []; - } - - createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { - const hydrationState = resolveHydrationState(params); - const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; - const stream = this.stream; - return { - pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions): void => { - const subscriptions = options.streams[stream.name] ?? []; - - if (!stream.subscribedToByDefault && !subscriptions.length) { - // The client is not subscribing to this stream, so don't query buckets related to it. - return; - } - - let hasExplicitDefaultSubscription = false; - for (const subscription of subscriptions) { - let subscriptionParams = options.globalParameters; - if (subscription.parameters != null) { - subscriptionParams = subscriptionParams.withAddedStreamParameters(subscription.parameters); - } else { - hasExplicitDefaultSubscription = true; - } - - this.queriersForSubscription(result, subscription, subscriptionParams, bucketPrefix); - } - - // If the stream is subscribed to by default and there is no explicit subscription that would match the default - // subscription, also include the default querier. - if (stream.subscribedToByDefault && !hasExplicitDefaultSubscription) { - this.queriersForSubscription(result, null, options.globalParameters, bucketPrefix); - } - } - }; - } - - private queriersForSubscription( - result: PendingQueriers, - subscription: RequestedStream | null, - params: RequestParameters, - bucketPrefix: string - ) { - const reason: BucketInclusionReason = subscription != null ? { subscription: subscription.opaque_id } : 'default'; - - try { - const querier = this.variant.querier(this.stream, reason, params, bucketPrefix); - if (querier) { - result.queriers.push(querier); - } - } catch (e) { - result.errors.push({ - descriptor: this.stream.name, - message: `Error evaluating bucket ids: ${e.message}`, - subscription: subscription ?? undefined - }); - } - } -} diff --git a/packages/sync-rules/src/streams/variant.ts b/packages/sync-rules/src/streams/variant.ts index 35e016330..61cc6c353 100644 --- a/packages/sync-rules/src/streams/variant.ts +++ b/packages/sync-rules/src/streams/variant.ts @@ -1,10 +1,15 @@ import { BucketInclusionReason, ResolvedBucket } from '../BucketDescription.js'; -import { BucketParameterQuerier, ParameterLookup } from '../BucketParameterQuerier.js'; +import { BucketParameterQuerier, ParameterLookup, PendingQueriers } from '../BucketParameterQuerier.js'; import { + BucketDataSourceDefinition, BucketParameterLookupSource, BucketParameterLookupSourceDefinition, + BucketParameterQuerierSource, + BucketParameterQuerierSourceDefinition, CreateSourceParams } from '../BucketSource.js'; +import { resolveHydrationState } from '../HydrationState.js'; +import { GetQuerierOptions, RequestedStream } from '../index.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { TablePattern } from '../TablePattern.js'; import { @@ -18,7 +23,7 @@ import { } from '../types.js'; import { isJsonValue, JSONBucketNameSerialize, normalizeParameterValue } from '../utils.js'; import { BucketParameter, SubqueryEvaluator } from './parameter.js'; -import { SyncStream } from './stream.js'; +import { SyncStream, SyncStreamDataSource } from './stream.js'; import { cartesianProduct } from './utils.js'; /** @@ -75,8 +80,12 @@ export class StreamVariant { return `${streamName}|${this.id}`; } - lookupSources(streamName: string): BucketParameterLookupSourceDefinition[] { - return this.subqueries.flatMap((subquery) => subquery.lookupSources(streamName)); + lookupSources(): BucketParameterLookupSourceDefinition[] { + return this.subqueries.flatMap((subquery) => subquery.lookupSources()); + } + + querierSource(stream: SyncStream, dataSource: SyncStreamDataSource): BucketParameterQuerierSourceDefinition { + return new SyncStreamParameterQuerierSource(stream, this, dataSource); } /** @@ -132,7 +141,8 @@ export class StreamVariant { stream: SyncStream, reason: BucketInclusionReason, params: RequestParameters, - bucketPrefix: string + bucketPrefix: string, + hydratedSubqueries: HydratedSubqueries ): BucketParameterQuerier | null { const instantiation = this.partiallyEvaluateParameters(params); if (instantiation == null) { @@ -161,7 +171,11 @@ export class StreamVariant { } for (const subquery of this.subqueries) { - subqueryToLookups.set(subquery, subquery.lookupsForRequest(params)); + const subqueryLookup = hydratedSubqueries.get(subquery); + if (subqueryLookup == null) { + throw new Error('Internal error, missing subquery lookup'); + } + subqueryToLookups.set(subquery, subqueryLookup(params)); } const staticBuckets: ResolvedBucket[] = []; @@ -331,3 +345,82 @@ export interface SubqueryRequestFilter { matches(params: RequestParameters, results: SqliteJsonValue[]): boolean; } export type RequestFilter = StaticRequestFilter | SubqueryRequestFilter; + +export class SyncStreamParameterQuerierSource implements BucketParameterQuerierSourceDefinition { + constructor( + private stream: SyncStream, + private variant: StreamVariant, + public readonly querierDataSource: BucketDataSourceDefinition + ) {} + + /** + * Not relevant for sync streams. + */ + get bucketParameters() { + return []; + } + + createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { + const hydrationState = resolveHydrationState(params); + const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; + const stream = this.stream; + + const hydratedSubqueries: HydratedSubqueries = new Map( + this.variant.subqueries.map((s) => [s, s.hydrateLookupsForRequest(hydrationState)]) + ); + + return { + pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions): void => { + const subscriptions = options.streams[stream.name] ?? []; + + if (!stream.subscribedToByDefault && !subscriptions.length) { + // The client is not subscribing to this stream, so don't query buckets related to it. + return; + } + + let hasExplicitDefaultSubscription = false; + for (const subscription of subscriptions) { + let subscriptionParams = options.globalParameters; + if (subscription.parameters != null) { + subscriptionParams = subscriptionParams.withAddedStreamParameters(subscription.parameters); + } else { + hasExplicitDefaultSubscription = true; + } + + this.queriersForSubscription(result, subscription, subscriptionParams, bucketPrefix, hydratedSubqueries); + } + + // If the stream is subscribed to by default and there is no explicit subscription that would match the default + // subscription, also include the default querier. + if (stream.subscribedToByDefault && !hasExplicitDefaultSubscription) { + this.queriersForSubscription(result, null, options.globalParameters, bucketPrefix, hydratedSubqueries); + } + } + }; + } + + private queriersForSubscription( + result: PendingQueriers, + subscription: RequestedStream | null, + params: RequestParameters, + bucketPrefix: string, + hydratedSubqueries: HydratedSubqueries + ) { + const reason: BucketInclusionReason = subscription != null ? { subscription: subscription.opaque_id } : 'default'; + + try { + const querier = this.variant.querier(this.stream, reason, params, bucketPrefix, hydratedSubqueries); + if (querier) { + result.queriers.push(querier); + } + } catch (e) { + result.errors.push({ + descriptor: this.stream.name, + message: `Error evaluating bucket ids: ${e.message}`, + subscription: subscription ?? undefined + }); + } + } +} + +type HydratedSubqueries = Map ParameterLookup[]>; diff --git a/packages/sync-rules/test/src/parameter_queries.test.ts b/packages/sync-rules/test/src/parameter_queries.test.ts index 653eb56ed..483f4e919 100644 --- a/packages/sync-rules/test/src/parameter_queries.test.ts +++ b/packages/sync-rules/test/src/parameter_queries.test.ts @@ -2,8 +2,13 @@ import { describe, expect, test } from 'vitest'; import { ParameterLookup, SqlParameterQuery } from '../../src/index.js'; import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; import { BASIC_SCHEMA, EMPTY_DATA_SOURCE, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; +import { ParameterLookupScope } from '../../src/HydrationState.js'; describe('parameter queries', () => { + const MYBUCKET_1: ParameterLookupScope = { + lookupName: 'mybucket', + queryId: '1' + }; test('token_parameters IN query', function () { const sql = 'SELECT id as group_id FROM groups WHERE token_parameters.user_id IN groups.user_ids'; const query = SqlParameterQuery.fromSql( @@ -14,9 +19,11 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'group1', user_ids: JSON.stringify(['user1', 'user2']) })).toEqual([ + expect( + query.evaluateParameterRow(MYBUCKET_1, { id: 'group1', user_ids: JSON.stringify(['user1', 'user2']) }) + ).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1']), bucketParameters: [ { group_id: 'group1' @@ -24,7 +31,7 @@ describe('parameter queries', () => { ] }, { - lookup: ParameterLookup.normalized('mybucket', '1', ['user2']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['user2']), bucketParameters: [ { group_id: 'group1' @@ -34,11 +41,12 @@ describe('parameter queries', () => { ]); expect( query.getLookups( + MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1' }) ) - ).toEqual([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + ).toEqual([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); }); test('IN token_parameters query', function () { @@ -51,9 +59,9 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'region1', name: 'colorado' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'region1', name: 'colorado' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['colorado']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['colorado']), bucketParameters: [ { region_id: 'region1' @@ -63,13 +71,14 @@ describe('parameter queries', () => { ]); expect( query.getLookups( + MYBUCKET_1, normalizeTokenParameters({ region_names: JSON.stringify(['colorado', 'texas']) }) ) ).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['colorado']), - ParameterLookup.normalized('mybucket', '1', ['texas']) + ParameterLookup.normalized(MYBUCKET_1, ['colorado']), + ParameterLookup.normalized(MYBUCKET_1, ['texas']) ]); }); @@ -85,9 +94,9 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); // Note: We don't need to worry about numeric vs decimal types in the lookup - JSONB handles normalization for us. - expect(query.evaluateParameterRow({ int1: 314n, float1: 3.14, float2: 314 })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { int1: 314n, float1: 3.14, float2: 314 })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', [314n, 3.14, 314]), + lookup: ParameterLookup.normalized(MYBUCKET_1, [314n, 3.14, 314]), bucketParameters: [{ int1: 314n, float1: 3.14, float2: 314 }] } @@ -95,8 +104,8 @@ describe('parameter queries', () => { // Similarly, we don't need to worry about the types here. // This test just checks the current behavior. - expect(query.getLookups(normalizeTokenParameters({ int1: 314n, float1: 3.14, float2: 314 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [314n, 3.14, 314n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ int1: 314n, float1: 3.14, float2: 314 }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, [314n, 3.14, 314n]) ]); // We _do_ need to care about the bucket string representation. @@ -128,16 +137,17 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ + MYBUCKET_1; + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['test_param']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['test_param']), bucketParameters: [{ id: 'test_id' }] } ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['test']) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['test']) ]); }); @@ -152,16 +162,16 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['test_param']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['test_param']), bucketParameters: [{ id: 'test_id' }] } ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['TEST']) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['TEST']) ]); }); @@ -176,17 +186,17 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['test_param']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['test_param']), bucketParameters: [{ id: 'test_id' }] } ]); - expect(query.getLookups(normalizeTokenParameters({ some_param: { description: 'test_description' } }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['test_description']) - ]); + expect( + query.getLookups(MYBUCKET_1, normalizeTokenParameters({ some_param: { description: 'test_description' } })) + ).toEqual([ParameterLookup.normalized(MYBUCKET_1, ['test_description'])]); }); test('token parameter and binary operator', () => { @@ -200,8 +210,8 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(normalizeTokenParameters({ some_param: 3 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [5n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ some_param: 3 }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, [5n]) ]); }); @@ -216,11 +226,11 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(normalizeTokenParameters({ some_param: null }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [1n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ some_param: null }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, [1n]) ]); - expect(query.getLookups(normalizeTokenParameters({ some_param: 'test' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [0n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ some_param: 'test' }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, [0n]) ]); }); @@ -235,18 +245,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', [1n]), + lookup: ParameterLookup.normalized(MYBUCKET_1, [1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [0n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, [0n]) ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [1n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, [1n]) ]); }); @@ -261,18 +271,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', [1n]), + lookup: ParameterLookup.normalized(MYBUCKET_1, [1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [1n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, [1n]) ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [0n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, [0n]) ]); }); @@ -287,18 +297,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', [1n]), + lookup: ParameterLookup.normalized(MYBUCKET_1, [1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [0n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, [0n]) ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [1n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, [1n]) ]); }); @@ -313,18 +323,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', [1n]), + lookup: ParameterLookup.normalized(MYBUCKET_1, [1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [1n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, [1n]) ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', [0n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', is_admin: 123 }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, [0n]) ]); }); @@ -339,18 +349,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1', 1n]), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 1n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]) ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 0n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['user1', 0n]) ]); }); @@ -365,18 +375,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1', 1n]), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 1n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]) ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 0n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['user1', 0n]) ]); }); @@ -391,11 +401,11 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1']) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['user1']) ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 123 }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['123']) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 123 }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['123']) ]); }); @@ -410,15 +420,15 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1', role: null })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1', role: null })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', []), + lookup: ParameterLookup.normalized(MYBUCKET_1, []), bucketParameters: [{ id: 'user1' }] } ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', []) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, []) ]); }); @@ -436,19 +446,19 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1', 1n]), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 1n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]) ]); // Would not match any actual lookups - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 0n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['user1', 0n]) ]); }); @@ -464,16 +474,16 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1', 1n]), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]), bucketParameters: [{ user_id: 'user1' }] } ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['user1', 1n]) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]) ]); expect( @@ -496,9 +506,9 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ userId: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { userId: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1']), bucketParameters: [{ user_id: 'user1' }] } @@ -522,10 +532,10 @@ describe('parameter queries', () => { { message: `Unquoted identifiers are converted to lower-case. Use "userId" instead.` } ]); - expect(query.evaluateParameterRow({ userId: 'user1' })).toEqual([]); - expect(query.evaluateParameterRow({ userid: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { userId: 'user1' })).toEqual([]); + expect(query.evaluateParameterRow(MYBUCKET_1, { userid: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1']), bucketParameters: [{ user_id: 'user1' }] } @@ -613,15 +623,15 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'workspace1', visibility: 'public' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'workspace1', visibility: 'public' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', []), + lookup: ParameterLookup.normalized(MYBUCKET_1, []), bucketParameters: [{ workspace_id: 'workspace1' }] } ]); - expect(query.evaluateParameterRow({ id: 'workspace1', visibility: 'private' })).toEqual([]); + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'workspace1', visibility: 'private' })).toEqual([]); }); test('multiple different functions on token_parameter with AND', () => { @@ -637,16 +647,16 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['test_param', 'test_param']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['test_param', 'test_param']), bucketParameters: [{ id: 'test_id' }] } ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['TEST', 'test']) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['TEST', 'test']) ]); }); @@ -663,19 +673,21 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'test_id', filter_param1: 'test1', filter_param2: 'test2' })).toEqual([ + expect( + query.evaluateParameterRow(MYBUCKET_1, { id: 'test_id', filter_param1: 'test1', filter_param2: 'test2' }) + ).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['test1']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['test1']), bucketParameters: [{ id: 'test_id' }] }, { - lookup: ParameterLookup.normalized('mybucket', '1', ['test2']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['test2']), bucketParameters: [{ id: 'test_id' }] } ]); - expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['TEST']) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['TEST']) ]); }); @@ -692,14 +704,14 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'group1', category: 'red' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'group1', category: 'red' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['red']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['red']), bucketParameters: [{}] } ]); - expect(query.getLookups(normalizeTokenParameters({}, { category_id: 'red' }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['red']) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({}, { category_id: 'red' }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['red']) ]); }); @@ -716,8 +728,8 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['red']) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['red']) ]); }); @@ -734,8 +746,8 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['red']) + expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ + ParameterLookup.normalized(MYBUCKET_1, ['red']) ]); }); @@ -753,9 +765,9 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'region1', name: 'colorado' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'region1', name: 'colorado' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['colorado']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['colorado']), bucketParameters: [ { region_id: 'region1' @@ -765,6 +777,7 @@ describe('parameter queries', () => { ]); expect( query.getLookups( + MYBUCKET_1, normalizeTokenParameters( {}, { @@ -773,8 +786,8 @@ describe('parameter queries', () => { ) ) ).toEqual([ - ParameterLookup.normalized('mybucket', '1', ['colorado']), - ParameterLookup.normalized('mybucket', '1', ['texas']) + ParameterLookup.normalized(MYBUCKET_1, ['colorado']), + ParameterLookup.normalized(MYBUCKET_1, ['texas']) ]); }); @@ -788,14 +801,14 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1']), bucketParameters: [{ id: 'user1' }] } ]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }, { other_id: 'red' }); - expect(query.getLookups(requestParams)).toEqual([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + expect(query.getLookups(MYBUCKET_1, requestParams)).toEqual([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); }); test('request.parameters() in SELECT', function () { @@ -809,14 +822,14 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized('mybucket', '1', ['user1']), + lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1']), bucketParameters: [{ id: 'user1' }] } ]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }, { other_id: 'red' }); - expect(query.getLookups(requestParams)).toEqual([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + expect(query.getLookups(MYBUCKET_1, requestParams)).toEqual([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); }); test('request.jwt()', function () { @@ -831,7 +844,7 @@ describe('parameter queries', () => { expect(query.errors).toEqual([]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }); - expect(query.getLookups(requestParams)).toEqual([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + expect(query.getLookups(MYBUCKET_1, requestParams)).toEqual([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); }); test('request.user_id()', function () { @@ -846,7 +859,7 @@ describe('parameter queries', () => { expect(query.errors).toEqual([]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }); - expect(query.getLookups(requestParams)).toEqual([ParameterLookup.normalized('mybucket', '1', ['user1'])]); + expect(query.getLookups(MYBUCKET_1, requestParams)).toEqual([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); }); test('invalid OR in parameter queries', () => { diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index d4cc22e64..5f6285834 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -22,8 +22,18 @@ import { syncStreamFromSql } from '../../src/index.js'; import { normalizeQuerierOptions, PARSE_OPTIONS, TestSourceTable } from './util.js'; +import { ParameterLookupScope } from '../../src/HydrationState.js'; describe('streams', () => { + const STREAM_0: ParameterLookupScope = { + lookupName: 'stream', + queryId: '0' + }; + const STREAM_1: ParameterLookupScope = { + lookupName: 'stream', + queryId: '1' + }; + test('refuses edition: 1', () => { expect(() => syncStreamFromSql('stream', 'SELECT * FROM comments', { @@ -228,7 +238,7 @@ describe('streams', () => { }) ).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', ['u1']), + lookup: ParameterLookup.normalized(STREAM_0, ['u1']), bucketParameters: [ { result: 'i1' @@ -238,7 +248,7 @@ describe('streams', () => { ]); function getParameterSets(lookups: ParameterLookup[]) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['u1'])]); + expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['u1'])]); return [{ result: 'i1' }]; } @@ -266,7 +276,7 @@ describe('streams', () => { .evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) ).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', ['user1']), + lookup: ParameterLookup.normalized(STREAM_0, ['user1']), bucketParameters: [ { result: 'issue_id' @@ -282,7 +292,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['user1'])]); + expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -303,7 +313,7 @@ describe('streams', () => { .evaluateParameterRow(USERS, { id: 'u', is_admin: 1n }) ).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', ['u']), + lookup: ParameterLookup.normalized(STREAM_0, ['u']), bucketParameters: [ { result: 'u' @@ -322,7 +332,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'u' }, getParameterSets: (lookups: ParameterLookup[]) => { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['u'])]); + expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['u'])]); return [{ result: 'u' }]; } }) @@ -333,7 +343,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'u2' }, getParameterSets: (lookups: ParameterLookup[]) => { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['u2'])]); + expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['u2'])]); return []; } }) @@ -355,7 +365,7 @@ describe('streams', () => { expect(source.evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' })).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', ['b']), + lookup: ParameterLookup.normalized(STREAM_0, ['b']), bucketParameters: [ { result: 'a' @@ -363,7 +373,7 @@ describe('streams', () => { ] }, { - lookup: ParameterLookup.normalized('stream', '1', ['a']), + lookup: ParameterLookup.normalized(STREAM_1, ['a']), bucketParameters: [ { result: 'b' @@ -376,10 +386,10 @@ describe('streams', () => { expect(lookups).toHaveLength(1); const [lookup] = lookups; if (lookup.values[1] == '0') { - expect(lookup).toStrictEqual(ParameterLookup.normalized('stream', '0', ['a'])); + expect(lookup).toStrictEqual(ParameterLookup.normalized(STREAM_0, ['a'])); return []; } else { - expect(lookup).toStrictEqual(ParameterLookup.normalized('stream', '1', ['a'])); + expect(lookup).toStrictEqual(ParameterLookup.normalized(STREAM_1, ['a'])); return [{ result: 'b' }]; } } @@ -419,7 +429,7 @@ describe('streams', () => { getParameterSets(lookups) { expect(lookups).toHaveLength(1); const [lookup] = lookups; - expect(lookup).toStrictEqual(ParameterLookup.normalized('stream', '0', ['a'])); + expect(lookup).toStrictEqual(ParameterLookup.normalized(STREAM_0, ['a'])); return [{ result: 'i1' }, { result: 'i2' }]; } }) @@ -466,7 +476,7 @@ describe('streams', () => { .evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' }) ).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', ['b']), + lookup: ParameterLookup.normalized(STREAM_0, ['b']), bucketParameters: [ { result: 'a' @@ -483,7 +493,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['user1'])]); + expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -625,7 +635,7 @@ describe('streams', () => { .evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) ).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', ['user1']), + lookup: ParameterLookup.normalized(STREAM_0, ['user1']), bucketParameters: [ { result: 'issue_id' @@ -641,7 +651,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['user1'])]); + expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -696,7 +706,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['user1'])]); + expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -706,7 +716,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1', is_admin: true }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', ['user1'])]); + expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -756,7 +766,7 @@ describe('streams', () => { .evaluateParameterRow(accountMember, row) ).toStrictEqual([ { - lookup: ParameterLookup.normalized('account_member', '0', ['id']), + lookup: ParameterLookup.normalized({ lookupName: 'account_member', queryId: '0' }, ['id']), bucketParameters: [ { result: 'account_id' @@ -770,7 +780,9 @@ describe('streams', () => { token: { sub: 'id' }, parameters: {}, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('account_member', '0', ['id'])]); + expect(lookups).toStrictEqual([ + ParameterLookup.normalized({ lookupName: 'account_member', queryId: '0' }, ['id']) + ]); return [{ result: 'account_id' }]; } }) @@ -844,7 +856,7 @@ WHERE }) ).toStrictEqual([ { - lookup: ParameterLookup.normalized('stream', '0', [1n, 'foo']), + lookup: ParameterLookup.normalized(STREAM_0, [1n, 'foo']), bucketParameters: [ { result: 'foo' @@ -852,7 +864,7 @@ WHERE ] }, { - lookup: ParameterLookup.normalized('stream', '0', [2n, 'foo']), + lookup: ParameterLookup.normalized(STREAM_0, [2n, 'foo']), bucketParameters: [ { result: 'foo' @@ -866,7 +878,7 @@ WHERE token: { sub: 'user1', haystack_id: 1 }, parameters: { project: 'foo' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized('stream', '0', [1n, 'foo'])]); + expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, [1n, 'foo'])]); return [{ result: 'foo' }]; } }) diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 21d3839ce..d4d3fcb6e 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -110,7 +110,7 @@ bucket_definitions: expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ { bucketParameters: [{}], - lookup: ParameterLookup.normalized('mybucket', '1', ['user1']) + lookup: ParameterLookup.normalized({ lookupName: 'mybucket', queryId: '1' }, ['user1']) } ]); expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 0 })).toEqual([]); @@ -962,10 +962,10 @@ bucket_definitions: expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ hasDynamicBuckets: true, parameterQueryLookups: [ - ParameterLookup.normalized('mybucket', '2', ['user1']), - ParameterLookup.normalized('by_list', '1', ['user1']), + ParameterLookup.normalized({ lookupName: 'mybucket', queryId: '2' }, ['user1']), + ParameterLookup.normalized({ lookupName: 'by_list', queryId: '1' }, ['user1']), // These are not filtered out yet, due to how the lookups are structured internally - ParameterLookup.normalized('admin_only', '1', [1]) + ParameterLookup.normalized({ lookupName: 'admin_only', queryId: '1' }, [1]) ], staticBuckets: [ { From bf7fb60dfb620bb4ad5ba487709b5657d1b203d1 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 21:09:05 +0200 Subject: [PATCH 20/33] More minor refactoring. --- packages/sync-rules/src/BucketSource.ts | 5 +- packages/sync-rules/src/HydrationState.ts | 30 +- packages/sync-rules/src/SqlParameterQuery.ts | 11 +- packages/sync-rules/src/streams/filter.ts | 9 +- .../test/src/parameter_queries.test.ts | 258 +++++++++--------- 5 files changed, 146 insertions(+), 167 deletions(-) diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index b53ccfe09..75ea63385 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -1,6 +1,6 @@ import { BucketParameterQuerier, ParameterLookup, PendingQueriers } from './BucketParameterQuerier.js'; import { ColumnDefinition } from './ExpressionType.js'; -import { HydrationState } from './HydrationState.js'; +import { HydrationState, ParameterLookupScope } from './HydrationState.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { GetQuerierOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; @@ -107,8 +107,7 @@ export interface BucketParameterLookupSourceDefinition { * * This defines the default values if no transformations are applied. */ - defaultLookupName: string; - defaultQueryId: string; + readonly defaultLookupScope: ParameterLookupScope; getSourceTables(): Set; createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource; diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts index ed31e0aaf..b668d573e 100644 --- a/packages/sync-rules/src/HydrationState.ts +++ b/packages/sync-rules/src/HydrationState.ts @@ -45,37 +45,16 @@ export const DEFAULT_HYDRATION_STATE: HydrationState = { }; }, getParameterLookupScope(source) { - return { - lookupName: source.defaultLookupName, - queryId: source.defaultQueryId - }; + return source.defaultLookupScope; } }; export function versionedHydrationState(version: number) { - return new VersionedHydrationState((bucketId: string) => { + return new BucketIdTransformerHydrationState((bucketId: string) => { return `${version}#${bucketId}`; }); } -export class VersionedHydrationState implements HydrationState { - constructor(private transformer: BucketIdTransformer) {} - - getBucketSourceState(source: BucketDataSourceDefinition): BucketSourceState { - return { - bucketPrefix: this.transformer(source.defaultBucketPrefix) - }; - } - - getParameterLookupScope(source: BucketParameterLookupSourceDefinition): ParameterLookupScope { - // No transformations applied here - return { - lookupName: source.defaultLookupName, - queryId: source.defaultQueryId - }; - } -} - export class BucketIdTransformerHydrationState implements HydrationState { constructor(private transformer: BucketIdTransformer) {} @@ -87,10 +66,7 @@ export class BucketIdTransformerHydrationState implements HydrationState { getParameterLookupScope(source: BucketParameterLookupSourceDefinition): ParameterLookupScope { // No transformations applied here - return { - lookupName: source.defaultLookupName, - queryId: source.defaultQueryId - }; + return source.defaultLookupScope; } } diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 9bd4303b4..dd66811df 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -331,12 +331,11 @@ export class SqlParameterQuery this.querierDataSource = options.querierDataSource; } - public get defaultLookupName(): string { - return this.descriptorName; - } - - public get defaultQueryId(): string { - return this.queryId; + public get defaultLookupScope(): ParameterLookupScope { + return { + lookupName: this.descriptorName, + queryId: this.queryId + }; } tableSyncsParameters(table: SourceTableInterface): boolean { diff --git a/packages/sync-rules/src/streams/filter.ts b/packages/sync-rules/src/streams/filter.ts index 38f188ca9..a7c40a889 100644 --- a/packages/sync-rules/src/streams/filter.ts +++ b/packages/sync-rules/src/streams/filter.ts @@ -539,12 +539,15 @@ export class SubqueryParameterLookupSource implements BucketParameterLookupSourc private parameterTable: TablePattern, private column: RowValueClause, private innerVariant: StreamVariant, - public readonly defaultQueryId: string, + private defaultQueryId: string, private streamName: string ) {} - get defaultLookupName() { - return this.streamName; + public get defaultLookupScope() { + return { + lookupName: this.streamName, + queryId: this.defaultQueryId + }; } getSourceTables(): Set { diff --git a/packages/sync-rules/test/src/parameter_queries.test.ts b/packages/sync-rules/test/src/parameter_queries.test.ts index 483f4e919..8ca201913 100644 --- a/packages/sync-rules/test/src/parameter_queries.test.ts +++ b/packages/sync-rules/test/src/parameter_queries.test.ts @@ -5,10 +5,12 @@ import { BASIC_SCHEMA, EMPTY_DATA_SOURCE, normalizeTokenParameters, PARSE_OPTION import { ParameterLookupScope } from '../../src/HydrationState.js'; describe('parameter queries', () => { - const MYBUCKET_1: ParameterLookupScope = { - lookupName: 'mybucket', - queryId: '1' + // Specifically different from mybucket/1, to make sure this is being used. + const TEST_SCOPE: ParameterLookupScope = { + lookupName: 'test', + queryId: '42' }; + test('token_parameters IN query', function () { const sql = 'SELECT id as group_id FROM groups WHERE token_parameters.user_id IN groups.user_ids'; const query = SqlParameterQuery.fromSql( @@ -20,10 +22,10 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect( - query.evaluateParameterRow(MYBUCKET_1, { id: 'group1', user_ids: JSON.stringify(['user1', 'user2']) }) + query.evaluateParameterRow(TEST_SCOPE, { id: 'group1', user_ids: JSON.stringify(['user1', 'user2']) }) ).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1']), bucketParameters: [ { group_id: 'group1' @@ -31,7 +33,7 @@ describe('parameter queries', () => { ] }, { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['user2']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['user2']), bucketParameters: [ { group_id: 'group1' @@ -41,12 +43,12 @@ describe('parameter queries', () => { ]); expect( query.getLookups( - MYBUCKET_1, + TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1' }) ) - ).toEqual([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); + ).toEqual([ParameterLookup.normalized(TEST_SCOPE, ['user1'])]); }); test('IN token_parameters query', function () { @@ -59,9 +61,9 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'region1', name: 'colorado' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'region1', name: 'colorado' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['colorado']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['colorado']), bucketParameters: [ { region_id: 'region1' @@ -71,14 +73,14 @@ describe('parameter queries', () => { ]); expect( query.getLookups( - MYBUCKET_1, + TEST_SCOPE, normalizeTokenParameters({ region_names: JSON.stringify(['colorado', 'texas']) }) ) ).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['colorado']), - ParameterLookup.normalized(MYBUCKET_1, ['texas']) + ParameterLookup.normalized(TEST_SCOPE, ['colorado']), + ParameterLookup.normalized(TEST_SCOPE, ['texas']) ]); }); @@ -94,9 +96,9 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); // Note: We don't need to worry about numeric vs decimal types in the lookup - JSONB handles normalization for us. - expect(query.evaluateParameterRow(MYBUCKET_1, { int1: 314n, float1: 3.14, float2: 314 })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { int1: 314n, float1: 3.14, float2: 314 })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, [314n, 3.14, 314]), + lookup: ParameterLookup.normalized(TEST_SCOPE, [314n, 3.14, 314]), bucketParameters: [{ int1: 314n, float1: 3.14, float2: 314 }] } @@ -104,8 +106,8 @@ describe('parameter queries', () => { // Similarly, we don't need to worry about the types here. // This test just checks the current behavior. - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ int1: 314n, float1: 3.14, float2: 314 }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, [314n, 3.14, 314n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ int1: 314n, float1: 3.14, float2: 314 }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, [314n, 3.14, 314n]) ]); // We _do_ need to care about the bucket string representation. @@ -137,17 +139,17 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - MYBUCKET_1; - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'test_id', filter_param: 'test_param' })).toEqual([ + TEST_SCOPE; + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['test_param']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['test_param']), bucketParameters: [{ id: 'test_id' }] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['test']) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['test']) ]); }); @@ -162,16 +164,16 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['test_param']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['test_param']), bucketParameters: [{ id: 'test_id' }] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['TEST']) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['TEST']) ]); }); @@ -186,17 +188,17 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['test_param']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['test_param']), bucketParameters: [{ id: 'test_id' }] } ]); expect( - query.getLookups(MYBUCKET_1, normalizeTokenParameters({ some_param: { description: 'test_description' } })) - ).toEqual([ParameterLookup.normalized(MYBUCKET_1, ['test_description'])]); + query.getLookups(TEST_SCOPE, normalizeTokenParameters({ some_param: { description: 'test_description' } })) + ).toEqual([ParameterLookup.normalized(TEST_SCOPE, ['test_description'])]); }); test('token parameter and binary operator', () => { @@ -210,8 +212,8 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ some_param: 3 }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, [5n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ some_param: 3 }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, [5n]) ]); }); @@ -226,11 +228,11 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ some_param: null }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, [1n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ some_param: null }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, [1n]) ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ some_param: 'test' }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, [0n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ some_param: 'test' }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, [0n]) ]); }); @@ -245,18 +247,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, [1n]), + lookup: ParameterLookup.normalized(TEST_SCOPE, [1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, [0n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, [0n]) ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, [1n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, [1n]) ]); }); @@ -271,18 +273,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, [1n]), + lookup: ParameterLookup.normalized(TEST_SCOPE, [1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, [1n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, [1n]) ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, [0n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, [0n]) ]); }); @@ -297,18 +299,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, [1n]), + lookup: ParameterLookup.normalized(TEST_SCOPE, [1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, [0n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, [0n]) ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, [1n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, [1n]) ]); }); @@ -323,18 +325,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, [1n]), + lookup: ParameterLookup.normalized(TEST_SCOPE, [1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, [1n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, [1n]) ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', is_admin: 123 }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, [0n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', is_admin: 123 }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, [0n]) ]); }); @@ -349,18 +351,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]) ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['user1', 0n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['user1', 0n]) ]); }); @@ -375,18 +377,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]) ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['user1', 0n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['user1', 0n]) ]); }); @@ -401,11 +403,11 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['user1']) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['user1']) ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 123 }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['123']) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 123 }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['123']) ]); }); @@ -420,15 +422,15 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1', role: null })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1', role: null })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, []), + lookup: ParameterLookup.normalized(TEST_SCOPE, []), bucketParameters: [{ id: 'user1' }] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, []) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, []) ]); }); @@ -446,19 +448,19 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]) ]); // Would not match any actual lookups - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['user1', 0n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['user1', 0n]) ]); }); @@ -474,16 +476,16 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]), bucketParameters: [{ user_id: 'user1' }] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['user1', 1n]) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]) ]); expect( @@ -506,9 +508,9 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { userId: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { userId: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1']), bucketParameters: [{ user_id: 'user1' }] } @@ -532,10 +534,10 @@ describe('parameter queries', () => { { message: `Unquoted identifiers are converted to lower-case. Use "userId" instead.` } ]); - expect(query.evaluateParameterRow(MYBUCKET_1, { userId: 'user1' })).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { userid: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { userId: 'user1' })).toEqual([]); + expect(query.evaluateParameterRow(TEST_SCOPE, { userid: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1']), bucketParameters: [{ user_id: 'user1' }] } @@ -623,15 +625,15 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'workspace1', visibility: 'public' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'workspace1', visibility: 'public' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, []), + lookup: ParameterLookup.normalized(TEST_SCOPE, []), bucketParameters: [{ workspace_id: 'workspace1' }] } ]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'workspace1', visibility: 'private' })).toEqual([]); + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'workspace1', visibility: 'private' })).toEqual([]); }); test('multiple different functions on token_parameter with AND', () => { @@ -647,16 +649,16 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['test_param', 'test_param']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['test_param', 'test_param']), bucketParameters: [{ id: 'test_id' }] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['TEST', 'test']) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['TEST', 'test']) ]); }); @@ -674,20 +676,20 @@ describe('parameter queries', () => { expect(query.errors).toEqual([]); expect( - query.evaluateParameterRow(MYBUCKET_1, { id: 'test_id', filter_param1: 'test1', filter_param2: 'test2' }) + query.evaluateParameterRow(TEST_SCOPE, { id: 'test_id', filter_param1: 'test1', filter_param2: 'test2' }) ).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['test1']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['test1']), bucketParameters: [{ id: 'test_id' }] }, { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['test2']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['test2']), bucketParameters: [{ id: 'test_id' }] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['TEST']) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['TEST']) ]); }); @@ -704,14 +706,14 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'group1', category: 'red' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'group1', category: 'red' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['red']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['red']), bucketParameters: [{}] } ]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({}, { category_id: 'red' }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['red']) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({}, { category_id: 'red' }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['red']) ]); }); @@ -728,8 +730,8 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['red']) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['red']) ]); }); @@ -746,8 +748,8 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(MYBUCKET_1, normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['red']) + expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ + ParameterLookup.normalized(TEST_SCOPE, ['red']) ]); }); @@ -765,9 +767,9 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'region1', name: 'colorado' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'region1', name: 'colorado' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['colorado']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['colorado']), bucketParameters: [ { region_id: 'region1' @@ -777,7 +779,7 @@ describe('parameter queries', () => { ]); expect( query.getLookups( - MYBUCKET_1, + TEST_SCOPE, normalizeTokenParameters( {}, { @@ -786,8 +788,8 @@ describe('parameter queries', () => { ) ) ).toEqual([ - ParameterLookup.normalized(MYBUCKET_1, ['colorado']), - ParameterLookup.normalized(MYBUCKET_1, ['texas']) + ParameterLookup.normalized(TEST_SCOPE, ['colorado']), + ParameterLookup.normalized(TEST_SCOPE, ['texas']) ]); }); @@ -801,14 +803,14 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1']), bucketParameters: [{ id: 'user1' }] } ]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }, { other_id: 'red' }); - expect(query.getLookups(MYBUCKET_1, requestParams)).toEqual([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); + expect(query.getLookups(TEST_SCOPE, requestParams)).toEqual([ParameterLookup.normalized(TEST_SCOPE, ['user1'])]); }); test('request.parameters() in SELECT', function () { @@ -822,14 +824,14 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(MYBUCKET_1, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(MYBUCKET_1, ['user1']), + lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1']), bucketParameters: [{ id: 'user1' }] } ]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }, { other_id: 'red' }); - expect(query.getLookups(MYBUCKET_1, requestParams)).toEqual([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); + expect(query.getLookups(TEST_SCOPE, requestParams)).toEqual([ParameterLookup.normalized(TEST_SCOPE, ['user1'])]); }); test('request.jwt()', function () { @@ -844,7 +846,7 @@ describe('parameter queries', () => { expect(query.errors).toEqual([]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }); - expect(query.getLookups(MYBUCKET_1, requestParams)).toEqual([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); + expect(query.getLookups(TEST_SCOPE, requestParams)).toEqual([ParameterLookup.normalized(TEST_SCOPE, ['user1'])]); }); test('request.user_id()', function () { @@ -859,7 +861,7 @@ describe('parameter queries', () => { expect(query.errors).toEqual([]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }); - expect(query.getLookups(MYBUCKET_1, requestParams)).toEqual([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); + expect(query.getLookups(TEST_SCOPE, requestParams)).toEqual([ParameterLookup.normalized(TEST_SCOPE, ['user1'])]); }); test('invalid OR in parameter queries', () => { From 8a6b40df07ba5544bac399b5cbe802f97abdf997 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 21:14:01 +0200 Subject: [PATCH 21/33] Some comments. --- packages/sync-rules/src/SqlParameterQuery.ts | 1 - packages/sync-rules/src/streams/parameter.ts | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index dd66811df..ebc1a93b9 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -360,7 +360,6 @@ export class SqlParameterQuery } createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { - // FIXME: Use HydrationState for lookups. const hydrationState = resolveHydrationState(params); const lookupState = hydrationState.getParameterLookupScope(this); return { diff --git a/packages/sync-rules/src/streams/parameter.ts b/packages/sync-rules/src/streams/parameter.ts index 75c6b44e4..f1527b25a 100644 --- a/packages/sync-rules/src/streams/parameter.ts +++ b/packages/sync-rules/src/streams/parameter.ts @@ -41,6 +41,11 @@ export interface SubqueryEvaluator { parameterTable: TablePattern; lookupSources(): BucketParameterLookupSourceDefinition[]; + // TODO: Is there a better design here? + // This is used for parameter _queries_. But the queries need to know which lookup scopes to + // use, and each querier may use multiple lookup sources, each with their own scope. + // This implementation here does "hydration" on each subquery, which gives us hydrated function call. + // Should this maybe be part of a higher-level class instead of just a function, i.e. a hydrated subquery? hydrateLookupsForRequest(hydrationState: HydrationState): (params: RequestParameters) => ParameterLookup[]; } From 1eecae9b5cf1ec9f18acd26faf04807a2380a0b3 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 10 Dec 2025 21:17:42 +0200 Subject: [PATCH 22/33] Import cleanup. --- packages/sync-rules/src/BaseSqlDataQuery.ts | 3 +-- packages/sync-rules/src/HydrationState.ts | 4 ++-- packages/sync-rules/src/SqlDataQuery.ts | 4 ++-- packages/sync-rules/src/SqlParameterQuery.ts | 3 +-- .../sync-rules/src/StaticSqlParameterQuery.ts | 10 ++-------- .../src/TableValuedFunctionSqlParameterQuery.ts | 3 +-- packages/sync-rules/src/streams/parameter.ts | 13 ++----------- packages/sync-rules/src/streams/stream.ts | 16 ++-------------- packages/sync-rules/src/streams/variant.ts | 15 ++------------- 9 files changed, 15 insertions(+), 56 deletions(-) diff --git a/packages/sync-rules/src/BaseSqlDataQuery.ts b/packages/sync-rules/src/BaseSqlDataQuery.ts index 7bec16db8..355a0970f 100644 --- a/packages/sync-rules/src/BaseSqlDataQuery.ts +++ b/packages/sync-rules/src/BaseSqlDataQuery.ts @@ -3,9 +3,9 @@ import { SqlRuleError } from './errors.js'; import { ColumnDefinition } from './ExpressionType.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; +import { castAsText } from './sql_functions.js'; import { TablePattern } from './TablePattern.js'; import { - BucketIdTransformer, EvaluationResult, QueryParameters, QuerySchema, @@ -15,7 +15,6 @@ import { SqliteRow } from './types.js'; import { filterJsonRow } from './utils.js'; -import { castAsText } from './sql_functions.js'; export interface RowValueExtractor { extract(tables: QueryParameters, into: SqliteRow): void; diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts index b668d573e..377904582 100644 --- a/packages/sync-rules/src/HydrationState.ts +++ b/packages/sync-rules/src/HydrationState.ts @@ -1,5 +1,5 @@ -import { BucketDataSource, BucketDataSourceDefinition, BucketParameterLookupSourceDefinition } from './BucketSource.js'; -import { BucketIdTransformer, CompatibilityContext, CompatibilityOption, CreateSourceParams } from './index.js'; +import { BucketDataSourceDefinition, BucketParameterLookupSourceDefinition } from './BucketSource.js'; +import { BucketIdTransformer, CreateSourceParams } from './index.js'; export interface BucketSourceState { /** The prefix is the bucket name before the parameters. */ diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index bec1daad2..63b0dd069 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -1,6 +1,7 @@ import { JSONBig } from '@powersync/service-jsonbig'; import { parse } from 'pgsql-ast-parser'; import { BaseSqlDataQuery, BaseSqlDataQueryOptions, RowValueExtractor } from './BaseSqlDataQuery.js'; +import { CompatibilityContext } from './compatibility.js'; import { SqlRuleError } from './errors.js'; import { ExpressionType } from './ExpressionType.js'; import { SourceTableInterface } from './SourceTableInterface.js'; @@ -9,9 +10,8 @@ import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; import { SyncRulesOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; import { TableQuerySchema } from './TableQuerySchema.js'; -import { BucketIdTransformer, EvaluationResult, ParameterMatchClause, QuerySchema, SqliteRow } from './types.js'; +import { EvaluationResult, ParameterMatchClause, QuerySchema, SqliteRow } from './types.js'; import { getBucketId, isSelectStatement } from './utils.js'; -import { CompatibilityContext } from './compatibility.js'; export interface SqlDataQueryOptions extends BaseSqlDataQueryOptions { filter: ParameterMatchClause; diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index ebc1a93b9..640a12aef 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -19,6 +19,7 @@ import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; +import { ParameterLookupScope, resolveHydrationState } from './HydrationState.js'; import { BucketDataSourceDefinition, GetQuerierOptions } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; @@ -28,7 +29,6 @@ import { TablePattern } from './TablePattern.js'; import { TableQuerySchema } from './TableQuerySchema.js'; import { TableValuedFunctionSqlParameterQuery } from './TableValuedFunctionSqlParameterQuery.js'; import { - BucketIdTransformer, EvaluatedParameters, EvaluatedParametersResult, InputParameter, @@ -44,7 +44,6 @@ import { } from './types.js'; import { filterJsonRow, getBucketId, isJsonValue, isSelectStatement, normalizeParameterValue } from './utils.js'; import { DetectRequestParameters } from './validators.js'; -import { ParameterLookupScope, HydrationState, resolveHydrationState } from './HydrationState.js'; export interface SqlParameterQueryOptions { sourceTable: TablePattern; diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index 9643cd606..fa05c7e4c 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -7,21 +7,15 @@ import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; +import { resolveHydrationState } from './HydrationState.js'; import { BucketDataSourceDefinition, GetQuerierOptions } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; import { checkUnsupportedFeatures, isClauseError, sqliteBool } from './sql_support.js'; import { TablePattern } from './TablePattern.js'; -import { - BucketIdTransformer, - ParameterValueClause, - QueryParseOptions, - RequestParameters, - SqliteJsonValue -} from './types.js'; +import { ParameterValueClause, QueryParseOptions, RequestParameters, SqliteJsonValue } from './types.js'; import { getBucketId, isJsonValue } from './utils.js'; import { DetectRequestParameters } from './validators.js'; -import { resolveHydrationState } from './HydrationState.js'; export interface StaticSqlParameterQueryOptions { sql: string; diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index 767d62242..d0a042c39 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -6,6 +6,7 @@ import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; +import { resolveHydrationState } from './HydrationState.js'; import { BucketDataSourceDefinition, BucketParameterQuerier, GetQuerierOptions, PendingQueriers } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; @@ -13,7 +14,6 @@ import { checkUnsupportedFeatures, isClauseError, sqliteBool } from './sql_suppo import { TablePattern } from './TablePattern.js'; import { generateTableValuedFunctions, TableValuedFunction } from './TableValuedFunctions.js'; import { - BucketIdTransformer, ParameterValueClause, ParameterValueSet, QueryParseOptions, @@ -23,7 +23,6 @@ import { } from './types.js'; import { getBucketId, isJsonValue } from './utils.js'; import { DetectRequestParameters } from './validators.js'; -import { resolveHydrationState } from './HydrationState.js'; export interface TableValuedFunctionSqlParameterQueryOptions { sql: string; diff --git a/packages/sync-rules/src/streams/parameter.ts b/packages/sync-rules/src/streams/parameter.ts index f1527b25a..74820bf0c 100644 --- a/packages/sync-rules/src/streams/parameter.ts +++ b/packages/sync-rules/src/streams/parameter.ts @@ -1,17 +1,8 @@ import { ParameterLookup } from '../BucketParameterQuerier.js'; -import { BucketParameterLookupSource, BucketParameterLookupSourceDefinition } from '../BucketSource.js'; +import { BucketParameterLookupSourceDefinition } from '../BucketSource.js'; import { HydrationState } from '../HydrationState.js'; -import { SourceTableInterface } from '../SourceTableInterface.js'; import { TablePattern } from '../TablePattern.js'; -import { - EvaluateRowOptions, - ParameterValueSet, - RequestParameters, - SqliteJsonValue, - SqliteRow, - SqliteValue, - TableRow -} from '../types.js'; +import { ParameterValueSet, RequestParameters, SqliteJsonValue, SqliteValue, TableRow } from '../types.js'; /** * A source of parameterization, causing data from the source table to be distributed into multiple buckets instead of diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index 89496c75f..a651574d3 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -1,12 +1,9 @@ import { BaseSqlDataQuery } from '../BaseSqlDataQuery.js'; -import { BucketInclusionReason, BucketPriority, DEFAULT_BUCKET_PRIORITY } from '../BucketDescription.js'; -import { PendingQueriers } from '../BucketParameterQuerier.js'; +import { BucketPriority, DEFAULT_BUCKET_PRIORITY } from '../BucketDescription.js'; import { BucketDataSource, BucketDataSourceDefinition, - BucketParameterLookupSource, BucketParameterLookupSourceDefinition, - BucketParameterQuerierSource, BucketParameterQuerierSourceDefinition, BucketSource, BucketSourceType, @@ -15,17 +12,8 @@ import { import { ColumnDefinition } from '../ExpressionType.js'; import { resolveHydrationState } from '../HydrationState.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; -import { GetQuerierOptions, RequestedStream } from '../SqlSyncRules.js'; import { TablePattern } from '../TablePattern.js'; -import { - BucketIdTransformer, - EvaluatedParametersResult, - EvaluateRowOptions, - EvaluationResult, - RequestParameters, - SourceSchema, - TableRow -} from '../types.js'; +import { EvaluateRowOptions, EvaluationResult, SourceSchema, TableRow } from '../types.js'; import { StreamVariant } from './variant.js'; export class SyncStream implements BucketSource { diff --git a/packages/sync-rules/src/streams/variant.ts b/packages/sync-rules/src/streams/variant.ts index 61cc6c353..feb636cba 100644 --- a/packages/sync-rules/src/streams/variant.ts +++ b/packages/sync-rules/src/streams/variant.ts @@ -2,7 +2,6 @@ import { BucketInclusionReason, ResolvedBucket } from '../BucketDescription.js'; import { BucketParameterQuerier, ParameterLookup, PendingQueriers } from '../BucketParameterQuerier.js'; import { BucketDataSourceDefinition, - BucketParameterLookupSource, BucketParameterLookupSourceDefinition, BucketParameterQuerierSource, BucketParameterQuerierSourceDefinition, @@ -10,18 +9,8 @@ import { } from '../BucketSource.js'; import { resolveHydrationState } from '../HydrationState.js'; import { GetQuerierOptions, RequestedStream } from '../index.js'; -import { SourceTableInterface } from '../SourceTableInterface.js'; -import { TablePattern } from '../TablePattern.js'; -import { - BucketIdTransformer, - EvaluatedParametersResult, - EvaluateRowOptions, - RequestParameters, - SqliteJsonValue, - SqliteRow, - TableRow -} from '../types.js'; -import { isJsonValue, JSONBucketNameSerialize, normalizeParameterValue } from '../utils.js'; +import { RequestParameters, SqliteJsonValue, TableRow } from '../types.js'; +import { isJsonValue, JSONBucketNameSerialize } from '../utils.js'; import { BucketParameter, SubqueryEvaluator } from './parameter.js'; import { SyncStream, SyncStreamDataSource } from './stream.js'; import { cartesianProduct } from './utils.js'; From ae08fd3e4b77903a6a9723dced96289afeb01d00 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 11 Dec 2025 10:01:43 +0200 Subject: [PATCH 23/33] Remove BucketIdTransformer. --- .../implementation/MongoPersistedSyncRules.ts | 3 +- .../PostgresPersistedSyncRulesContent.ts | 3 +- .../src/test-utils/general-utils.ts | 3 +- .../test/src/sync/BucketChecksumState.test.ts | 13 ++--- packages/sync-rules/src/BucketSource.ts | 20 ++----- packages/sync-rules/src/HydrationState.ts | 53 +++++++++---------- .../sync-rules/src/SqlBucketDescriptor.ts | 3 +- packages/sync-rules/src/SqlParameterQuery.ts | 6 +-- packages/sync-rules/src/SqlSyncRules.ts | 30 +++++------ .../sync-rules/src/StaticSqlParameterQuery.ts | 3 +- .../TableValuedFunctionSqlParameterQuery.ts | 3 +- packages/sync-rules/src/streams/filter.ts | 16 +++--- packages/sync-rules/src/streams/stream.ts | 3 +- packages/sync-rules/src/streams/variant.ts | 3 +- packages/sync-rules/src/types.ts | 14 ----- packages/sync-rules/src/utils.ts | 1 - .../sync-rules/test/src/compatibility.test.ts | 12 +++-- packages/sync-rules/test/src/streams.test.ts | 39 ++++++-------- .../sync-rules/test/src/sync_rules.test.ts | 45 ++++++++-------- packages/sync-rules/test/src/util.ts | 13 +++-- 20 files changed, 123 insertions(+), 163 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts index 131c35b9a..77796b143 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts @@ -1,6 +1,7 @@ import { SqlSyncRules, HydratedSyncRules } from '@powersync/service-sync-rules'; import { storage } from '@powersync/service-core'; +import { versionedHydrationState } from '@powersync/service-sync-rules/src/HydrationState.js'; export class MongoPersistedSyncRules implements storage.PersistedSyncRules { public readonly slot_name: string; @@ -15,6 +16,6 @@ export class MongoPersistedSyncRules implements storage.PersistedSyncRules { } hydratedSyncRules(): HydratedSyncRules { - return this.sync_rules.hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${this.id}`) }); + return this.sync_rules.hydrate({ hydrationState: versionedHydrationState(this.id) }); } } diff --git a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts index e2dd67184..2548b03b7 100644 --- a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +++ b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts @@ -4,6 +4,7 @@ import { storage } from '@powersync/service-core'; import { SqlSyncRules } from '@powersync/service-sync-rules'; import { models } from '../../types/types.js'; +import { versionedHydrationState } from '@powersync/service-sync-rules/src/HydrationState.js'; export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncRulesContent { public readonly slot_name: string; @@ -38,7 +39,7 @@ export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncR sync_rules: SqlSyncRules.fromYaml(this.sync_rules_content, options), hydratedSyncRules() { return this.sync_rules.hydrate({ - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${this.id}`) + hydrationState: versionedHydrationState(this.id) }); } }; diff --git a/packages/service-core-tests/src/test-utils/general-utils.ts b/packages/service-core-tests/src/test-utils/general-utils.ts index 5f48f38f4..fb79c5fae 100644 --- a/packages/service-core-tests/src/test-utils/general-utils.ts +++ b/packages/service-core-tests/src/test-utils/general-utils.ts @@ -1,5 +1,6 @@ import { storage, utils } from '@powersync/service-core'; import { GetQuerierOptions, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules'; +import { versionedHydrationState } from '@powersync/service-sync-rules/src/HydrationState.js'; import * as bson from 'bson'; export const ZERO_LSN = '0/0'; @@ -27,7 +28,7 @@ export function testRules(content: string): storage.PersistedSyncRulesContent { sync_rules: SqlSyncRules.fromYaml(content, options), slot_name: 'test', hydratedSyncRules() { - return this.sync_rules.hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); + return this.sync_rules.hydrate({ hydrationState: versionedHydrationState(1) }); } }; }, diff --git a/packages/service-core/test/src/sync/BucketChecksumState.test.ts b/packages/service-core/test/src/sync/BucketChecksumState.test.ts index 2d3fbde45..48cdd24fc 100644 --- a/packages/service-core/test/src/sync/BucketChecksumState.test.ts +++ b/packages/service-core/test/src/sync/BucketChecksumState.test.ts @@ -12,8 +12,9 @@ import { WatchFilterEvent } from '@/index.js'; import { JSONBig } from '@powersync/service-jsonbig'; -import { SqliteJsonRow, ParameterLookup, SqlSyncRules, RequestJwtPayload } from '@powersync/service-sync-rules'; -import { describe, expect, test, beforeEach } from 'vitest'; +import { ParameterLookup, RequestJwtPayload, SqliteJsonRow, SqlSyncRules } from '@powersync/service-sync-rules'; +import { versionedHydrationState } from '@powersync/service-sync-rules/src/HydrationState.js'; +import { beforeEach, describe, expect, test } from 'vitest'; describe('BucketChecksumState', () => { // Single global[] bucket. @@ -25,7 +26,7 @@ bucket_definitions: data: [] `, { defaultSchema: 'public' } - ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); + ).hydrate({ hydrationState: versionedHydrationState(1) }); // global[1] and global[2] const SYNC_RULES_GLOBAL_TWO = SqlSyncRules.fromYaml( @@ -38,7 +39,7 @@ bucket_definitions: data: [] `, { defaultSchema: 'public' } - ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('2') }); + ).hydrate({ hydrationState: versionedHydrationState(2) }); // by_project[n] const SYNC_RULES_DYNAMIC = SqlSyncRules.fromYaml( @@ -49,7 +50,7 @@ bucket_definitions: data: [] `, { defaultSchema: 'public' } - ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('3') }); + ).hydrate({ hydrationState: versionedHydrationState(3) }); const syncContext = new SyncContext({ maxBuckets: 100, @@ -614,7 +615,7 @@ config: const rules = SqlSyncRules.fromYaml(source, { defaultSchema: 'public' - }).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); + }).hydrate({ hydrationState: versionedHydrationState(1) }); return new BucketChecksumState({ syncContext, diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 75ea63385..1754a6a10 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -1,24 +1,13 @@ import { BucketParameterQuerier, ParameterLookup, PendingQueriers } from './BucketParameterQuerier.js'; import { ColumnDefinition } from './ExpressionType.js'; -import { HydrationState, ParameterLookupScope } from './HydrationState.js'; +import { DEFAULT_HYDRATION_STATE, HydrationState, ParameterLookupScope } from './HydrationState.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { GetQuerierOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; -import { - BucketIdTransformer, - EvaluatedParametersResult, - EvaluateRowOptions, - EvaluationResult, - SourceSchema, - SqliteRow -} from './types.js'; +import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, SourceSchema, SqliteRow } from './types.js'; export interface CreateSourceParams { - hydrationState?: HydrationState; - /** - * @deprecated Use hydrationState instead. - */ - bucketIdTransformer?: BucketIdTransformer; + hydrationState: HydrationState; } /** @@ -227,7 +216,8 @@ export function mergeParameterQuerierSources(sources: BucketParameterQuerierSour * it is useful to have a single merged source that can evaluate everything. */ export function debugHydratedMergedSource(bucketSource: BucketSource, params?: CreateSourceParams): DebugMergedSource { - const resolvedParams = params ?? { bucketIdTransformer: (id: string) => id }; + const hydrationState = params?.hydrationState ?? DEFAULT_HYDRATION_STATE; + const resolvedParams = { hydrationState }; const dataSource = mergeDataSources( bucketSource.dataSources.map((source) => source.createDataSource(resolvedParams)) ); diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts index 377904582..bdcde34b3 100644 --- a/packages/sync-rules/src/HydrationState.ts +++ b/packages/sync-rules/src/HydrationState.ts @@ -1,5 +1,4 @@ import { BucketDataSourceDefinition, BucketParameterLookupSourceDefinition } from './BucketSource.js'; -import { BucketIdTransformer, CreateSourceParams } from './index.js'; export interface BucketSourceState { /** The prefix is the bucket name before the parameters. */ @@ -49,33 +48,29 @@ export const DEFAULT_HYDRATION_STATE: HydrationState = { } }; -export function versionedHydrationState(version: number) { - return new BucketIdTransformerHydrationState((bucketId: string) => { - return `${version}#${bucketId}`; - }); -} - -export class BucketIdTransformerHydrationState implements HydrationState { - constructor(private transformer: BucketIdTransformer) {} - - getBucketSourceState(source: BucketDataSourceDefinition): BucketSourceState { - return { - bucketPrefix: this.transformer(source.defaultBucketPrefix) - }; - } - - getParameterLookupScope(source: BucketParameterLookupSourceDefinition): ParameterLookupScope { - // No transformations applied here - return source.defaultLookupScope; - } -} +/** + * Transforms bucket ids generated when evaluating the row by e.g. encoding version information. + * + * Because buckets are recreated on a sync rule redeploy, it makes sense to use different bucket ids (otherwise, clients + * may run into checksum errors causing a sync to take longer than necessary or breaking progress). + * + * So, this transformer receives the original bucket id as generated by defined sync rules, and can prepend a version + * identifier. + * + * Note that this transformation has not been present in older versions of the sync service. To preserve backwards + * compatibility, sync rules will not use this without an opt-in. + */ +export function versionedHydrationState(version: number): HydrationState { + return { + getBucketSourceState(source: BucketDataSourceDefinition): BucketSourceState { + return { + bucketPrefix: `${version}#${source.defaultBucketPrefix}` + }; + }, -export function resolveHydrationState(params: CreateSourceParams): HydrationState { - if (params.hydrationState) { - return params.hydrationState; - } else if (params.bucketIdTransformer) { - return new BucketIdTransformerHydrationState(params.bucketIdTransformer); - } else { - return DEFAULT_HYDRATION_STATE; - } + getParameterLookupScope(source: BucketParameterLookupSourceDefinition): ParameterLookupScope { + // No transformations applied here + return source.defaultLookupScope; + } + }; } diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index d3e23e996..88993d790 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -6,7 +6,6 @@ import { CreateSourceParams } from './BucketSource.js'; import { ColumnDefinition } from './ExpressionType.js'; -import { resolveHydrationState } from './HydrationState.js'; import { IdSequence } from './IdSequence.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SqlDataQuery } from './SqlDataQuery.js'; @@ -158,7 +157,7 @@ export class BucketDefinitionDataSource implements BucketDataSourceDefinition { } createDataSource(params: CreateSourceParams): BucketDataSource { - const hydrationState = resolveHydrationState(params); + const hydrationState = params.hydrationState; const bucketPrefix = hydrationState.getBucketSourceState(this).bucketPrefix; return { evaluateRow: (options) => { diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 640a12aef..586331331 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -19,7 +19,7 @@ import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; -import { ParameterLookupScope, resolveHydrationState } from './HydrationState.js'; +import { ParameterLookupScope } from './HydrationState.js'; import { BucketDataSourceDefinition, GetQuerierOptions } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; @@ -346,7 +346,7 @@ export class SqlParameterQuery } createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { - const hydrationState = resolveHydrationState(params); + const hydrationState = params.hydrationState; const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; const lookupState = hydrationState.getParameterLookupScope(this); @@ -359,7 +359,7 @@ export class SqlParameterQuery } createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { - const hydrationState = resolveHydrationState(params); + const hydrationState = params.hydrationState; const lookupState = hydrationState.getParameterLookupScope(this); return { evaluateParameterRow: (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 10d7b9fb4..a8f4791da 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -5,11 +5,13 @@ import { BucketDataSourceDefinition, BucketParameterLookupSourceDefinition, BucketParameterQuerierSourceDefinition, - BucketSource + BucketSource, + CreateSourceParams } from './BucketSource.js'; import { CompatibilityContext, CompatibilityEdition, CompatibilityOption } from './compatibility.js'; import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js'; import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; +import { DEFAULT_HYDRATION_STATE } from './HydrationState.js'; import { validateSyncRulesSchema } from './json_schema.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { QueryParseResult, SqlBucketDescriptor } from './SqlBucketDescriptor.js'; @@ -17,7 +19,6 @@ import { syncStreamFromSql } from './streams/from_sql.js'; import { HydratedSyncRules } from './SyncRules.js'; import { TablePattern } from './TablePattern.js'; import { - BucketIdTransformer, QueryParseOptions, RequestParameters, SourceSchema, @@ -388,21 +389,20 @@ export class SqlSyncRules { /** * Hydrate the sync rule definitions with persisted state into runnable sync rules. * - * Right now this is just the bucketIdTransformer, but this is expected to expand in the future to support - * incremental sync rule reprocessing. - * - * @param params.bucketIdTransformer A function that transforms bucket ids based on persisted state. May omit for tests. + * @param params.hydrationState Transforms bucket ids based on persisted state. May omit for tests. */ - hydrate(params?: { bucketIdTransformer?: BucketIdTransformer }): HydratedSyncRules { - const bucketIdTransformer = this.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) - ? (params?.bucketIdTransformer ?? ((id: string) => id)) - : (id: string) => id; + hydrate(params?: CreateSourceParams): HydratedSyncRules { + let hydrationState = params?.hydrationState; + if (hydrationState == null || !this.compatibility.isEnabled(CompatibilityOption.versionedBucketIds)) { + hydrationState = DEFAULT_HYDRATION_STATE; + } + const resolvedParams = { hydrationState }; return new HydratedSyncRules({ definition: this, - createParams: { bucketIdTransformer }, - bucketDataSources: this.bucketDataSources.map((d) => d.createDataSource({ bucketIdTransformer })), + createParams: resolvedParams, + bucketDataSources: this.bucketDataSources.map((d) => d.createDataSource(resolvedParams)), bucketParameterLookupSources: this.bucketParameterLookupSources.map((d) => - d.createParameterLookupSource({ bucketIdTransformer }) + d.createParameterLookupSource(resolvedParams) ), eventDescriptors: this.eventDescriptors, compatibility: this.compatibility @@ -488,8 +488,4 @@ export class SqlSyncRules { } } } - - static versionedBucketIdTransformer(version: string) { - return (bucketId: string) => `${version}#${bucketId}`; - } } diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index fa05c7e4c..7d68b78df 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -7,7 +7,6 @@ import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; -import { resolveHydrationState } from './HydrationState.js'; import { BucketDataSourceDefinition, GetQuerierOptions } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; @@ -180,7 +179,7 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi } createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { - const hydrationState = resolveHydrationState(params); + const hydrationState = params.hydrationState; const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; return { pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index d0a042c39..3ca810dbf 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -6,7 +6,6 @@ import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; -import { resolveHydrationState } from './HydrationState.js'; import { BucketDataSourceDefinition, BucketParameterQuerier, GetQuerierOptions, PendingQueriers } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; @@ -230,7 +229,7 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer } createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { - const hydrationState = resolveHydrationState(params); + const hydrationState = params.hydrationState; const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; return { pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { diff --git a/packages/sync-rules/src/streams/filter.ts b/packages/sync-rules/src/streams/filter.ts index a7c40a889..691f6acee 100644 --- a/packages/sync-rules/src/streams/filter.ts +++ b/packages/sync-rules/src/streams/filter.ts @@ -1,3 +1,6 @@ +import { ParameterLookup } from '../BucketParameterQuerier.js'; +import { SqlTools } from '../sql_filters.js'; +import { checkJsonArray, OPERATOR_NOT } from '../sql_functions.js'; import { isParameterValueClause, isRowValueClause, SQLITE_TRUE, sqliteBool } from '../sql_support.js'; import { TablePattern } from '../TablePattern.js'; import { @@ -10,21 +13,18 @@ import { SqliteRow } from '../types.js'; import { isJsonValue, normalizeParameterValue } from '../utils.js'; -import { SqlTools } from '../sql_filters.js'; -import { checkJsonArray, OPERATOR_NOT } from '../sql_functions.js'; -import { ParameterLookup } from '../BucketParameterQuerier.js'; -import { StreamVariant } from './variant.js'; -import { SubqueryEvaluator } from './parameter.js'; -import { cartesianProduct } from './utils.js'; import { NodeLocation } from 'pgsql-ast-parser'; import { BucketParameterLookupSource, BucketParameterLookupSourceDefinition, CreateSourceParams } from '../BucketSource.js'; +import { HydrationState, ParameterLookupScope } from '../HydrationState.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; -import { HydrationState, ParameterLookupScope, resolveHydrationState } from '../HydrationState.js'; +import { SubqueryEvaluator } from './parameter.js'; +import { cartesianProduct } from './utils.js'; +import { StreamVariant } from './variant.js'; /** * An intermediate representation of a `WHERE` clause for stream queries. @@ -600,7 +600,7 @@ export class SubqueryParameterLookupSource implements BucketParameterLookupSourc } createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { - const hydrationState = resolveHydrationState(params); + const hydrationState = params.hydrationState; const lookupScope = hydrationState.getParameterLookupScope(this); return { evaluateParameterRow: (sourceTable, row) => { diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index a651574d3..c7d45e29b 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -10,7 +10,6 @@ import { CreateSourceParams } from '../BucketSource.js'; import { ColumnDefinition } from '../ExpressionType.js'; -import { resolveHydrationState } from '../HydrationState.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { TablePattern } from '../TablePattern.js'; import { EvaluateRowOptions, EvaluationResult, SourceSchema, TableRow } from '../types.js'; @@ -104,7 +103,7 @@ export class SyncStreamDataSource implements BucketDataSourceDefinition { } createDataSource(params: CreateSourceParams): BucketDataSource { - const hydrationState = resolveHydrationState(params); + const hydrationState = params.hydrationState; const bucketPrefix = hydrationState.getBucketSourceState(this).bucketPrefix; return { evaluateRow: (options: EvaluateRowOptions): EvaluationResult[] => { diff --git a/packages/sync-rules/src/streams/variant.ts b/packages/sync-rules/src/streams/variant.ts index feb636cba..a6cebbd4e 100644 --- a/packages/sync-rules/src/streams/variant.ts +++ b/packages/sync-rules/src/streams/variant.ts @@ -7,7 +7,6 @@ import { BucketParameterQuerierSourceDefinition, CreateSourceParams } from '../BucketSource.js'; -import { resolveHydrationState } from '../HydrationState.js'; import { GetQuerierOptions, RequestedStream } from '../index.js'; import { RequestParameters, SqliteJsonValue, TableRow } from '../types.js'; import { isJsonValue, JSONBucketNameSerialize } from '../utils.js'; @@ -350,7 +349,7 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS } createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { - const hydrationState = resolveHydrationState(params); + const hydrationState = params.hydrationState; const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; const stream = this.stream; diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index 1730ef803..502a7ec77 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -285,20 +285,6 @@ export interface InputParameter { parametersToLookupValue(parameters: ParameterValueSet): SqliteValue; } -/** - * Transforms bucket ids generated when evaluating the row by e.g. encoding version information. - * - * Because buckets are recreated on a sync rule redeploy, it makes sense to use different bucket ids (otherwise, clients - * may run into checksum errors causing a sync to take longer than necessary or breaking progress). - * - * So, this transformer receives the original bucket id as generated by defined sync rules, and can prepend a version - * identifier. - * - * Note that this transformation has not been present in older versions of the sync service. To preserve backwards - * compatibility, sync rules will not use this function without an opt-in. - */ -export type BucketIdTransformer = (regularId: string) => string; - export interface EvaluateRowOptions extends TableRow {} /** diff --git a/packages/sync-rules/src/utils.ts b/packages/sync-rules/src/utils.ts index e66d8367b..1ae02026f 100644 --- a/packages/sync-rules/src/utils.ts +++ b/packages/sync-rules/src/utils.ts @@ -4,7 +4,6 @@ import { CompatibilityContext } from './compatibility.js'; import { SyncRuleProcessingError as SyncRulesProcessingError } from './errors.js'; import { SQLITE_FALSE, SQLITE_TRUE } from './sql_support.js'; import { - BucketIdTransformer, DatabaseInputRow, DatabaseInputValue, SqliteInputRow, diff --git a/packages/sync-rules/test/src/compatibility.test.ts b/packages/sync-rules/test/src/compatibility.test.ts index 624468a58..aded28073 100644 --- a/packages/sync-rules/test/src/compatibility.test.ts +++ b/packages/sync-rules/test/src/compatibility.test.ts @@ -2,6 +2,8 @@ import { describe, expect, test } from 'vitest'; import { DateTimeValue, SqlSyncRules, toSyncRulesValue } from '../../src/index.js'; import { ASSETS, normalizeQuerierOptions, PARSE_OPTIONS } from './util.js'; +import { version } from 'node:process'; +import { versionedHydrationState } from '../../src/HydrationState.js'; describe('compatibility options', () => { describe('timestamps', () => { @@ -70,7 +72,7 @@ config: edition: 2 `, PARSE_OPTIONS - ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); + ).hydrate({ hydrationState: versionedHydrationState(1) }); expect( rules.evaluateRow({ @@ -108,7 +110,7 @@ config: versioned_bucket_ids: false `, PARSE_OPTIONS - ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); + ).hydrate({ hydrationState: versionedHydrationState(1) }); expect( rules.evaluateRow({ @@ -145,7 +147,7 @@ config: versioned_bucket_ids: true `, PARSE_OPTIONS - ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); + ).hydrate({ hydrationState: versionedHydrationState(1) }); expect( rules.evaluateRow({ @@ -169,7 +171,7 @@ config: edition: 2 `, PARSE_OPTIONS - ).hydrate({ bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') }); + ).hydrate({ hydrationState: versionedHydrationState(1) }); expect( rules.evaluateRow({ @@ -270,7 +272,7 @@ config: } const rules = SqlSyncRules.fromYaml(syncRules, PARSE_OPTIONS).hydrate({ - bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer('1') + hydrationState: versionedHydrationState(1) }); expect( rules.evaluateRow({ diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index 5f6285834..713f2effa 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -4,6 +4,7 @@ import { BucketParameterQuerier, CompatibilityContext, CompatibilityEdition, + CreateSourceParams, debugHydratedMergedSource, DEFAULT_TAG, GetBucketParameterQuerierResult, @@ -22,7 +23,7 @@ import { syncStreamFromSql } from '../../src/index.js'; import { normalizeQuerierOptions, PARSE_OPTIONS, TestSourceTable } from './util.js'; -import { ParameterLookupScope } from '../../src/HydrationState.js'; +import { ParameterLookupScope, versionedHydrationState } from '../../src/HydrationState.js'; describe('streams', () => { const STREAM_0: ParameterLookupScope = { @@ -49,9 +50,7 @@ describe('streams', () => { expect(desc.variants).toHaveLength(1); expect(evaluateBucketIds(desc, COMMENTS, { id: 'foo' })).toStrictEqual(['1#stream|0[]']); expect( - desc.dataSources[0] - .createDataSource({ bucketIdTransformer }) - .evaluateRow({ sourceTable: USERS, record: { id: 'foo' } }) + desc.dataSources[0].createDataSource(hydrationParams).evaluateRow({ sourceTable: USERS, record: { id: 'foo' } }) ).toHaveLength(0); }); @@ -82,7 +81,7 @@ describe('streams', () => { test('legacy token parameter', async () => { const desc = parseStream(`SELECT * FROM issues WHERE owner_id = auth.parameter('$.parameters.test')`); - const source = debugHydratedMergedSource(desc, { bucketIdTransformer }); + const source = debugHydratedMergedSource(desc, hydrationParams); const queriers: BucketParameterQuerier[] = []; const errors: QuerierError[] = []; @@ -232,7 +231,7 @@ describe('streams', () => { ]); expect( - debugHydratedMergedSource(desc, { bucketIdTransformer }).evaluateParameterRow(ISSUES, { + debugHydratedMergedSource(desc, hydrationParams).evaluateParameterRow(ISSUES, { id: 'i1', owner_id: 'u1' }) @@ -272,7 +271,7 @@ describe('streams', () => { expect(lookup.tableSyncsParameters(ISSUES)).toBe(true); expect( lookup - .createParameterLookupSource({ bucketIdTransformer }) + .createParameterLookupSource(hydrationParams) .evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) ).toStrictEqual([ { @@ -308,9 +307,7 @@ describe('streams', () => { expect(lookup.tableSyncsParameters(USERS)).toBe(true); expect( - lookup - .createParameterLookupSource({ bucketIdTransformer }) - .evaluateParameterRow(USERS, { id: 'u', is_admin: 1n }) + lookup.createParameterLookupSource(hydrationParams).evaluateParameterRow(USERS, { id: 'u', is_admin: 1n }) ).toStrictEqual([ { lookup: ParameterLookup.normalized(STREAM_0, ['u']), @@ -322,9 +319,7 @@ describe('streams', () => { } ]); expect( - lookup - .createParameterLookupSource({ bucketIdTransformer }) - .evaluateParameterRow(USERS, { id: 'u', is_admin: 0n }) + lookup.createParameterLookupSource(hydrationParams).evaluateParameterRow(USERS, { id: 'u', is_admin: 0n }) ).toStrictEqual([]); // Should return bucket id for admin users @@ -361,7 +356,7 @@ describe('streams', () => { '1#stream|1["a"]' ]); - const source = debugHydratedMergedSource(desc, { bucketIdTransformer }); + const source = debugHydratedMergedSource(desc, hydrationParams); expect(source.evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' })).toStrictEqual([ { @@ -471,9 +466,7 @@ describe('streams', () => { expect(lookup.tableSyncsParameters(FRIENDS)).toBe(true); expect( - lookup - .createParameterLookupSource({ bucketIdTransformer }) - .evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' }) + lookup.createParameterLookupSource(hydrationParams).evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' }) ).toStrictEqual([ { lookup: ParameterLookup.normalized(STREAM_0, ['b']), @@ -631,7 +624,7 @@ describe('streams', () => { expect( desc.parameterLookupSources[0] - .createParameterLookupSource({ bucketIdTransformer }) + .createParameterLookupSource(hydrationParams) .evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) ).toStrictEqual([ { @@ -762,7 +755,7 @@ describe('streams', () => { // Ensure lookup steps work. expect( stream.parameterLookupSources[0] - .createParameterLookupSource({ bucketIdTransformer }) + .createParameterLookupSource(hydrationParams) .evaluateParameterRow(accountMember, row) ).toStrictEqual([ { @@ -848,7 +841,7 @@ WHERE expect( desc.parameterLookupSources[0] - .createParameterLookupSource({ bucketIdTransformer }) + .createParameterLookupSource(hydrationParams) .evaluateParameterRow(projectInvitation, { project: 'foo', appliedTo: '[1,2]', @@ -945,10 +938,10 @@ const options: StreamParseOptions = { compatibility: new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS) }; -const bucketIdTransformer = SqlSyncRules.versionedBucketIdTransformer('1'); +const hydrationParams: CreateSourceParams = { hydrationState: versionedHydrationState(1) }; function evaluateBucketIds(stream: SyncStream, sourceTable: SourceTableInterface, record: SqliteRow) { - return debugHydratedMergedSource(stream, { bucketIdTransformer }) + return debugHydratedMergedSource(stream, hydrationParams) .evaluateRow({ sourceTable, record }) .map((r) => { if ('error' in r) { @@ -984,7 +977,7 @@ async function createQueriers( }; for (let querier of stream.parameterQuerierSources) { - querier.createParameterQuerierSource({ bucketIdTransformer }).pushBucketParameterQueriers(pending, querierOptions); + querier.createParameterQuerierSource(hydrationParams).pushBucketParameterQueriers(pending, querierOptions); } return { querier: mergeBucketParameterQueriers(queriers), errors }; diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index d4d3fcb6e..599642686 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { ParameterLookup, SqlParameterQuery, SqlSyncRules } from '../../src/index.js'; +import { CreateSourceParams, ParameterLookup, SqlParameterQuery, SqlSyncRules } from '../../src/index.js'; import { ASSETS, @@ -12,9 +12,10 @@ import { } from './util.js'; import { SqlBucketDescriptor } from '../../src/SqlBucketDescriptor.js'; import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; +import { DEFAULT_HYDRATION_STATE } from '../../src/HydrationState.js'; describe('sync rules', () => { - const bucketIdTransformer = SqlSyncRules.versionedBucketIdTransformer(''); + const hydrationParams: CreateSourceParams = { hydrationState: DEFAULT_HYDRATION_STATE }; test('parse empty sync rules', () => { const rules = SqlSyncRules.fromYaml('bucket_definitions: {}', PARSE_OPTIONS); @@ -33,7 +34,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); const bucket = rules.bucketSources[0] as SqlBucketDescriptor; expect(bucket.name).toEqual('mybucket'); expect(bucket.bucketParameters).toEqual([]); @@ -72,7 +73,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect(rules.bucketParameterLookupSources).toEqual([]); const parameterSource = rules.bucketParameterQuerierSources[0]; expect(parameterSource.bucketParameters).toEqual([]); @@ -106,7 +107,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ { bucketParameters: [{}], @@ -127,7 +128,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); const parameterSource = rules.bucketParameterQuerierSources[0]; const bucketData = rules.bucketDataSources[0]; expect(parameterSource.bucketParameters).toEqual(['user_id', 'device_id']); @@ -174,7 +175,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); const bucketParameters = rules.bucketParameterQuerierSources[0]; const bucketData = rules.bucketDataSources[0]; expect(bucketParameters.bucketParameters).toEqual(['user_id']); @@ -317,7 +318,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); const bucketParameters = rules.bucketParameterQuerierSources[0]; expect(bucketParameters.bucketParameters).toEqual(['user_id']); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ @@ -355,7 +356,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); const bucketParameters = rules.bucketParameterQuerierSources[0]; expect(bucketParameters.bucketParameters).toEqual(['user_id']); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ @@ -391,7 +392,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect( hydrated.evaluateRow({ sourceTable: ASSETS, @@ -425,7 +426,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect( hydrated.evaluateRow({ @@ -470,7 +471,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect( hydrated.evaluateRow({ @@ -547,7 +548,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect(hydrated.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset1' } })).toEqual([ { @@ -575,7 +576,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect( hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ int1: 314, float1: 3.14, float2: 314 })).querier ).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[314,3.14,314]', priority: 3 }] }); @@ -608,7 +609,7 @@ bucket_definitions: PARSE_OPTIONS ); expect(rules.errors).toEqual([]); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'test' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["TEST"]', priority: 3 }], hasDynamicBuckets: false @@ -626,7 +627,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect( hydrated.evaluateRow({ @@ -667,7 +668,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect( hydrated.evaluateRow({ @@ -701,7 +702,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect( hydrated.evaluateRow({ @@ -737,7 +738,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect( hydrated.evaluateRow({ @@ -866,7 +867,7 @@ bucket_definitions: expect(rules.errors).toEqual([]); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [ { bucket: 'highprio[]', priority: 0 }, @@ -892,7 +893,7 @@ bucket_definitions: expect(rules.errors).toEqual([]); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [ { bucket: 'highprio[]', priority: 0 }, @@ -957,7 +958,7 @@ bucket_definitions: const bucketParameters = rules.bucketParameterQuerierSources[0]; expect(bucketParameters.bucketParameters).toEqual(['user_id']); - const hydrated = rules.hydrate({ bucketIdTransformer }); + const hydrated = rules.hydrate(hydrationParams); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ hasDynamicBuckets: true, diff --git a/packages/sync-rules/test/src/util.ts b/packages/sync-rules/test/src/util.ts index 7e714815f..72649fa81 100644 --- a/packages/sync-rules/test/src/util.ts +++ b/packages/sync-rules/test/src/util.ts @@ -1,19 +1,18 @@ import { + BucketDataSource, + BucketDataSourceDefinition, + ColumnDefinition, CompatibilityContext, - BucketIdTransformer, + CreateSourceParams, DEFAULT_TAG, GetQuerierOptions, RequestedStream, RequestJwtPayload, RequestParameters, + SourceSchema, SourceTableInterface, StaticSchema, - BucketDataSourceDefinition, - TablePattern, - CreateSourceParams, - BucketDataSource, - SourceSchema, - ColumnDefinition + TablePattern } from '../../src/index.js'; export class TestSourceTable implements SourceTableInterface { From 1c1d55f5194b6e789ec84b46a2d5c7427a787ddb Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 11 Dec 2025 10:34:49 +0200 Subject: [PATCH 24/33] Add a stream test with custom hydrationState. --- packages/sync-rules/test/src/streams.test.ts | 151 ++++++++++++++++--- 1 file changed, 127 insertions(+), 24 deletions(-) diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index 713f2effa..2f5a96d42 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -1,5 +1,6 @@ /// import { describe, expect, test } from 'vitest'; +import { HydrationState, ParameterLookupScope, versionedHydrationState } from '../../src/HydrationState.js'; import { BucketParameterQuerier, CompatibilityContext, @@ -7,6 +8,7 @@ import { CreateSourceParams, debugHydratedMergedSource, DEFAULT_TAG, + EvaluationResult, GetBucketParameterQuerierResult, GetQuerierOptions, mergeBucketParameterQueriers, @@ -16,14 +18,12 @@ import { SourceTableInterface, SqliteJsonRow, SqliteRow, - SqlSyncRules, StaticSchema, StreamParseOptions, SyncStream, syncStreamFromSql } from '../../src/index.js'; import { normalizeQuerierOptions, PARSE_OPTIONS, TestSourceTable } from './util.js'; -import { ParameterLookupScope, versionedHydrationState } from '../../src/HydrationState.js'; describe('streams', () => { const STREAM_0: ParameterLookupScope = { @@ -878,6 +878,108 @@ WHERE ).toStrictEqual(['1#stream|0["foo"]']); }); }); + + test('variants with custom hydrationState', async () => { + // Convoluted example, but want to test specific variant usage. + // This test that bucket prefix and lookup scope mappings are correctly applied for each variant. + const desc = parseStream(` + SELECT * FROM comments WHERE + issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) OR -- stream|0 + issue_id IN (SELECT id FROM issues WHERE name = subscription.parameter('issue_name')) OR -- stream|1 + label = subscription.parameter('comment_label') OR -- stream|2 + auth.parameter('is_admin') -- stream|3 + `); + + const hydrationState: HydrationState = { + getBucketSourceState(source) { + return { bucketPrefix: `${source.defaultBucketPrefix}.test` }; + }, + getParameterLookupScope(source) { + return { + lookupName: `${source.defaultLookupScope.lookupName}.test`, + queryId: `${source.defaultLookupScope.queryId}.test` + }; + } + }; + + const hydrated = debugHydratedMergedSource(desc, { hydrationState }); + + expect( + bucketIds(hydrated.evaluateRow({ sourceTable: COMMENTS, record: { id: 'c', issue_id: 'i1', label: 'l1' } })) + ).toStrictEqual(['stream|0.test["i1"]', 'stream|1.test["i1"]', 'stream|2.test["l1"]', 'stream|3.test[]']); + + expect( + hydrated.evaluateParameterRow(ISSUES, { + id: 'i1', + owner_id: 'u1', + name: 'myname' + }) + ).toStrictEqual([ + { + lookup: ParameterLookup.normalized({ lookupName: 'stream.test', queryId: '0.test' }, ['u1']), + bucketParameters: [ + { + result: 'i1' + } + ] + }, + + { + lookup: ParameterLookup.normalized({ lookupName: 'stream.test', queryId: '1.test' }, ['myname']), + bucketParameters: [ + { + result: 'i1' + } + ] + } + ]); + + expect( + hydrated.evaluateParameterRow(ISSUES, { + id: 'i1', + owner_id: 'u1' + }) + ).toStrictEqual([ + { + lookup: ParameterLookup.normalized({ lookupName: 'stream.test', queryId: '0.test' }, ['u1']), + bucketParameters: [ + { + result: 'i1' + } + ] + } + ]); + + function getParameterSets(lookups: ParameterLookup[]) { + return lookups.flatMap((lookup) => { + if (JSON.stringify(lookup.values) == JSON.stringify(['stream.test', '1.test', null])) { + return []; + } else if (JSON.stringify(lookup.values) == JSON.stringify(['stream.test', '0.test', 'u1'])) { + return [{ result: 'i1' }]; + } else if (JSON.stringify(lookup.values) == JSON.stringify(['stream.test', '1.test', 'myname'])) { + return [{ result: 'i2' }]; + } else { + throw new Error(`Unexpected lookup: ${JSON.stringify(lookup.values)}`); + } + }); + } + + expect( + await queryBucketIds(desc, { + hydrationState, + token: { sub: 'u1', is_admin: false }, + getParameterSets + }) + ).toStrictEqual(['stream|2.test[null]', 'stream|0.test["i1"]']); + expect( + await queryBucketIds(desc, { + hydrationState, + token: { sub: 'u1', is_admin: true }, + parameters: { comment_label: 'l1', issue_name: 'myname' }, + getParameterSets + }) + ).toStrictEqual(['stream|2.test["l1"]', 'stream|3.test[]', 'stream|0.test["i1"]', 'stream|1.test["i2"]']); + }); }); const USERS = new TestSourceTable('users'); @@ -941,24 +1043,28 @@ const options: StreamParseOptions = { const hydrationParams: CreateSourceParams = { hydrationState: versionedHydrationState(1) }; function evaluateBucketIds(stream: SyncStream, sourceTable: SourceTableInterface, record: SqliteRow) { - return debugHydratedMergedSource(stream, hydrationParams) - .evaluateRow({ sourceTable, record }) - .map((r) => { - if ('error' in r) { - throw new Error(`Unexpected error evaluating row: ${r.error}`); - } + return bucketIds(debugHydratedMergedSource(stream, hydrationParams).evaluateRow({ sourceTable, record })); +} - return r.bucket; - }); +function bucketIds(result: EvaluationResult[]): string[] { + return result.map((r) => { + if ('error' in r) { + throw new Error(`Unexpected error evaluating row: ${r.error}`); + } + + return r.bucket; + }); } +interface TestQuerierOptions { + token?: Record; + parameters?: Record; + getParameterSets?: (lookups: ParameterLookup[]) => SqliteJsonRow[]; + hydrationState?: HydrationState; +} async function createQueriers( stream: SyncStream, - options?: { - token?: Record; - parameters?: Record; - getParameterSets?: (lookups: ParameterLookup[]) => SqliteJsonRow[]; - } + options?: TestQuerierOptions ): Promise { const queriers: BucketParameterQuerier[] = []; const errors: QuerierError[] = []; @@ -977,20 +1083,17 @@ async function createQueriers( }; for (let querier of stream.parameterQuerierSources) { - querier.createParameterQuerierSource(hydrationParams).pushBucketParameterQueriers(pending, querierOptions); + querier + .createParameterQuerierSource( + options?.hydrationState ? { hydrationState: options.hydrationState } : hydrationParams + ) + .pushBucketParameterQueriers(pending, querierOptions); } return { querier: mergeBucketParameterQueriers(queriers), errors }; } -async function queryBucketIds( - stream: SyncStream, - options?: { - token?: Record; - parameters?: Record; - getParameterSets?: (lookups: ParameterLookup[]) => SqliteJsonRow[]; - } -) { +async function queryBucketIds(stream: SyncStream, options?: TestQuerierOptions) { const { querier, errors } = await createQueriers(stream, options); expect(errors).toHaveLength(0); From f919d20b98b9d090715a7f6939eb105216c54c8b Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 11 Dec 2025 10:45:10 +0200 Subject: [PATCH 25/33] Add changeset. --- .changeset/fresh-geckos-develop.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .changeset/fresh-geckos-develop.md diff --git a/.changeset/fresh-geckos-develop.md b/.changeset/fresh-geckos-develop.md new file mode 100644 index 000000000..dd071398b --- /dev/null +++ b/.changeset/fresh-geckos-develop.md @@ -0,0 +1,13 @@ +--- +'@powersync/service-module-postgres-storage': minor +'@powersync/service-module-mongodb-storage': minor +'@powersync/service-core-tests': minor +'@powersync/service-module-postgres': minor +'@powersync/service-module-mongodb': minor +'@powersync/service-core': minor +'@powersync/service-module-mssql': minor +'@powersync/service-module-mysql': minor +'@powersync/service-sync-rules': minor +--- + +[Internal] Refactor sync rule representation to split out the parsed definitions from the hydrated state. From 1679f4a4122bb3b7f25bdd9fd32686a0ec96a529 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 12 Dec 2025 13:25:58 +0200 Subject: [PATCH 26/33] Remove need for hydration on BucketDataSource. --- packages/sync-rules/src/BaseSqlDataQuery.ts | 20 +++-- packages/sync-rules/src/BucketSource.ts | 87 +++++++++++-------- .../{SyncRules.ts => HydratedSyncRules.ts} | 30 ++++++- packages/sync-rules/src/HydrationState.ts | 12 +-- .../sync-rules/src/SqlBucketDescriptor.ts | 36 +++----- packages/sync-rules/src/SqlDataQuery.ts | 10 +-- packages/sync-rules/src/SqlParameterQuery.ts | 33 ++++--- packages/sync-rules/src/SqlSyncRules.ts | 8 +- .../sync-rules/src/StaticSqlParameterQuery.ts | 21 +++-- .../TableValuedFunctionSqlParameterQuery.ts | 25 +++--- packages/sync-rules/src/index.ts | 2 +- packages/sync-rules/src/streams/stream.ts | 55 +++++------- packages/sync-rules/src/streams/variant.ts | 35 ++++---- packages/sync-rules/src/types.ts | 51 +++++++++-- .../src/types/custom_sqlite_value.ts | 2 +- packages/sync-rules/src/utils.ts | 13 +-- .../sync-rules/test/src/data_queries.test.ts | 55 ++++-------- .../test/src/parameter_queries.test.ts | 20 ++--- .../test/src/static_parameter_queries.test.ts | 69 ++++++++------- packages/sync-rules/test/src/streams.test.ts | 6 +- .../src/table_valued_function_queries.test.ts | 54 +++++++++--- packages/sync-rules/test/src/util.ts | 5 +- 22 files changed, 367 insertions(+), 282 deletions(-) rename packages/sync-rules/src/{SyncRules.ts => HydratedSyncRules.ts} (84%) diff --git a/packages/sync-rules/src/BaseSqlDataQuery.ts b/packages/sync-rules/src/BaseSqlDataQuery.ts index 355a0970f..f83c5362a 100644 --- a/packages/sync-rules/src/BaseSqlDataQuery.ts +++ b/packages/sync-rules/src/BaseSqlDataQuery.ts @@ -6,9 +6,10 @@ import { AvailableTable, SqlTools } from './sql_filters.js'; import { castAsText } from './sql_functions.js'; import { TablePattern } from './TablePattern.js'; import { - EvaluationResult, QueryParameters, QuerySchema, + SourceEvaluatedRow, + SourceEvaluationResult, SourceSchema, SourceSchemaTable, SqliteJsonRow, @@ -24,7 +25,7 @@ export interface RowValueExtractor { export interface EvaluateRowOptions { table: SourceTableInterface; row: SqliteRow; - bucketIds: (params: QueryParameters) => string[]; + serializedBucketParameters: (params: QueryParameters) => string[]; } export interface BaseSqlDataQueryOptions { @@ -169,13 +170,14 @@ export class BaseSqlDataQuery { } } - evaluateRowWithOptions(options: EvaluateRowOptions): EvaluationResult[] { + evaluateRowWithOptions(options: EvaluateRowOptions): SourceEvaluationResult[] { try { - const { table, row, bucketIds } = options; + const { table, row, serializedBucketParameters } = options; const tables = { [this.table.nameInSchema]: this.addSpecialParameters(table, row) }; - const resolvedBucketIds = bucketIds(tables); - if (resolvedBucketIds.length == 0) { + // Array of _serialized_ parameters, one per output result. + const resolvedBucketParameters = serializedBucketParameters(tables); + if (resolvedBucketParameters.length == 0) { // Short-circuit: No need to transform the row if there are no matching buckets. return []; } @@ -193,13 +195,13 @@ export class BaseSqlDataQuery { } const outputTable = this.getOutputName(table.name); - return resolvedBucketIds.map((bucketId) => { + return resolvedBucketParameters.map((serializedBucketParameters) => { return { - bucket: bucketId, + serializedBucketParameters, table: outputTable, id: id, data - } as EvaluationResult; + } satisfies SourceEvaluatedRow; }); } catch (e) { return [{ error: e.message ?? `Evaluating data query failed` }]; diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 1754a6a10..2b81a0963 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -4,7 +4,17 @@ import { DEFAULT_HYDRATION_STATE, HydrationState, ParameterLookupScope } from '. import { SourceTableInterface } from './SourceTableInterface.js'; import { GetQuerierOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; -import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, SourceSchema, SqliteRow } from './types.js'; +import { + EvaluatedParametersResult, + EvaluatedRow, + EvaluateRowOptions, + EvaluationResult, + isEvaluationError, + SourceEvaluationResult, + SourceSchema, + SqliteRow +} from './types.js'; +import { buildBucketName } from './utils.js'; export interface CreateSourceParams { hydrationState: HydrationState; @@ -30,7 +40,7 @@ export interface BucketSource { * Specifically, bucket definitions would always have a single data source, while stream definitions may have * one per variant. */ - readonly dataSources: BucketDataSourceDefinition[]; + readonly dataSources: BucketDataSource[]; /** * BucketParameterQuerierSource describing the parameter queries / stream subqueries in this bucket/stream definition. @@ -57,8 +67,11 @@ export interface HydratedBucketSource { /** * Encodes a static definition of a bucket source, as parsed from sync rules or stream definitions. + * + * This does not require any "hydration" itself: All results are independent of bucket names. + * The higher-level HydratedSyncRules will use a HydrationState to generate bucket names. */ -export interface BucketDataSourceDefinition { +export interface BucketDataSource { /** * Bucket prefix if no transformations are defined. * @@ -70,10 +83,16 @@ export interface BucketDataSourceDefinition { * For debug use only. */ readonly bucketParameters: string[]; + getSourceTables(): Set; - createDataSource(params: CreateSourceParams): BucketDataSource; tableSyncsData(table: SourceTableInterface): boolean; + /** + * Given a row as it appears in a table that affects sync data, return buckets, logical table names and transformed + * data for rows to add to buckets. + */ + evaluateRow(options: EvaluateRowOptions): SourceEvaluationResult[]; + /** * Given a static schema, infer all logical tables and associated columns that appear in buckets defined by this * source. @@ -121,29 +140,11 @@ export interface BucketParameterQuerierSourceDefinition { * * Note that queriers do not persist data themselves; they only resolve which buckets to load based on request parameters. */ - readonly querierDataSource: BucketDataSourceDefinition; + readonly querierDataSource: BucketDataSource; createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource; } -/** - * An interface declaring - * - * - which buckets the sync service should create when processing change streams from the database. - * - how data in source tables maps to data in buckets (e.g. when we're not selecting all columns). - * - which buckets a given connection has access to. - * - * There are two ways to define bucket sources: Via sync rules made up of parameter and data queries, and via stream - * definitions that only consist of a single query. - */ -export interface BucketDataSource { - /** - * Given a row as it appears in a table that affects sync data, return buckets, logical table names and transformed - * data for rows to add to buckets. - */ - evaluateRow(options: EvaluateRowOptions): EvaluationResult[]; -} - export interface BucketParameterLookupSource { /** * Given a row in a source table that affects sync parameters, returns a structure to index which buckets rows should @@ -165,10 +166,9 @@ export interface BucketParameterQuerierSource { pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void; } -export interface DebugMergedSource - extends BucketDataSource, - BucketParameterLookupSource, - BucketParameterQuerierSource {} +export interface DebugMergedSource extends BucketParameterLookupSource, BucketParameterQuerierSource { + evaluateRow(options: EvaluateRowOptions): EvaluationResult[]; +} export enum BucketSourceType { SYNC_RULE, @@ -177,14 +177,31 @@ export enum BucketSourceType { export type ResultSetDescription = { name: string; columns: ColumnDefinition[] }; -export function mergeDataSources(sources: BucketDataSource[]): BucketDataSource { +export function hydrateEvaluateRow( + hydrationState: HydrationState, + source: BucketDataSource +): (options: EvaluateRowOptions) => EvaluationResult[] { + const scope = hydrationState.getBucketSourceScope(source); + return (options: EvaluateRowOptions): EvaluationResult[] => { + return source.evaluateRow(options).map((result) => { + if (isEvaluationError(result)) { + return result; + } + return { + bucket: buildBucketName(scope, result.serializedBucketParameters), + id: result.id, + table: result.table, + data: result.data + } satisfies EvaluatedRow; + }); + }; +} + +export function mergeDataSources(hydrationState: HydrationState, sources: BucketDataSource[]) { + const withScope = sources.map((source) => hydrateEvaluateRow(hydrationState, source)); return { evaluateRow(options: EvaluateRowOptions): EvaluationResult[] { - let results: EvaluationResult[] = []; - for (let source of sources) { - results.push(...source.evaluateRow(options)); - } - return results; + return withScope.flatMap((source) => source(options)); } }; } @@ -218,9 +235,7 @@ export function mergeParameterQuerierSources(sources: BucketParameterQuerierSour export function debugHydratedMergedSource(bucketSource: BucketSource, params?: CreateSourceParams): DebugMergedSource { const hydrationState = params?.hydrationState ?? DEFAULT_HYDRATION_STATE; const resolvedParams = { hydrationState }; - const dataSource = mergeDataSources( - bucketSource.dataSources.map((source) => source.createDataSource(resolvedParams)) - ); + const dataSource = mergeDataSources(hydrationState, bucketSource.dataSources); const parameterLookupSource = mergeParameterLookupSources( bucketSource.parameterLookupSources.map((source) => source.createParameterLookupSource(resolvedParams)) ); diff --git a/packages/sync-rules/src/SyncRules.ts b/packages/sync-rules/src/HydratedSyncRules.ts similarity index 84% rename from packages/sync-rules/src/SyncRules.ts rename to packages/sync-rules/src/HydratedSyncRules.ts index 4e2491526..b4a1c0473 100644 --- a/packages/sync-rules/src/SyncRules.ts +++ b/packages/sync-rules/src/HydratedSyncRules.ts @@ -1,11 +1,10 @@ import { BucketDataSource, BucketParameterLookupSource, - BucketParameterQuerierSource, - BucketParameterQuerierSourceDefinition, CreateSourceParams, HydratedBucketSource } from './BucketSource.js'; +import { BucketDataScope } from './HydrationState.js'; import { BucketParameterQuerier, CompatibilityContext, @@ -35,6 +34,7 @@ export class HydratedSyncRules { bucketSources: HydratedBucketSource[] = []; bucketDataSources: BucketDataSource[]; bucketParameterLookupSources: BucketParameterLookupSource[]; + bucketSourceHydration: Map = new Map(); eventDescriptors: SqlEventDescriptor[] = []; compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; @@ -53,6 +53,13 @@ export class HydratedSyncRules { this.bucketParameterLookupSources = params.bucketParameterLookupSources; this.definition = params.definition; + const hydrationState = params.createParams.hydrationState; + + for (let source of this.bucketDataSources) { + const state = hydrationState.getBucketSourceScope(source); + this.bucketSourceHydration.set(source, state); + } + if (params.eventDescriptors) { this.eventDescriptors = params.eventDescriptors; } @@ -107,7 +114,24 @@ export class HydratedSyncRules { evaluateRowWithErrors(options: EvaluateRowOptions): { results: EvaluatedRow[]; errors: EvaluationError[] } { let rawResults: EvaluationResult[] = []; for (let source of this.bucketDataSources) { - rawResults.push(...source.evaluateRow(options)); + const sourceResults = source.evaluateRow(options); + if (sourceResults.length == 0) { + continue; + } + const bucketPrefix = this.bucketSourceHydration.get(source)!.bucketPrefix; + rawResults.push( + ...sourceResults.map((sourceRow) => { + if (isEvaluationError(sourceRow)) { + return sourceRow; + } + return { + bucket: bucketPrefix + sourceRow.serializedBucketParameters, + id: sourceRow.id, + table: sourceRow.table, + data: sourceRow.data + } satisfies EvaluatedRow; + }) + ); } const results = rawResults.filter(isEvaluatedRow) as EvaluatedRow[]; diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts index bdcde34b3..4a52e305c 100644 --- a/packages/sync-rules/src/HydrationState.ts +++ b/packages/sync-rules/src/HydrationState.ts @@ -1,6 +1,6 @@ -import { BucketDataSourceDefinition, BucketParameterLookupSourceDefinition } from './BucketSource.js'; +import { BucketDataSource, BucketParameterLookupSourceDefinition } from './BucketSource.js'; -export interface BucketSourceState { +export interface BucketDataScope { /** The prefix is the bucket name before the parameters. */ bucketPrefix: string; } @@ -18,13 +18,13 @@ export interface ParameterLookupScope { * both to re-use mappings across hydrations of different sync rule versions, or to generate new mappings. */ export interface HydrationState< - T extends BucketSourceState = BucketSourceState, + T extends BucketDataScope = BucketDataScope, U extends ParameterLookupScope = ParameterLookupScope > { /** * Given a bucket data source definition, get the bucket prefix to use for it. */ - getBucketSourceState(source: BucketDataSourceDefinition): T; + getBucketSourceScope(source: BucketDataSource): T; /** * Given a bucket parameter lookup definition, get the persistence name to use. @@ -38,7 +38,7 @@ export interface HydrationState< * This is the legacy default behavior with no bucket versioning. */ export const DEFAULT_HYDRATION_STATE: HydrationState = { - getBucketSourceState(source: BucketDataSourceDefinition) { + getBucketSourceScope(source: BucketDataSource) { return { bucketPrefix: source.defaultBucketPrefix }; @@ -62,7 +62,7 @@ export const DEFAULT_HYDRATION_STATE: HydrationState = { */ export function versionedHydrationState(version: number): HydrationState { return { - getBucketSourceState(source: BucketDataSourceDefinition): BucketSourceState { + getBucketSourceScope(source: BucketDataSource): BucketDataScope { return { bucketPrefix: `${version}#${source.defaultBucketPrefix}` }; diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 88993d790..f74254d68 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -1,10 +1,4 @@ -import { - BucketDataSource, - BucketDataSourceDefinition, - BucketSource, - BucketSourceType, - CreateSourceParams -} from './BucketSource.js'; +import { BucketDataSource, BucketSource, BucketSourceType } from './BucketSource.js'; import { ColumnDefinition } from './ExpressionType.js'; import { IdSequence } from './IdSequence.js'; import { SourceTableInterface } from './SourceTableInterface.js'; @@ -16,7 +10,7 @@ import { TablePattern } from './TablePattern.js'; import { TableValuedFunctionSqlParameterQuery } from './TableValuedFunctionSqlParameterQuery.js'; import { CompatibilityContext } from './compatibility.js'; import { SqlRuleError } from './errors.js'; -import { EvaluationResult, QueryParseOptions, SourceSchema } from './types.js'; +import { EvaluateRowOptions, QueryParseOptions, SourceEvaluationResult, SourceSchema } from './types.js'; export interface QueryParseResult { /** @@ -142,7 +136,7 @@ export class SqlBucketDescriptor implements BucketSource { } } -export class BucketDefinitionDataSource implements BucketDataSourceDefinition { +export class BucketDefinitionDataSource implements BucketDataSource { constructor(private descriptor: SqlBucketDescriptor) {} /** @@ -156,22 +150,16 @@ export class BucketDefinitionDataSource implements BucketDataSourceDefinition { return this.descriptor.name; } - createDataSource(params: CreateSourceParams): BucketDataSource { - const hydrationState = params.hydrationState; - const bucketPrefix = hydrationState.getBucketSourceState(this).bucketPrefix; - return { - evaluateRow: (options) => { - let results: EvaluationResult[] = []; - for (let query of this.descriptor.dataQueries) { - if (!query.applies(options.sourceTable)) { - continue; - } - - results.push(...query.evaluateRow(options.sourceTable, options.record, bucketPrefix)); - } - return results; + evaluateRow(options: EvaluateRowOptions) { + let results: SourceEvaluationResult[] = []; + for (let query of this.descriptor.dataQueries) { + if (!query.applies(options.sourceTable)) { + continue; } - }; + + results.push(...query.evaluateRow(options.sourceTable, options.record)); + } + return results; } getSourceTables(): Set { diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index 63b0dd069..b533694b9 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -10,8 +10,8 @@ import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; import { SyncRulesOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; import { TableQuerySchema } from './TableQuerySchema.js'; -import { EvaluationResult, ParameterMatchClause, QuerySchema, SqliteRow } from './types.js'; -import { getBucketId, isSelectStatement } from './utils.js'; +import { ParameterMatchClause, QuerySchema, SourceEvaluationResult, SqliteRow } from './types.js'; +import { isSelectStatement, serializeBucketParameters } from './utils.js'; export interface SqlDataQueryOptions extends BaseSqlDataQueryOptions { filter: ParameterMatchClause; @@ -190,13 +190,13 @@ export class SqlDataQuery extends BaseSqlDataQuery { this.filter = options.filter; } - evaluateRow(table: SourceTableInterface, row: SqliteRow, bucketPrefix: string): EvaluationResult[] { + evaluateRow(table: SourceTableInterface, row: SqliteRow): SourceEvaluationResult[] { return this.evaluateRowWithOptions({ table, row, - bucketIds: (tables) => { + serializedBucketParameters: (tables) => { const bucketParameters = this.filter.filterRow(tables); - return bucketParameters.map((params) => getBucketId(bucketPrefix, this.bucketParameters, params)); + return bucketParameters.map((params) => serializeBucketParameters(this.bucketParameters, params)); } }); } diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 586331331..7a219e69f 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -19,8 +19,8 @@ import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; -import { ParameterLookupScope } from './HydrationState.js'; -import { BucketDataSourceDefinition, GetQuerierOptions } from './index.js'; +import { BucketDataScope, ParameterLookupScope } from './HydrationState.js'; +import { BucketDataSource, GetQuerierOptions } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; @@ -42,7 +42,14 @@ import { SqliteJsonValue, SqliteRow } from './types.js'; -import { filterJsonRow, getBucketId, isJsonValue, isSelectStatement, normalizeParameterValue } from './utils.js'; +import { + buildBucketName, + filterJsonRow, + isJsonValue, + isSelectStatement, + normalizeParameterValue, + serializeBucketParameters +} from './utils.js'; import { DetectRequestParameters } from './validators.js'; export interface SqlParameterQueryOptions { @@ -59,7 +66,7 @@ export interface SqlParameterQueryOptions { bucketParameters: string[]; queryId: string; tools: SqlTools; - querierDataSource: BucketDataSourceDefinition; + querierDataSource: BucketDataSource; errors?: SqlRuleError[]; } @@ -77,7 +84,7 @@ export class SqlParameterQuery sql: string, options: QueryParseOptions, queryId: string, - querierDataSource: BucketDataSourceDefinition + querierDataSource: BucketDataSource ): SqlParameterQuery | StaticSqlParameterQuery | TableValuedFunctionSqlParameterQuery { const parsed = parse(sql, { locationTracking: true }); const schema = options?.schema; @@ -308,7 +315,7 @@ export class SqlParameterQuery readonly queryId: string; readonly tools: SqlTools; - readonly querierDataSource: BucketDataSourceDefinition; + readonly querierDataSource: BucketDataSource; readonly errors: SqlRuleError[]; @@ -347,12 +354,12 @@ export class SqlParameterQuery createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { const hydrationState = params.hydrationState; - const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; + const bucketScope = hydrationState.getBucketSourceScope(this.querierDataSource); const lookupState = hydrationState.getParameterLookupScope(this); return { pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { - const q = this.getBucketParameterQuerier(options.globalParameters, ['default'], bucketPrefix, lookupState); + const q = this.getBucketParameterQuerier(options.globalParameters, ['default'], bucketScope, lookupState); result.queriers.push(q); } }; @@ -422,7 +429,7 @@ export class SqlParameterQuery resolveBucketDescriptions( bucketParameters: SqliteJsonRow[], parameters: RequestParameters, - bucketPrefix: string + bucketScope: BucketDataScope ): BucketDescription[] { // Filters have already been applied and gotten us the set of bucketParameters - don't attempt to filter again. // We _do_ need to evaluate the output columns here, using a combination of precomputed bucketParameters, @@ -446,8 +453,10 @@ export class SqlParameterQuery } } + const serializedParameters = serializeBucketParameters(this.bucketParameters, result); + return { - bucket: getBucketId(bucketPrefix, this.bucketParameters, result), + bucket: buildBucketName(bucketScope, serializedParameters), priority: this.priority }; }) @@ -533,7 +542,7 @@ export class SqlParameterQuery getBucketParameterQuerier( requestParameters: RequestParameters, reasons: BucketInclusionReason[], - bucketPrefix: string, + bucketDataScope: BucketDataScope, scope: ParameterLookupScope ): BucketParameterQuerier { const lookups = this.getLookups(scope, requestParameters); @@ -554,7 +563,7 @@ export class SqlParameterQuery parameterQueryLookups: lookups, queryDynamicBucketDescriptions: async (source: ParameterLookupSource) => { const bucketParameters = await source.getParameterSets(lookups); - return this.resolveBucketDescriptions(bucketParameters, requestParameters, bucketPrefix).map((bucket) => ({ + return this.resolveBucketDescriptions(bucketParameters, requestParameters, bucketDataScope).map((bucket) => ({ ...bucket, definition: this.descriptorName, inclusion_reasons: reasons diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index f42d1e67a..6fbdad975 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -2,7 +2,7 @@ import { isScalar, LineCounter, parseDocument, Scalar, YAMLMap, YAMLSeq } from ' import { isValidPriority } from './BucketDescription.js'; import { BucketParameterQuerier, QuerierError } from './BucketParameterQuerier.js'; import { - BucketDataSourceDefinition, + BucketDataSource, BucketParameterLookupSourceDefinition, BucketParameterQuerierSourceDefinition, BucketSource, @@ -21,7 +21,7 @@ import { validateSyncRulesSchema } from './json_schema.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { QueryParseResult, SqlBucketDescriptor } from './SqlBucketDescriptor.js'; import { syncStreamFromSql } from './streams/from_sql.js'; -import { HydratedSyncRules } from './SyncRules.js'; +import { HydratedSyncRules } from './HydratedSyncRules.js'; import { TablePattern } from './TablePattern.js'; import { QueryParseOptions, @@ -89,7 +89,7 @@ export interface GetBucketParameterQuerierResult { } export class SqlSyncRules { - bucketDataSources: BucketDataSourceDefinition[] = []; + bucketDataSources: BucketDataSource[] = []; bucketParameterLookupSources: BucketParameterLookupSourceDefinition[] = []; bucketParameterQuerierSources: BucketParameterQuerierSourceDefinition[] = []; bucketSources: BucketSource[] = []; @@ -416,7 +416,7 @@ export class SqlSyncRules { return new HydratedSyncRules({ definition: this, createParams: resolvedParams, - bucketDataSources: this.bucketDataSources.map((d) => d.createDataSource(resolvedParams)), + bucketDataSources: this.bucketDataSources, bucketParameterLookupSources: this.bucketParameterLookupSources.map((d) => d.createParameterLookupSource(resolvedParams) ), diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index 7d68b78df..370fdbcf8 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -7,13 +7,14 @@ import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; -import { BucketDataSourceDefinition, GetQuerierOptions } from './index.js'; +import { BucketDataScope } from './HydrationState.js'; +import { BucketDataSource, GetQuerierOptions } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; import { checkUnsupportedFeatures, isClauseError, sqliteBool } from './sql_support.js'; import { TablePattern } from './TablePattern.js'; import { ParameterValueClause, QueryParseOptions, RequestParameters, SqliteJsonValue } from './types.js'; -import { getBucketId, isJsonValue } from './utils.js'; +import { buildBucketName, isJsonValue, serializeBucketParameters } from './utils.js'; import { DetectRequestParameters } from './validators.js'; export interface StaticSqlParameterQueryOptions { @@ -24,7 +25,7 @@ export interface StaticSqlParameterQueryOptions { bucketParameters: string[]; queryId: string; filter: ParameterValueClause | undefined; - querierDataSource: BucketDataSourceDefinition; + querierDataSource: BucketDataSource; errors?: SqlRuleError[]; } @@ -41,7 +42,7 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi q: SelectFromStatement, options: QueryParseOptions, queryId: string, - querierDataSource: BucketDataSourceDefinition + querierDataSource: BucketDataSource ) { let errors: SqlRuleError[] = []; @@ -154,7 +155,7 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi */ readonly filter: ParameterValueClause | undefined; - public readonly querierDataSource: BucketDataSourceDefinition; + public readonly querierDataSource: BucketDataSource; readonly errors: SqlRuleError[]; @@ -180,10 +181,10 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { const hydrationState = params.hydrationState; - const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; + const bucketScope = hydrationState.getBucketSourceScope(this.querierDataSource); return { pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { - const staticBuckets = this.getStaticBucketDescriptions(options.globalParameters, bucketPrefix).map((desc) => { + const staticBuckets = this.getStaticBucketDescriptions(options.globalParameters, bucketScope).map((desc) => { return { ...desc, definition: this.descriptorName, @@ -205,7 +206,7 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi }; } - getStaticBucketDescriptions(parameters: RequestParameters, bucketPrefix: string): BucketDescription[] { + getStaticBucketDescriptions(parameters: RequestParameters, bucketSourceScope: BucketDataScope): BucketDescription[] { if (this.filter == null) { // Error in filter clause return []; @@ -227,9 +228,11 @@ export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefi } } + const serializedParamters = serializeBucketParameters(this.bucketParameters, result); + return [ { - bucket: getBucketId(bucketPrefix, this.bucketParameters, result), + bucket: buildBucketName(bucketSourceScope, serializedParamters), priority: this.priority } ]; diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index 3ca810dbf..5e8f311e6 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -6,7 +6,8 @@ import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; -import { BucketDataSourceDefinition, BucketParameterQuerier, GetQuerierOptions, PendingQueriers } from './index.js'; +import { BucketDataScope } from './HydrationState.js'; +import { BucketDataSource, BucketParameterQuerier, GetQuerierOptions, PendingQueriers } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; import { checkUnsupportedFeatures, isClauseError, sqliteBool } from './sql_support.js'; @@ -20,7 +21,7 @@ import { SqliteJsonValue, SqliteRow } from './types.js'; -import { getBucketId, isJsonValue } from './utils.js'; +import { buildBucketName, isJsonValue, serializeBucketParameters } from './utils.js'; import { DetectRequestParameters } from './validators.js'; export interface TableValuedFunctionSqlParameterQueryOptions { @@ -35,7 +36,7 @@ export interface TableValuedFunctionSqlParameterQueryOptions { callClause: ParameterValueClause | undefined; function: TableValuedFunction; callTable: AvailableTable; - querierDataSource: BucketDataSourceDefinition; + querierDataSource: BucketDataSource; errors: SqlRuleError[]; } @@ -57,7 +58,7 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer q: SelectFromStatement, options: QueryParseOptions, queryId: string, - querierDataSource: BucketDataSourceDefinition + querierDataSource: BucketDataSource ): TableValuedFunctionSqlParameterQuery { const compatibility = options.compatibility; let errors: SqlRuleError[] = []; @@ -199,7 +200,7 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer */ readonly callTable: AvailableTable; - public readonly querierDataSource: BucketDataSourceDefinition; + public readonly querierDataSource: BucketDataSource; readonly errors: SqlRuleError[]; @@ -230,10 +231,10 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { const hydrationState = params.hydrationState; - const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; + const bucketScope = hydrationState.getBucketSourceScope(this.querierDataSource); return { pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { - const staticBuckets = this.getStaticBucketDescriptions(options.globalParameters, bucketPrefix).map((desc) => { + const staticBuckets = this.getStaticBucketDescriptions(options.globalParameters, bucketScope).map((desc) => { return { ...desc, definition: this.descriptorName, @@ -255,7 +256,7 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer }; } - getStaticBucketDescriptions(parameters: RequestParameters, bucketPrefix: string): BucketDescription[] { + getStaticBucketDescriptions(parameters: RequestParameters, scope: BucketDataScope): BucketDescription[] { if (this.filter == null || this.callClause == null) { // Error in filter clause return []; @@ -265,7 +266,7 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer const rows = this.function.call([valueString]); let total: BucketDescription[] = []; for (let row of rows) { - const description = this.getIndividualBucketDescription(row, parameters, bucketPrefix); + const description = this.getIndividualBucketDescription(row, parameters, scope); if (description !== null) { total.push(description); } @@ -276,7 +277,7 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer private getIndividualBucketDescription( row: SqliteRow, parameters: RequestParameters, - bucketPrefix: string + bucketScope: BucketDataScope ): BucketDescription | null { const mergedParams: ParameterValueSet = { ...parameters, @@ -303,8 +304,10 @@ export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuer } } + const serializedBucketParameters = serializeBucketParameters(this.bucketParameters, result); + return { - bucket: getBucketId(bucketPrefix, this.bucketParameters, result), + bucket: buildBucketName(bucketScope, serializedBucketParameters), priority: this.priority }; } diff --git a/packages/sync-rules/src/index.ts b/packages/sync-rules/src/index.ts index 08b82f11d..4609714fd 100644 --- a/packages/sync-rules/src/index.ts +++ b/packages/sync-rules/src/index.ts @@ -27,4 +27,4 @@ export * from './types.js'; export * from './types/custom_sqlite_value.js'; export * from './types/time.js'; export * from './utils.js'; -export * from './SyncRules.js'; +export * from './HydratedSyncRules.js'; diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index c7d45e29b..0db285a50 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -2,17 +2,15 @@ import { BaseSqlDataQuery } from '../BaseSqlDataQuery.js'; import { BucketPriority, DEFAULT_BUCKET_PRIORITY } from '../BucketDescription.js'; import { BucketDataSource, - BucketDataSourceDefinition, BucketParameterLookupSourceDefinition, BucketParameterQuerierSourceDefinition, BucketSource, - BucketSourceType, - CreateSourceParams + BucketSourceType } from '../BucketSource.js'; import { ColumnDefinition } from '../ExpressionType.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { TablePattern } from '../TablePattern.js'; -import { EvaluateRowOptions, EvaluationResult, SourceSchema, TableRow } from '../types.js'; +import { EvaluateRowOptions, SourceEvaluationResult, SourceSchema, TableRow } from '../types.js'; import { StreamVariant } from './variant.js'; export class SyncStream implements BucketSource { @@ -22,7 +20,7 @@ export class SyncStream implements BucketSource { variants: StreamVariant[]; data: BaseSqlDataQuery; - public readonly dataSources: BucketDataSourceDefinition[]; + public readonly dataSources: BucketDataSource[]; public readonly parameterLookupSources: BucketParameterLookupSourceDefinition[]; public readonly parameterQuerierSources: BucketParameterQuerierSourceDefinition[]; @@ -63,7 +61,7 @@ export class SyncStream implements BucketSource { } } -export class SyncStreamDataSource implements BucketDataSourceDefinition { +export class SyncStreamDataSource implements BucketDataSource { constructor( private stream: SyncStream, private data: BaseSqlDataQuery, @@ -102,32 +100,25 @@ export class SyncStreamDataSource implements BucketDataSourceDefinition { result[this.data.table!.sqlName].push(r); } - createDataSource(params: CreateSourceParams): BucketDataSource { - const hydrationState = params.hydrationState; - const bucketPrefix = hydrationState.getBucketSourceState(this).bucketPrefix; - return { - evaluateRow: (options: EvaluateRowOptions): EvaluationResult[] => { - if (!this.data.applies(options.sourceTable)) { - return []; - } - - const stream = this.stream; - const row: TableRow = { - sourceTable: options.sourceTable, - record: options.record - }; - - // There is some duplication in work here when there are multiple variants on a stream: - // Each variant does the same row transformation (only the filters / bucket ids differ). - // However, architecturally we do need to be able to evaluate each variant separately. - return this.data.evaluateRowWithOptions({ - table: options.sourceTable, - row: options.record, - bucketIds: () => { - return this.variant.bucketIdsForRow(bucketPrefix, row); - } - }); - } + evaluateRow(options: EvaluateRowOptions): SourceEvaluationResult[] { + if (!this.data.applies(options.sourceTable)) { + return []; + } + + const row: TableRow = { + sourceTable: options.sourceTable, + record: options.record }; + + // There is some duplication in work here when there are multiple variants on a stream: + // Each variant does the same row transformation (only the filters / bucket ids differ). + // However, architecturally we do need to be able to evaluate each variant separately. + return this.data.evaluateRowWithOptions({ + table: options.sourceTable, + row: options.record, + serializedBucketParameters: () => { + return this.variant.bucketParametersForRow(row); + } + }); } } diff --git a/packages/sync-rules/src/streams/variant.ts b/packages/sync-rules/src/streams/variant.ts index a6cebbd4e..1b9000eb7 100644 --- a/packages/sync-rules/src/streams/variant.ts +++ b/packages/sync-rules/src/streams/variant.ts @@ -1,15 +1,16 @@ import { BucketInclusionReason, ResolvedBucket } from '../BucketDescription.js'; import { BucketParameterQuerier, ParameterLookup, PendingQueriers } from '../BucketParameterQuerier.js'; import { - BucketDataSourceDefinition, + BucketDataSource, BucketParameterLookupSourceDefinition, BucketParameterQuerierSource, BucketParameterQuerierSourceDefinition, CreateSourceParams } from '../BucketSource.js'; +import { BucketDataScope } from '../HydrationState.js'; import { GetQuerierOptions, RequestedStream } from '../index.js'; import { RequestParameters, SqliteJsonValue, TableRow } from '../types.js'; -import { isJsonValue, JSONBucketNameSerialize } from '../utils.js'; +import { buildBucketName, isJsonValue, JSONBucketNameSerialize } from '../utils.js'; import { BucketParameter, SubqueryEvaluator } from './parameter.js'; import { SyncStream, SyncStreamDataSource } from './stream.js'; import { cartesianProduct } from './utils.js'; @@ -79,8 +80,8 @@ export class StreamVariant { /** * Given a row in the table this stream selects from, returns all ids of buckets to which that row belongs to. */ - bucketIdsForRow(bucketPrefix: string, options: TableRow): string[] { - return this.instantiationsForRow(options).map((values) => this.buildBucketId(bucketPrefix, values)); + bucketParametersForRow(options: TableRow): string[] { + return this.instantiationsForRow(options).map((values) => this.serializeBucketParameters(values)); } /** @@ -129,7 +130,7 @@ export class StreamVariant { stream: SyncStream, reason: BucketInclusionReason, params: RequestParameters, - bucketPrefix: string, + bucketScope: BucketDataScope, hydratedSubqueries: HydratedSubqueries ): BucketParameterQuerier | null { const instantiation = this.partiallyEvaluateParameters(params); @@ -171,7 +172,7 @@ export class StreamVariant { // When we have no dynamic parameters, the partial evaluation is a full instantiation. const instantiations = this.cartesianProductOfParameterInstantiations(instantiation as SqliteJsonValue[][]); for (const instantiation of instantiations) { - staticBuckets.push(this.resolveBucket(stream, instantiation, reason, bucketPrefix)); + staticBuckets.push(this.resolveBucket(stream, instantiation, reason, bucketScope)); } } @@ -216,7 +217,7 @@ export class StreamVariant { perParameterInstantiation as SqliteJsonValue[][] ); - return Promise.resolve(product.map((e) => variant.resolveBucket(stream, e, reason, bucketPrefix))); + return Promise.resolve(product.map((e) => variant.resolveBucket(stream, e, reason, bucketScope))); } }; } @@ -287,24 +288,24 @@ export class StreamVariant { * @param transformer A transformer adding version information to the inner id. * @returns The generated bucket id */ - private buildBucketId(bucketPrefix: string, instantiation: SqliteJsonValue[]) { + private serializeBucketParameters(instantiation: SqliteJsonValue[]) { if (instantiation.length != this.parameters.length) { throw Error('Internal error, instantiation length mismatch'); } - return `${bucketPrefix}${JSONBucketNameSerialize.stringify(instantiation)}`; + return JSONBucketNameSerialize.stringify(instantiation); } private resolveBucket( stream: SyncStream, instantiation: SqliteJsonValue[], reason: BucketInclusionReason, - bucketPrefix: string + bucketScope: BucketDataScope ): ResolvedBucket { return { definition: stream.name, inclusion_reasons: [reason], - bucket: this.buildBucketId(bucketPrefix, instantiation), + bucket: buildBucketName(bucketScope, this.serializeBucketParameters(instantiation)), priority: stream.priority }; } @@ -338,7 +339,7 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS constructor( private stream: SyncStream, private variant: StreamVariant, - public readonly querierDataSource: BucketDataSourceDefinition + public readonly querierDataSource: BucketDataSource ) {} /** @@ -350,7 +351,7 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { const hydrationState = params.hydrationState; - const bucketPrefix = hydrationState.getBucketSourceState(this.querierDataSource).bucketPrefix; + const bucketScope = hydrationState.getBucketSourceScope(this.querierDataSource); const stream = this.stream; const hydratedSubqueries: HydratedSubqueries = new Map( @@ -375,13 +376,13 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS hasExplicitDefaultSubscription = true; } - this.queriersForSubscription(result, subscription, subscriptionParams, bucketPrefix, hydratedSubqueries); + this.queriersForSubscription(result, subscription, subscriptionParams, bucketScope, hydratedSubqueries); } // If the stream is subscribed to by default and there is no explicit subscription that would match the default // subscription, also include the default querier. if (stream.subscribedToByDefault && !hasExplicitDefaultSubscription) { - this.queriersForSubscription(result, null, options.globalParameters, bucketPrefix, hydratedSubqueries); + this.queriersForSubscription(result, null, options.globalParameters, bucketScope, hydratedSubqueries); } } }; @@ -391,13 +392,13 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS result: PendingQueriers, subscription: RequestedStream | null, params: RequestParameters, - bucketPrefix: string, + bucketScope: BucketDataScope, hydratedSubqueries: HydratedSubqueries ) { const reason: BucketInclusionReason = subscription != null ? { subscription: subscription.opaque_id } : 'default'; try { - const querier = this.variant.querier(this.stream, reason, params, bucketPrefix, hydratedSubqueries); + const querier = this.variant.querier(this.stream, reason, params, bucketScope, hydratedSubqueries); if (querier) { result.queriers.push(querier); } diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index 502a7ec77..aeca806f1 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -1,14 +1,14 @@ import { JSONBig, JsonContainer } from '@powersync/service-jsonbig'; +import { BucketPriority } from './BucketDescription.js'; +import { ParameterLookup } from './BucketParameterQuerier.js'; +import { CompatibilityContext } from './compatibility.js'; import { ColumnDefinition } from './ExpressionType.js'; +import { RequestFunctionCall } from './request_functions.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SyncRulesOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; -import { toSyncRulesParameters } from './utils.js'; -import { BucketPriority } from './BucketDescription.js'; -import { ParameterLookup } from './BucketParameterQuerier.js'; import { CustomSqliteValue } from './types/custom_sqlite_value.js'; -import { CompatibilityContext } from './compatibility.js'; -import { RequestFunctionCall } from './request_functions.js'; +import { toSyncRulesParameters } from './utils.js'; export interface QueryParseOptions extends SyncRulesOptions { accept_potentially_dangerous_queries?: boolean; @@ -48,23 +48,60 @@ export interface EvaluatedRow { data: SqliteJsonRow; } +/** + * Bucket data as evaluated by the BucketDataSource. + * + * The bucket name must still be resolved, external to this. + */ +export interface SourceEvaluatedRow { + /** + * Serialized evaluated parameters used to generate the bucket id. Serialized as a JSON array. + * + * Examples: + * [] // no bucket parameters + * [1] // single numeric parameter + * [1,"foo"] // multiple parameters + * + * The bucket name is derived by using concetenating these parameters with the generated bucket name. + */ + serializedBucketParameters: string; + + /** Output table - may be different from input table. */ + table: string; + + /** + * Convenience attribute. Must match data.id. + */ + id: string; + + /** Must be JSON-serializable. */ + data: SqliteJsonRow; +} + export interface EvaluationError { error: string; } -export function isEvaluationError(e: any): e is EvaluationError { - return typeof e.error == 'string'; +export function isEvaluationError( + e: EvaluationResult | SourceEvaluationResult | EvaluatedParametersResult +): e is EvaluationError { + return typeof (e as EvaluationError).error == 'string'; } export function isEvaluatedRow(e: EvaluationResult): e is EvaluatedRow { return typeof (e as EvaluatedRow).bucket == 'string'; } +export function isSourceEvaluatedRow(e: SourceEvaluationResult): e is SourceEvaluatedRow { + return typeof (e as SourceEvaluatedRow).serializedBucketParameters == 'string'; +} + export function isEvaluatedParameters(e: EvaluatedParametersResult): e is EvaluatedParameters { return 'lookup' in e; } export type EvaluationResult = EvaluatedRow | EvaluationError; +export type SourceEvaluationResult = SourceEvaluatedRow | EvaluationError; export interface RequestJwtPayload { /** diff --git a/packages/sync-rules/src/types/custom_sqlite_value.ts b/packages/sync-rules/src/types/custom_sqlite_value.ts index d26af5390..53ef1cc44 100644 --- a/packages/sync-rules/src/types/custom_sqlite_value.ts +++ b/packages/sync-rules/src/types/custom_sqlite_value.ts @@ -1,7 +1,7 @@ import { JSONBig } from '@powersync/service-jsonbig'; import { CompatibilityContext } from '../compatibility.js'; -import { SqliteValue, EvaluatedRow, SqliteInputValue, DatabaseInputValue } from '../types.js'; import { SqliteValueType } from '../ExpressionType.js'; +import { EvaluatedRow, SqliteValue } from '../types.js'; /** * A value that decays into a {@link SqliteValue} in a context-specific way. diff --git a/packages/sync-rules/src/utils.ts b/packages/sync-rules/src/utils.ts index 1ae02026f..cd5ed1566 100644 --- a/packages/sync-rules/src/utils.ts +++ b/packages/sync-rules/src/utils.ts @@ -2,6 +2,7 @@ import { JSONBig, JsonContainer, Replacer, stringifyRaw } from '@powersync/servi import { SelectFromStatement, Statement } from 'pgsql-ast-parser'; import { CompatibilityContext } from './compatibility.js'; import { SyncRuleProcessingError as SyncRulesProcessingError } from './errors.js'; +import { BucketDataScope } from './HydrationState.js'; import { SQLITE_FALSE, SQLITE_TRUE } from './sql_support.js'; import { DatabaseInputRow, @@ -19,14 +20,14 @@ export function isSelectStatement(q: Statement): q is SelectFromStatement { return q.type == 'select'; } -export function getBucketId( - bucketPrefix: string, - bucketParameters: string[], - params: Record -): string { +export function buildBucketName(scope: BucketDataScope, serializedParameters: string): string { + return scope.bucketPrefix + serializedParameters; +} + +export function serializeBucketParameters(bucketParameters: string[], params: Record): string { // Important: REAL and INTEGER values matching the same number needs the same representation in the bucket name. const paramArray = bucketParameters.map((name) => params[`bucket.${name}`]); - return `${bucketPrefix}${JSONBucketNameSerialize.stringify(paramArray)}`; + return JSONBucketNameSerialize.stringify(paramArray); } const DEPTH_LIMIT = 10; diff --git a/packages/sync-rules/test/src/data_queries.test.ts b/packages/sync-rules/test/src/data_queries.test.ts index b029e7304..09e615095 100644 --- a/packages/sync-rules/test/src/data_queries.test.ts +++ b/packages/sync-rules/test/src/data_queries.test.ts @@ -1,38 +1,23 @@ import { describe, expect, test } from 'vitest'; -import { CompatibilityContext, ExpressionType, SqlDataQuery, SqlSyncRules } from '../../src/index.js'; -import { ASSETS, BASIC_SCHEMA, identityBucketTransformer, PARSE_OPTIONS } from './util.js'; +import { CompatibilityContext, ExpressionType, SqlDataQuery } from '../../src/index.js'; +import { ASSETS, BASIC_SCHEMA, PARSE_OPTIONS } from './util.js'; describe('data queries', () => { - test('uses bucket id transformer', function () { - const sql = 'SELECT * FROM assets WHERE assets.org_id = bucket.org_id'; - const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); - expect(query.errors).toEqual([]); - - expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, '1#mybucket')).toEqual([ - { - bucket: '1#mybucket["org1"]', - table: 'assets', - id: 'asset1', - data: { id: 'asset1', org_id: 'org1' } - } - ]); - }); - test('bucket parameters = query', function () { const sql = 'SELECT * FROM assets WHERE assets.org_id = bucket.org_id'; const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, 'mybucket')).toEqual([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' })).toEqual([ { - bucket: 'mybucket["org1"]', + serializedBucketParameters: '["org1"]', table: 'assets', id: 'asset1', data: { id: 'asset1', org_id: 'org1' } } ]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null }, 'mybucket')).toEqual([]); + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null })).toEqual([]); }); test('bucket parameters IN query', function () { @@ -40,22 +25,20 @@ describe('data queries', () => { const query = SqlDataQuery.fromSql(['category'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect( - query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'green']) }, 'mybucket') - ).toMatchObject([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'green']) })).toMatchObject([ { - bucket: 'mybucket["red"]', + serializedBucketParameters: '["red"]', table: 'assets', id: 'asset1' }, { - bucket: 'mybucket["green"]', + serializedBucketParameters: '["green"]', table: 'assets', id: 'asset1' } ]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null }, 'mybucket')).toEqual([]); + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null })).toEqual([]); }); test('static IN data query', function () { @@ -63,19 +46,15 @@ describe('data queries', () => { const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect( - query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'green']) }, 'mybucket') - ).toMatchObject([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'green']) })).toMatchObject([ { - bucket: 'mybucket[]', + serializedBucketParameters: '[]', table: 'assets', id: 'asset1' } ]); - expect( - query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'blue']) }, 'mybucket') - ).toEqual([]); + expect(query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'blue']) })).toEqual([]); }); test('data IN static query', function () { @@ -83,15 +62,15 @@ describe('data queries', () => { const query = SqlDataQuery.fromSql([], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'good' }, 'mybucket')).toMatchObject([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'good' })).toMatchObject([ { - bucket: 'mybucket[]', + serializedBucketParameters: '[]', table: 'assets', id: 'asset1' } ]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'bad' }, 'mybucket')).toEqual([]); + expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'bad' })).toEqual([]); }); test('table alias', function () { @@ -99,9 +78,9 @@ describe('data queries', () => { const query = SqlDataQuery.fromSql(['org_id'], sql, PARSE_OPTIONS, compatibility); expect(query.errors).toEqual([]); - expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' }, 'mybucket')).toEqual([ + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' })).toEqual([ { - bucket: 'mybucket["org1"]', + serializedBucketParameters: '["org1"]', table: 'others', id: 'asset1', data: { id: 'asset1', org_id: 'org1' } diff --git a/packages/sync-rules/test/src/parameter_queries.test.ts b/packages/sync-rules/test/src/parameter_queries.test.ts index 8ca201913..8c2fc6150 100644 --- a/packages/sync-rules/test/src/parameter_queries.test.ts +++ b/packages/sync-rules/test/src/parameter_queries.test.ts @@ -1,8 +1,8 @@ import { describe, expect, test } from 'vitest'; +import { ParameterLookupScope } from '../../src/HydrationState.js'; import { ParameterLookup, SqlParameterQuery } from '../../src/index.js'; import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; import { BASIC_SCHEMA, EMPTY_DATA_SOURCE, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; -import { ParameterLookupScope } from '../../src/HydrationState.js'; describe('parameter queries', () => { // Specifically different from mybucket/1, to make sure this is being used. @@ -112,19 +112,15 @@ describe('parameter queries', () => { // We _do_ need to care about the bucket string representation. expect( - query.resolveBucketDescriptions( - [{ int1: 314, float1: 3.14, float2: 314 }], - normalizeTokenParameters({}), - 'mybucket' - ) + query.resolveBucketDescriptions([{ int1: 314, float1: 3.14, float2: 314 }], normalizeTokenParameters({}), { + bucketPrefix: 'mybucket' + }) ).toEqual([{ bucket: 'mybucket[314,3.14,314]', priority: 3 }]); expect( - query.resolveBucketDescriptions( - [{ int1: 314n, float1: 3.14, float2: 314 }], - normalizeTokenParameters({}), - 'mybucket' - ) + query.resolveBucketDescriptions([{ int1: 314n, float1: 3.14, float2: 314 }], normalizeTokenParameters({}), { + bucketPrefix: 'mybucket' + }) ).toEqual([{ bucket: 'mybucket[314,3.14,314]', priority: 3 }]); }); @@ -492,7 +488,7 @@ describe('parameter queries', () => { query.resolveBucketDescriptions( [{ user_id: 'user1' }], normalizeTokenParameters({ user_id: 'user1', is_admin: true }), - 'mybucket' + { bucketPrefix: 'mybucket' } ) ).toEqual([{ bucket: 'mybucket["user1",1]', priority: 3 }]); }); diff --git a/packages/sync-rules/test/src/static_parameter_queries.test.ts b/packages/sync-rules/test/src/static_parameter_queries.test.ts index bb78ca4a2..efec0eabc 100644 --- a/packages/sync-rules/test/src/static_parameter_queries.test.ts +++ b/packages/sync-rules/test/src/static_parameter_queries.test.ts @@ -1,9 +1,14 @@ import { describe, expect, test } from 'vitest'; +import { BucketDataScope } from '../../src/HydrationState.js'; import { RequestParameters, SqlParameterQuery } from '../../src/index.js'; import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; import { EMPTY_DATA_SOURCE, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; describe('static parameter queries', () => { + const MYBUCKET_SCOPE: BucketDataScope = { + bucketPrefix: 'mybucket' + }; + test('basic query', function () { const sql = 'SELECT token_parameters.user_id'; const query = SqlParameterQuery.fromSql( @@ -15,7 +20,7 @@ describe('static parameter queries', () => { ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters!).toEqual(['user_id']); - expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), 'mybucket')).toEqual([ + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), MYBUCKET_SCOPE)).toEqual([ { bucket: 'mybucket["user1"]', priority: 3 } ]); }); @@ -31,9 +36,11 @@ describe('static parameter queries', () => { ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters!).toEqual(['user_id']); - expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), '1#mybucket')).toEqual([ - { bucket: '1#mybucket["user1"]', priority: 3 } - ]); + expect( + query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), { + bucketPrefix: '1#mybucket' + }) + ).toEqual([{ bucket: '1#mybucket["user1"]', priority: 3 }]); }); test('global query', function () { @@ -47,7 +54,7 @@ describe('static parameter queries', () => { ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.bucketParameters!).toEqual([]); - expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), 'mybucket')).toEqual([ + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), MYBUCKET_SCOPE)).toEqual([ { bucket: 'mybucket[]', priority: 3 } ]); }); @@ -63,10 +70,10 @@ describe('static parameter queries', () => { ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1', is_admin: true }), 'mybucket') + query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1', is_admin: true }), MYBUCKET_SCOPE) ).toEqual([{ bucket: 'mybucket["user1"]', priority: 3 }]); expect( - query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1', is_admin: false }), 'mybucket') + query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1', is_admin: false }), MYBUCKET_SCOPE) ).toEqual([]); }); @@ -80,7 +87,7 @@ describe('static parameter queries', () => { EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), 'mybucket')).toEqual([ + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), MYBUCKET_SCOPE)).toEqual([ { bucket: 'mybucket["USER1"]', priority: 3 } ]); expect(query.bucketParameters!).toEqual(['upper_id']); @@ -96,10 +103,10 @@ describe('static parameter queries', () => { EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'admin' }), 'mybucket')).toEqual([ + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'admin' }), MYBUCKET_SCOPE)).toEqual([ { bucket: 'mybucket[]', priority: 3 } ]); - expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'user' }), 'mybucket')).toEqual([]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'user' }), MYBUCKET_SCOPE)).toEqual([]); }); test('comparison in filter clause', function () { @@ -112,12 +119,12 @@ describe('static parameter queries', () => { EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't1' }), 'mybucket')).toEqual([ - { bucket: 'mybucket[]', priority: 3 } - ]); - expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't2' }), 'mybucket')).toEqual( - [] - ); + expect( + query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't1' }), MYBUCKET_SCOPE) + ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); + expect( + query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't2' }), MYBUCKET_SCOPE) + ).toEqual([]); }); test('request.parameters()', function () { @@ -134,9 +141,9 @@ describe('static parameter queries', () => { ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getStaticBucketDescriptions(normalizeTokenParameters({}, { org_id: 'test' }), 'mybucket')).toEqual([ - { bucket: 'mybucket["test"]', priority: 3 } - ]); + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({}, { org_id: 'test' }), MYBUCKET_SCOPE)).toEqual( + [{ bucket: 'mybucket["test"]', priority: 3 }] + ); }); test('request.jwt()', function () { @@ -151,7 +158,7 @@ describe('static parameter queries', () => { expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['user_id']); - expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), 'mybucket')).toEqual([ + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), MYBUCKET_SCOPE)).toEqual([ { bucket: 'mybucket["user1"]', priority: 3 } ]); }); @@ -168,7 +175,7 @@ describe('static parameter queries', () => { expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['user_id']); - expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), 'mybucket')).toEqual([ + expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }), MYBUCKET_SCOPE)).toEqual([ { bucket: 'mybucket["user1"]', priority: 3 } ]); }); @@ -183,7 +190,7 @@ describe('static parameter queries', () => { EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([ + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), MYBUCKET_SCOPE)).toEqual([ { bucket: 'mybucket[]', priority: 3 } ]); }); @@ -198,7 +205,7 @@ describe('static parameter queries', () => { EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([ + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), MYBUCKET_SCOPE)).toEqual([ { bucket: 'mybucket[]', priority: 3 } ]); }); @@ -213,7 +220,7 @@ describe('static parameter queries', () => { EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([]); + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), MYBUCKET_SCOPE)).toEqual([]); }); test('static IN expression', function () { @@ -226,7 +233,7 @@ describe('static parameter queries', () => { EMPTY_DATA_SOURCE ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([ + expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), MYBUCKET_SCOPE)).toEqual([ { bucket: 'mybucket[]', priority: 3 } ]); }); @@ -245,13 +252,13 @@ describe('static parameter queries', () => { expect( query.getStaticBucketDescriptions( new RequestParameters({ sub: '', permissions: ['write', 'read:users'] }, {}), - 'mybucket' + MYBUCKET_SCOPE ) ).toEqual([{ bucket: 'mybucket[1]', priority: 3 }]); expect( query.getStaticBucketDescriptions( new RequestParameters({ sub: '', permissions: ['write', 'write:users'] }, {}), - 'mybucket' + MYBUCKET_SCOPE ) ).toEqual([{ bucket: 'mybucket[0]', priority: 3 }]); }); @@ -270,13 +277,13 @@ describe('static parameter queries', () => { expect( query.getStaticBucketDescriptions( new RequestParameters({ sub: '', permissions: ['write', 'read:users'] }, {}), - 'mybucket' + MYBUCKET_SCOPE ) ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); expect( query.getStaticBucketDescriptions( new RequestParameters({ sub: '', permissions: ['write', 'write:users'] }, {}), - 'mybucket' + MYBUCKET_SCOPE ) ).toEqual([]); }); @@ -292,10 +299,10 @@ describe('static parameter queries', () => { ) as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '', role: 'superuser' }, {}), 'mybucket') + query.getStaticBucketDescriptions(new RequestParameters({ sub: '', role: 'superuser' }, {}), MYBUCKET_SCOPE) ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '', role: 'superadmin' }, {}), 'mybucket') + query.getStaticBucketDescriptions(new RequestParameters({ sub: '', role: 'superadmin' }, {}), MYBUCKET_SCOPE) ).toEqual([]); }); diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index 207a15139..4128bcab8 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -49,9 +49,7 @@ describe('streams', () => { expect(desc.variants).toHaveLength(1); expect(evaluateBucketIds(desc, COMMENTS, { id: 'foo' })).toStrictEqual(['1#stream|0[]']); - expect( - desc.dataSources[0].createDataSource(hydrationParams).evaluateRow({ sourceTable: USERS, record: { id: 'foo' } }) - ).toHaveLength(0); + expect(desc.dataSources[0].evaluateRow({ sourceTable: USERS, record: { id: 'foo' } })).toHaveLength(0); }); test('row condition', () => { @@ -891,7 +889,7 @@ WHERE `); const hydrationState: HydrationState = { - getBucketSourceState(source) { + getBucketSourceScope(source) { return { bucketPrefix: `${source.defaultBucketPrefix}.test` }; }, getParameterLookupScope(source) { diff --git a/packages/sync-rules/test/src/table_valued_function_queries.test.ts b/packages/sync-rules/test/src/table_valued_function_queries.test.ts index 9de631f25..306fb6935 100644 --- a/packages/sync-rules/test/src/table_valued_function_queries.test.ts +++ b/packages/sync-rules/test/src/table_valued_function_queries.test.ts @@ -26,7 +26,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['v']); expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3, null] }), 'mybucket') + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3, null] }), { + bucketPrefix: 'mybucket' + }) ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, @@ -55,7 +57,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['v']); expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3, null] }), 'mybucket') + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3, null] }), { + bucketPrefix: 'mybucket' + }) ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, @@ -76,7 +80,11 @@ describe('table-valued function queries', () => { expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); - expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([ + expect( + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), { + bucketPrefix: 'mybucket' + }) + ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, { bucket: 'mybucket[3]', priority: 3 } @@ -95,7 +103,11 @@ describe('table-valued function queries', () => { expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); - expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([]); + expect( + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), { + bucketPrefix: 'mybucket' + }) + ).toEqual([]); }); test('json_each(array param not present)', function () { @@ -113,7 +125,11 @@ describe('table-valued function queries', () => { expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); - expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([]); + expect( + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), { + bucketPrefix: 'mybucket' + }) + ).toEqual([]); }); test('json_each(array param not present, ifnull)', function () { @@ -131,7 +147,11 @@ describe('table-valued function queries', () => { expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['v']); - expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([]); + expect( + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), { + bucketPrefix: 'mybucket' + }) + ).toEqual([]); }); test('json_each on json_keys', function () { @@ -146,7 +166,11 @@ describe('table-valued function queries', () => { expect(query.errors).toEqual([]); expect(query.bucketParameters).toEqual(['value']); - expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), 'mybucket')).toEqual([ + expect( + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}), { + bucketPrefix: 'mybucket' + }) + ).toEqual([ { bucket: 'mybucket["a"]', priority: 3 }, { bucket: 'mybucket["b"]', priority: 3 }, { bucket: 'mybucket["c"]', priority: 3 } @@ -169,7 +193,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['value']); expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), 'mybucket') + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), { + bucketPrefix: 'mybucket' + }) ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, @@ -193,7 +219,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['value']); expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), 'mybucket') + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), { + bucketPrefix: 'mybucket' + }) ).toEqual([ { bucket: 'mybucket[1]', priority: 3 }, { bucket: 'mybucket[2]', priority: 3 }, @@ -217,7 +245,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['v']); expect( - query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), 'mybucket') + query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }), { + bucketPrefix: 'mybucket' + }) ).toEqual([ { bucket: 'mybucket[2]', priority: 3 }, { bucket: 'mybucket[3]', priority: 3 } @@ -252,7 +282,9 @@ describe('table-valued function queries', () => { }, {} ), - 'mybucket' + { + bucketPrefix: 'mybucket' + } ) ).toEqual([{ bucket: 'mybucket[1]', priority: 3 }]); }); diff --git a/packages/sync-rules/test/src/util.ts b/packages/sync-rules/test/src/util.ts index 72649fa81..9ece687f9 100644 --- a/packages/sync-rules/test/src/util.ts +++ b/packages/sync-rules/test/src/util.ts @@ -1,6 +1,5 @@ import { BucketDataSource, - BucketDataSourceDefinition, ColumnDefinition, CompatibilityContext, CreateSourceParams, @@ -91,14 +90,14 @@ export function identityBucketTransformer(id: string) { /** * Empty data source that can be used for testing parameter queries, where most of the functionality here is not used. */ -export const EMPTY_DATA_SOURCE: BucketDataSourceDefinition = { +export const EMPTY_DATA_SOURCE: BucketDataSource = { defaultBucketPrefix: 'mybucket', bucketParameters: [], // These are not used in the tests. getSourceTables: function (): Set { return new Set(); }, - createDataSource: function (params: CreateSourceParams): BucketDataSource { + evaluateRow(options) { throw new Error('Function not implemented.'); }, tableSyncsData: function (table: SourceTableInterface): boolean { From 448e84c72ff90c7f5b6124b33596e0047ec531df Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 12 Dec 2025 13:35:11 +0200 Subject: [PATCH 27/33] Rename defaultBucketPrefix -> uniqueName. --- packages/sync-rules/src/BucketSource.ts | 6 +++--- packages/sync-rules/src/HydrationState.ts | 4 ++-- packages/sync-rules/src/SqlBucketDescriptor.ts | 2 +- packages/sync-rules/src/streams/stream.ts | 2 +- packages/sync-rules/test/src/streams.test.ts | 2 +- packages/sync-rules/test/src/util.ts | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 2b81a0963..9a3890372 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -73,11 +73,11 @@ export interface HydratedBucketSource { */ export interface BucketDataSource { /** - * Bucket prefix if no transformations are defined. + * Unique name of the data source within a sync rules version. * - * Transformations may use this as a base, or may generate an entirely different prefix. + * This may be used as the basis for bucketPrefix (or it could be ignored). */ - readonly defaultBucketPrefix: string; + readonly uniqueName: string; /** * For debug use only. diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts index 4a52e305c..149e82630 100644 --- a/packages/sync-rules/src/HydrationState.ts +++ b/packages/sync-rules/src/HydrationState.ts @@ -40,7 +40,7 @@ export interface HydrationState< export const DEFAULT_HYDRATION_STATE: HydrationState = { getBucketSourceScope(source: BucketDataSource) { return { - bucketPrefix: source.defaultBucketPrefix + bucketPrefix: source.uniqueName }; }, getParameterLookupScope(source) { @@ -64,7 +64,7 @@ export function versionedHydrationState(version: number): HydrationState { return { getBucketSourceScope(source: BucketDataSource): BucketDataScope { return { - bucketPrefix: `${version}#${source.defaultBucketPrefix}` + bucketPrefix: `${version}#${source.uniqueName}` }; }, diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index f74254d68..d914f19f2 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -146,7 +146,7 @@ export class BucketDefinitionDataSource implements BucketDataSource { return this.descriptor.bucketParameters; } - public get defaultBucketPrefix(): string { + public get uniqueName(): string { return this.descriptor.name; } diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index 0db285a50..118b24ec8 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -75,7 +75,7 @@ export class SyncStreamDataSource implements BucketDataSource { return []; } - public get defaultBucketPrefix(): string { + public get uniqueName(): string { return this.variant.defaultBucketPrefix(this.stream.name); } diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index 4128bcab8..313d1fc11 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -890,7 +890,7 @@ WHERE const hydrationState: HydrationState = { getBucketSourceScope(source) { - return { bucketPrefix: `${source.defaultBucketPrefix}.test` }; + return { bucketPrefix: `${source.uniqueName}.test` }; }, getParameterLookupScope(source) { return { diff --git a/packages/sync-rules/test/src/util.ts b/packages/sync-rules/test/src/util.ts index 9ece687f9..61478218a 100644 --- a/packages/sync-rules/test/src/util.ts +++ b/packages/sync-rules/test/src/util.ts @@ -91,7 +91,7 @@ export function identityBucketTransformer(id: string) { * Empty data source that can be used for testing parameter queries, where most of the functionality here is not used. */ export const EMPTY_DATA_SOURCE: BucketDataSource = { - defaultBucketPrefix: 'mybucket', + uniqueName: 'mybucket', bucketParameters: [], // These are not used in the tests. getSourceTables: function (): Set { From 7f60a4b93c89476a52280e8b90b3516a392c40ca Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 12 Dec 2025 14:56:06 +0200 Subject: [PATCH 28/33] Remove hydration for ParameterLookupSource. --- .../implementation/MongoSyncBucketStorage.ts | 14 +- .../src/storage/PostgresSyncRulesStorage.ts | 5 +- .../register-data-storage-parameter-tests.ts | 34 ++- .../register-parameter-compacting-tests.ts | 6 +- packages/service-core-tests/tsconfig.json | 3 + .../src/storage/SyncRulesBucketStorage.ts | 4 +- packages/service-core/src/storage/bson.ts | 6 +- .../test/src/sync/BucketChecksumState.test.ts | 10 +- packages/sync-rules/src/BaseSqlDataQuery.ts | 8 +- .../sync-rules/src/BucketParameterQuerier.ts | 29 +- packages/sync-rules/src/BucketSource.ts | 91 +++--- packages/sync-rules/src/HydratedSyncRules.ts | 68 ++--- .../sync-rules/src/SqlBucketDescriptor.ts | 4 +- packages/sync-rules/src/SqlDataQuery.ts | 4 +- packages/sync-rules/src/SqlParameterQuery.ts | 46 ++- packages/sync-rules/src/SqlSyncRules.ts | 4 +- packages/sync-rules/src/streams/filter.ts | 39 +-- packages/sync-rules/src/streams/parameter.ts | 6 +- packages/sync-rules/src/streams/stream.ts | 4 +- packages/sync-rules/src/streams/variant.ts | 8 +- packages/sync-rules/src/types.ts | 26 +- .../test/src/parameter_queries.test.ts | 269 +++++++++--------- packages/sync-rules/test/src/streams.test.ts | 111 +++----- .../sync-rules/test/src/sync_rules.test.ts | 16 +- 24 files changed, 407 insertions(+), 408 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 8ab803726..eb217683f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -10,7 +10,6 @@ import { BroadcastIterable, CHECKPOINT_INVALIDATE_ALL, CheckpointChanges, - CompactOptions, deserializeParameterLookup, GetCheckpointChangesOptions, InternalOpId, @@ -25,10 +24,11 @@ import { WatchWriteCheckpointOptions } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; -import { ParameterLookup, SqliteJsonRow, SqlSyncRules, HydratedSyncRules } from '@powersync/service-sync-rules'; +import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { LRUCache } from 'lru-cache'; import * as timers from 'timers/promises'; +import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; import { PowerSyncMongo } from './db.js'; import { BucketDataDocument, BucketDataKey, BucketStateDocument, SourceKey, SourceTableDocument } from './models.js'; @@ -37,7 +37,6 @@ import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; import { MongoCompactor } from './MongoCompactor.js'; import { MongoParameterCompactor } from './MongoParameterCompactor.js'; import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js'; -import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js'; export interface MongoSyncBucketStorageOptions { checksumOptions?: MongoChecksumOptions; @@ -293,7 +292,10 @@ export class MongoSyncBucketStorage return result!; } - async getParameterSets(checkpoint: MongoReplicationCheckpoint, lookups: ParameterLookup[]): Promise { + async getParameterSets( + checkpoint: MongoReplicationCheckpoint, + lookups: ScopedParameterLookup[] + ): Promise { return this.db.client.withSession({ snapshot: true }, async (session) => { // Set the session's snapshot time to the checkpoint's snapshot time. // An alternative would be to create the session when the checkpoint is created, but managing @@ -1025,7 +1027,7 @@ class MongoReplicationCheckpoint implements ReplicationCheckpoint { public snapshotTime: mongo.Timestamp ) {} - async getParameterSets(lookups: ParameterLookup[]): Promise { + async getParameterSets(lookups: ScopedParameterLookup[]): Promise { return this.storage.getParameterSets(this, lookups); } } @@ -1034,7 +1036,7 @@ class EmptyReplicationCheckpoint implements ReplicationCheckpoint { readonly checkpoint: InternalOpId = 0n; readonly lsn: string | null = null; - async getParameterSets(lookups: ParameterLookup[]): Promise { + async getParameterSets(lookups: ScopedParameterLookup[]): Promise { return []; } } diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index faccb530a..c7a3c2c29 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -4,7 +4,6 @@ import { BucketChecksum, CHECKPOINT_INVALIDATE_ALL, CheckpointChanges, - CompactOptions, GetCheckpointChangesOptions, InternalOpId, internalToExternalOpId, @@ -376,7 +375,7 @@ export class PostgresSyncRulesStorage async getParameterSets( checkpoint: ReplicationCheckpoint, - lookups: sync_rules.ParameterLookup[] + lookups: sync_rules.ScopedParameterLookup[] ): Promise { const rows = await this.db.sql` SELECT DISTINCT @@ -881,7 +880,7 @@ class PostgresReplicationCheckpoint implements storage.ReplicationCheckpoint { public readonly lsn: string | null ) {} - getParameterSets(lookups: sync_rules.ParameterLookup[]): Promise { + getParameterSets(lookups: sync_rules.ScopedParameterLookup[]): Promise { return this.storage.getParameterSets(this, lookups); } } diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index ff39db906..c65ee68d6 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -1,5 +1,5 @@ import { storage } from '@powersync/service-core'; -import { ParameterLookup, RequestParameters } from '@powersync/service-sync-rules'; +import { RequestParameters, ScopedParameterLookup } from '@powersync/service-sync-rules'; import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; import { TEST_TABLE } from './util.js'; @@ -60,7 +60,7 @@ bucket_definitions: }); const checkpoint = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); + const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); expect(parameters).toEqual([ { group_id: 'group1a' @@ -108,7 +108,7 @@ bucket_definitions: }); const checkpoint2 = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint2.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); + const parameters = await checkpoint2.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); expect(parameters).toEqual([ { group_id: 'group2' @@ -116,7 +116,7 @@ bucket_definitions: ]); // Use the checkpoint to get older data if relevant - const parameters2 = await checkpoint1.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); + const parameters2 = await checkpoint1.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); expect(parameters2).toEqual([ { group_id: 'group1' @@ -185,8 +185,8 @@ bucket_definitions: // association of `list1`::`todo2` const checkpoint = await bucketStorage.getCheckpoint(); const parameters = await checkpoint.getParameterSets([ - ParameterLookup.normalized(MYBUCKET_1, ['list1']), - ParameterLookup.normalized(MYBUCKET_1, ['list2']) + ScopedParameterLookup.direct(MYBUCKET_1, ['list1']), + ScopedParameterLookup.direct(MYBUCKET_1, ['list2']) ]); expect(parameters.sort((a, b) => (a.todo_id as string).localeCompare(b.todo_id as string))).toEqual([ @@ -233,11 +233,15 @@ bucket_definitions: const checkpoint = await bucketStorage.getCheckpoint(); - const parameters1 = await checkpoint.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, [314n, 314, 3.14])]); + const parameters1 = await checkpoint.getParameterSets([ + ScopedParameterLookup.direct(MYBUCKET_1, [314n, 314, 3.14]) + ]); expect(parameters1).toEqual([TEST_PARAMS]); - const parameters2 = await checkpoint.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, [314, 314n, 3.14])]); + const parameters2 = await checkpoint.getParameterSets([ + ScopedParameterLookup.direct(MYBUCKET_1, [314, 314n, 3.14]) + ]); expect(parameters2).toEqual([TEST_PARAMS]); - const parameters3 = await checkpoint.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, [314n, 314, 3])]); + const parameters3 = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, [314n, 314, 3])]); expect(parameters3).toEqual([]); }); @@ -291,7 +295,7 @@ bucket_definitions: const checkpoint = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint.getParameterSets([ - ParameterLookup.normalized(MYBUCKET_1, [1152921504606846976n]) + ScopedParameterLookup.direct(MYBUCKET_1, [1152921504606846976n]) ]); expect(parameters1).toEqual([TEST_PARAMS]); }); @@ -332,7 +336,7 @@ bucket_definitions: const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; const lookups = querier.parameterQueryLookups; - expect(lookups).toEqual([ParameterLookup.normalized({ lookupName: 'by_workspace', queryId: '1' }, ['u1'])]); + expect(lookups).toEqual([ScopedParameterLookup.direct({ lookupName: 'by_workspace', queryId: '1' }, ['u1'])]); const parameter_sets = await checkpoint.getParameterSets(lookups); expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]); @@ -405,7 +409,7 @@ bucket_definitions: const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; const lookups = querier.parameterQueryLookups; - expect(lookups).toEqual([ParameterLookup.normalized({ lookupName: 'by_public_workspace', queryId: '1' }, [])]); + expect(lookups).toEqual([ScopedParameterLookup.direct({ lookupName: 'by_public_workspace', queryId: '1' }, [])]); const parameter_sets = await checkpoint.getParameterSets(lookups); parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); @@ -507,8 +511,8 @@ bucket_definitions: const lookups = querier.parameterQueryLookups; expect(lookups).toEqual([ - ParameterLookup.normalized({ lookupName: 'by_workspace', queryId: '1' }, []), - ParameterLookup.normalized({ lookupName: 'by_workspace', queryId: '2' }, ['u1']) + ScopedParameterLookup.direct({ lookupName: 'by_workspace', queryId: '1' }, []), + ScopedParameterLookup.direct({ lookupName: 'by_workspace', queryId: '2' }, ['u1']) ]); const parameter_sets = await checkpoint.getParameterSets(lookups); @@ -558,7 +562,7 @@ bucket_definitions: const checkpoint = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint.getParameterSets([ParameterLookup.normalized(MYBUCKET_1, ['user1'])]); + const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); expect(parameters).toEqual([]); }); diff --git a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts index 0b63b62ba..10263fc7d 100644 --- a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts @@ -1,5 +1,5 @@ import { storage } from '@powersync/service-core'; -import { ParameterLookup } from '@powersync/service-sync-rules'; +import { ScopedParameterLookup } from '@powersync/service-sync-rules'; import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; @@ -40,7 +40,7 @@ bucket_definitions: await batch.commit('1/1'); }); - const lookup = ParameterLookup.normalized({ lookupName: 'test', queryId: '1' }, ['t1']); + const lookup = ScopedParameterLookup.direct({ lookupName: 'test', queryId: '1' }, ['t1']); const checkpoint1 = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint1.getParameterSets([lookup]); @@ -151,7 +151,7 @@ bucket_definitions: await batch.commit('3/1'); }); - const lookup = ParameterLookup.normalized({ lookupName: 'test', queryId: '1' }, ['u1']); + const lookup = ScopedParameterLookup.direct({ lookupName: 'test', queryId: '1' }, ['u1']); const checkpoint1 = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint1.getParameterSets([lookup]); diff --git a/packages/service-core-tests/tsconfig.json b/packages/service-core-tests/tsconfig.json index d2e14207b..cde5249d4 100644 --- a/packages/service-core-tests/tsconfig.json +++ b/packages/service-core-tests/tsconfig.json @@ -29,6 +29,9 @@ }, { "path": "../../libs/lib-services" + }, + { + "path": "../sync-rules" } ] } diff --git a/packages/service-core/src/storage/SyncRulesBucketStorage.ts b/packages/service-core/src/storage/SyncRulesBucketStorage.ts index 9621059c7..175427449 100644 --- a/packages/service-core/src/storage/SyncRulesBucketStorage.ts +++ b/packages/service-core/src/storage/SyncRulesBucketStorage.ts @@ -1,5 +1,5 @@ import { Logger, ObserverClient } from '@powersync/lib-services-framework'; -import { ParameterLookup, SqliteJsonRow, HydratedSyncRules } from '@powersync/service-sync-rules'; +import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; import * as util from '../util/util-index.js'; import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js'; import { BucketStorageFactory } from './BucketStorageFactory.js'; @@ -284,7 +284,7 @@ export interface ReplicationCheckpoint { * * This gets parameter sets specific to this checkpoint. */ - getParameterSets(lookups: ParameterLookup[]): Promise; + getParameterSets(lookups: ScopedParameterLookup[]): Promise; } export interface WatchWriteCheckpointOptions { diff --git a/packages/service-core/src/storage/bson.ts b/packages/service-core/src/storage/bson.ts index db4b732be..ad7ee3e16 100644 --- a/packages/service-core/src/storage/bson.ts +++ b/packages/service-core/src/storage/bson.ts @@ -1,6 +1,6 @@ import * as bson from 'bson'; -import { ParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules'; +import { ScopedParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules'; import { ReplicaId } from './BucketStorageBatch.js'; type NodeBuffer = Buffer; @@ -27,11 +27,11 @@ export const BSON_DESERIALIZE_DATA_OPTIONS: bson.DeserializeOptions = { * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers. * @param lookup */ -export const serializeLookupBuffer = (lookup: ParameterLookup): NodeBuffer => { +export const serializeLookupBuffer = (lookup: ScopedParameterLookup): NodeBuffer => { return bson.serialize({ l: lookup.values }) as NodeBuffer; }; -export const serializeLookup = (lookup: ParameterLookup) => { +export const serializeLookup = (lookup: ScopedParameterLookup) => { return new bson.Binary(serializeLookupBuffer(lookup)); }; diff --git a/packages/service-core/test/src/sync/BucketChecksumState.test.ts b/packages/service-core/test/src/sync/BucketChecksumState.test.ts index 48cdd24fc..26c986521 100644 --- a/packages/service-core/test/src/sync/BucketChecksumState.test.ts +++ b/packages/service-core/test/src/sync/BucketChecksumState.test.ts @@ -12,7 +12,7 @@ import { WatchFilterEvent } from '@/index.js'; import { JSONBig } from '@powersync/service-jsonbig'; -import { ParameterLookup, RequestJwtPayload, SqliteJsonRow, SqlSyncRules } from '@powersync/service-sync-rules'; +import { RequestJwtPayload, ScopedParameterLookup, SqliteJsonRow, SqlSyncRules } from '@powersync/service-sync-rules'; import { versionedHydrationState } from '@powersync/service-sync-rules/src/HydrationState.js'; import { beforeEach, describe, expect, test } from 'vitest'; @@ -505,7 +505,7 @@ bucket_definitions: const line = (await state.buildNextCheckpointLine({ base: storage.makeCheckpoint(1n, (lookups) => { - expect(lookups).toEqual([ParameterLookup.normalized({ lookupName: 'by_project', queryId: '1' }, ['u1'])]); + expect(lookups).toEqual([ScopedParameterLookup.direct({ lookupName: 'by_project', queryId: '1' }, ['u1'])]); return [{ id: 1 }, { id: 2 }]; }), writeCheckpoint: null, @@ -566,7 +566,7 @@ bucket_definitions: // Now we get a new line const line2 = (await state.buildNextCheckpointLine({ base: storage.makeCheckpoint(2n, (lookups) => { - expect(lookups).toEqual([ParameterLookup.normalized({ lookupName: 'by_project', queryId: '1' }, ['u1'])]); + expect(lookups).toEqual([ScopedParameterLookup.direct({ lookupName: 'by_project', queryId: '1' }, ['u1'])]); return [{ id: 1 }, { id: 2 }, { id: 3 }]; }), writeCheckpoint: null, @@ -854,12 +854,12 @@ class MockBucketChecksumStateStorage implements BucketChecksumStateStorage { makeCheckpoint( opId: InternalOpId, - parameters?: (lookups: ParameterLookup[]) => SqliteJsonRow[] + parameters?: (lookups: ScopedParameterLookup[]) => SqliteJsonRow[] ): ReplicationCheckpoint { return { checkpoint: opId, lsn: String(opId), - getParameterSets: async (lookups: ParameterLookup[]) => { + getParameterSets: async (lookups: ScopedParameterLookup[]) => { if (parameters == null) { throw new Error(`getParametersSets not defined for checkpoint ${opId}`); } diff --git a/packages/sync-rules/src/BaseSqlDataQuery.ts b/packages/sync-rules/src/BaseSqlDataQuery.ts index f83c5362a..24ab1138b 100644 --- a/packages/sync-rules/src/BaseSqlDataQuery.ts +++ b/packages/sync-rules/src/BaseSqlDataQuery.ts @@ -8,8 +8,8 @@ import { TablePattern } from './TablePattern.js'; import { QueryParameters, QuerySchema, - SourceEvaluatedRow, - SourceEvaluationResult, + UnscopedEvaluatedRow, + UnscopedEvaluationResult, SourceSchema, SourceSchemaTable, SqliteJsonRow, @@ -170,7 +170,7 @@ export class BaseSqlDataQuery { } } - evaluateRowWithOptions(options: EvaluateRowOptions): SourceEvaluationResult[] { + evaluateRowWithOptions(options: EvaluateRowOptions): UnscopedEvaluationResult[] { try { const { table, row, serializedBucketParameters } = options; @@ -201,7 +201,7 @@ export class BaseSqlDataQuery { table: outputTable, id: id, data - } satisfies SourceEvaluatedRow; + } satisfies UnscopedEvaluatedRow; }); } catch (e) { return [{ error: e.message ?? `Evaluating data query failed` }]; diff --git a/packages/sync-rules/src/BucketParameterQuerier.ts b/packages/sync-rules/src/BucketParameterQuerier.ts index 025744beb..5ef89a592 100644 --- a/packages/sync-rules/src/BucketParameterQuerier.ts +++ b/packages/sync-rules/src/BucketParameterQuerier.ts @@ -26,7 +26,7 @@ export interface BucketParameterQuerier { */ readonly hasDynamicBuckets: boolean; - readonly parameterQueryLookups: ParameterLookup[]; + readonly parameterQueryLookups: ScopedParameterLookup[]; /** * These buckets depend on parameter storage, and needs to be retrieved dynamically for each checkpoint. @@ -59,7 +59,7 @@ export interface PendingQueriers { } export interface ParameterLookupSource { - getParameterSets: (lookups: ParameterLookup[]) => Promise; + getParameterSets: (lookups: ScopedParameterLookup[]) => Promise; } export interface QueryBucketDescriptorOptions extends ParameterLookupSource { @@ -89,12 +89,19 @@ export function mergeBucketParameterQueriers(queriers: BucketParameterQuerier[]) * * Other query types are not supported yet. */ -export class ParameterLookup { +export class ScopedParameterLookup { // bucket definition name, parameter query index, ...lookup values readonly values: SqliteJsonValue[]; - static normalized(scope: ParameterLookupScope, values: SqliteJsonValue[]): ParameterLookup { - return new ParameterLookup([scope.lookupName, scope.queryId, ...values.map(normalizeParameterValue)]); + static normalized(scope: ParameterLookupScope, lookup: UnscopedParameterLookup): ScopedParameterLookup { + return new ScopedParameterLookup([scope.lookupName, scope.queryId, ...lookup.lookupValues]); + } + + /** + * Primarily for test fixtures. + */ + static direct(scope: ParameterLookupScope, values: SqliteJsonValue[]): ScopedParameterLookup { + return new ScopedParameterLookup([scope.lookupName, scope.queryId, ...values.map(normalizeParameterValue)]); } /** @@ -105,3 +112,15 @@ export class ParameterLookup { this.values = values; } } + +export class UnscopedParameterLookup { + readonly lookupValues: SqliteJsonValue[]; + + static normalized(values: SqliteJsonValue[]): UnscopedParameterLookup { + return new UnscopedParameterLookup(values.map(normalizeParameterValue)); + } + + constructor(lookupValues: SqliteJsonValue[]) { + this.lookupValues = lookupValues; + } +} diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 9a3890372..9e132c077 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -1,4 +1,9 @@ -import { BucketParameterQuerier, ParameterLookup, PendingQueriers } from './BucketParameterQuerier.js'; +import { + BucketParameterQuerier, + UnscopedParameterLookup, + PendingQueriers, + ScopedParameterLookup +} from './BucketParameterQuerier.js'; import { ColumnDefinition } from './ExpressionType.js'; import { DEFAULT_HYDRATION_STATE, HydrationState, ParameterLookupScope } from './HydrationState.js'; import { SourceTableInterface } from './SourceTableInterface.js'; @@ -10,9 +15,11 @@ import { EvaluateRowOptions, EvaluationResult, isEvaluationError, - SourceEvaluationResult, + UnscopedEvaluationResult, SourceSchema, - SqliteRow + SqliteRow, + UnscopedEvaluatedParametersResult, + EvaluatedParameters } from './types.js'; import { buildBucketName } from './utils.js'; @@ -65,6 +72,12 @@ export interface HydratedBucketSource { readonly parameterQuerierSources: BucketParameterQuerierSource[]; } +export type ScopedEvaluateRow = (options: EvaluateRowOptions) => EvaluationResult[]; +export type ScopedEvaluateParameterRow = ( + sourceTable: SourceTableInterface, + row: SqliteRow +) => EvaluatedParametersResult[]; + /** * Encodes a static definition of a bucket source, as parsed from sync rules or stream definitions. * @@ -91,7 +104,7 @@ export interface BucketDataSource { * Given a row as it appears in a table that affects sync data, return buckets, logical table names and transformed * data for rows to add to buckets. */ - evaluateRow(options: EvaluateRowOptions): SourceEvaluationResult[]; + evaluateRow(options: EvaluateRowOptions): UnscopedEvaluationResult[]; /** * Given a static schema, infer all logical tables and associated columns that appear in buckets defined by this @@ -118,7 +131,15 @@ export interface BucketParameterLookupSourceDefinition { readonly defaultLookupScope: ParameterLookupScope; getSourceTables(): Set; - createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource; + + /** + * Given a row in a source table that affects sync parameters, returns a structure to index which buckets rows should + * be associated with. + * + * The returned {@link UnscopedParameterLookup} can be referenced by {@link pushBucketParameterQueriers} to allow the storage + * system to find buckets. + */ + evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): UnscopedEvaluatedParametersResult[]; /** Whether the table possibly affects the buckets resolved by this source. */ tableSyncsParameters(table: SourceTableInterface): boolean; @@ -145,17 +166,6 @@ export interface BucketParameterQuerierSourceDefinition { createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource; } -export interface BucketParameterLookupSource { - /** - * Given a row in a source table that affects sync parameters, returns a structure to index which buckets rows should - * be associated with. - * - * The returned {@link ParameterLookup} can be referenced by {@link pushBucketParameterQueriers} to allow the storage - * system to find buckets. - */ - evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[]; -} - export interface BucketParameterQuerierSource { /** * Reports {@link BucketParameterQuerier}s resolving buckets that a specific stream request should have access to. @@ -166,8 +176,9 @@ export interface BucketParameterQuerierSource { pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void; } -export interface DebugMergedSource extends BucketParameterLookupSource, BucketParameterQuerierSource { - evaluateRow(options: EvaluateRowOptions): EvaluationResult[]; +export interface DebugMergedSource extends BucketParameterQuerierSource { + evaluateRow: ScopedEvaluateRow; + evaluateParameterRow: ScopedEvaluateParameterRow; } export enum BucketSourceType { @@ -177,10 +188,7 @@ export enum BucketSourceType { export type ResultSetDescription = { name: string; columns: ColumnDefinition[] }; -export function hydrateEvaluateRow( - hydrationState: HydrationState, - source: BucketDataSource -): (options: EvaluateRowOptions) => EvaluationResult[] { +export function hydrateEvaluateRow(hydrationState: HydrationState, source: BucketDataSource): ScopedEvaluateRow { const scope = hydrationState.getBucketSourceScope(source); return (options: EvaluateRowOptions): EvaluationResult[] => { return source.evaluateRow(options).map((result) => { @@ -197,7 +205,28 @@ export function hydrateEvaluateRow( }; } -export function mergeDataSources(hydrationState: HydrationState, sources: BucketDataSource[]) { +export function hydrateEvaluateParameterRow( + hydrationState: HydrationState, + source: BucketParameterLookupSourceDefinition +): ScopedEvaluateParameterRow { + const scope = hydrationState.getParameterLookupScope(source); + return (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { + return source.evaluateParameterRow(sourceTable, row).map((result) => { + if (isEvaluationError(result)) { + return result; + } + return { + bucketParameters: result.bucketParameters, + lookup: ScopedParameterLookup.normalized(scope, result.lookup) + } satisfies EvaluatedParameters; + }); + }; +} + +export function mergeDataSources( + hydrationState: HydrationState, + sources: BucketDataSource[] +): { evaluateRow: ScopedEvaluateRow } { const withScope = sources.map((source) => hydrateEvaluateRow(hydrationState, source)); return { evaluateRow(options: EvaluateRowOptions): EvaluationResult[] { @@ -206,14 +235,14 @@ export function mergeDataSources(hydrationState: HydrationState, sources: Bucket }; } -export function mergeParameterLookupSources(sources: BucketParameterLookupSource[]): BucketParameterLookupSource { +export function mergeParameterLookupSources( + hydrationState: HydrationState, + sources: BucketParameterLookupSourceDefinition[] +): { evaluateParameterRow: ScopedEvaluateParameterRow } { + const withScope = sources.map((source) => hydrateEvaluateParameterRow(hydrationState, source)); return { evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] { - let results: EvaluatedParametersResult[] = []; - for (let source of sources) { - results.push(...source.evaluateParameterRow(sourceTable, row)); - } - return results; + return withScope.flatMap((source) => source(sourceTable, row)); } }; } @@ -236,9 +265,7 @@ export function debugHydratedMergedSource(bucketSource: BucketSource, params?: C const hydrationState = params?.hydrationState ?? DEFAULT_HYDRATION_STATE; const resolvedParams = { hydrationState }; const dataSource = mergeDataSources(hydrationState, bucketSource.dataSources); - const parameterLookupSource = mergeParameterLookupSources( - bucketSource.parameterLookupSources.map((source) => source.createParameterLookupSource(resolvedParams)) - ); + const parameterLookupSource = mergeParameterLookupSources(hydrationState, bucketSource.parameterLookupSources); const parameterQuerierSource = mergeParameterQuerierSources( bucketSource.parameterQuerierSources.map((source) => source.createParameterQuerierSource(resolvedParams)) ); diff --git a/packages/sync-rules/src/HydratedSyncRules.ts b/packages/sync-rules/src/HydratedSyncRules.ts index b4a1c0473..e0db33d16 100644 --- a/packages/sync-rules/src/HydratedSyncRules.ts +++ b/packages/sync-rules/src/HydratedSyncRules.ts @@ -1,12 +1,10 @@ +import { Scope } from 'ajv/dist/compile/codegen/scope.js'; +import { BucketDataSource, CreateSourceParams, HydratedBucketSource } from './BucketSource.js'; +import { BucketDataScope, ParameterLookupScope } from './HydrationState.js'; import { - BucketDataSource, - BucketParameterLookupSource, - CreateSourceParams, - HydratedBucketSource -} from './BucketSource.js'; -import { BucketDataScope } from './HydrationState.js'; -import { + BucketParameterLookupSourceDefinition, BucketParameterQuerier, + buildBucketName, CompatibilityContext, EvaluatedParameters, EvaluatedRow, @@ -17,7 +15,11 @@ import { isEvaluatedRow, isEvaluationError, mergeBucketParameterQueriers, + mergeDataSources, + mergeParameterLookupSources, QuerierError, + ScopedEvaluateParameterRow, + ScopedEvaluateRow, SqlEventDescriptor, SqliteInputValue, SqliteValue, @@ -32,33 +34,30 @@ import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, Sqlite */ export class HydratedSyncRules { bucketSources: HydratedBucketSource[] = []; - bucketDataSources: BucketDataSource[]; - bucketParameterLookupSources: BucketParameterLookupSource[]; - bucketSourceHydration: Map = new Map(); - eventDescriptors: SqlEventDescriptor[] = []; compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; readonly definition: SqlSyncRules; + private readonly innerEvaluateRow: ScopedEvaluateRow; + private readonly innerEvaluateParameterRow: ScopedEvaluateParameterRow; + constructor(params: { definition: SqlSyncRules; createParams: CreateSourceParams; bucketDataSources: BucketDataSource[]; - bucketParameterLookupSources: BucketParameterLookupSource[]; + bucketParameterLookupSources: BucketParameterLookupSourceDefinition[]; eventDescriptors?: SqlEventDescriptor[]; compatibility?: CompatibilityContext; }) { - this.bucketDataSources = params.bucketDataSources; - this.bucketParameterLookupSources = params.bucketParameterLookupSources; - this.definition = params.definition; - const hydrationState = params.createParams.hydrationState; - for (let source of this.bucketDataSources) { - const state = hydrationState.getBucketSourceScope(source); - this.bucketSourceHydration.set(source, state); - } + this.definition = params.definition; + this.innerEvaluateRow = mergeDataSources(hydrationState, params.bucketDataSources).evaluateRow; + this.innerEvaluateParameterRow = mergeParameterLookupSources( + hydrationState, + params.bucketParameterLookupSources + ).evaluateParameterRow; if (params.eventDescriptors) { this.eventDescriptors = params.eventDescriptors; @@ -112,28 +111,7 @@ export class HydratedSyncRules { } evaluateRowWithErrors(options: EvaluateRowOptions): { results: EvaluatedRow[]; errors: EvaluationError[] } { - let rawResults: EvaluationResult[] = []; - for (let source of this.bucketDataSources) { - const sourceResults = source.evaluateRow(options); - if (sourceResults.length == 0) { - continue; - } - const bucketPrefix = this.bucketSourceHydration.get(source)!.bucketPrefix; - rawResults.push( - ...sourceResults.map((sourceRow) => { - if (isEvaluationError(sourceRow)) { - return sourceRow; - } - return { - bucket: bucketPrefix + sourceRow.serializedBucketParameters, - id: sourceRow.id, - table: sourceRow.table, - data: sourceRow.data - } satisfies EvaluatedRow; - }) - ); - } - + const rawResults: EvaluationResult[] = this.innerEvaluateRow(options); const results = rawResults.filter(isEvaluatedRow) as EvaluatedRow[]; const errors = rawResults.filter(isEvaluationError) as EvaluationError[]; @@ -155,11 +133,7 @@ export class HydratedSyncRules { table: SourceTableInterface, row: SqliteRow ): { results: EvaluatedParameters[]; errors: EvaluationError[] } { - let rawResults: EvaluatedParametersResult[] = []; - for (let source of this.bucketParameterLookupSources) { - rawResults.push(...source.evaluateParameterRow(table, row)); - } - + const rawResults: EvaluatedParametersResult[] = this.innerEvaluateParameterRow(table, row); const results = rawResults.filter(isEvaluatedParameters) as EvaluatedParameters[]; const errors = rawResults.filter(isEvaluationError) as EvaluationError[]; return { results, errors }; diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index d914f19f2..bfcdf97d4 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -10,7 +10,7 @@ import { TablePattern } from './TablePattern.js'; import { TableValuedFunctionSqlParameterQuery } from './TableValuedFunctionSqlParameterQuery.js'; import { CompatibilityContext } from './compatibility.js'; import { SqlRuleError } from './errors.js'; -import { EvaluateRowOptions, QueryParseOptions, SourceEvaluationResult, SourceSchema } from './types.js'; +import { EvaluateRowOptions, QueryParseOptions, UnscopedEvaluationResult, SourceSchema } from './types.js'; export interface QueryParseResult { /** @@ -151,7 +151,7 @@ export class BucketDefinitionDataSource implements BucketDataSource { } evaluateRow(options: EvaluateRowOptions) { - let results: SourceEvaluationResult[] = []; + let results: UnscopedEvaluationResult[] = []; for (let query of this.descriptor.dataQueries) { if (!query.applies(options.sourceTable)) { continue; diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index b533694b9..389189a23 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -10,7 +10,7 @@ import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; import { SyncRulesOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; import { TableQuerySchema } from './TableQuerySchema.js'; -import { ParameterMatchClause, QuerySchema, SourceEvaluationResult, SqliteRow } from './types.js'; +import { ParameterMatchClause, QuerySchema, UnscopedEvaluationResult, SqliteRow } from './types.js'; import { isSelectStatement, serializeBucketParameters } from './utils.js'; export interface SqlDataQueryOptions extends BaseSqlDataQueryOptions { @@ -190,7 +190,7 @@ export class SqlDataQuery extends BaseSqlDataQuery { this.filter = options.filter; } - evaluateRow(table: SourceTableInterface, row: SqliteRow): SourceEvaluationResult[] { + evaluateRow(table: SourceTableInterface, row: SqliteRow): UnscopedEvaluationResult[] { return this.evaluateRowWithOptions({ table, row, diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 7a219e69f..be7306e8a 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -7,12 +7,11 @@ import { } from './BucketDescription.js'; import { BucketParameterQuerier, - ParameterLookup, + UnscopedParameterLookup, ParameterLookupSource, PendingQueriers } from './BucketParameterQuerier.js'; import { - BucketParameterLookupSource, BucketParameterLookupSourceDefinition, BucketParameterQuerierSource, BucketParameterQuerierSourceDefinition, @@ -20,7 +19,13 @@ import { } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; import { BucketDataScope, ParameterLookupScope } from './HydrationState.js'; -import { BucketDataSource, GetQuerierOptions } from './index.js'; +import { + BucketDataSource, + GetQuerierOptions, + ScopedParameterLookup, + UnscopedEvaluatedParameters, + UnscopedEvaluatedParametersResult +} from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; @@ -365,30 +370,19 @@ export class SqlParameterQuery }; } - createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { - const hydrationState = params.hydrationState; - const lookupState = hydrationState.getParameterLookupScope(this); - return { - evaluateParameterRow: (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { - if (this.tableSyncsParameters(sourceTable)) { - return this.evaluateParameterRow(lookupState, row); - } else { - return []; - } - } - }; - } - /** * Given a replicated row, results an array of bucket parameter rows to persist. */ - evaluateParameterRow(scope: ParameterLookupScope, row: SqliteRow): EvaluatedParametersResult[] { + evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): UnscopedEvaluatedParametersResult[] { + if (!this.tableSyncsParameters(sourceTable)) { + return []; + } const tables = { [this.table.nameInSchema]: row }; try { const filterParameters = this.filter.filterRow(tables); - let result: EvaluatedParametersResult[] = []; + let result: UnscopedEvaluatedParametersResult[] = []; for (let filterParamSet of filterParameters) { let lookupValues: SqliteJsonValue[] = []; lookupValues.push( @@ -399,9 +393,9 @@ export class SqlParameterQuery const data = this.transformRows(row); - const role: EvaluatedParameters = { + const role: UnscopedEvaluatedParameters = { bucketParameters: data.map((row) => filterJsonRow(row)), - lookup: ParameterLookup.normalized(scope, lookupValues) + lookup: UnscopedParameterLookup.normalized(lookupValues) }; result.push(role); } @@ -468,7 +462,7 @@ export class SqlParameterQuery * * Each lookup is [bucket definition name, parameter query index, ...lookup values] */ - getLookups(scope: ParameterLookupScope, parameters: RequestParameters): ParameterLookup[] { + getLookups(parameters: RequestParameters): UnscopedParameterLookup[] { if (!this.expandedInputParameter) { let lookupValues: SqliteJsonValue[] = []; @@ -489,7 +483,7 @@ export class SqlParameterQuery if (!valid) { return []; } - return [ParameterLookup.normalized(scope, lookupValues)]; + return [UnscopedParameterLookup.normalized(lookupValues)]; } else { const arrayString = this.expandedInputParameter.parametersToLookupValue(parameters); @@ -533,9 +527,9 @@ export class SqlParameterQuery return null; } - return ParameterLookup.normalized(scope, lookupValues); + return UnscopedParameterLookup.normalized(lookupValues); }) - .filter((lookup) => lookup != null) as ParameterLookup[]; + .filter((lookup) => lookup != null) as UnscopedParameterLookup[]; } } @@ -545,7 +539,7 @@ export class SqlParameterQuery bucketDataScope: BucketDataScope, scope: ParameterLookupScope ): BucketParameterQuerier { - const lookups = this.getLookups(scope, requestParameters); + const lookups = this.getLookups(requestParameters).map((lookup) => ScopedParameterLookup.normalized(scope, lookup)); if (lookups.length == 0) { // This typically happens when the query is pre-filtered using a where clause // on the parameters, and does not depend on the database state. diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 6fbdad975..8e654fc31 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -417,9 +417,7 @@ export class SqlSyncRules { definition: this, createParams: resolvedParams, bucketDataSources: this.bucketDataSources, - bucketParameterLookupSources: this.bucketParameterLookupSources.map((d) => - d.createParameterLookupSource(resolvedParams) - ), + bucketParameterLookupSources: this.bucketParameterLookupSources, eventDescriptors: this.eventDescriptors, compatibility: this.compatibility }); diff --git a/packages/sync-rules/src/streams/filter.ts b/packages/sync-rules/src/streams/filter.ts index 691f6acee..20cea5914 100644 --- a/packages/sync-rules/src/streams/filter.ts +++ b/packages/sync-rules/src/streams/filter.ts @@ -1,4 +1,4 @@ -import { ParameterLookup } from '../BucketParameterQuerier.js'; +import { ScopedParameterLookup, UnscopedParameterLookup } from '../BucketParameterQuerier.js'; import { SqlTools } from '../sql_filters.js'; import { checkJsonArray, OPERATOR_NOT } from '../sql_functions.js'; import { isParameterValueClause, isRowValueClause, SQLITE_TRUE, sqliteBool } from '../sql_support.js'; @@ -10,16 +10,13 @@ import { RequestParameters, RowValueClause, SqliteJsonValue, - SqliteRow + SqliteRow, + UnscopedEvaluatedParametersResult } from '../types.js'; import { isJsonValue, normalizeParameterValue } from '../utils.js'; import { NodeLocation } from 'pgsql-ast-parser'; -import { - BucketParameterLookupSource, - BucketParameterLookupSourceDefinition, - CreateSourceParams -} from '../BucketSource.js'; +import { BucketParameterLookupSourceDefinition, CreateSourceParams } from '../BucketSource.js'; import { HydrationState, ParameterLookupScope } from '../HydrationState.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { SubqueryEvaluator } from './parameter.js'; @@ -269,7 +266,7 @@ export class Subquery { let lookupSources: BucketParameterLookupSourceDefinition[] = []; let lookupsForRequest: (( hydrationState: HydrationState - ) => (parameters: RequestParameters) => ParameterLookup[])[] = []; + ) => (parameters: RequestParameters) => ScopedParameterLookup[])[] = []; for (let [variant, id] of innerVariants) { const source = new SubqueryParameterLookupSource(this.table, column, variant, id, context.streamName); @@ -277,10 +274,10 @@ export class Subquery { lookupsForRequest.push((hydrationState: HydrationState) => { const scope = hydrationState.getParameterLookupScope(source); return (parameters: RequestParameters) => { - const lookups: ParameterLookup[] = []; + const lookups: ScopedParameterLookup[] = []; const instantiations = variant.findStaticInstantiations(parameters); for (const instantiation of instantiations) { - lookups.push(ParameterLookup.normalized(scope, instantiation)); + lookups.push(ScopedParameterLookup.normalized(scope, UnscopedParameterLookup.normalized(instantiation))); } return lookups; }; @@ -295,7 +292,7 @@ export class Subquery { hydrateLookupsForRequest(hydrationState: HydrationState) { const hydrated = lookupsForRequest.map((fn) => fn(hydrationState)); return (parameters: RequestParameters) => { - const lookups: ParameterLookup[] = []; + const lookups: ScopedParameterLookup[] = []; for (const getLookups of hydrated) { lookups.push(...getLookups(parameters)); } @@ -569,11 +566,7 @@ export class SubqueryParameterLookupSource implements BucketParameterLookupSourc * @param sourceTable A table we depend on in a subquery. * @param row Row data to index. */ - evaluateParameterRow( - lookupScope: ParameterLookupScope, - sourceTable: SourceTableInterface, - row: SqliteRow - ): EvaluatedParametersResult[] { + evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): UnscopedEvaluatedParametersResult[] { if (this.parameterTable.matches(sourceTable)) { // Theoretically we're doing duplicate work by doing this for each innerVariant in a subquery. // In practice, we don't have more than one innerVariant per subquery right now, so this is fine. @@ -582,9 +575,9 @@ export class SubqueryParameterLookupSource implements BucketParameterLookupSourc return []; } - const lookups: ParameterLookup[] = []; + const lookups: UnscopedParameterLookup[] = []; for (const instantiation of this.innerVariant.instantiationsForRow({ sourceTable, record: row })) { - lookups.push(ParameterLookup.normalized(lookupScope, instantiation)); + lookups.push(UnscopedParameterLookup.normalized(instantiation)); } // The row of the subquery. Since we only support subqueries with a single column, we unconditionally name the @@ -599,16 +592,6 @@ export class SubqueryParameterLookupSource implements BucketParameterLookupSourc return []; } - createParameterLookupSource(params: CreateSourceParams): BucketParameterLookupSource { - const hydrationState = params.hydrationState; - const lookupScope = hydrationState.getParameterLookupScope(this); - return { - evaluateParameterRow: (sourceTable, row) => { - return this.evaluateParameterRow(lookupScope, sourceTable, row); - } - }; - } - tableSyncsParameters(table: SourceTableInterface): boolean { return this.parameterTable.matches(table); } diff --git a/packages/sync-rules/src/streams/parameter.ts b/packages/sync-rules/src/streams/parameter.ts index 74820bf0c..ae856a957 100644 --- a/packages/sync-rules/src/streams/parameter.ts +++ b/packages/sync-rules/src/streams/parameter.ts @@ -1,4 +1,4 @@ -import { ParameterLookup } from '../BucketParameterQuerier.js'; +import { ScopedParameterLookup, UnscopedParameterLookup } from '../BucketParameterQuerier.js'; import { BucketParameterLookupSourceDefinition } from '../BucketSource.js'; import { HydrationState } from '../HydrationState.js'; import { TablePattern } from '../TablePattern.js'; @@ -37,11 +37,11 @@ export interface SubqueryEvaluator { // use, and each querier may use multiple lookup sources, each with their own scope. // This implementation here does "hydration" on each subquery, which gives us hydrated function call. // Should this maybe be part of a higher-level class instead of just a function, i.e. a hydrated subquery? - hydrateLookupsForRequest(hydrationState: HydrationState): (params: RequestParameters) => ParameterLookup[]; + hydrateLookupsForRequest(hydrationState: HydrationState): (params: RequestParameters) => ScopedParameterLookup[]; } export interface SubqueryLookups { - lookups: ParameterLookup[]; + lookups: UnscopedParameterLookup[]; /** * The value that the single column in the subquery evaluated to. */ diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index 118b24ec8..4a84038c0 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -10,7 +10,7 @@ import { import { ColumnDefinition } from '../ExpressionType.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { TablePattern } from '../TablePattern.js'; -import { EvaluateRowOptions, SourceEvaluationResult, SourceSchema, TableRow } from '../types.js'; +import { EvaluateRowOptions, UnscopedEvaluationResult, SourceSchema, TableRow } from '../types.js'; import { StreamVariant } from './variant.js'; export class SyncStream implements BucketSource { @@ -100,7 +100,7 @@ export class SyncStreamDataSource implements BucketDataSource { result[this.data.table!.sqlName].push(r); } - evaluateRow(options: EvaluateRowOptions): SourceEvaluationResult[] { + evaluateRow(options: EvaluateRowOptions): UnscopedEvaluationResult[] { if (!this.data.applies(options.sourceTable)) { return []; } diff --git a/packages/sync-rules/src/streams/variant.ts b/packages/sync-rules/src/streams/variant.ts index 1b9000eb7..d2fc1f71b 100644 --- a/packages/sync-rules/src/streams/variant.ts +++ b/packages/sync-rules/src/streams/variant.ts @@ -1,5 +1,5 @@ import { BucketInclusionReason, ResolvedBucket } from '../BucketDescription.js'; -import { BucketParameterQuerier, ParameterLookup, PendingQueriers } from '../BucketParameterQuerier.js'; +import { BucketParameterQuerier, UnscopedParameterLookup, PendingQueriers } from '../BucketParameterQuerier.js'; import { BucketDataSource, BucketParameterLookupSourceDefinition, @@ -8,7 +8,7 @@ import { CreateSourceParams } from '../BucketSource.js'; import { BucketDataScope } from '../HydrationState.js'; -import { GetQuerierOptions, RequestedStream } from '../index.js'; +import { GetQuerierOptions, RequestedStream, ScopedParameterLookup } from '../index.js'; import { RequestParameters, SqliteJsonValue, TableRow } from '../types.js'; import { buildBucketName, isJsonValue, JSONBucketNameSerialize } from '../utils.js'; import { BucketParameter, SubqueryEvaluator } from './parameter.js'; @@ -145,7 +145,7 @@ export class StreamVariant { const dynamicRequestFilters: SubqueryRequestFilter[] = this.requestFilters.filter((f) => f.type == 'dynamic'); const dynamicParameters: ResolvedDynamicParameter[] = []; - const subqueryToLookups = new Map(); + const subqueryToLookups = new Map(); for (let i = 0; i < this.parameters.length; i++) { const parameter = this.parameters[i]; @@ -412,4 +412,4 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS } } -type HydratedSubqueries = Map ParameterLookup[]>; +type HydratedSubqueries = Map ScopedParameterLookup[]>; diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index aeca806f1..44bc7ee30 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -1,6 +1,6 @@ import { JSONBig, JsonContainer } from '@powersync/service-jsonbig'; import { BucketPriority } from './BucketDescription.js'; -import { ParameterLookup } from './BucketParameterQuerier.js'; +import { ScopedParameterLookup, UnscopedParameterLookup } from './BucketParameterQuerier.js'; import { CompatibilityContext } from './compatibility.js'; import { ColumnDefinition } from './ExpressionType.js'; import { RequestFunctionCall } from './request_functions.js'; @@ -21,7 +21,18 @@ export interface StreamParseOptions extends QueryParseOptions { } export interface EvaluatedParameters { - lookup: ParameterLookup; + lookup: ScopedParameterLookup; + + /** + * Parameters used to generate bucket id. May be incomplete. + * + * JSON-serializable. + */ + bucketParameters: Record[]; +} + +export interface UnscopedEvaluatedParameters { + lookup: UnscopedParameterLookup; /** * Parameters used to generate bucket id. May be incomplete. @@ -32,6 +43,7 @@ export interface EvaluatedParameters { } export type EvaluatedParametersResult = EvaluatedParameters | EvaluationError; +export type UnscopedEvaluatedParametersResult = UnscopedEvaluatedParameters | EvaluationError; export interface EvaluatedRow { bucket: string; @@ -53,7 +65,7 @@ export interface EvaluatedRow { * * The bucket name must still be resolved, external to this. */ -export interface SourceEvaluatedRow { +export interface UnscopedEvaluatedRow { /** * Serialized evaluated parameters used to generate the bucket id. Serialized as a JSON array. * @@ -83,7 +95,7 @@ export interface EvaluationError { } export function isEvaluationError( - e: EvaluationResult | SourceEvaluationResult | EvaluatedParametersResult + e: EvaluationResult | UnscopedEvaluationResult | EvaluatedParametersResult | UnscopedEvaluatedParametersResult ): e is EvaluationError { return typeof (e as EvaluationError).error == 'string'; } @@ -92,8 +104,8 @@ export function isEvaluatedRow(e: EvaluationResult): e is EvaluatedRow { return typeof (e as EvaluatedRow).bucket == 'string'; } -export function isSourceEvaluatedRow(e: SourceEvaluationResult): e is SourceEvaluatedRow { - return typeof (e as SourceEvaluatedRow).serializedBucketParameters == 'string'; +export function isSourceEvaluatedRow(e: UnscopedEvaluationResult): e is UnscopedEvaluatedRow { + return typeof (e as UnscopedEvaluatedRow).serializedBucketParameters == 'string'; } export function isEvaluatedParameters(e: EvaluatedParametersResult): e is EvaluatedParameters { @@ -101,7 +113,7 @@ export function isEvaluatedParameters(e: EvaluatedParametersResult): e is Evalua } export type EvaluationResult = EvaluatedRow | EvaluationError; -export type SourceEvaluationResult = SourceEvaluatedRow | EvaluationError; +export type UnscopedEvaluationResult = UnscopedEvaluatedRow | EvaluationError; export interface RequestJwtPayload { /** diff --git a/packages/sync-rules/test/src/parameter_queries.test.ts b/packages/sync-rules/test/src/parameter_queries.test.ts index 8c2fc6150..7c00b161c 100644 --- a/packages/sync-rules/test/src/parameter_queries.test.ts +++ b/packages/sync-rules/test/src/parameter_queries.test.ts @@ -1,15 +1,20 @@ import { describe, expect, test } from 'vitest'; -import { ParameterLookupScope } from '../../src/HydrationState.js'; -import { ParameterLookup, SqlParameterQuery } from '../../src/index.js'; +import { UnscopedParameterLookup, SqlParameterQuery, SourceTableInterface } from '../../src/index.js'; import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; import { BASIC_SCHEMA, EMPTY_DATA_SOURCE, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; describe('parameter queries', () => { - // Specifically different from mybucket/1, to make sure this is being used. - const TEST_SCOPE: ParameterLookupScope = { - lookupName: 'test', - queryId: '42' - }; + const table = (name: string): SourceTableInterface => ({ + connectionTag: 'default', + name, + schema: PARSE_OPTIONS.defaultSchema + }); + + const TABLE_USERS = table('users'); + const TABLE_REGIONS = table('regions'); + const TABLE_WORKSPACES = table('workspaces'); + const TABLE_POSTS = table('posts'); + const TABLE_GROUPS = table('groups'); test('token_parameters IN query', function () { const sql = 'SELECT id as group_id FROM groups WHERE token_parameters.user_id IN groups.user_ids'; @@ -22,10 +27,10 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); expect( - query.evaluateParameterRow(TEST_SCOPE, { id: 'group1', user_ids: JSON.stringify(['user1', 'user2']) }) + query.evaluateParameterRow(TABLE_GROUPS, { id: 'group1', user_ids: JSON.stringify(['user1', 'user2']) }) ).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [ { group_id: 'group1' @@ -33,7 +38,7 @@ describe('parameter queries', () => { ] }, { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['user2']), + lookup: UnscopedParameterLookup.normalized(['user2']), bucketParameters: [ { group_id: 'group1' @@ -43,12 +48,11 @@ describe('parameter queries', () => { ]); expect( query.getLookups( - TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1' }) ) - ).toEqual([ParameterLookup.normalized(TEST_SCOPE, ['user1'])]); + ).toEqual([UnscopedParameterLookup.normalized(['user1'])]); }); test('IN token_parameters query', function () { @@ -61,9 +65,9 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'region1', name: 'colorado' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_REGIONS, { id: 'region1', name: 'colorado' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['colorado']), + lookup: UnscopedParameterLookup.normalized(['colorado']), bucketParameters: [ { region_id: 'region1' @@ -73,15 +77,11 @@ describe('parameter queries', () => { ]); expect( query.getLookups( - TEST_SCOPE, normalizeTokenParameters({ region_names: JSON.stringify(['colorado', 'texas']) }) ) - ).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['colorado']), - ParameterLookup.normalized(TEST_SCOPE, ['texas']) - ]); + ).toEqual([UnscopedParameterLookup.normalized(['colorado']), UnscopedParameterLookup.normalized(['texas'])]); }); test('queried numeric parameters', () => { @@ -96,9 +96,9 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); // Note: We don't need to worry about numeric vs decimal types in the lookup - JSONB handles normalization for us. - expect(query.evaluateParameterRow(TEST_SCOPE, { int1: 314n, float1: 3.14, float2: 314 })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { int1: 314n, float1: 3.14, float2: 314 })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, [314n, 3.14, 314]), + lookup: UnscopedParameterLookup.normalized([314n, 3.14, 314]), bucketParameters: [{ int1: 314n, float1: 3.14, float2: 314 }] } @@ -106,8 +106,8 @@ describe('parameter queries', () => { // Similarly, we don't need to worry about the types here. // This test just checks the current behavior. - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ int1: 314n, float1: 3.14, float2: 314 }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, [314n, 3.14, 314n]) + expect(query.getLookups(normalizeTokenParameters({ int1: 314n, float1: 3.14, float2: 314 }))).toEqual([ + UnscopedParameterLookup.normalized([314n, 3.14, 314n]) ]); // We _do_ need to care about the bucket string representation. @@ -135,17 +135,16 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - TEST_SCOPE; - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['test_param']), + lookup: UnscopedParameterLookup.normalized(['test_param']), bucketParameters: [{ id: 'test_id' }] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['test']) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([ + UnscopedParameterLookup.normalized(['test']) ]); }); @@ -160,16 +159,16 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['test_param']), + lookup: UnscopedParameterLookup.normalized(['test_param']), bucketParameters: [{ id: 'test_id' }] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['TEST']) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([ + UnscopedParameterLookup.normalized(['TEST']) ]); }); @@ -184,17 +183,17 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['test_param']), + lookup: UnscopedParameterLookup.normalized(['test_param']), bucketParameters: [{ id: 'test_id' }] } ]); - expect( - query.getLookups(TEST_SCOPE, normalizeTokenParameters({ some_param: { description: 'test_description' } })) - ).toEqual([ParameterLookup.normalized(TEST_SCOPE, ['test_description'])]); + expect(query.getLookups(normalizeTokenParameters({ some_param: { description: 'test_description' } }))).toEqual([ + UnscopedParameterLookup.normalized(['test_description']) + ]); }); test('token parameter and binary operator', () => { @@ -208,8 +207,8 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ some_param: 3 }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, [5n]) + expect(query.getLookups(normalizeTokenParameters({ some_param: 3 }))).toEqual([ + UnscopedParameterLookup.normalized([5n]) ]); }); @@ -224,11 +223,11 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ some_param: null }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, [1n]) + expect(query.getLookups(normalizeTokenParameters({ some_param: null }))).toEqual([ + UnscopedParameterLookup.normalized([1n]) ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ some_param: 'test' }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, [0n]) + expect(query.getLookups(normalizeTokenParameters({ some_param: 'test' }))).toEqual([ + UnscopedParameterLookup.normalized([0n]) ]); }); @@ -243,18 +242,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, [1n]), + lookup: UnscopedParameterLookup.normalized([1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, [0n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + UnscopedParameterLookup.normalized([0n]) ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, [1n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + UnscopedParameterLookup.normalized([1n]) ]); }); @@ -269,18 +268,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, [1n]), + lookup: UnscopedParameterLookup.normalized([1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, [1n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + UnscopedParameterLookup.normalized([1n]) ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, [0n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + UnscopedParameterLookup.normalized([0n]) ]); }); @@ -295,18 +294,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, [1n]), + lookup: UnscopedParameterLookup.normalized([1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, [0n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + UnscopedParameterLookup.normalized([0n]) ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, [1n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + UnscopedParameterLookup.normalized([1n]) ]); }); @@ -321,18 +320,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, [1n]), + lookup: UnscopedParameterLookup.normalized([1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, [1n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ + UnscopedParameterLookup.normalized([1n]) ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', is_admin: 123 }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, [0n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: 123 }))).toEqual([ + UnscopedParameterLookup.normalized([0n]) ]); }); @@ -347,18 +346,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]), + lookup: UnscopedParameterLookup.normalized(['user1', 1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + UnscopedParameterLookup.normalized(['user1', 1n]) ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['user1', 0n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + UnscopedParameterLookup.normalized(['user1', 0n]) ]); }); @@ -373,18 +372,18 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]), + lookup: UnscopedParameterLookup.normalized(['user1', 1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + UnscopedParameterLookup.normalized(['user1', 1n]) ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['user1', 0n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + UnscopedParameterLookup.normalized(['user1', 0n]) ]); }); @@ -399,11 +398,11 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['user1']) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ + UnscopedParameterLookup.normalized(['user1']) ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 123 }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['123']) + expect(query.getLookups(normalizeTokenParameters({ user_id: 123 }))).toEqual([ + UnscopedParameterLookup.normalized(['123']) ]); }); @@ -418,15 +417,15 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1', role: null })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1', role: null })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, []), + lookup: UnscopedParameterLookup.normalized([]), bucketParameters: [{ id: 'user1' }] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, []) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ + UnscopedParameterLookup.normalized([]) ]); }); @@ -444,19 +443,19 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]), + lookup: UnscopedParameterLookup.normalized(['user1', 1n]), bucketParameters: [{}] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ + UnscopedParameterLookup.normalized(['user1', 1n]) ]); // Would not match any actual lookups - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['user1', 0n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ + UnscopedParameterLookup.normalized(['user1', 0n]) ]); }); @@ -472,16 +471,16 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]), + lookup: UnscopedParameterLookup.normalized(['user1', 1n]), bucketParameters: [{ user_id: 'user1' }] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['user1', 1n]) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ + UnscopedParameterLookup.normalized(['user1', 1n]) ]); expect( @@ -504,9 +503,9 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { userId: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { userId: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [{ user_id: 'user1' }] } @@ -530,10 +529,10 @@ describe('parameter queries', () => { { message: `Unquoted identifiers are converted to lower-case. Use "userId" instead.` } ]); - expect(query.evaluateParameterRow(TEST_SCOPE, { userId: 'user1' })).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { userid: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { userId: 'user1' })).toEqual([]); + expect(query.evaluateParameterRow(TABLE_USERS, { userid: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [{ user_id: 'user1' }] } @@ -621,15 +620,15 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'workspace1', visibility: 'public' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_WORKSPACES, { id: 'workspace1', visibility: 'public' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, []), + lookup: UnscopedParameterLookup.normalized([]), bucketParameters: [{ workspace_id: 'workspace1' }] } ]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'workspace1', visibility: 'private' })).toEqual([]); + expect(query.evaluateParameterRow(TABLE_WORKSPACES, { id: 'workspace1', visibility: 'private' })).toEqual([]); }); test('multiple different functions on token_parameter with AND', () => { @@ -645,16 +644,16 @@ describe('parameter queries', () => { ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'test_id', filter_param: 'test_param' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'test_id', filter_param: 'test_param' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['test_param', 'test_param']), + lookup: UnscopedParameterLookup.normalized(['test_param', 'test_param']), bucketParameters: [{ id: 'test_id' }] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['TEST', 'test']) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([ + UnscopedParameterLookup.normalized(['TEST', 'test']) ]); }); @@ -672,20 +671,20 @@ describe('parameter queries', () => { expect(query.errors).toEqual([]); expect( - query.evaluateParameterRow(TEST_SCOPE, { id: 'test_id', filter_param1: 'test1', filter_param2: 'test2' }) + query.evaluateParameterRow(TABLE_USERS, { id: 'test_id', filter_param1: 'test1', filter_param2: 'test2' }) ).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['test1']), + lookup: UnscopedParameterLookup.normalized(['test1']), bucketParameters: [{ id: 'test_id' }] }, { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['test2']), + lookup: UnscopedParameterLookup.normalized(['test2']), bucketParameters: [{ id: 'test_id' }] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({ user_id: 'test' }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['TEST']) + expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([ + UnscopedParameterLookup.normalized(['TEST']) ]); }); @@ -702,14 +701,14 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'group1', category: 'red' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_POSTS, { id: 'group1', category: 'red' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['red']), + lookup: UnscopedParameterLookup.normalized(['red']), bucketParameters: [{}] } ]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({}, { category_id: 'red' }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['red']) + expect(query.getLookups(normalizeTokenParameters({}, { category_id: 'red' }))).toEqual([ + UnscopedParameterLookup.normalized(['red']) ]); }); @@ -726,8 +725,8 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['red']) + expect(query.getLookups(normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ + UnscopedParameterLookup.normalized(['red']) ]); }); @@ -744,8 +743,8 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.getLookups(TEST_SCOPE, normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['red']) + expect(query.getLookups(normalizeTokenParameters({}, { details: { category: 'red' } }))).toEqual([ + UnscopedParameterLookup.normalized(['red']) ]); }); @@ -763,9 +762,9 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'region1', name: 'colorado' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_REGIONS, { id: 'region1', name: 'colorado' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['colorado']), + lookup: UnscopedParameterLookup.normalized(['colorado']), bucketParameters: [ { region_id: 'region1' @@ -775,7 +774,6 @@ describe('parameter queries', () => { ]); expect( query.getLookups( - TEST_SCOPE, normalizeTokenParameters( {}, { @@ -783,10 +781,7 @@ describe('parameter queries', () => { } ) ) - ).toEqual([ - ParameterLookup.normalized(TEST_SCOPE, ['colorado']), - ParameterLookup.normalized(TEST_SCOPE, ['texas']) - ]); + ).toEqual([UnscopedParameterLookup.normalized(['colorado']), UnscopedParameterLookup.normalized(['texas'])]); }); test('user_parameters in SELECT', function () { @@ -799,14 +794,14 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [{ id: 'user1' }] } ]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }, { other_id: 'red' }); - expect(query.getLookups(TEST_SCOPE, requestParams)).toEqual([ParameterLookup.normalized(TEST_SCOPE, ['user1'])]); + expect(query.getLookups(requestParams)).toEqual([UnscopedParameterLookup.normalized(['user1'])]); }); test('request.parameters() in SELECT', function () { @@ -820,14 +815,14 @@ describe('parameter queries', () => { EMPTY_DATA_SOURCE ) as SqlParameterQuery; expect(query.errors).toEqual([]); - expect(query.evaluateParameterRow(TEST_SCOPE, { id: 'user1' })).toEqual([ + expect(query.evaluateParameterRow(TABLE_USERS, { id: 'user1' })).toEqual([ { - lookup: ParameterLookup.normalized(TEST_SCOPE, ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [{ id: 'user1' }] } ]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }, { other_id: 'red' }); - expect(query.getLookups(TEST_SCOPE, requestParams)).toEqual([ParameterLookup.normalized(TEST_SCOPE, ['user1'])]); + expect(query.getLookups(requestParams)).toEqual([UnscopedParameterLookup.normalized(['user1'])]); }); test('request.jwt()', function () { @@ -842,7 +837,7 @@ describe('parameter queries', () => { expect(query.errors).toEqual([]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }); - expect(query.getLookups(TEST_SCOPE, requestParams)).toEqual([ParameterLookup.normalized(TEST_SCOPE, ['user1'])]); + expect(query.getLookups(requestParams)).toEqual([UnscopedParameterLookup.normalized(['user1'])]); }); test('request.user_id()', function () { @@ -857,7 +852,7 @@ describe('parameter queries', () => { expect(query.errors).toEqual([]); const requestParams = normalizeTokenParameters({ user_id: 'user1' }); - expect(query.getLookups(TEST_SCOPE, requestParams)).toEqual([ParameterLookup.normalized(TEST_SCOPE, ['user1'])]); + expect(query.getLookups(requestParams)).toEqual([UnscopedParameterLookup.normalized(['user1'])]); }); test('invalid OR in parameter queries', () => { diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index 313d1fc11..104039c4f 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -12,7 +12,7 @@ import { GetBucketParameterQuerierResult, GetQuerierOptions, mergeBucketParameterQueriers, - ParameterLookup, + UnscopedParameterLookup, QuerierError, RequestParameters, SourceTableInterface, @@ -21,7 +21,8 @@ import { StaticSchema, StreamParseOptions, SyncStream, - syncStreamFromSql + syncStreamFromSql, + ScopedParameterLookup } from '../../src/index.js'; import { normalizeQuerierOptions, PARSE_OPTIONS, TestSourceTable } from './util.js'; @@ -235,7 +236,7 @@ describe('streams', () => { }) ).toStrictEqual([ { - lookup: ParameterLookup.normalized(STREAM_0, ['u1']), + lookup: ScopedParameterLookup.direct(STREAM_0, ['u1']), bucketParameters: [ { result: 'i1' @@ -244,8 +245,8 @@ describe('streams', () => { } ]); - function getParameterSets(lookups: ParameterLookup[]) { - expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['u1'])]); + function getParameterSets(lookups: ScopedParameterLookup[]) { + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['u1'])]); return [{ result: 'i1' }]; } @@ -267,13 +268,9 @@ describe('streams', () => { const lookup = desc.parameterLookupSources[0]; expect(lookup.tableSyncsParameters(ISSUES)).toBe(true); - expect( - lookup - .createParameterLookupSource(hydrationParams) - .evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) - ).toStrictEqual([ + expect(lookup.evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' })).toStrictEqual([ { - lookup: ParameterLookup.normalized(STREAM_0, ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [ { result: 'issue_id' @@ -289,7 +286,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['user1'])]); + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -304,11 +301,9 @@ describe('streams', () => { expect(lookup.tableSyncsParameters(ISSUES)).toBe(false); expect(lookup.tableSyncsParameters(USERS)).toBe(true); - expect( - lookup.createParameterLookupSource(hydrationParams).evaluateParameterRow(USERS, { id: 'u', is_admin: 1n }) - ).toStrictEqual([ + expect(lookup.evaluateParameterRow(USERS, { id: 'u', is_admin: 1n })).toStrictEqual([ { - lookup: ParameterLookup.normalized(STREAM_0, ['u']), + lookup: UnscopedParameterLookup.normalized(['u']), bucketParameters: [ { result: 'u' @@ -316,16 +311,14 @@ describe('streams', () => { ] } ]); - expect( - lookup.createParameterLookupSource(hydrationParams).evaluateParameterRow(USERS, { id: 'u', is_admin: 0n }) - ).toStrictEqual([]); + expect(lookup.evaluateParameterRow(USERS, { id: 'u', is_admin: 0n })).toStrictEqual([]); // Should return bucket id for admin users expect( await queryBucketIds(desc, { token: { sub: 'u' }, - getParameterSets: (lookups: ParameterLookup[]) => { - expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['u'])]); + getParameterSets: (lookups: ScopedParameterLookup[]) => { + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['u'])]); return [{ result: 'u' }]; } }) @@ -335,8 +328,8 @@ describe('streams', () => { expect( await queryBucketIds(desc, { token: { sub: 'u2' }, - getParameterSets: (lookups: ParameterLookup[]) => { - expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['u2'])]); + getParameterSets: (lookups: ScopedParameterLookup[]) => { + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['u2'])]); return []; } }) @@ -358,7 +351,7 @@ describe('streams', () => { expect(source.evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' })).toStrictEqual([ { - lookup: ParameterLookup.normalized(STREAM_0, ['b']), + lookup: ScopedParameterLookup.direct(STREAM_0, ['b']), bucketParameters: [ { result: 'a' @@ -366,7 +359,7 @@ describe('streams', () => { ] }, { - lookup: ParameterLookup.normalized(STREAM_1, ['a']), + lookup: ScopedParameterLookup.direct(STREAM_1, ['a']), bucketParameters: [ { result: 'b' @@ -375,14 +368,14 @@ describe('streams', () => { } ]); - function getParameterSets(lookups: ParameterLookup[]) { + function getParameterSets(lookups: ScopedParameterLookup[]) { expect(lookups).toHaveLength(1); const [lookup] = lookups; if (lookup.values[1] == '0') { - expect(lookup).toStrictEqual(ParameterLookup.normalized(STREAM_0, ['a'])); + expect(lookup).toStrictEqual(ScopedParameterLookup.direct(STREAM_0, ['a'])); return []; } else { - expect(lookup).toStrictEqual(ParameterLookup.normalized(STREAM_1, ['a'])); + expect(lookup).toStrictEqual(ScopedParameterLookup.direct(STREAM_1, ['a'])); return [{ result: 'b' }]; } } @@ -422,7 +415,7 @@ describe('streams', () => { getParameterSets(lookups) { expect(lookups).toHaveLength(1); const [lookup] = lookups; - expect(lookup).toStrictEqual(ParameterLookup.normalized(STREAM_0, ['a'])); + expect(lookup).toStrictEqual(ScopedParameterLookup.direct(STREAM_0, ['a'])); return [{ result: 'i1' }, { result: 'i2' }]; } }) @@ -463,11 +456,9 @@ describe('streams', () => { const lookup = desc.parameterLookupSources[0]; expect(lookup.tableSyncsParameters(FRIENDS)).toBe(true); - expect( - lookup.createParameterLookupSource(hydrationParams).evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' }) - ).toStrictEqual([ + expect(lookup.evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' })).toStrictEqual([ { - lookup: ParameterLookup.normalized(STREAM_0, ['b']), + lookup: UnscopedParameterLookup.normalized(['b']), bucketParameters: [ { result: 'a' @@ -484,7 +475,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['user1'])]); + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -621,12 +612,10 @@ describe('streams', () => { ); expect( - desc.parameterLookupSources[0] - .createParameterLookupSource(hydrationParams) - .evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) + desc.parameterLookupSources[0].evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) ).toStrictEqual([ { - lookup: ParameterLookup.normalized(STREAM_0, ['user1']), + lookup: UnscopedParameterLookup.normalized(['user1']), bucketParameters: [ { result: 'issue_id' @@ -642,7 +631,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['user1'])]); + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -697,7 +686,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['user1'])]); + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -707,7 +696,7 @@ describe('streams', () => { await queryBucketIds(desc, { token: { sub: 'user1', is_admin: true }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, ['user1'])]); + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, ['user1'])]); return [{ result: 'issue_id' }]; } @@ -751,13 +740,9 @@ describe('streams', () => { expect(stream.parameterLookupSources[0].tableSyncsParameters(accountMember)).toBeTruthy(); // Ensure lookup steps work. - expect( - stream.parameterLookupSources[0] - .createParameterLookupSource(hydrationParams) - .evaluateParameterRow(accountMember, row) - ).toStrictEqual([ + expect(stream.parameterLookupSources[0].evaluateParameterRow(accountMember, row)).toStrictEqual([ { - lookup: ParameterLookup.normalized({ lookupName: 'account_member', queryId: '0' }, ['id']), + lookup: UnscopedParameterLookup.normalized(['id']), bucketParameters: [ { result: 'account_id' @@ -772,7 +757,7 @@ describe('streams', () => { parameters: {}, getParameterSets(lookups) { expect(lookups).toStrictEqual([ - ParameterLookup.normalized({ lookupName: 'account_member', queryId: '0' }, ['id']) + ScopedParameterLookup.direct({ lookupName: 'account_member', queryId: '0' }, ['id']) ]); return [{ result: 'account_id' }]; } @@ -838,16 +823,14 @@ WHERE expect(evaluateBucketIds(desc, scene, { _id: 'scene', project: 'foo' })).toStrictEqual(['1#stream|0["foo"]']); expect( - desc.parameterLookupSources[0] - .createParameterLookupSource(hydrationParams) - .evaluateParameterRow(projectInvitation, { - project: 'foo', - appliedTo: '[1,2]', - status: 'CLAIMED' - }) + desc.parameterLookupSources[0].evaluateParameterRow(projectInvitation, { + project: 'foo', + appliedTo: '[1,2]', + status: 'CLAIMED' + }) ).toStrictEqual([ { - lookup: ParameterLookup.normalized(STREAM_0, [1n, 'foo']), + lookup: UnscopedParameterLookup.normalized([1n, 'foo']), bucketParameters: [ { result: 'foo' @@ -855,7 +838,7 @@ WHERE ] }, { - lookup: ParameterLookup.normalized(STREAM_0, [2n, 'foo']), + lookup: UnscopedParameterLookup.normalized([2n, 'foo']), bucketParameters: [ { result: 'foo' @@ -869,7 +852,7 @@ WHERE token: { sub: 'user1', haystack_id: 1 }, parameters: { project: 'foo' }, getParameterSets(lookups) { - expect(lookups).toStrictEqual([ParameterLookup.normalized(STREAM_0, [1n, 'foo'])]); + expect(lookups).toStrictEqual([ScopedParameterLookup.direct(STREAM_0, [1n, 'foo'])]); return [{ result: 'foo' }]; } }) @@ -914,7 +897,7 @@ WHERE }) ).toStrictEqual([ { - lookup: ParameterLookup.normalized({ lookupName: 'stream.test', queryId: '0.test' }, ['u1']), + lookup: ScopedParameterLookup.direct({ lookupName: 'stream.test', queryId: '0.test' }, ['u1']), bucketParameters: [ { result: 'i1' @@ -923,7 +906,7 @@ WHERE }, { - lookup: ParameterLookup.normalized({ lookupName: 'stream.test', queryId: '1.test' }, ['myname']), + lookup: ScopedParameterLookup.direct({ lookupName: 'stream.test', queryId: '1.test' }, ['myname']), bucketParameters: [ { result: 'i1' @@ -939,7 +922,7 @@ WHERE }) ).toStrictEqual([ { - lookup: ParameterLookup.normalized({ lookupName: 'stream.test', queryId: '0.test' }, ['u1']), + lookup: ScopedParameterLookup.direct({ lookupName: 'stream.test', queryId: '0.test' }, ['u1']), bucketParameters: [ { result: 'i1' @@ -948,7 +931,7 @@ WHERE } ]); - function getParameterSets(lookups: ParameterLookup[]) { + function getParameterSets(lookups: ScopedParameterLookup[]) { return lookups.flatMap((lookup) => { if (JSON.stringify(lookup.values) == JSON.stringify(['stream.test', '1.test', null])) { return []; @@ -1057,7 +1040,7 @@ function bucketIds(result: EvaluationResult[]): string[] { interface TestQuerierOptions { token?: Record; parameters?: Record; - getParameterSets?: (lookups: ParameterLookup[]) => SqliteJsonRow[]; + getParameterSets?: (lookups: ScopedParameterLookup[]) => SqliteJsonRow[]; hydrationState?: HydrationState; } async function createQueriers( @@ -1095,7 +1078,7 @@ async function queryBucketIds(stream: SyncStream, options?: TestQuerierOptions) const { querier, errors } = await createQueriers(stream, options); expect(errors).toHaveLength(0); - async function getParameterSets(lookups: ParameterLookup[]): Promise { + async function getParameterSets(lookups: ScopedParameterLookup[]): Promise { const provided = options?.getParameterSets; if (provided) { return provided(lookups); diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 599642686..c0aa7d4c1 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -1,5 +1,11 @@ import { describe, expect, test } from 'vitest'; -import { CreateSourceParams, ParameterLookup, SqlParameterQuery, SqlSyncRules } from '../../src/index.js'; +import { + CreateSourceParams, + ScopedParameterLookup, + UnscopedParameterLookup, + SqlParameterQuery, + SqlSyncRules +} from '../../src/index.js'; import { ASSETS, @@ -111,7 +117,7 @@ bucket_definitions: expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ { bucketParameters: [{}], - lookup: ParameterLookup.normalized({ lookupName: 'mybucket', queryId: '1' }, ['user1']) + lookup: ScopedParameterLookup.direct({ lookupName: 'mybucket', queryId: '1' }, ['user1']) } ]); expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 0 })).toEqual([]); @@ -963,10 +969,10 @@ bucket_definitions: expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ hasDynamicBuckets: true, parameterQueryLookups: [ - ParameterLookup.normalized({ lookupName: 'mybucket', queryId: '2' }, ['user1']), - ParameterLookup.normalized({ lookupName: 'by_list', queryId: '1' }, ['user1']), + ScopedParameterLookup.direct({ lookupName: 'mybucket', queryId: '2' }, ['user1']), + ScopedParameterLookup.direct({ lookupName: 'by_list', queryId: '1' }, ['user1']), // These are not filtered out yet, due to how the lookups are structured internally - ParameterLookup.normalized({ lookupName: 'admin_only', queryId: '1' }, [1]) + ScopedParameterLookup.direct({ lookupName: 'admin_only', queryId: '1' }, [1]) ], staticBuckets: [ { From 68c69e66daa3e4c6b06c2449e5442ba0ecbf2d9f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 12 Dec 2025 15:06:46 +0200 Subject: [PATCH 29/33] Rename BucketParameterLookupSource -> ParameterIndexLookupCreator. --- packages/sync-rules/src/BucketSource.ts | 21 +++++++++++-------- packages/sync-rules/src/HydratedSyncRules.ts | 10 ++++----- packages/sync-rules/src/HydrationState.ts | 8 +++---- .../sync-rules/src/SqlBucketDescriptor.ts | 2 +- packages/sync-rules/src/SqlParameterQuery.ts | 8 +++---- packages/sync-rules/src/SqlSyncRules.ts | 10 ++++----- packages/sync-rules/src/streams/filter.ts | 14 ++++++------- packages/sync-rules/src/streams/parameter.ts | 4 ++-- packages/sync-rules/src/streams/stream.ts | 10 ++++----- packages/sync-rules/src/streams/variant.ts | 6 +++--- packages/sync-rules/test/src/streams.test.ts | 20 +++++++++++------- 11 files changed, 59 insertions(+), 54 deletions(-) diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 9e132c077..0486e4931 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -61,7 +61,7 @@ export interface BucketSource { * * The same source could in theory be present in multiple stream definitions. */ - readonly parameterLookupSources: BucketParameterLookupSourceDefinition[]; + readonly parameterIndexLookupCreators: ParameterIndexLookupCreator[]; debugRepresentation(): any; } @@ -118,11 +118,11 @@ export interface BucketDataSource { } /** - * A parameter lookup source defines how to extract parameter lookup values from parameter queries. + * This defines how to extract parameter index lookup values from parameter queries or stream subqueries. * - * This is only relevant for parameter queries that query tables. + * This is only relevant for parameter queries and subqueries that query tables. */ -export interface BucketParameterLookupSourceDefinition { +export interface ParameterIndexLookupCreator { /** * lookupName + queryId is used to uniquely identify parameter queries for parameter storage. * @@ -207,9 +207,9 @@ export function hydrateEvaluateRow(hydrationState: HydrationState, source: Bucke export function hydrateEvaluateParameterRow( hydrationState: HydrationState, - source: BucketParameterLookupSourceDefinition + source: ParameterIndexLookupCreator ): ScopedEvaluateParameterRow { - const scope = hydrationState.getParameterLookupScope(source); + const scope = hydrationState.getParameterIndexLookupScope(source); return (sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] => { return source.evaluateParameterRow(sourceTable, row).map((result) => { if (isEvaluationError(result)) { @@ -235,9 +235,9 @@ export function mergeDataSources( }; } -export function mergeParameterLookupSources( +export function mergeParameterIndexLookupCreators( hydrationState: HydrationState, - sources: BucketParameterLookupSourceDefinition[] + sources: ParameterIndexLookupCreator[] ): { evaluateParameterRow: ScopedEvaluateParameterRow } { const withScope = sources.map((source) => hydrateEvaluateParameterRow(hydrationState, source)); return { @@ -265,7 +265,10 @@ export function debugHydratedMergedSource(bucketSource: BucketSource, params?: C const hydrationState = params?.hydrationState ?? DEFAULT_HYDRATION_STATE; const resolvedParams = { hydrationState }; const dataSource = mergeDataSources(hydrationState, bucketSource.dataSources); - const parameterLookupSource = mergeParameterLookupSources(hydrationState, bucketSource.parameterLookupSources); + const parameterLookupSource = mergeParameterIndexLookupCreators( + hydrationState, + bucketSource.parameterIndexLookupCreators + ); const parameterQuerierSource = mergeParameterQuerierSources( bucketSource.parameterQuerierSources.map((source) => source.createParameterQuerierSource(resolvedParams)) ); diff --git a/packages/sync-rules/src/HydratedSyncRules.ts b/packages/sync-rules/src/HydratedSyncRules.ts index e0db33d16..d10a39643 100644 --- a/packages/sync-rules/src/HydratedSyncRules.ts +++ b/packages/sync-rules/src/HydratedSyncRules.ts @@ -2,7 +2,7 @@ import { Scope } from 'ajv/dist/compile/codegen/scope.js'; import { BucketDataSource, CreateSourceParams, HydratedBucketSource } from './BucketSource.js'; import { BucketDataScope, ParameterLookupScope } from './HydrationState.js'; import { - BucketParameterLookupSourceDefinition, + ParameterIndexLookupCreator, BucketParameterQuerier, buildBucketName, CompatibilityContext, @@ -16,7 +16,7 @@ import { isEvaluationError, mergeBucketParameterQueriers, mergeDataSources, - mergeParameterLookupSources, + mergeParameterIndexLookupCreators, QuerierError, ScopedEvaluateParameterRow, ScopedEvaluateRow, @@ -46,7 +46,7 @@ export class HydratedSyncRules { definition: SqlSyncRules; createParams: CreateSourceParams; bucketDataSources: BucketDataSource[]; - bucketParameterLookupSources: BucketParameterLookupSourceDefinition[]; + bucketParameterIndexLookupCreators: ParameterIndexLookupCreator[]; eventDescriptors?: SqlEventDescriptor[]; compatibility?: CompatibilityContext; }) { @@ -54,9 +54,9 @@ export class HydratedSyncRules { this.definition = params.definition; this.innerEvaluateRow = mergeDataSources(hydrationState, params.bucketDataSources).evaluateRow; - this.innerEvaluateParameterRow = mergeParameterLookupSources( + this.innerEvaluateParameterRow = mergeParameterIndexLookupCreators( hydrationState, - params.bucketParameterLookupSources + params.bucketParameterIndexLookupCreators ).evaluateParameterRow; if (params.eventDescriptors) { diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts index 149e82630..b83b7f0a2 100644 --- a/packages/sync-rules/src/HydrationState.ts +++ b/packages/sync-rules/src/HydrationState.ts @@ -1,4 +1,4 @@ -import { BucketDataSource, BucketParameterLookupSourceDefinition } from './BucketSource.js'; +import { BucketDataSource, ParameterIndexLookupCreator } from './BucketSource.js'; export interface BucketDataScope { /** The prefix is the bucket name before the parameters. */ @@ -29,7 +29,7 @@ export interface HydrationState< /** * Given a bucket parameter lookup definition, get the persistence name to use. */ - getParameterLookupScope(source: BucketParameterLookupSourceDefinition): U; + getParameterIndexLookupScope(source: ParameterIndexLookupCreator): U; } /** @@ -43,7 +43,7 @@ export const DEFAULT_HYDRATION_STATE: HydrationState = { bucketPrefix: source.uniqueName }; }, - getParameterLookupScope(source) { + getParameterIndexLookupScope(source) { return source.defaultLookupScope; } }; @@ -68,7 +68,7 @@ export function versionedHydrationState(version: number): HydrationState { }; }, - getParameterLookupScope(source: BucketParameterLookupSourceDefinition): ParameterLookupScope { + getParameterIndexLookupScope(source: ParameterIndexLookupCreator): ParameterLookupScope { // No transformations applied here return source.defaultLookupScope; } diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index bfcdf97d4..bde1904b0 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -54,7 +54,7 @@ export class SqlBucketDescriptor implements BucketSource { return [this.dataSource]; } - get parameterLookupSources() { + get parameterIndexLookupCreators() { return this.parameterQueries; } diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index be7306e8a..225cb4deb 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -12,7 +12,7 @@ import { PendingQueriers } from './BucketParameterQuerier.js'; import { - BucketParameterLookupSourceDefinition, + ParameterIndexLookupCreator, BucketParameterQuerierSource, BucketParameterQuerierSourceDefinition, CreateSourceParams @@ -81,9 +81,7 @@ export interface SqlParameterQueryOptions { * SELECT id as user_id FROM users WHERE users.user_id = token_parameters.user_id * SELECT id as user_id, token_parameters.is_admin as is_admin FROM users WHERE users.user_id = token_parameters.user_id */ -export class SqlParameterQuery - implements BucketParameterLookupSourceDefinition, BucketParameterQuerierSourceDefinition -{ +export class SqlParameterQuery implements ParameterIndexLookupCreator, BucketParameterQuerierSourceDefinition { static fromSql( descriptorName: string, sql: string, @@ -360,7 +358,7 @@ export class SqlParameterQuery createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { const hydrationState = params.hydrationState; const bucketScope = hydrationState.getBucketSourceScope(this.querierDataSource); - const lookupState = hydrationState.getParameterLookupScope(this); + const lookupState = hydrationState.getParameterIndexLookupScope(this); return { pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 8e654fc31..b417085fb 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -3,7 +3,7 @@ import { isValidPriority } from './BucketDescription.js'; import { BucketParameterQuerier, QuerierError } from './BucketParameterQuerier.js'; import { BucketDataSource, - BucketParameterLookupSourceDefinition, + ParameterIndexLookupCreator, BucketParameterQuerierSourceDefinition, BucketSource, CreateSourceParams @@ -90,7 +90,7 @@ export interface GetBucketParameterQuerierResult { export class SqlSyncRules { bucketDataSources: BucketDataSource[] = []; - bucketParameterLookupSources: BucketParameterLookupSourceDefinition[] = []; + bucketParameterLookupSources: ParameterIndexLookupCreator[] = []; bucketParameterQuerierSources: BucketParameterQuerierSourceDefinition[] = []; bucketSources: BucketSource[] = []; @@ -258,7 +258,7 @@ export class SqlSyncRules { rules.bucketSources.push(descriptor); rules.bucketDataSources.push(...descriptor.dataSources); - rules.bucketParameterLookupSources.push(...descriptor.parameterLookupSources); + rules.bucketParameterLookupSources.push(...descriptor.parameterIndexLookupCreators); rules.bucketParameterQuerierSources.push(...descriptor.parameterQuerierSources); } @@ -286,7 +286,7 @@ export class SqlSyncRules { const [parsed, errors] = syncStreamFromSql(key, q, queryOptions); rules.bucketSources.push(parsed); rules.bucketDataSources.push(...parsed.dataSources); - rules.bucketParameterLookupSources.push(...parsed.parameterLookupSources); + rules.bucketParameterLookupSources.push(...parsed.parameterIndexLookupCreators); rules.bucketParameterQuerierSources.push(...parsed.parameterQuerierSources); return { parsed: true, @@ -417,7 +417,7 @@ export class SqlSyncRules { definition: this, createParams: resolvedParams, bucketDataSources: this.bucketDataSources, - bucketParameterLookupSources: this.bucketParameterLookupSources, + bucketParameterIndexLookupCreators: this.bucketParameterLookupSources, eventDescriptors: this.eventDescriptors, compatibility: this.compatibility }); diff --git a/packages/sync-rules/src/streams/filter.ts b/packages/sync-rules/src/streams/filter.ts index 20cea5914..4ef22eab6 100644 --- a/packages/sync-rules/src/streams/filter.ts +++ b/packages/sync-rules/src/streams/filter.ts @@ -16,7 +16,7 @@ import { import { isJsonValue, normalizeParameterValue } from '../utils.js'; import { NodeLocation } from 'pgsql-ast-parser'; -import { BucketParameterLookupSourceDefinition, CreateSourceParams } from '../BucketSource.js'; +import { ParameterIndexLookupCreator, CreateSourceParams } from '../BucketSource.js'; import { HydrationState, ParameterLookupScope } from '../HydrationState.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { SubqueryEvaluator } from './parameter.js'; @@ -263,16 +263,16 @@ export class Subquery { const column = this.column; - let lookupSources: BucketParameterLookupSourceDefinition[] = []; + let lookupCreators: ParameterIndexLookupCreator[] = []; let lookupsForRequest: (( hydrationState: HydrationState ) => (parameters: RequestParameters) => ScopedParameterLookup[])[] = []; for (let [variant, id] of innerVariants) { const source = new SubqueryParameterLookupSource(this.table, column, variant, id, context.streamName); - lookupSources.push(source); + lookupCreators.push(source); lookupsForRequest.push((hydrationState: HydrationState) => { - const scope = hydrationState.getParameterLookupScope(source); + const scope = hydrationState.getParameterIndexLookupScope(source); return (parameters: RequestParameters) => { const lookups: ScopedParameterLookup[] = []; const instantiations = variant.findStaticInstantiations(parameters); @@ -286,8 +286,8 @@ export class Subquery { const evaluator: SubqueryEvaluator = { parameterTable: this.table, - lookupSources() { - return lookupSources; + indexLookupCreators() { + return lookupCreators; }, hydrateLookupsForRequest(hydrationState: HydrationState) { const hydrated = lookupsForRequest.map((fn) => fn(hydrationState)); @@ -531,7 +531,7 @@ export class EvaluateSimpleCondition extends FilterOperator { } } -export class SubqueryParameterLookupSource implements BucketParameterLookupSourceDefinition { +export class SubqueryParameterLookupSource implements ParameterIndexLookupCreator { constructor( private parameterTable: TablePattern, private column: RowValueClause, diff --git a/packages/sync-rules/src/streams/parameter.ts b/packages/sync-rules/src/streams/parameter.ts index ae856a957..e92936374 100644 --- a/packages/sync-rules/src/streams/parameter.ts +++ b/packages/sync-rules/src/streams/parameter.ts @@ -1,5 +1,5 @@ import { ScopedParameterLookup, UnscopedParameterLookup } from '../BucketParameterQuerier.js'; -import { BucketParameterLookupSourceDefinition } from '../BucketSource.js'; +import { ParameterIndexLookupCreator } from '../BucketSource.js'; import { HydrationState } from '../HydrationState.js'; import { TablePattern } from '../TablePattern.js'; import { ParameterValueSet, RequestParameters, SqliteJsonValue, SqliteValue, TableRow } from '../types.js'; @@ -31,7 +31,7 @@ export interface BucketParameter { export interface SubqueryEvaluator { parameterTable: TablePattern; - lookupSources(): BucketParameterLookupSourceDefinition[]; + indexLookupCreators(): ParameterIndexLookupCreator[]; // TODO: Is there a better design here? // This is used for parameter _queries_. But the queries need to know which lookup scopes to // use, and each querier may use multiple lookup sources, each with their own scope. diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index 4a84038c0..f33a1f2e0 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -2,7 +2,7 @@ import { BaseSqlDataQuery } from '../BaseSqlDataQuery.js'; import { BucketPriority, DEFAULT_BUCKET_PRIORITY } from '../BucketDescription.js'; import { BucketDataSource, - BucketParameterLookupSourceDefinition, + ParameterIndexLookupCreator, BucketParameterQuerierSourceDefinition, BucketSource, BucketSourceType @@ -21,7 +21,7 @@ export class SyncStream implements BucketSource { data: BaseSqlDataQuery; public readonly dataSources: BucketDataSource[]; - public readonly parameterLookupSources: BucketParameterLookupSourceDefinition[]; + public readonly parameterIndexLookupCreators: ParameterIndexLookupCreator[]; public readonly parameterQuerierSources: BucketParameterQuerierSourceDefinition[]; constructor(name: string, data: BaseSqlDataQuery, variants: StreamVariant[]) { @@ -32,15 +32,15 @@ export class SyncStream implements BucketSource { this.data = data; this.dataSources = []; - this.parameterLookupSources = []; + this.parameterIndexLookupCreators = []; this.parameterQuerierSources = []; for (let variant of variants) { const dataSource = new SyncStreamDataSource(this, data, variant); this.dataSources.push(dataSource); - const lookupSources = variant.lookupSources(); + const lookupCreators = variant.indexLookupCreators(); this.parameterQuerierSources.push(variant.querierSource(this, dataSource)); - this.parameterLookupSources.push(...lookupSources); + this.parameterIndexLookupCreators.push(...lookupCreators); } } diff --git a/packages/sync-rules/src/streams/variant.ts b/packages/sync-rules/src/streams/variant.ts index d2fc1f71b..f7cbcc02b 100644 --- a/packages/sync-rules/src/streams/variant.ts +++ b/packages/sync-rules/src/streams/variant.ts @@ -2,7 +2,7 @@ import { BucketInclusionReason, ResolvedBucket } from '../BucketDescription.js'; import { BucketParameterQuerier, UnscopedParameterLookup, PendingQueriers } from '../BucketParameterQuerier.js'; import { BucketDataSource, - BucketParameterLookupSourceDefinition, + ParameterIndexLookupCreator, BucketParameterQuerierSource, BucketParameterQuerierSourceDefinition, CreateSourceParams @@ -69,8 +69,8 @@ export class StreamVariant { return `${streamName}|${this.id}`; } - lookupSources(): BucketParameterLookupSourceDefinition[] { - return this.subqueries.flatMap((subquery) => subquery.lookupSources()); + indexLookupCreators(): ParameterIndexLookupCreator[] { + return this.subqueries.flatMap((subquery) => subquery.indexLookupCreators()); } querierSource(stream: SyncStream, dataSource: SyncStreamDataSource): BucketParameterQuerierSourceDefinition { diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index 104039c4f..071650b01 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -265,7 +265,7 @@ describe('streams', () => { const desc = parseStream( 'SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id())' ); - const lookup = desc.parameterLookupSources[0]; + const lookup = desc.parameterIndexLookupCreators[0]; expect(lookup.tableSyncsParameters(ISSUES)).toBe(true); expect(lookup.evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' })).toStrictEqual([ @@ -296,7 +296,7 @@ describe('streams', () => { test('parameter value in subquery', async () => { const desc = parseStream('SELECT * FROM issues WHERE auth.user_id() IN (SELECT id FROM users WHERE is_admin)'); - const lookup = desc.parameterLookupSources[0]; + const lookup = desc.parameterIndexLookupCreators[0]; expect(lookup.tableSyncsParameters(ISSUES)).toBe(false); expect(lookup.tableSyncsParameters(USERS)).toBe(true); @@ -453,7 +453,7 @@ describe('streams', () => { const desc = parseStream( 'SELECT * FROM comments WHERE tagged_users && (SELECT user_a FROM friends WHERE user_b = auth.user_id())' ); - const lookup = desc.parameterLookupSources[0]; + const lookup = desc.parameterIndexLookupCreators[0]; expect(lookup.tableSyncsParameters(FRIENDS)).toBe(true); expect(lookup.evaluateParameterRow(FRIENDS, { user_a: 'a', user_b: 'b' })).toStrictEqual([ @@ -612,7 +612,11 @@ describe('streams', () => { ); expect( - desc.parameterLookupSources[0].evaluateParameterRow(ISSUES, { id: 'issue_id', owner_id: 'user1', name: 'name' }) + desc.parameterIndexLookupCreators[0].evaluateParameterRow(ISSUES, { + id: 'issue_id', + owner_id: 'user1', + name: 'name' + }) ).toStrictEqual([ { lookup: UnscopedParameterLookup.normalized(['user1']), @@ -737,10 +741,10 @@ describe('streams', () => { const row = { id: 'id', account_id: 'account_id' }; expect(stream.dataSources[0].tableSyncsData(accountMember)).toBeTruthy(); - expect(stream.parameterLookupSources[0].tableSyncsParameters(accountMember)).toBeTruthy(); + expect(stream.parameterIndexLookupCreators[0].tableSyncsParameters(accountMember)).toBeTruthy(); // Ensure lookup steps work. - expect(stream.parameterLookupSources[0].evaluateParameterRow(accountMember, row)).toStrictEqual([ + expect(stream.parameterIndexLookupCreators[0].evaluateParameterRow(accountMember, row)).toStrictEqual([ { lookup: UnscopedParameterLookup.normalized(['id']), bucketParameters: [ @@ -823,7 +827,7 @@ WHERE expect(evaluateBucketIds(desc, scene, { _id: 'scene', project: 'foo' })).toStrictEqual(['1#stream|0["foo"]']); expect( - desc.parameterLookupSources[0].evaluateParameterRow(projectInvitation, { + desc.parameterIndexLookupCreators[0].evaluateParameterRow(projectInvitation, { project: 'foo', appliedTo: '[1,2]', status: 'CLAIMED' @@ -875,7 +879,7 @@ WHERE getBucketSourceScope(source) { return { bucketPrefix: `${source.uniqueName}.test` }; }, - getParameterLookupScope(source) { + getParameterIndexLookupScope(source) { return { lookupName: `${source.defaultLookupScope.lookupName}.test`, queryId: `${source.defaultLookupScope.queryId}.test` From f8f432dc7ea653dd4fbfe53a1511c64b85067c56 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 12 Dec 2025 15:52:46 +0200 Subject: [PATCH 30/33] Merge BucketParameterQuerierSourceDefinition into HydratedBucketSource. --- packages/sync-rules/src/BucketSource.ts | 77 ++++--------- packages/sync-rules/src/HydratedSyncRules.ts | 12 +- .../sync-rules/src/SqlBucketDescriptor.ts | 34 +++++- packages/sync-rules/src/SqlParameterQuery.ts | 16 +-- packages/sync-rules/src/SqlSyncRules.ts | 13 +-- .../sync-rules/src/StaticSqlParameterQuery.ts | 10 +- .../TableValuedFunctionSqlParameterQuery.ts | 16 +-- packages/sync-rules/src/streams/stream.ts | 33 ++++-- packages/sync-rules/src/streams/variant.ts | 106 ++++++++---------- packages/sync-rules/test/src/streams.test.ts | 11 +- .../sync-rules/test/src/sync_rules.test.ts | 22 ++-- 11 files changed, 156 insertions(+), 194 deletions(-) diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 0486e4931..482fb5f5b 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -49,13 +49,6 @@ export interface BucketSource { */ readonly dataSources: BucketDataSource[]; - /** - * BucketParameterQuerierSource describing the parameter queries / stream subqueries in this bucket/stream definition. - * - * The same source could in theory be present in multiple stream definitions. - */ - readonly parameterQuerierSources: BucketParameterQuerierSourceDefinition[]; - /** * BucketParameterLookupSource describing the parameter tables used in this bucket/stream definition. * @@ -64,12 +57,26 @@ export interface BucketSource { readonly parameterIndexLookupCreators: ParameterIndexLookupCreator[]; debugRepresentation(): any; + + hydrate(params: CreateSourceParams): HydratedBucketSource; } -export interface HydratedBucketSource { - readonly definition: BucketSource; +/** + * Internal interface for individual queriers. This is not used on its in the public API directly, apart + * from in HydratedBucketSource. Everywhere else it is just to standardize the internal functions that we re-use. + */ +export interface BucketParameterQuerierSource { + /** + * Reports {@link BucketParameterQuerier}s resolving buckets that a specific stream request should have access to. + * + * @param result The target array to insert queriers and errors into. + * @param options Options, including parameters that may affect the buckets loaded by this source. + */ + pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void; +} - readonly parameterQuerierSources: BucketParameterQuerierSource[]; +export interface HydratedBucketSource extends BucketParameterQuerierSource { + readonly definition: BucketSource; } export type ScopedEvaluateRow = (options: EvaluateRowOptions) => EvaluationResult[]; @@ -145,38 +152,7 @@ export interface ParameterIndexLookupCreator { tableSyncsParameters(table: SourceTableInterface): boolean; } -/** - * Parameter querier source definitions define how to bucket parameter queries are evaluated. - * - * This may use request data only, or it may use parameter lookup data persisted by a BucketParameterLookupSourceDefinition. - */ -export interface BucketParameterQuerierSourceDefinition { - /** - * For debug use only. - */ - readonly bucketParameters: string[]; - - /** - * The data source linked to this querier. This determines the bucket names that the querier generates. - * - * Note that queriers do not persist data themselves; they only resolve which buckets to load based on request parameters. - */ - readonly querierDataSource: BucketDataSource; - - createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource; -} - -export interface BucketParameterQuerierSource { - /** - * Reports {@link BucketParameterQuerier}s resolving buckets that a specific stream request should have access to. - * - * @param result The target array to insert queriers and errors into. - * @param options Options, including parameters that may affect the buckets loaded by this source. - */ - pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void; -} - -export interface DebugMergedSource extends BucketParameterQuerierSource { +export interface DebugMergedSource extends HydratedBucketSource { evaluateRow: ScopedEvaluateRow; evaluateParameterRow: ScopedEvaluateParameterRow; } @@ -247,16 +223,6 @@ export function mergeParameterIndexLookupCreators( }; } -export function mergeParameterQuerierSources(sources: BucketParameterQuerierSource[]): BucketParameterQuerierSource { - return { - pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void { - for (let source of sources) { - source.pushBucketParameterQueriers(result, options); - } - } - }; -} - /** * For production purposes, we typically need to operate on the different sources separately. However, for debugging, * it is useful to have a single merged source that can evaluate everything. @@ -269,12 +235,11 @@ export function debugHydratedMergedSource(bucketSource: BucketSource, params?: C hydrationState, bucketSource.parameterIndexLookupCreators ); - const parameterQuerierSource = mergeParameterQuerierSources( - bucketSource.parameterQuerierSources.map((source) => source.createParameterQuerierSource(resolvedParams)) - ); + const hydratedBucketSource = bucketSource.hydrate(resolvedParams); return { + definition: bucketSource, evaluateParameterRow: parameterLookupSource.evaluateParameterRow.bind(parameterLookupSource), evaluateRow: dataSource.evaluateRow.bind(dataSource), - pushBucketParameterQueriers: parameterQuerierSource.pushBucketParameterQueriers.bind(parameterQuerierSource) + pushBucketParameterQueriers: hydratedBucketSource.pushBucketParameterQueriers.bind(hydratedBucketSource) }; } diff --git a/packages/sync-rules/src/HydratedSyncRules.ts b/packages/sync-rules/src/HydratedSyncRules.ts index d10a39643..36ccb8a1f 100644 --- a/packages/sync-rules/src/HydratedSyncRules.ts +++ b/packages/sync-rules/src/HydratedSyncRules.ts @@ -66,13 +66,7 @@ export class HydratedSyncRules { this.compatibility = params.compatibility; } - for (let definition of this.definition.bucketSources) { - const hydratedBucketSource: HydratedBucketSource = { definition: definition, parameterQuerierSources: [] }; - this.bucketSources.push(hydratedBucketSource); - for (let querier of definition.parameterQuerierSources) { - hydratedBucketSource.parameterQuerierSources.push(querier.createParameterQuerierSource(params.createParams)); - } - } + this.bucketSources = this.definition.bucketSources.map((source) => source.hydrate(params.createParams)); } // These methods do not depend on hydration, so we can just forward them to the definition. @@ -149,9 +143,7 @@ export class HydratedSyncRules { (source.definition.subscribedToByDefault && options.hasDefaultStreams) || source.definition.name in options.streams ) { - for (let querier of source.parameterQuerierSources) { - querier.pushBucketParameterQueriers(pending, options); - } + source.pushBucketParameterQueriers(pending, options); } } diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index bde1904b0..04eb7f590 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -1,16 +1,23 @@ -import { BucketDataSource, BucketSource, BucketSourceType } from './BucketSource.js'; +import { PendingQueriers } from './BucketParameterQuerier.js'; +import { + BucketDataSource, + BucketSource, + BucketSourceType, + CreateSourceParams, + HydratedBucketSource +} from './BucketSource.js'; import { ColumnDefinition } from './ExpressionType.js'; import { IdSequence } from './IdSequence.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SqlDataQuery } from './SqlDataQuery.js'; import { SqlParameterQuery } from './SqlParameterQuery.js'; -import { SyncRulesOptions } from './SqlSyncRules.js'; +import { GetQuerierOptions, SyncRulesOptions } from './SqlSyncRules.js'; import { StaticSqlParameterQuery } from './StaticSqlParameterQuery.js'; import { TablePattern } from './TablePattern.js'; import { TableValuedFunctionSqlParameterQuery } from './TableValuedFunctionSqlParameterQuery.js'; import { CompatibilityContext } from './compatibility.js'; import { SqlRuleError } from './errors.js'; -import { EvaluateRowOptions, QueryParseOptions, UnscopedEvaluationResult, SourceSchema } from './types.js'; +import { EvaluateRowOptions, QueryParseOptions, SourceSchema, UnscopedEvaluationResult } from './types.js'; export interface QueryParseResult { /** @@ -134,6 +141,27 @@ export class SqlBucketDescriptor implements BucketSource { }) }; } + + hydrate(params: CreateSourceParams): HydratedBucketSource { + const hydratedParameterQueriers = this.parameterQueries.map((querier) => + querier.createParameterQuerierSource(params) + ); + const hydratedGlobalParameterQueriers = this.globalParameterQueries.map((querier) => + querier.createParameterQuerierSource(params) + ); + + return { + definition: this, + pushBucketParameterQueriers: (result: PendingQueriers, options: GetQuerierOptions) => { + for (let querier of hydratedParameterQueriers) { + querier.pushBucketParameterQueriers(result, options); + } + for (let querier of hydratedGlobalParameterQueriers) { + querier.pushBucketParameterQueriers(result, options); + } + } + }; + } } export class BucketDefinitionDataSource implements BucketDataSource { diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 225cb4deb..218724573 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -7,20 +7,16 @@ import { } from './BucketDescription.js'; import { BucketParameterQuerier, - UnscopedParameterLookup, ParameterLookupSource, - PendingQueriers + PendingQueriers, + UnscopedParameterLookup } from './BucketParameterQuerier.js'; -import { - ParameterIndexLookupCreator, - BucketParameterQuerierSource, - BucketParameterQuerierSourceDefinition, - CreateSourceParams -} from './BucketSource.js'; +import { CreateSourceParams, ParameterIndexLookupCreator } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; import { BucketDataScope, ParameterLookupScope } from './HydrationState.js'; import { BucketDataSource, + BucketParameterQuerierSource, GetQuerierOptions, ScopedParameterLookup, UnscopedEvaluatedParameters, @@ -34,8 +30,6 @@ import { TablePattern } from './TablePattern.js'; import { TableQuerySchema } from './TableQuerySchema.js'; import { TableValuedFunctionSqlParameterQuery } from './TableValuedFunctionSqlParameterQuery.js'; import { - EvaluatedParameters, - EvaluatedParametersResult, InputParameter, ParameterMatchClause, ParameterValueClause, @@ -81,7 +75,7 @@ export interface SqlParameterQueryOptions { * SELECT id as user_id FROM users WHERE users.user_id = token_parameters.user_id * SELECT id as user_id, token_parameters.is_admin as is_admin FROM users WHERE users.user_id = token_parameters.user_id */ -export class SqlParameterQuery implements ParameterIndexLookupCreator, BucketParameterQuerierSourceDefinition { +export class SqlParameterQuery implements ParameterIndexLookupCreator { static fromSql( descriptorName: string, sql: string, diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index b417085fb..5a6adfd24 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -1,13 +1,7 @@ import { isScalar, LineCounter, parseDocument, Scalar, YAMLMap, YAMLSeq } from 'yaml'; import { isValidPriority } from './BucketDescription.js'; import { BucketParameterQuerier, QuerierError } from './BucketParameterQuerier.js'; -import { - BucketDataSource, - ParameterIndexLookupCreator, - BucketParameterQuerierSourceDefinition, - BucketSource, - CreateSourceParams -} from './BucketSource.js'; +import { BucketDataSource, BucketSource, CreateSourceParams, ParameterIndexLookupCreator } from './BucketSource.js'; import { CompatibilityContext, CompatibilityEdition, @@ -16,12 +10,12 @@ import { } from './compatibility.js'; import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js'; import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; +import { HydratedSyncRules } from './HydratedSyncRules.js'; import { DEFAULT_HYDRATION_STATE } from './HydrationState.js'; import { validateSyncRulesSchema } from './json_schema.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { QueryParseResult, SqlBucketDescriptor } from './SqlBucketDescriptor.js'; import { syncStreamFromSql } from './streams/from_sql.js'; -import { HydratedSyncRules } from './HydratedSyncRules.js'; import { TablePattern } from './TablePattern.js'; import { QueryParseOptions, @@ -91,7 +85,6 @@ export interface GetBucketParameterQuerierResult { export class SqlSyncRules { bucketDataSources: BucketDataSource[] = []; bucketParameterLookupSources: ParameterIndexLookupCreator[] = []; - bucketParameterQuerierSources: BucketParameterQuerierSourceDefinition[] = []; bucketSources: BucketSource[] = []; eventDescriptors: SqlEventDescriptor[] = []; @@ -259,7 +252,6 @@ export class SqlSyncRules { rules.bucketSources.push(descriptor); rules.bucketDataSources.push(...descriptor.dataSources); rules.bucketParameterLookupSources.push(...descriptor.parameterIndexLookupCreators); - rules.bucketParameterQuerierSources.push(...descriptor.parameterQuerierSources); } for (const entry of streamMap?.items ?? []) { @@ -287,7 +279,6 @@ export class SqlSyncRules { rules.bucketSources.push(parsed); rules.bucketDataSources.push(...parsed.dataSources); rules.bucketParameterLookupSources.push(...parsed.parameterIndexLookupCreators); - rules.bucketParameterQuerierSources.push(...parsed.parameterQuerierSources); return { parsed: true, errors diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index 370fdbcf8..7a55dbb92 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -1,14 +1,10 @@ import { SelectFromStatement } from 'pgsql-ast-parser'; import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY, ResolvedBucket } from './BucketDescription.js'; import { BucketParameterQuerier, PendingQueriers } from './BucketParameterQuerier.js'; -import { - BucketParameterQuerierSource, - BucketParameterQuerierSourceDefinition, - CreateSourceParams -} from './BucketSource.js'; +import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; import { BucketDataScope } from './HydrationState.js'; -import { BucketDataSource, GetQuerierOptions } from './index.js'; +import { BucketDataSource, BucketParameterQuerierSource, GetQuerierOptions } from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; import { checkUnsupportedFeatures, isClauseError, sqliteBool } from './sql_support.js'; @@ -35,7 +31,7 @@ export interface StaticSqlParameterQueryOptions { * SELECT token_parameters.user_id * SELECT token_parameters.user_id as user_id WHERE token_parameters.is_admin */ -export class StaticSqlParameterQuery implements BucketParameterQuerierSourceDefinition { +export class StaticSqlParameterQuery { static fromSql( descriptorName: string, sql: string, diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index 5e8f311e6..d4e9bfff7 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -1,13 +1,15 @@ import { FromCall, SelectFromStatement } from 'pgsql-ast-parser'; import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY, ResolvedBucket } from './BucketDescription.js'; -import { - BucketParameterQuerierSource, - BucketParameterQuerierSourceDefinition, - CreateSourceParams -} from './BucketSource.js'; +import { CreateSourceParams } from './BucketSource.js'; import { SqlRuleError } from './errors.js'; import { BucketDataScope } from './HydrationState.js'; -import { BucketDataSource, BucketParameterQuerier, GetQuerierOptions, PendingQueriers } from './index.js'; +import { + BucketDataSource, + BucketParameterQuerier, + BucketParameterQuerierSource, + GetQuerierOptions, + PendingQueriers +} from './index.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { AvailableTable, SqlTools } from './sql_filters.js'; import { checkUnsupportedFeatures, isClauseError, sqliteBool } from './sql_support.js'; @@ -50,7 +52,7 @@ export interface TableValuedFunctionSqlParameterQueryOptions { * * This can currently not be combined with parameter table queries or multiple table-valued functions. */ -export class TableValuedFunctionSqlParameterQuery implements BucketParameterQuerierSourceDefinition { +export class TableValuedFunctionSqlParameterQuery { static fromSql( descriptorName: string, sql: string, diff --git a/packages/sync-rules/src/streams/stream.ts b/packages/sync-rules/src/streams/stream.ts index f33a1f2e0..3394cfa51 100644 --- a/packages/sync-rules/src/streams/stream.ts +++ b/packages/sync-rules/src/streams/stream.ts @@ -3,9 +3,11 @@ import { BucketPriority, DEFAULT_BUCKET_PRIORITY } from '../BucketDescription.js import { BucketDataSource, ParameterIndexLookupCreator, - BucketParameterQuerierSourceDefinition, BucketSource, - BucketSourceType + BucketSourceType, + CreateSourceParams, + HydratedBucketSource, + BucketParameterQuerierSource } from '../BucketSource.js'; import { ColumnDefinition } from '../ExpressionType.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; @@ -17,29 +19,27 @@ export class SyncStream implements BucketSource { name: string; subscribedToByDefault: boolean; priority: BucketPriority; - variants: StreamVariant[]; + variants: { variant: StreamVariant; dataSource: SyncStreamDataSource }[]; data: BaseSqlDataQuery; public readonly dataSources: BucketDataSource[]; public readonly parameterIndexLookupCreators: ParameterIndexLookupCreator[]; - public readonly parameterQuerierSources: BucketParameterQuerierSourceDefinition[]; constructor(name: string, data: BaseSqlDataQuery, variants: StreamVariant[]) { this.name = name; this.subscribedToByDefault = false; this.priority = DEFAULT_BUCKET_PRIORITY; - this.variants = variants; + this.variants = []; this.data = data; this.dataSources = []; this.parameterIndexLookupCreators = []; - this.parameterQuerierSources = []; for (let variant of variants) { const dataSource = new SyncStreamDataSource(this, data, variant); this.dataSources.push(dataSource); + this.variants.push({ variant, dataSource }); const lookupCreators = variant.indexLookupCreators(); - this.parameterQuerierSources.push(variant.querierSource(this, dataSource)); this.parameterIndexLookupCreators.push(...lookupCreators); } } @@ -52,13 +52,30 @@ export class SyncStream implements BucketSource { return { name: this.name, type: BucketSourceType[BucketSourceType.SYNC_STREAM], - variants: this.variants.map((v) => v.debugRepresentation()), + variants: this.variants.map(({ variant }) => variant.debugRepresentation()), data: { table: this.data.sourceTable, columns: this.data.columnOutputNames() } }; } + + hydrate(params: CreateSourceParams): HydratedBucketSource { + let queriers: BucketParameterQuerierSource[] = []; + for (let { variant, dataSource } of this.variants) { + const querier = variant.createParameterQuerierSource(params, this, dataSource); + queriers.push(querier); + } + + return { + definition: this, + pushBucketParameterQueriers(result, options) { + for (let querier of queriers) { + querier.pushBucketParameterQueriers(result, options); + } + } + }; + } } export class SyncStreamDataSource implements BucketDataSource { diff --git a/packages/sync-rules/src/streams/variant.ts b/packages/sync-rules/src/streams/variant.ts index f7cbcc02b..faa448f6f 100644 --- a/packages/sync-rules/src/streams/variant.ts +++ b/packages/sync-rules/src/streams/variant.ts @@ -1,14 +1,8 @@ import { BucketInclusionReason, ResolvedBucket } from '../BucketDescription.js'; -import { BucketParameterQuerier, UnscopedParameterLookup, PendingQueriers } from '../BucketParameterQuerier.js'; -import { - BucketDataSource, - ParameterIndexLookupCreator, - BucketParameterQuerierSource, - BucketParameterQuerierSourceDefinition, - CreateSourceParams -} from '../BucketSource.js'; +import { BucketParameterQuerier, PendingQueriers } from '../BucketParameterQuerier.js'; +import { BucketDataSource, BucketParameterQuerierSource, ParameterIndexLookupCreator } from '../BucketSource.js'; import { BucketDataScope } from '../HydrationState.js'; -import { GetQuerierOptions, RequestedStream, ScopedParameterLookup } from '../index.js'; +import { CreateSourceParams, GetQuerierOptions, RequestedStream, ScopedParameterLookup } from '../index.js'; import { RequestParameters, SqliteJsonValue, TableRow } from '../types.js'; import { buildBucketName, isJsonValue, JSONBucketNameSerialize } from '../utils.js'; import { BucketParameter, SubqueryEvaluator } from './parameter.js'; @@ -73,10 +67,6 @@ export class StreamVariant { return this.subqueries.flatMap((subquery) => subquery.indexLookupCreators()); } - querierSource(stream: SyncStream, dataSource: SyncStreamDataSource): BucketParameterQuerierSourceDefinition { - return new SyncStreamParameterQuerierSource(stream, this, dataSource); - } - /** * Given a row in the table this stream selects from, returns all ids of buckets to which that row belongs to. */ @@ -309,53 +299,17 @@ export class StreamVariant { priority: stream.priority }; } -} -/** - * A stateless filter condition that only depends on the request itself, e.g. `WHERE token_parameters.is_admin`. - */ -export interface StaticRequestFilter { - type: 'static'; - matches(params: RequestParameters): boolean; -} - -/** - * A filter condition that depends on parameters and an evaluated subquery, e.g. - * `WHERE request.user_id() IN (SELECT id FROM users WHERE ...)`. - */ -export interface SubqueryRequestFilter { - type: 'dynamic'; - subquery: SubqueryEvaluator; - - /** - * Checks whether the parameter matches values from the subquery. - * - * @param results The values that the subquery evaluates to. - */ - matches(params: RequestParameters, results: SqliteJsonValue[]): boolean; -} -export type RequestFilter = StaticRequestFilter | SubqueryRequestFilter; - -export class SyncStreamParameterQuerierSource implements BucketParameterQuerierSourceDefinition { - constructor( - private stream: SyncStream, - private variant: StreamVariant, - public readonly querierDataSource: BucketDataSource - ) {} - - /** - * Not relevant for sync streams. - */ - get bucketParameters() { - return []; - } - createParameterQuerierSource(params: CreateSourceParams): BucketParameterQuerierSource { + createParameterQuerierSource( + params: CreateSourceParams, + stream: SyncStream, + querierDataSource: BucketDataSource + ): BucketParameterQuerierSource { const hydrationState = params.hydrationState; - const bucketScope = hydrationState.getBucketSourceScope(this.querierDataSource); - const stream = this.stream; + const bucketScope = hydrationState.getBucketSourceScope(querierDataSource); const hydratedSubqueries: HydratedSubqueries = new Map( - this.variant.subqueries.map((s) => [s, s.hydrateLookupsForRequest(hydrationState)]) + this.subqueries.map((s) => [s, s.hydrateLookupsForRequest(hydrationState)]) ); return { @@ -376,19 +330,27 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS hasExplicitDefaultSubscription = true; } - this.queriersForSubscription(result, subscription, subscriptionParams, bucketScope, hydratedSubqueries); + this.queriersForSubscription( + stream, + result, + subscription, + subscriptionParams, + bucketScope, + hydratedSubqueries + ); } // If the stream is subscribed to by default and there is no explicit subscription that would match the default // subscription, also include the default querier. if (stream.subscribedToByDefault && !hasExplicitDefaultSubscription) { - this.queriersForSubscription(result, null, options.globalParameters, bucketScope, hydratedSubqueries); + this.queriersForSubscription(stream, result, null, options.globalParameters, bucketScope, hydratedSubqueries); } } }; } private queriersForSubscription( + stream: SyncStream, result: PendingQueriers, subscription: RequestedStream | null, params: RequestParameters, @@ -398,18 +360,42 @@ export class SyncStreamParameterQuerierSource implements BucketParameterQuerierS const reason: BucketInclusionReason = subscription != null ? { subscription: subscription.opaque_id } : 'default'; try { - const querier = this.variant.querier(this.stream, reason, params, bucketScope, hydratedSubqueries); + const querier = this.querier(stream, reason, params, bucketScope, hydratedSubqueries); if (querier) { result.queriers.push(querier); } } catch (e) { result.errors.push({ - descriptor: this.stream.name, + descriptor: stream.name, message: `Error evaluating bucket ids: ${e.message}`, subscription: subscription ?? undefined }); } } } +/** + * A stateless filter condition that only depends on the request itself, e.g. `WHERE token_parameters.is_admin`. + */ +export interface StaticRequestFilter { + type: 'static'; + matches(params: RequestParameters): boolean; +} + +/** + * A filter condition that depends on parameters and an evaluated subquery, e.g. + * `WHERE request.user_id() IN (SELECT id FROM users WHERE ...)`. + */ +export interface SubqueryRequestFilter { + type: 'dynamic'; + subquery: SubqueryEvaluator; + + /** + * Checks whether the parameter matches values from the subquery. + * + * @param results The values that the subquery evaluates to. + */ + matches(params: RequestParameters, results: SqliteJsonValue[]): boolean; +} +export type RequestFilter = StaticRequestFilter | SubqueryRequestFilter; type HydratedSubqueries = Map ScopedParameterLookup[]>; diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts index 071650b01..dc8b59b85 100644 --- a/packages/sync-rules/test/src/streams.test.ts +++ b/packages/sync-rules/test/src/streams.test.ts @@ -1067,13 +1067,10 @@ async function createQueriers( streams: { [stream.name]: [{ opaque_id: 0, parameters: options?.parameters ?? null }] } }; - for (let querier of stream.parameterQuerierSources) { - querier - .createParameterQuerierSource( - options?.hydrationState ? { hydrationState: options.hydrationState } : hydrationParams - ) - .pushBucketParameterQueriers(pending, querierOptions); - } + const hydrated = stream.hydrate( + options?.hydrationState ? { hydrationState: options.hydrationState } : hydrationParams + ); + hydrated.pushBucketParameterQueriers(pending, querierOptions); return { querier: mergeBucketParameterQueriers(queriers), errors }; } diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index c0aa7d4c1..11620a9ff 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -26,7 +26,6 @@ describe('sync rules', () => { test('parse empty sync rules', () => { const rules = SqlSyncRules.fromYaml('bucket_definitions: {}', PARSE_OPTIONS); expect(rules.bucketParameterLookupSources).toEqual([]); - expect(rules.bucketParameterQuerierSources).toEqual([]); expect(rules.bucketDataSources).toEqual([]); }); @@ -81,11 +80,10 @@ bucket_definitions: ); const hydrated = rules.hydrate(hydrationParams); expect(rules.bucketParameterLookupSources).toEqual([]); - const parameterSource = rules.bucketParameterQuerierSources[0]; - expect(parameterSource.bucketParameters).toEqual([]); // Internal API, subject to change - const parameterQuery = parameterSource as StaticSqlParameterQuery; + const parameterQuery = (rules.bucketSources[0] as SqlBucketDescriptor) + .globalParameterQueries[0] as StaticSqlParameterQuery; expect(parameterQuery.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 1n }))).toEqual(1n); expect(parameterQuery.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 0n }))).toEqual(0n); @@ -135,9 +133,7 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate(hydrationParams); - const parameterSource = rules.bucketParameterQuerierSources[0]; const bucketData = rules.bucketDataSources[0]; - expect(parameterSource.bucketParameters).toEqual(['user_id', 'device_id']); expect(bucketData.bucketParameters).toEqual(['user_id', 'device_id']); expect( hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' })) @@ -182,9 +178,7 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate(hydrationParams); - const bucketParameters = rules.bucketParameterQuerierSources[0]; const bucketData = rules.bucketDataSources[0]; - expect(bucketParameters.bucketParameters).toEqual(['user_id']); expect(bucketData.bucketParameters).toEqual(['user_id']); expect( hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier.staticBuckets @@ -325,8 +319,8 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate(hydrationParams); - const bucketParameters = rules.bucketParameterQuerierSources[0]; - expect(bucketParameters.bucketParameters).toEqual(['user_id']); + const bucketData = rules.bucketDataSources[0]; + expect(bucketData.bucketParameters).toEqual(['user_id']); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false @@ -363,8 +357,8 @@ bucket_definitions: PARSE_OPTIONS ); const hydrated = rules.hydrate(hydrationParams); - const bucketParameters = rules.bucketParameterQuerierSources[0]; - expect(bucketParameters.bucketParameters).toEqual(['user_id']); + const bucketData = rules.bucketDataSources[0]; + expect(bucketData.bucketParameters).toEqual(['user_id']); expect(hydrated.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false @@ -961,8 +955,8 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucketParameters = rules.bucketParameterQuerierSources[0]; - expect(bucketParameters.bucketParameters).toEqual(['user_id']); + const bucket1data = rules.bucketDataSources[0]; + expect(bucket1data.bucketParameters).toEqual(['user_id']); const hydrated = rules.hydrate(hydrationParams); From b3a5ec3b820aa2a572efe7019c9fc2828f48e77b Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 15 Dec 2025 14:22:37 +0200 Subject: [PATCH 31/33] Remove HydrationState generics. We can re-add it if we actually need it later. --- packages/sync-rules/src/HydrationState.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts index b83b7f0a2..f836a62b4 100644 --- a/packages/sync-rules/src/HydrationState.ts +++ b/packages/sync-rules/src/HydrationState.ts @@ -17,19 +17,16 @@ export interface ParameterLookupScope { * This is what keeps track of bucket name and parameter lookup mappings for hydration. This can be used * both to re-use mappings across hydrations of different sync rule versions, or to generate new mappings. */ -export interface HydrationState< - T extends BucketDataScope = BucketDataScope, - U extends ParameterLookupScope = ParameterLookupScope -> { +export interface HydrationState { /** * Given a bucket data source definition, get the bucket prefix to use for it. */ - getBucketSourceScope(source: BucketDataSource): T; + getBucketSourceScope(source: BucketDataSource): BucketDataScope; /** * Given a bucket parameter lookup definition, get the persistence name to use. */ - getParameterIndexLookupScope(source: ParameterIndexLookupCreator): U; + getParameterIndexLookupScope(source: ParameterIndexLookupCreator): ParameterLookupScope; } /** From 46fc9aca11461250e7e7eb7a857751a9c91fa23e Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 15 Dec 2025 14:38:47 +0200 Subject: [PATCH 32/33] Add some hydration tests. --- .../test/src/parameter_queries.test.ts | 95 ++++++++++++++++++- .../test/src/static_parameter_queries.test.ts | 70 +++++++++++++- 2 files changed, 161 insertions(+), 4 deletions(-) diff --git a/packages/sync-rules/test/src/parameter_queries.test.ts b/packages/sync-rules/test/src/parameter_queries.test.ts index 7c00b161c..81c5f690a 100644 --- a/packages/sync-rules/test/src/parameter_queries.test.ts +++ b/packages/sync-rules/test/src/parameter_queries.test.ts @@ -1,7 +1,19 @@ -import { describe, expect, test } from 'vitest'; -import { UnscopedParameterLookup, SqlParameterQuery, SourceTableInterface } from '../../src/index.js'; +import { beforeEach, describe, expect, test } from 'vitest'; +import { + UnscopedParameterLookup, + SqlParameterQuery, + SourceTableInterface, + debugHydratedMergedSource, + BucketParameterQuerier, + QuerierError, + GetQuerierOptions, + RequestParameters, + ScopedParameterLookup, + mergeParameterIndexLookupCreators +} from '../../src/index.js'; import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; import { BASIC_SCHEMA, EMPTY_DATA_SOURCE, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; +import { HydrationState } from '../../src/HydrationState.js'; describe('parameter queries', () => { const table = (name: string): SourceTableInterface => ({ @@ -855,6 +867,85 @@ describe('parameter queries', () => { expect(query.getLookups(requestParams)).toEqual([UnscopedParameterLookup.normalized(['user1'])]); }); + describe('custom hydrationState', function () { + const hydrationState: HydrationState = { + getBucketSourceScope(source) { + return { bucketPrefix: `${source.uniqueName}-test` }; + }, + getParameterIndexLookupScope(source) { + return { + lookupName: `${source.defaultLookupScope.lookupName}.test`, + queryId: `${source.defaultLookupScope.queryId}.test` + }; + } + }; + + let query: SqlParameterQuery; + + beforeEach(() => { + const sql = 'SELECT id as group_id FROM groups WHERE token_parameters.user_id IN groups.user_ids'; + query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + 'myquery', + EMPTY_DATA_SOURCE + ) as SqlParameterQuery; + + expect(query.errors).toEqual([]); + }); + + test('for lookups', function () { + const merged = mergeParameterIndexLookupCreators(hydrationState, [query]); + const result = merged.evaluateParameterRow(TABLE_GROUPS, { + id: 'group1', + user_ids: JSON.stringify(['test-user', 'other-user']) + }); + expect(result).toEqual([ + { + lookup: ScopedParameterLookup.direct({ lookupName: 'mybucket.test', queryId: 'myquery.test' }, ['test-user']), + bucketParameters: [{ group_id: 'group1' }] + }, + { + lookup: ScopedParameterLookup.direct({ lookupName: 'mybucket.test', queryId: 'myquery.test' }, [ + 'other-user' + ]), + bucketParameters: [{ group_id: 'group1' }] + } + ]); + }); + + test('for queries', function () { + const hydrated = query.createParameterQuerierSource({ hydrationState }); + + const queriers: BucketParameterQuerier[] = []; + const errors: QuerierError[] = []; + const pending = { queriers, errors }; + + const querierOptions: GetQuerierOptions = { + hasDefaultStreams: true, + globalParameters: new RequestParameters( + { + sub: 'test-user' + }, + {} + ), + streams: {} + }; + + hydrated.pushBucketParameterQueriers(pending, querierOptions); + + expect(errors).toEqual([]); + expect(queriers.length).toBe(1); + + const querier = queriers[0]; + + expect(querier.parameterQueryLookups).toEqual([ + ScopedParameterLookup.direct({ lookupName: 'mybucket.test', queryId: 'myquery.test' }, ['test-user']) + ]); + }); + }); + test('invalid OR in parameter queries', () => { // Supporting this case is more tricky. We can do this by effectively denormalizing the OR clause // into separate queries, but it's a significant change. For now, developers should do that manually. diff --git a/packages/sync-rules/test/src/static_parameter_queries.test.ts b/packages/sync-rules/test/src/static_parameter_queries.test.ts index efec0eabc..62566d4eb 100644 --- a/packages/sync-rules/test/src/static_parameter_queries.test.ts +++ b/packages/sync-rules/test/src/static_parameter_queries.test.ts @@ -1,6 +1,13 @@ import { describe, expect, test } from 'vitest'; -import { BucketDataScope } from '../../src/HydrationState.js'; -import { RequestParameters, SqlParameterQuery } from '../../src/index.js'; +import { BucketDataScope, HydrationState } from '../../src/HydrationState.js'; +import { + BucketParameterQuerier, + GetQuerierOptions, + QuerierError, + RequestParameters, + ScopedParameterLookup, + SqlParameterQuery +} from '../../src/index.js'; import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; import { EMPTY_DATA_SOURCE, normalizeTokenParameters, PARSE_OPTIONS } from './util.js'; @@ -365,4 +372,63 @@ describe('static parameter queries', () => { "select request.parameters() ->> 'project_id' as project_id where request.jwt() ->> 'role' = 'authenticated'" ); }); + + test('custom hydrationState for buckets', function () { + const sql = 'SELECT token_parameters.user_id'; + const query = SqlParameterQuery.fromSql( + 'mybucket', + sql, + PARSE_OPTIONS, + '1', + EMPTY_DATA_SOURCE + ) as StaticSqlParameterQuery; + + expect(query.errors).toEqual([]); + + const hydrationState: HydrationState = { + getBucketSourceScope(source) { + return { bucketPrefix: `${source.uniqueName}-test` }; + }, + getParameterIndexLookupScope(source) { + return { + lookupName: `${source.defaultLookupScope.lookupName}.test`, + queryId: `${source.defaultLookupScope.queryId}.test` + }; + } + }; + + // Internal API + const hydrated = query.createParameterQuerierSource({ hydrationState }); + + const queriers: BucketParameterQuerier[] = []; + const errors: QuerierError[] = []; + const pending = { queriers, errors }; + + const querierOptions: GetQuerierOptions = { + hasDefaultStreams: true, + globalParameters: new RequestParameters( + { + sub: 'test-user' + }, + {} + ), + streams: {} + }; + + hydrated.pushBucketParameterQueriers(pending, querierOptions); + + expect(errors).toEqual([]); + expect(queriers).toMatchObject([ + { + staticBuckets: [ + { + bucket: 'mybucket-test["test-user"]', + definition: 'mybucket', + inclusion_reasons: ['default'], + priority: 3 + } + ] + } + ]); + }); }); From ea0c1c0778747fde5c4f725e6be2fe77cac2de51 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 15 Dec 2025 14:48:11 +0200 Subject: [PATCH 33/33] Add "end-to-end" sync rules test. --- .../sync-rules/test/src/sync_rules.test.ts | 84 ++++++++++++++++--- 1 file changed, 74 insertions(+), 10 deletions(-) diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 11620a9ff..80a73bce8 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -1,12 +1,9 @@ import { describe, expect, test } from 'vitest'; -import { - CreateSourceParams, - ScopedParameterLookup, - UnscopedParameterLookup, - SqlParameterQuery, - SqlSyncRules -} from '../../src/index.js'; +import { CreateSourceParams, ScopedParameterLookup, SqlSyncRules } from '../../src/index.js'; +import { DEFAULT_HYDRATION_STATE, HydrationState } from '../../src/HydrationState.js'; +import { SqlBucketDescriptor } from '../../src/SqlBucketDescriptor.js'; +import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; import { ASSETS, BASIC_SCHEMA, @@ -16,9 +13,6 @@ import { normalizeQuerierOptions, normalizeTokenParameters } from './util.js'; -import { SqlBucketDescriptor } from '../../src/SqlBucketDescriptor.js'; -import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; -import { DEFAULT_HYDRATION_STATE } from '../../src/HydrationState.js'; describe('sync rules', () => { const hydrationParams: CreateSourceParams = { hydrationState: DEFAULT_HYDRATION_STATE }; @@ -166,6 +160,76 @@ bucket_definitions: ).toEqual([]); }); + test('bucket with parameters with custom hydrationState', () => { + // "end-to-end" test with custom hydrationState. + // We don't test complex details here, but do cover bucket names and parameter lookup scope. + const rules = SqlSyncRules.fromYaml( + ` +config: + edition: 2 +bucket_definitions: + mybucket: + parameters: + - SELECT token_parameters.user_id as user_id + - SELECT users.id as user_id FROM users WHERE users.id = token_parameters.user_id AND users.is_admin + data: + - SELECT id, description FROM assets WHERE assets.user_id = bucket.user_id AND NOT assets.archived + `, + PARSE_OPTIONS + ); + const hydrationState: HydrationState = { + getBucketSourceScope(source) { + return { bucketPrefix: `${source.uniqueName}-test` }; + }, + getParameterIndexLookupScope(source) { + return { + lookupName: `${source.defaultLookupScope.lookupName}.test`, + queryId: `${source.defaultLookupScope.queryId}.test` + }; + } + }; + const hydrated = rules.hydrate({ hydrationState }); + const querier = hydrated.getBucketParameterQuerier( + normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' }) + ); + expect(querier.errors).toEqual([]); + expect(querier.querier.staticBuckets).toEqual([ + { + bucket: 'mybucket-test["user1"]', + definition: 'mybucket', + inclusion_reasons: ['default'], + priority: 3 + } + ]); + expect(querier.querier.parameterQueryLookups).toEqual([ + ScopedParameterLookup.direct({ lookupName: 'mybucket.test', queryId: '2.test' }, ['user1']) + ]); + + expect(hydrated.evaluateParameterRow(USERS, { id: 'user1', is_admin: 1 })).toEqual([ + { + bucketParameters: [{ user_id: 'user1' }], + lookup: ScopedParameterLookup.direct({ lookupName: 'mybucket.test', queryId: '2.test' }, ['user1']) + } + ]); + + expect( + hydrated.evaluateRow({ + sourceTable: ASSETS, + record: { id: 'asset1', description: 'test', user_id: 'user1', device_id: 'device1' } + }) + ).toEqual([ + { + bucket: 'mybucket-test["user1"]', + id: 'asset1', + data: { + id: 'asset1', + description: 'test' + }, + table: 'assets' + } + ]); + }); + test('parse bucket with parameters and OR condition', () => { const rules = SqlSyncRules.fromYaml( `