WIP: Persist index data

src/Umbraco.Cms.Search.Core/Constants.cs

@@ -41,4 +41,9 @@ public static class FieldNames
 
         public const string Tags = $"{FieldPrefix}Tags";
     }
+
+    public static class Persistence
+    {
+        public const string DocumentTableName = Umbraco.Cms.Core.Constants.DatabaseSchema.TableNamePrefix + "SearchDocument";
+    }
 }

src/Umbraco.Cms.Search.Core/Models/Persistence/Document.cs

@@ -0,0 +1,17 @@
+using Umbraco.Cms.Core.Models;
+using Umbraco.Cms.Search.Core.Models.Indexing;
+
+namespace Umbraco.Cms.Search.Core.Models.Persistence;
+
+public class Document
+{
+    public required Guid DocumentKey { get; set; }
+
+    public required IndexField[] Fields { get; set; }
+
+    public UmbracoObjectTypes ObjectType { get; set; }
+
+    public required Variation[] Variations { get; set; }
+
+    public ContentProtection? Protection { get; set; }
+}

src/Umbraco.Cms.Search.Core/NotificationHandlers/RebuildIndexesNotificationHandler.cs

@@ -48,7 +48,7 @@ public void Handle(LanguageDeletedNotification notification)
         {
             if (indexRegistration.ContainedObjectTypes.Contains(UmbracoObjectTypes.Document))
             {
-                _contentIndexingService.Rebuild(indexRegistration.IndexAlias);
+                _contentIndexingService.Rebuild(indexRegistration.IndexAlias, false);
             }
         }
     }
@@ -76,7 +76,7 @@ private void RebuildByObjectType<T>(IEnumerable<ContentTypeChange<T>> changes, U
             {
                 if (indexRegistration.ContainedObjectTypes.Contains(objectType))
                 {
-                    _contentIndexingService.Rebuild(indexRegistration.IndexAlias);
+                    _contentIndexingService.Rebuild(indexRegistration.IndexAlias, false);
                 }
             }
         }

src/Umbraco.Cms.Search.Core/Persistence/DocumentDto.cs

@@ -0,0 +1,31 @@
+using NPoco;
+using Umbraco.Cms.Infrastructure.Persistence.DatabaseAnnotations;
+
+namespace Umbraco.Cms.Search.Core.Persistence;
+
+[TableName(Constants.Persistence.DocumentTableName)]
+[PrimaryKey("id")]
+[ExplicitColumns]
+public class DocumentDto
+{
+    [Column("id")]
+    [PrimaryKeyColumn(AutoIncrement = true)]
+    public int Id { get; set; }
+
+    [Column("documentKey")]
+    public required Guid DocumentKey { get; set; }
+
+    [Column("changeStrategy")]
+    public required string ChangeStrategy { get; set; }
+
+    [Column("fields")]
+    [SpecialDbType(SpecialDbTypes.NVARCHARMAX)]
+    public required string Fields { get; set; }
+
+    [Column("objectType")]
+    public required string ObjectType { get; set; }
+
+    [Column("variations")]
+    [SpecialDbType(SpecialDbTypes.NVARCHARMAX)]
+    public required string Variations { get; set; }
+}

src/Umbraco.Cms.Search.Core/Persistence/DocumentRepository.cs

@@ -0,0 +1,130 @@
+using System.Text.Json;
+using NPoco;
+using Umbraco.Cms.Core.Models;
+using Umbraco.Cms.Infrastructure.Persistence;
+using Umbraco.Cms.Infrastructure.Scoping;
+using Umbraco.Cms.Search.Core.Models.Indexing;
+using Umbraco.Cms.Search.Core.Models.Persistence;
+using Umbraco.Extensions;
+
+namespace Umbraco.Cms.Search.Core.Persistence;
+
+public class DocumentRepository : IDocumentRepository
+{
+    private readonly IScopeAccessor _scopeAccessor;
+
+    public DocumentRepository(IScopeAccessor scopeAccessor) => _scopeAccessor = scopeAccessor;
+
+    public async Task AddAsync(Document document, string changeStrategy)
+    {
+        if (_scopeAccessor.AmbientScope is null)
+        {
+            throw new InvalidOperationException("Cannot add document as there is no ambient scope.");
+        }
+
+        DocumentDto dto = ToDto(document, changeStrategy);
+        await _scopeAccessor.AmbientScope.Database.InsertAsync(dto);
+    }
+
+    public async Task<Document?> GetAsync(Guid id, string changeStrategy)
+    {
+        Sql<ISqlContext>? sql = _scopeAccessor.AmbientScope?.Database.SqlContext.Sql()
+            .Select<DocumentDto>()
+            .From<DocumentDto>()
+            .Where<DocumentDto>(x => x.DocumentKey == id && x.ChangeStrategy == changeStrategy);
+
+        if (_scopeAccessor.AmbientScope is null)
+        {
+            throw new InvalidOperationException("Cannot add document as there is no ambient scope.");
+        }
+
+        DocumentDto? documentDto = await _scopeAccessor.AmbientScope.Database.FirstOrDefaultAsync<DocumentDto>(sql);
+
+        return ToDocument(documentDto);
+    }
+
+    public async Task<IReadOnlyDictionary<Guid, Document>> GetManyAsync(IEnumerable<Guid> ids, string changeStrategy)
+    {
+        if (_scopeAccessor.AmbientScope is null)
+        {
+            throw new InvalidOperationException("Cannot get documents as there is no ambient scope.");
+        }
+
+        Guid[] idsArray = ids as Guid[] ?? ids.ToArray();
+        if (idsArray.Length == 0)
+        {
+            return new Dictionary<Guid, Document>();
+        }
+
+        Sql<ISqlContext> sql = _scopeAccessor.AmbientScope.Database.SqlContext.Sql()
+            .Select<DocumentDto>()
+            .From<DocumentDto>()
+            .Where<DocumentDto>(x => x.ChangeStrategy == changeStrategy)
+            .WhereIn<DocumentDto>(x => x.DocumentKey, idsArray);
+
+        List<DocumentDto> documentDtos = await _scopeAccessor.AmbientScope.Database.FetchAsync<DocumentDto>(sql);
+
+        return documentDtos
+            .Select(ToDocument)
+            .Where(doc => doc is not null)
+            .ToDictionary(doc => doc!.DocumentKey, doc => doc!);
+    }
+
+    public async Task DeleteAsync(Guid id, string changeStrategy)
+    {
+        if (_scopeAccessor.AmbientScope is null)
+        {
+            throw new InvalidOperationException("Cannot add document as there is no ambient scope.");
+        }
+
+        Sql<ISqlContext> sql = _scopeAccessor.AmbientScope!.Database.SqlContext.Sql()
+            .Delete<DocumentDto>()
+            .Where<DocumentDto>(x => x.DocumentKey == id && x.ChangeStrategy == changeStrategy);
+
+        await _scopeAccessor.AmbientScope?.Database.ExecuteAsync(sql)!;
+    }
+
+    public async Task<IEnumerable<Document>> GetByChangeStrategyAsync(string changeStrategy)
+    {
+        if (_scopeAccessor.AmbientScope is null)
+        {
+            throw new InvalidOperationException("Cannot get documents as there is no ambient scope.");
+        }
+
+        Sql<ISqlContext> sql = _scopeAccessor.AmbientScope.Database.SqlContext.Sql()
+            .Select<DocumentDto>()
+            .From<DocumentDto>()
+            .Where<DocumentDto>(x => x.ChangeStrategy == changeStrategy);
+
+        List<DocumentDto> documentDtos = await _scopeAccessor.AmbientScope.Database.FetchAsync<DocumentDto>(sql);
+
+        return documentDtos.Select(ToDocument).WhereNotNull();
+    }
+
+
+    private DocumentDto ToDto(Document document, string changeStrategy) =>
+        new()
+        {
+            DocumentKey = document.DocumentKey,
+            ChangeStrategy = changeStrategy,
+            Fields = JsonSerializer.Serialize(document.Fields),
+            ObjectType = document.ObjectType.ToString(),
+            Variations = JsonSerializer.Serialize(document.Variations),
+        };
+
+    private Document? ToDocument(DocumentDto? dto)
+    {
+        if (dto is null)
+        {
+            return null;
+        }
+
+        return new()
+        {
+            DocumentKey = dto.DocumentKey,
+            Fields = JsonSerializer.Deserialize<IndexField[]>(dto.Fields) ?? [],
+            ObjectType = Enum.Parse<UmbracoObjectTypes>(dto.ObjectType),
+            Variations = JsonSerializer.Deserialize<Variation[]>(dto.Variations) ?? [],
+        };
+    }
+}

src/Umbraco.Cms.Search.Core/Persistence/IDocumentRepository.cs

@@ -0,0 +1,16 @@
+using Umbraco.Cms.Search.Core.Models.Persistence;
+
+namespace Umbraco.Cms.Search.Core.Persistence;
+
+public interface IDocumentRepository
+{
+    public Task AddAsync(Document document, string changeStrategy);
+
+    public Task<Document?> GetAsync(Guid id, string changeStrategy);
+
+    public Task<IReadOnlyDictionary<Guid, Document>> GetManyAsync(IEnumerable<Guid> ids, string changeStrategy);
+
+    public Task DeleteAsync(Guid id, string changeStrategy);
+
+    public Task<IEnumerable<Document>> GetByChangeStrategyAsync(string changeStrategy);
+}

src/Umbraco.Cms.Search.Core/Persistence/Migration/CustomMigrationPlan.cs

@@ -0,0 +1,15 @@
+using Umbraco.Cms.Core.Packaging;
+
+namespace Umbraco.Cms.Search.Core.Persistence.Migration;
+
+public class CustomPackageMigrationPlan : PackageMigrationPlan
+{
+    public CustomPackageMigrationPlan() : base("Umbraco CMS Search")
+    {
+    }
+
+    protected override void DefinePlan()
+    {
+        To<CustomPackageMigration>(new Guid("4FD681BE-E27E-4688-922B-29EDCDCB8A49"));
+    }
+}

src/Umbraco.Cms.Search.Core/Persistence/Migration/CustomPackageMigration.cs

@@ -0,0 +1,49 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Umbraco.Cms.Core.Configuration.Models;
+using Umbraco.Cms.Core.IO;
+using Umbraco.Cms.Core.PropertyEditors;
+using Umbraco.Cms.Core.Services;
+using Umbraco.Cms.Core.Strings;
+using Umbraco.Cms.Infrastructure.Migrations;
+using Umbraco.Cms.Infrastructure.Packaging;
+
+namespace Umbraco.Cms.Search.Core.Persistence.Migration;
+
+public class CustomPackageMigration : AsyncPackageMigrationBase
+{
+    public CustomPackageMigration(
+        IPackagingService packagingService,
+        IMediaService mediaService,
+        MediaFileManager mediaFileManager,
+        MediaUrlGeneratorCollection mediaUrlGenerators,
+        IShortStringHelper shortStringHelper,
+        IContentTypeBaseServiceProvider contentTypeBaseServiceProvider,
+        IMigrationContext context,
+        IOptions<PackageMigrationSettings> packageMigrationsSettings)
+        : base(
+            packagingService,
+            mediaService,
+            mediaFileManager,
+            mediaUrlGenerators,
+            shortStringHelper,
+            contentTypeBaseServiceProvider,
+            context,
+            packageMigrationsSettings)
+    {
+    }
+
+    protected override Task MigrateAsync()
+    {
+        if (TableExists(Constants.Persistence.DocumentTableName) == false)
+        {
+            Create.Table<DocumentDto>().Do();
+        }
+        else
+        {
+            Logger.LogDebug("The database table {DbTable} already exists, skipping", Constants.Persistence.DocumentTableName);
+        }
+
+        return Task.CompletedTask;
+    }
+}

src/Umbraco.Cms.Search.Core/Services/ContentIndexing/ContentIndexingService.cs

@@ -31,7 +31,7 @@ public ContentIndexingService(
     public void Handle(IEnumerable<ContentChange> changes)
         => _backgroundTaskQueue.QueueBackgroundWorkItem(async cancellationToken => await HandleAsync(changes, cancellationToken));
 
-    public void Rebuild(string indexAlias)
+    public void Rebuild(string indexAlias, bool useDatabase = false)
     {
         IndexRegistration? indexRegistration = _indexOptions.GetIndexRegistration(indexAlias);
         if (indexRegistration is null)
@@ -40,7 +40,7 @@ public void Rebuild(string indexAlias)
             return;
         }
 
-        _backgroundTaskQueue.QueueBackgroundWorkItem(async cancellationToken => await RebuildAsync(indexRegistration, cancellationToken));
+        _backgroundTaskQueue.QueueBackgroundWorkItem(async cancellationToken => await RebuildAsync(indexRegistration, cancellationToken, useDatabase));
     }
 
     private async Task HandleAsync(IEnumerable<ContentChange> changes, CancellationToken cancellationToken)
@@ -76,15 +76,15 @@ private async Task HandleAsync(IEnumerable<ContentChange> changes, CancellationT
         }
     }
 
-    private async Task RebuildAsync(IndexRegistration indexRegistration, CancellationToken cancellationToken)
+    private async Task RebuildAsync(IndexRegistration indexRegistration, CancellationToken cancellationToken, bool useDatabase)
     {
         if (TryGetContentChangeStrategy(indexRegistration.ContentChangeStrategy, out IContentChangeStrategy? contentChangeStrategy) is false
             || TryGetIndexer(indexRegistration.Indexer, out IIndexer? indexer) is false)
         {
             return;
         }
 
-        await contentChangeStrategy.RebuildAsync(new IndexInfo(indexRegistration.IndexAlias, indexRegistration.ContainedObjectTypes, indexer), cancellationToken);
+        await contentChangeStrategy.RebuildAsync(new IndexInfo(indexRegistration.IndexAlias, indexRegistration.ContainedObjectTypes, indexer), cancellationToken, useDatabase);
     }
 
     private bool TryGetContentChangeStrategy(Type type, [NotNullWhen(true)] out IContentChangeStrategy? contentChangeStrategy)

src/Umbraco.Cms.Search.Core/Services/ContentIndexing/DocumentService.cs

@@ -0,0 +1,40 @@
+using Umbraco.Cms.Core.Scoping;
+using Umbraco.Cms.Core.Services;
+using Umbraco.Cms.Search.Core.Models.Persistence;
+using Umbraco.Cms.Search.Core.Persistence;
+
+namespace Umbraco.Cms.Search.Core.Services.ContentIndexing;
+
+public class DocumentService : IDocumentService
+{
+    private readonly ICoreScopeProvider _scopeProvider;
+    private readonly IDocumentRepository _documentRepository;
+
+    public DocumentService(ICoreScopeProvider scopeProvider, IDocumentRepository documentRepository, IContentIndexingDataCollectionService contentIndexingDataCollectionService, IContentProtectionProvider contentProtectionProvider, IContentService contentService)
+    {
+        _scopeProvider = scopeProvider;
+        _documentRepository = documentRepository;
+    }
+
+    public async Task AddAsync(Document document, string changeStrategy)
+    {
+        using ICoreScope scope = _scopeProvider.CreateCoreScope();
+        await _documentRepository.AddAsync(document, changeStrategy);
+        scope.Complete();
+    }
+
+    public async Task DeleteAsync(Guid id, string changeStrategy)
+    {
+        using ICoreScope scope = _scopeProvider.CreateCoreScope();
+        await _documentRepository.DeleteAsync(id, changeStrategy);
+        scope.Complete();
+    }
+
+    public async Task<IEnumerable<Document>> GetByChangeStrategyAsync(string changeStrategy)
+    {
+        using ICoreScope scope = _scopeProvider.CreateCoreScope();
+        IEnumerable<Document> documents = await _documentRepository.GetByChangeStrategyAsync(changeStrategy);
+        scope.Complete();
+        return documents;
+    }
+}