diff --git a/Directory.Packages.props b/Directory.Packages.props index e68816e..71d690d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,6 +6,7 @@ + @@ -15,4 +16,4 @@ - + \ No newline at end of file diff --git a/src/Umbraco.Cms.Search.Core/Constants.cs b/src/Umbraco.Cms.Search.Core/Constants.cs index c5893b8..e6dca81 100644 --- a/src/Umbraco.Cms.Search.Core/Constants.cs +++ b/src/Umbraco.Cms.Search.Core/Constants.cs @@ -41,4 +41,9 @@ public static class FieldNames public const string Tags = $"{FieldPrefix}Tags"; } + + public static class Persistence + { + public const string DocumentTableName = Umbraco.Cms.Core.Constants.DatabaseSchema.TableNamePrefix + "SearchDocument"; + } } diff --git a/src/Umbraco.Cms.Search.Core/Models/Persistence/Document.cs b/src/Umbraco.Cms.Search.Core/Models/Persistence/Document.cs new file mode 100644 index 0000000..9013dca --- /dev/null +++ b/src/Umbraco.Cms.Search.Core/Models/Persistence/Document.cs @@ -0,0 +1,17 @@ +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Search.Core.Models.Indexing; + +namespace Umbraco.Cms.Search.Core.Models.Persistence; + +public class Document +{ + public required Guid DocumentKey { get; set; } + + public required IndexField[] Fields { get; set; } + + public UmbracoObjectTypes ObjectType { get; set; } + + public required Variation[] Variations { get; set; } + + public ContentProtection? Protection { get; set; } +} diff --git a/src/Umbraco.Cms.Search.Core/NotificationHandlers/RebuildIndexesNotificationHandler.cs b/src/Umbraco.Cms.Search.Core/NotificationHandlers/RebuildIndexesNotificationHandler.cs index 49d4fd3..8c0d32a 100644 --- a/src/Umbraco.Cms.Search.Core/NotificationHandlers/RebuildIndexesNotificationHandler.cs +++ b/src/Umbraco.Cms.Search.Core/NotificationHandlers/RebuildIndexesNotificationHandler.cs @@ -48,7 +48,7 @@ public void Handle(LanguageDeletedNotification notification) { if (indexRegistration.ContainedObjectTypes.Contains(UmbracoObjectTypes.Document)) { - _contentIndexingService.Rebuild(indexRegistration.IndexAlias); + _contentIndexingService.Rebuild(indexRegistration.IndexAlias, false); } } } @@ -76,7 +76,7 @@ private void RebuildByObjectType(IEnumerable> changes, U { if (indexRegistration.ContainedObjectTypes.Contains(objectType)) { - _contentIndexingService.Rebuild(indexRegistration.IndexAlias); + _contentIndexingService.Rebuild(indexRegistration.IndexAlias, false); } } } diff --git a/src/Umbraco.Cms.Search.Core/Persistence/DocumentDto.cs b/src/Umbraco.Cms.Search.Core/Persistence/DocumentDto.cs new file mode 100644 index 0000000..74056c6 --- /dev/null +++ b/src/Umbraco.Cms.Search.Core/Persistence/DocumentDto.cs @@ -0,0 +1,29 @@ +using NPoco; +using Umbraco.Cms.Infrastructure.Persistence.DatabaseAnnotations; + +namespace Umbraco.Cms.Search.Core.Persistence; + +[TableName(Constants.Persistence.DocumentTableName)] +[PrimaryKey("id")] +[ExplicitColumns] +public class DocumentDto +{ + [Column("id")] + [PrimaryKeyColumn(AutoIncrement = true)] + public int Id { get; set; } + + [Column("documentKey")] + public required Guid DocumentKey { get; set; } + + [Column("changeStrategy")] + public required string ChangeStrategy { get; set; } + + [Column("fields")] + public required byte[] Fields { get; set; } + + [Column("objectType")] + public required string ObjectType { get; set; } + + [Column("variations")] + public required byte[] Variations { get; set; } +} diff --git a/src/Umbraco.Cms.Search.Core/Persistence/DocumentRepository.cs b/src/Umbraco.Cms.Search.Core/Persistence/DocumentRepository.cs new file mode 100644 index 0000000..9668394 --- /dev/null +++ b/src/Umbraco.Cms.Search.Core/Persistence/DocumentRepository.cs @@ -0,0 +1,115 @@ +using MessagePack; +using MessagePack.Resolvers; +using NPoco; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Infrastructure.Persistence; +using Umbraco.Cms.Infrastructure.Scoping; +using Umbraco.Cms.Search.Core.Models.Indexing; +using Umbraco.Cms.Search.Core.Models.Persistence; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Search.Core.Persistence; + +public class DocumentRepository : IDocumentRepository +{ + private readonly IScopeAccessor _scopeAccessor; + private readonly MessagePackSerializerOptions _options; + + public DocumentRepository(IScopeAccessor scopeAccessor) + { + _scopeAccessor = scopeAccessor; + + MessagePackSerializerOptions defaultOptions = ContractlessStandardResolver.Options; + IFormatterResolver resolver = CompositeResolver.Create(defaultOptions.Resolver); + _options = defaultOptions + .WithResolver(resolver) + .WithCompression(MessagePackCompression.Lz4BlockArray) + .WithSecurity(MessagePackSecurity.UntrustedData); + } + + public async Task AddAsync(Document document, string changeStrategy) + { + if (_scopeAccessor.AmbientScope is null) + { + throw new InvalidOperationException("Cannot add document as there is no ambient scope."); + } + + DocumentDto dto = ToDto(document, changeStrategy); + await _scopeAccessor.AmbientScope.Database.InsertAsync(dto); + } + + public async Task GetAsync(Guid id, string changeStrategy) + { + Sql? sql = _scopeAccessor.AmbientScope?.Database.SqlContext.Sql() + .Select() + .From() + .Where(x => x.DocumentKey == id && x.ChangeStrategy == changeStrategy); + + if (_scopeAccessor.AmbientScope is null) + { + throw new InvalidOperationException("Cannot add document as there is no ambient scope."); + } + + DocumentDto? documentDto = await _scopeAccessor.AmbientScope.Database.FirstOrDefaultAsync(sql); + + return ToDocument(documentDto); + } + + public async Task DeleteAsync(Guid id, string changeStrategy) + { + if (_scopeAccessor.AmbientScope is null) + { + throw new InvalidOperationException("Cannot add document as there is no ambient scope."); + } + + Sql sql = _scopeAccessor.AmbientScope!.Database.SqlContext.Sql() + .Delete() + .Where(x => x.DocumentKey == id && x.ChangeStrategy == changeStrategy); + + await _scopeAccessor.AmbientScope?.Database.ExecuteAsync(sql)!; + } + + public async Task> GetByChangeStrategyAsync(string changeStrategy) + { + if (_scopeAccessor.AmbientScope is null) + { + throw new InvalidOperationException("Cannot get documents as there is no ambient scope."); + } + + Sql sql = _scopeAccessor.AmbientScope.Database.SqlContext.Sql() + .Select() + .From() + .Where(x => x.ChangeStrategy == changeStrategy); + + List documentDtos = await _scopeAccessor.AmbientScope.Database.FetchAsync(sql); + + return documentDtos.Select(ToDocument).WhereNotNull(); + } + + + private DocumentDto ToDto(Document document, string changeStrategy) => + new() + { + DocumentKey = document.DocumentKey, + ChangeStrategy = changeStrategy, + Fields = MessagePackSerializer.Serialize(document.Fields, _options), + ObjectType = document.ObjectType.ToString(), + Variations = MessagePackSerializer.Serialize(document.Variations, _options), + }; + + private Document? ToDocument(DocumentDto? dto) + { + if (dto is null) + { + return null; + } + + return new() + { + DocumentKey = dto.DocumentKey, + Fields = MessagePackSerializer.Deserialize(dto.Fields, _options) ?? [], + ObjectType = Enum.Parse(dto.ObjectType), + Variations = MessagePackSerializer.Deserialize(dto.Variations, _options) ?? [], + }; + } +} diff --git a/src/Umbraco.Cms.Search.Core/Persistence/IDocumentRepository.cs b/src/Umbraco.Cms.Search.Core/Persistence/IDocumentRepository.cs new file mode 100644 index 0000000..46c78bf --- /dev/null +++ b/src/Umbraco.Cms.Search.Core/Persistence/IDocumentRepository.cs @@ -0,0 +1,14 @@ +using Umbraco.Cms.Search.Core.Models.Persistence; + +namespace Umbraco.Cms.Search.Core.Persistence; + +public interface IDocumentRepository +{ + public Task AddAsync(Document document, string changeStrategy); + + public Task GetAsync(Guid id, string changeStrategy); + + public Task DeleteAsync(Guid id, string changeStrategy); + + public Task> GetByChangeStrategyAsync(string changeStrategy); +} diff --git a/src/Umbraco.Cms.Search.Core/Persistence/Migration/CustomMigrationPlan.cs b/src/Umbraco.Cms.Search.Core/Persistence/Migration/CustomMigrationPlan.cs new file mode 100644 index 0000000..9d98269 --- /dev/null +++ b/src/Umbraco.Cms.Search.Core/Persistence/Migration/CustomMigrationPlan.cs @@ -0,0 +1,15 @@ +using Umbraco.Cms.Core.Packaging; + +namespace Umbraco.Cms.Search.Core.Persistence.Migration; + +public class CustomPackageMigrationPlan : PackageMigrationPlan +{ + public CustomPackageMigrationPlan() : base("Umbraco CMS Search") + { + } + + protected override void DefinePlan() + { + To(new Guid("4FD681BE-E27E-4688-922B-29EDCDCB8A49")); + } +} diff --git a/src/Umbraco.Cms.Search.Core/Persistence/Migration/CustomPackageMigration.cs b/src/Umbraco.Cms.Search.Core/Persistence/Migration/CustomPackageMigration.cs new file mode 100644 index 0000000..f95cf23 --- /dev/null +++ b/src/Umbraco.Cms.Search.Core/Persistence/Migration/CustomPackageMigration.cs @@ -0,0 +1,49 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.IO; +using Umbraco.Cms.Core.PropertyEditors; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Strings; +using Umbraco.Cms.Infrastructure.Migrations; +using Umbraco.Cms.Infrastructure.Packaging; + +namespace Umbraco.Cms.Search.Core.Persistence.Migration; + +public class CustomPackageMigration : AsyncPackageMigrationBase +{ + public CustomPackageMigration( + IPackagingService packagingService, + IMediaService mediaService, + MediaFileManager mediaFileManager, + MediaUrlGeneratorCollection mediaUrlGenerators, + IShortStringHelper shortStringHelper, + IContentTypeBaseServiceProvider contentTypeBaseServiceProvider, + IMigrationContext context, + IOptions packageMigrationsSettings) + : base( + packagingService, + mediaService, + mediaFileManager, + mediaUrlGenerators, + shortStringHelper, + contentTypeBaseServiceProvider, + context, + packageMigrationsSettings) + { + } + + protected override Task MigrateAsync() + { + if (TableExists(Constants.Persistence.DocumentTableName) == false) + { + Create.Table().Do(); + } + else + { + Logger.LogDebug("The database table {DbTable} already exists, skipping", Constants.Persistence.DocumentTableName); + } + + return Task.CompletedTask; + } +} diff --git a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/ContentIndexingService.cs b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/ContentIndexingService.cs index fe6c20f..aca9b17 100644 --- a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/ContentIndexingService.cs +++ b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/ContentIndexingService.cs @@ -31,7 +31,7 @@ public ContentIndexingService( public void Handle(IEnumerable changes) => _backgroundTaskQueue.QueueBackgroundWorkItem(async cancellationToken => await HandleAsync(changes, cancellationToken)); - public void Rebuild(string indexAlias) + public void Rebuild(string indexAlias, bool useDatabase = false) { IndexRegistration? indexRegistration = _indexOptions.GetIndexRegistration(indexAlias); if (indexRegistration is null) @@ -40,7 +40,7 @@ public void Rebuild(string indexAlias) return; } - _backgroundTaskQueue.QueueBackgroundWorkItem(async cancellationToken => await RebuildAsync(indexRegistration, cancellationToken)); + _backgroundTaskQueue.QueueBackgroundWorkItem(async cancellationToken => await RebuildAsync(indexRegistration, cancellationToken, useDatabase)); } private async Task HandleAsync(IEnumerable changes, CancellationToken cancellationToken) @@ -76,7 +76,7 @@ private async Task HandleAsync(IEnumerable changes, CancellationT } } - private async Task RebuildAsync(IndexRegistration indexRegistration, CancellationToken cancellationToken) + private async Task RebuildAsync(IndexRegistration indexRegistration, CancellationToken cancellationToken, bool useDatabase) { if (TryGetContentChangeStrategy(indexRegistration.ContentChangeStrategy, out IContentChangeStrategy? contentChangeStrategy) is false || TryGetIndexer(indexRegistration.Indexer, out IIndexer? indexer) is false) @@ -84,7 +84,7 @@ private async Task RebuildAsync(IndexRegistration indexRegistration, Cancellatio return; } - await contentChangeStrategy.RebuildAsync(new IndexInfo(indexRegistration.IndexAlias, indexRegistration.ContainedObjectTypes, indexer), cancellationToken); + await contentChangeStrategy.RebuildAsync(new IndexInfo(indexRegistration.IndexAlias, indexRegistration.ContainedObjectTypes, indexer), cancellationToken, useDatabase); } private bool TryGetContentChangeStrategy(Type type, [NotNullWhen(true)] out IContentChangeStrategy? contentChangeStrategy) diff --git a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/DocumentService.cs b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/DocumentService.cs new file mode 100644 index 0000000..721e96b --- /dev/null +++ b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/DocumentService.cs @@ -0,0 +1,40 @@ +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Search.Core.Models.Persistence; +using Umbraco.Cms.Search.Core.Persistence; + +namespace Umbraco.Cms.Search.Core.Services.ContentIndexing; + +public class DocumentService : IDocumentService +{ + private readonly ICoreScopeProvider _scopeProvider; + private readonly IDocumentRepository _documentRepository; + + public DocumentService(ICoreScopeProvider scopeProvider, IDocumentRepository documentRepository, IContentIndexingDataCollectionService contentIndexingDataCollectionService, IContentProtectionProvider contentProtectionProvider, IContentService contentService) + { + _scopeProvider = scopeProvider; + _documentRepository = documentRepository; + } + + public async Task AddAsync(Document document, string changeStrategy) + { + using ICoreScope scope = _scopeProvider.CreateCoreScope(); + await _documentRepository.AddAsync(document, changeStrategy); + scope.Complete(); + } + + public async Task DeleteAsync(Guid id, string changeStrategy) + { + using ICoreScope scope = _scopeProvider.CreateCoreScope(); + await _documentRepository.DeleteAsync(id, changeStrategy); + scope.Complete(); + } + + public async Task> GetByChangeStrategyAsync(string changeStrategy) + { + using ICoreScope scope = _scopeProvider.CreateCoreScope(); + IEnumerable documents = await _documentRepository.GetByChangeStrategyAsync(changeStrategy); + scope.Complete(); + return documents; + } +} diff --git a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/DraftContentChangeStrategy.cs b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/DraftContentChangeStrategy.cs index 07e641c..a86177c 100644 --- a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/DraftContentChangeStrategy.cs +++ b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/DraftContentChangeStrategy.cs @@ -1,42 +1,36 @@ using Microsoft.Extensions.Logging; using Umbraco.Cms.Core; -using Umbraco.Cms.Core.Events; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Infrastructure.Persistence; -using Umbraco.Cms.Search.Core.Extensions; using Umbraco.Cms.Search.Core.Models.Indexing; -using Umbraco.Cms.Search.Core.Notifications; -using Umbraco.Extensions; namespace Umbraco.Cms.Search.Core.Services.ContentIndexing; internal sealed class DraftContentChangeStrategy : ContentChangeStrategyBase, IDraftContentChangeStrategy { - private readonly IContentIndexingDataCollectionService _contentIndexingDataCollectionService; private readonly IContentService _contentService; private readonly IMediaService _mediaService; private readonly IMemberService _memberService; - private readonly IEventAggregator _eventAggregator; + private readonly IIndexingService _indexingService; + private const string StrategyName = "DraftContentChangeStrategy"; protected override bool SupportsTrashedContent => true; public DraftContentChangeStrategy( - IContentIndexingDataCollectionService contentIndexingDataCollectionService, IContentService contentService, IMediaService mediaService, IMemberService memberService, - IEventAggregator eventAggregator, IUmbracoDatabaseFactory umbracoDatabaseFactory, IIdKeyMap idKeyMap, - ILogger logger) + ILogger logger, + IIndexingService indexingService) : base(umbracoDatabaseFactory, idKeyMap, logger) { - _contentIndexingDataCollectionService = contentIndexingDataCollectionService; _contentService = contentService; _mediaService = mediaService; _memberService = memberService; - _eventAggregator = eventAggregator; + _indexingService = indexingService; } public async Task HandleAsync(IEnumerable indexInfos, IEnumerable changes, CancellationToken cancellationToken) @@ -68,7 +62,7 @@ change.ContentState is ContentState.Draft await RemoveFromIndexAsync(indexInfosAsArray, pendingRemovals); pendingRemovals.Clear(); - var updated = await UpdateIndexAsync(indexInfosAsArray, change, content, cancellationToken); + var updated = await HandleContentChangeAsync(indexInfosAsArray, change, content, cancellationToken); if (updated is false) { pendingRemovals.Add(change); @@ -79,78 +73,29 @@ change.ContentState is ContentState.Draft await RemoveFromIndexAsync(indexInfosAsArray, pendingRemovals); } - public async Task RebuildAsync(IndexInfo indexInfo, CancellationToken cancellationToken) + public async Task RebuildAsync(IndexInfo indexInfo, CancellationToken cancellationToken, bool useDatabase = false) { await indexInfo.Indexer.ResetAsync(indexInfo.IndexAlias); - await RebuildAsync( - indexInfo, - UmbracoObjectTypes.Document, - () => _contentService.GetRootContent(), - (pageIndex, pageSize) => _contentService.GetPagedChildren(Cms.Core.Constants.System.RecycleBinContent, pageIndex, pageSize, out _), - cancellationToken); - - if (cancellationToken.IsCancellationRequested) + if (useDatabase) { - LogIndexRebuildCancellation(indexInfo); - return; - } - - await RebuildAsync( - indexInfo, - UmbracoObjectTypes.Media, - () => _mediaService.GetRootMedia(), - (pageIndex, pageSize) => _mediaService.GetPagedChildren(Cms.Core.Constants.System.RecycleBinMedia, pageIndex, pageSize, out _), - cancellationToken); - - if (cancellationToken.IsCancellationRequested) - { - LogIndexRebuildCancellation(indexInfo); - return; - } - - if (indexInfo.ContainedObjectTypes.Contains(UmbracoObjectTypes.Member) is false) - { - return; - } - - IMember[] members; - var pageIndex = 0; - do - { - if (cancellationToken.IsCancellationRequested) - { - break; - } - - members = _memberService.GetAll(pageIndex, ContentEnumerationPageSize, out _, "sortOrder", Direction.Ascending).ToArray(); - foreach (IMember member in members) - { - if (cancellationToken.IsCancellationRequested) - { - break; - } - await UpdateIndexAsync([indexInfo], ContentChange.Member(member.Key, ChangeImpact.Refresh, ContentState.Draft), member, cancellationToken); - } - pageIndex++; + await _indexingService.RebuildFromRepositoryAsync(indexInfo, StrategyName, cancellationToken); } - while (members.Length == ContentEnumerationPageSize); - - if (cancellationToken.IsCancellationRequested) + else { - LogIndexRebuildCancellation(indexInfo); + await RebuildFromMemoryAsync(indexInfo, cancellationToken); } } - private async Task UpdateIndexAsync(IndexInfo[] indexInfos, ContentChange change, IContentBase content, CancellationToken cancellationToken) + private async Task HandleContentChangeAsync(IndexInfo[] indexInfos, ContentChange change, IContentBase content, CancellationToken cancellationToken) { IndexInfo[] applicableIndexInfos = indexInfos.Where(info => info.ContainedObjectTypes.Contains(change.ObjectType)).ToArray(); - if(applicableIndexInfos.Length is 0) + if (applicableIndexInfos.Length is 0) { return true; } - var result = await UpdateIndexAsync(applicableIndexInfos, content, change.ObjectType, cancellationToken); + var result = await _indexingService.IndexContentAsync(indexInfos, content, StrategyName, false, cancellationToken); if (change.ChangeImpact is ChangeImpact.RefreshWithDescendants) { @@ -164,7 +109,7 @@ await EnumerateDescendantsByPath( .GetPagedDescendants(id, pageIndex, pageSize, out _, query, ordering) .ToArray(), async descendants => - await UpdateIndexDescendantsAsync(applicableIndexInfos, descendants, change.ObjectType, cancellationToken)); + await HandleDescendantChangesAsync(applicableIndexInfos, descendants, cancellationToken)); break; case UmbracoObjectTypes.Media: await EnumerateDescendantsByPath( @@ -174,7 +119,7 @@ await EnumerateDescendantsByPath( .GetPagedDescendants(id, pageIndex, pageSize, out _, query, ordering) .ToArray(), async descendants => - await UpdateIndexDescendantsAsync(applicableIndexInfos, descendants, change.ObjectType, cancellationToken)); + await HandleDescendantChangesAsync(applicableIndexInfos, descendants, cancellationToken)); break; } } @@ -182,52 +127,15 @@ await EnumerateDescendantsByPath( return result; } - private async Task UpdateIndexDescendantsAsync(IndexInfo[] indexInfos, T[] descendants, UmbracoObjectTypes objectType, CancellationToken cancellationToken) + private async Task HandleDescendantChangesAsync(IndexInfo[] indexInfos, T[] descendants, CancellationToken cancellationToken) where T : IContentBase { foreach (T descendant in descendants) { - await UpdateIndexAsync(indexInfos, descendant, objectType, cancellationToken); + await _indexingService.IndexContentAsync(indexInfos, descendant, StrategyName, false, cancellationToken); } } - private async Task UpdateIndexAsync(IndexInfo[] indexInfos, IContentBase content, UmbracoObjectTypes objectType, CancellationToken cancellationToken) - { - IndexField[]? fields = (await _contentIndexingDataCollectionService.CollectAsync(content, false, cancellationToken))?.ToArray(); - if (fields is null) - { - return false; - } - - string?[] cultures = content.AvailableCultures(); - - Variation[] variations = content.ContentType.VariesBySegment() - ? cultures - .SelectMany(culture => content - .Properties - .SelectMany(property => property.Values.Where(value => value.Culture.InvariantEquals(culture))) - .DistinctBy(value => value.Segment).Select(value => value.Segment) - .Select(segment => new Variation(culture, segment))) - .ToArray() - : cultures - .Select(culture => new Variation(culture, null)) - .ToArray(); - - foreach (IndexInfo indexInfo in indexInfos) - { - var notification = new IndexingNotification(indexInfo, content.Key, UmbracoObjectTypes.Document, variations, fields); - if (await _eventAggregator.PublishCancelableAsync(notification)) - { - // the indexing operation was cancelled for this index; continue with the rest of the indexes - continue; - } - - await indexInfo.Indexer.AddOrUpdateAsync(indexInfo.IndexAlias, content.Key, objectType, variations, notification.Fields, null); - } - - return true; - } - private async Task RemoveFromIndexAsync(IndexInfo[] indexInfos, IReadOnlyCollection contentChanges) { if (contentChanges.Count is 0) @@ -241,7 +149,7 @@ private async Task RemoveFromIndexAsync(IndexInfo[] indexInfos, IReadOnlyCollect .Where(change => indexInfo.ContainedObjectTypes.Contains(change.ObjectType)) .Select(change => change.Id) .ToArray(); - await indexInfo.Indexer.DeleteAsync(indexInfo.IndexAlias, keys); + await _indexingService.RemoveAsync([indexInfo], StrategyName, keys); } } @@ -254,7 +162,7 @@ private async Task RemoveFromIndexAsync(IndexInfo[] indexInfos, IReadOnlyCollect _ => throw new ArgumentOutOfRangeException(nameof(change.ObjectType)) }; - private async Task RebuildAsync( + private async Task RebuildContentFromMemoryAsync( IndexInfo indexInfo, UmbracoObjectTypes objectType, Func> getContentAtRoot, @@ -272,8 +180,6 @@ private async Task RebuildAsync( return; } - IndexInfo[] indexInfos = [indexInfo]; - foreach (IContentBase rootContent in getContentAtRoot()) { if (cancellationToken.IsCancellationRequested) @@ -281,7 +187,8 @@ private async Task RebuildAsync( break; } - await UpdateIndexAsync(indexInfos, GetContentChange(rootContent), rootContent, cancellationToken); + await _indexingService.IndexContentAsync([indexInfo], rootContent, StrategyName, false, cancellationToken); + await IndexDescendantsAsync(indexInfo, rootContent, objectType, cancellationToken); } if (cancellationToken.IsCancellationRequested) @@ -300,30 +207,120 @@ private async Task RebuildAsync( } contentInRecycleBin = getPagedContentAtRecycleBinRoot(pageIndex, ContentEnumerationPageSize).ToArray(); + foreach (IContentBase content in contentInRecycleBin) { if (cancellationToken.IsCancellationRequested) { break; } - await UpdateIndexAsync(indexInfos, GetContentChange(content), content, cancellationToken); + + await _indexingService.IndexContentAsync([indexInfo], content, StrategyName, false, cancellationToken); } + pageIndex++; } while (contentInRecycleBin.Length == ContentEnumerationPageSize); + } - return; + private async Task IndexDescendantsAsync(IndexInfo indexInfo, IContentBase content, UmbracoObjectTypes objectType, CancellationToken cancellationToken) + { + switch (objectType) + { + case UmbracoObjectTypes.Document: + await EnumerateDescendantsByPath( + objectType, + content.Key, + (id, pageIndex, pageSize, query, ordering) => _contentService + .GetPagedDescendants(id, pageIndex, pageSize, out _, query, ordering) + .ToArray(), + async descendants => + { + foreach (IContent descendant in descendants) + { + await _indexingService.IndexContentAsync([indexInfo], descendant, StrategyName, false, cancellationToken); + } + }); + break; + case UmbracoObjectTypes.Media: + await EnumerateDescendantsByPath( + objectType, + content.Key, + (id, pageIndex, pageSize, query, ordering) => _mediaService + .GetPagedDescendants(id, pageIndex, pageSize, out _, query, ordering) + .ToArray(), + async descendants => + { + foreach (IMedia descendant in descendants) + { + await _indexingService.IndexContentAsync([indexInfo], descendant, StrategyName, false, cancellationToken); + } + }); + break; + } + } + + private async Task RebuildFromMemoryAsync(IndexInfo indexInfo, CancellationToken cancellationToken) + { + await RebuildContentFromMemoryAsync( + indexInfo, + UmbracoObjectTypes.Document, + () => _contentService.GetRootContent(), + (pageIndex, pageSize) => _contentService.GetPagedChildren(Cms.Core.Constants.System.RecycleBinContent, pageIndex, pageSize, out _), + cancellationToken); - ContentChange GetContentChange(IContentBase content) + if (cancellationToken.IsCancellationRequested) + { + LogIndexRebuildCancellation(indexInfo); + return; + } + + await RebuildContentFromMemoryAsync( + indexInfo, + UmbracoObjectTypes.Media, + () => _mediaService.GetRootMedia(), + (pageIndex, pageSize) => _mediaService.GetPagedChildren(Cms.Core.Constants.System.RecycleBinMedia, pageIndex, pageSize, out _), + cancellationToken); + + if (cancellationToken.IsCancellationRequested) { - ContentChange contentChange = objectType switch + LogIndexRebuildCancellation(indexInfo); + return; + } + + if (indexInfo.ContainedObjectTypes.Contains(UmbracoObjectTypes.Member) is false) + { + return; + } + + IMember[] members; + var pageIndex = 0; + do + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + members = _memberService.GetAll(pageIndex, ContentEnumerationPageSize, out _, "sortOrder", Direction.Ascending).ToArray(); + + foreach (IMember member in members) { - UmbracoObjectTypes.Document => ContentChange.Document(content.Key, ChangeImpact.RefreshWithDescendants, ContentState.Draft), - UmbracoObjectTypes.Media => ContentChange.Media(content.Key, ChangeImpact.RefreshWithDescendants, ContentState.Draft), - UmbracoObjectTypes.Member => ContentChange.Member(content.Key, ChangeImpact.Refresh, ContentState.Draft), - _ => throw new ArgumentOutOfRangeException(nameof(objectType), objectType, "This strategy only supports documents, media and members") - }; - return contentChange; + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await _indexingService.IndexContentAsync([indexInfo], member, StrategyName, false, cancellationToken); + } + + pageIndex++; + } + while (members.Length == ContentEnumerationPageSize); + + if (cancellationToken.IsCancellationRequested) + { + LogIndexRebuildCancellation(indexInfo); } } } diff --git a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IContentChangeStrategy.cs b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IContentChangeStrategy.cs index 7ca70e8..edd126a 100644 --- a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IContentChangeStrategy.cs +++ b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IContentChangeStrategy.cs @@ -6,5 +6,5 @@ public interface IContentChangeStrategy { Task HandleAsync(IEnumerable indexInfos, IEnumerable changes, CancellationToken cancellationToken); - Task RebuildAsync(IndexInfo indexInfo, CancellationToken cancellationToken); + Task RebuildAsync(IndexInfo indexInfo, CancellationToken cancellationToken, bool useDatabase = false); } diff --git a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IContentIndexingService.cs b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IContentIndexingService.cs index 9ac3707..66911d0 100644 --- a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IContentIndexingService.cs +++ b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IContentIndexingService.cs @@ -6,5 +6,5 @@ public interface IContentIndexingService { void Handle(IEnumerable changes); - void Rebuild(string indexAlias); + void Rebuild(string indexAlias, bool useDatabase = false); } diff --git a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IDocumentService.cs b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IDocumentService.cs new file mode 100644 index 0000000..31c856d --- /dev/null +++ b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IDocumentService.cs @@ -0,0 +1,12 @@ +using Umbraco.Cms.Search.Core.Models.Persistence; + +namespace Umbraco.Cms.Search.Core.Services.ContentIndexing; + +public interface IDocumentService +{ + Task AddAsync(Document document, string changeStrategy); + + Task DeleteAsync(Guid id, string changeStrategy); + + Task> GetByChangeStrategyAsync(string changeStrategy); +} diff --git a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IIndexingService.cs b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IIndexingService.cs new file mode 100644 index 0000000..ba00df3 --- /dev/null +++ b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IIndexingService.cs @@ -0,0 +1,13 @@ +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Search.Core.Models.Indexing; + +namespace Umbraco.Cms.Search.Core.Services.ContentIndexing; + +public interface IIndexingService +{ + Task IndexContentAsync(IndexInfo[] indexInfos, IContentBase content, string changeStrategy, bool published, CancellationToken cancellationToken); + + Task RemoveAsync(IndexInfo [] indexInfos, string changeStrategy, Guid[] documentKeys); + + Task RebuildFromRepositoryAsync(IndexInfo indexInfo, string changeStrategy, CancellationToken cancellationToken); +} diff --git a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IndexingService.cs b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IndexingService.cs new file mode 100644 index 0000000..68c69aa --- /dev/null +++ b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/IndexingService.cs @@ -0,0 +1,195 @@ +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Search.Core.Extensions; +using Umbraco.Cms.Search.Core.Models.Indexing; +using Umbraco.Cms.Search.Core.Models.Persistence; +using Umbraco.Cms.Search.Core.Notifications; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Search.Core.Services.ContentIndexing; + +public class IndexingService : IIndexingService +{ + private readonly IContentIndexingDataCollectionService _contentIndexingDataCollectionService; + private readonly IContentProtectionProvider _contentProtectionProvider; + private readonly IContentService _contentService; + private readonly IDocumentService _documentService; + private readonly IEventAggregator _eventAggregator; + + public IndexingService(IContentIndexingDataCollectionService contentIndexingDataCollectionService, IContentProtectionProvider contentProtectionProvider, IContentService contentService, IDocumentService documentService, IEventAggregator eventAggregator) + { + _contentIndexingDataCollectionService = contentIndexingDataCollectionService; + _contentProtectionProvider = contentProtectionProvider; + _contentService = contentService; + _documentService = documentService; + _eventAggregator = eventAggregator; + } + + public async Task IndexContentAsync(IndexInfo[] indexInfos, IContentBase content, string changeStrategy, bool published, CancellationToken cancellationToken) + { + // fetch the doc from service, make sure not to use database here, as it will be deleted + Document? document = await CalculateDocumentAsync(content, published, cancellationToken); + + if (document is null) + { + return false; + } + + // Delete old entry and persist new fields to database + await _documentService.DeleteAsync(content.Key, changeStrategy); + await _documentService.AddAsync(document, changeStrategy); + UmbracoObjectTypes objectType = content.ObjectType(); + + foreach (IndexInfo indexInfo in indexInfos) + { + var notification = new IndexingNotification(indexInfo, content.Key, objectType, document.Variations, document.Fields); + if (await _eventAggregator.PublishCancelableAsync(notification)) + { + // the indexing operation was cancelled for this index; continue with the rest of the indexes + continue; + } + + await indexInfo.Indexer.AddOrUpdateAsync(indexInfo.IndexAlias, content.Key, objectType, document.Variations, notification.Fields, document.Protection); + } + + return document.Variations.Length != 0; + } + + public async Task RemoveAsync(IndexInfo[] indexInfos, string changeStrategy, Guid[] documentKeys) + { + foreach (IndexInfo indexInfo in indexInfos) + { + await indexInfo.Indexer.DeleteAsync(indexInfo.IndexAlias, documentKeys); + } + + foreach (Guid documentKey in documentKeys) + { + await _documentService.DeleteAsync(documentKey, changeStrategy); + } + } + + public async Task RebuildFromRepositoryAsync(IndexInfo indexInfo, string changeStrategy, CancellationToken cancellationToken) + { + IEnumerable documents = await _documentService.GetByChangeStrategyAsync(changeStrategy); + foreach (Document document in documents) + { + var notification = new IndexingNotification(indexInfo, document.DocumentKey, UmbracoObjectTypes.Document, document.Variations, document.Fields); + if (await _eventAggregator.PublishCancelableAsync(notification)) + { + return; + } + + + await indexInfo.Indexer.AddOrUpdateAsync(indexInfo.IndexAlias, document.DocumentKey, UmbracoObjectTypes.Document, document.Variations, notification.Fields, document.Protection); + } + } + + private async Task CalculateDocumentAsync(IContentBase content, bool published, CancellationToken cancellationToken) + { + IEnumerable? fields = await _contentIndexingDataCollectionService.CollectAsync(content, published, cancellationToken); + if (fields is null) + { + return null; + } + + Variation[] variations; + ContentProtection? protection; + + if (published) + { + variations = RoutablePublishedVariations(content); + protection = await GetProtection(content); + + // the fields collection is for all published variants of the content - but it's not certain that a published + // variant is also routable, because the published routing state can be broken at ancestor level. + fields = fields.Where(field => variations.Any(v => (field.Culture is null || v.Culture == field.Culture) && (field.Segment is null || v.Segment == field.Segment))).ToArray(); + } + else + { + variations = Variations(content); + protection = null; + } + + return new Document + { + DocumentKey = content.Key, + Fields = fields.ToArray(), + ObjectType = content.ObjectType(), + Variations = variations, + Protection = protection, + }; + } + + // NOTE: for the time being, segments are not individually publishable, but it will likely happen at some point, + // so this method deals with variations - not cultures. + private Variation[] RoutablePublishedVariations(IContentBase content) + { + if (content.IsPublished() is false) + { + return []; + } + + var variesByCulture = content.VariesByCulture(); + + // if the content varies by culture, the indexable cultures are the published + // cultures - otherwise "null" represents "no culture" + var cultures = content.PublishedCultures(); + + // now iterate all ancestors and make sure all cultures are published all the way up the tree + foreach (var ancestorId in content.AncestorIds()) + { + IContent? ancestor = _contentService.GetById(ancestorId); + if (ancestor is null || ancestor.Published is false) + { + // no published ancestor => don't index anything + cultures = []; + } + else if (variesByCulture && ancestor.VariesByCulture()) + { + // both the content and the ancestor are culture variant => only index the published cultures they have in common + cultures = cultures.Intersect(ancestor.PublishedCultures).ToArray(); + } + + // if we've already run out of cultures to index, there is no reason to iterate the ancestors any further + if (cultures.Any() == false) + { + break; + } + } + + // for now, segments are not individually routable, so we only need to deal with cultures and append all known segments + if (content.Properties.Any(p => p.PropertyType.VariesBySegment()) is false) + { + // no segment variant properties - just return the found cultures + return cultures.Select(c => new Variation(c, null)).ToArray(); + } + + // segments are not "known" - we can only determine segment variation by looking at the property values + return cultures.SelectMany(culture => content + .Properties + .SelectMany(property => property.Values.Where(value => value.Culture.InvariantEquals(culture))) + .DistinctBy(value => value.Segment).Select(value => value.Segment) + .Select(segment => new Variation(culture, segment))) + .ToArray(); + } + + private async Task GetProtection(IContentBase content) => await _contentProtectionProvider.GetContentProtectionAsync(content); + + private Variation[] Variations(IContentBase content) + { + string?[] cultures = content.AvailableCultures(); + + return content.ContentType.VariesBySegment() + ? cultures + .SelectMany(culture => content + .Properties + .SelectMany(property => property.Values.Where(value => value.Culture.InvariantEquals(culture))) + .DistinctBy(value => value.Segment).Select(value => value.Segment) + .Select(segment => new Variation(culture, segment))) + .ToArray() + : cultures + .Select(culture => new Variation(culture, null)) + .ToArray(); + } +} diff --git a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/PublishedContentChangeStrategy.cs b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/PublishedContentChangeStrategy.cs index 7711494..8ed9055 100644 --- a/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/PublishedContentChangeStrategy.cs +++ b/src/Umbraco.Cms.Search.Core/Services/ContentIndexing/PublishedContentChangeStrategy.cs @@ -1,40 +1,31 @@ using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core.Events; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Infrastructure.Persistence; -using Umbraco.Cms.Search.Core.Extensions; using Umbraco.Cms.Search.Core.Models.Indexing; -using Umbraco.Cms.Search.Core.Notifications; -using Umbraco.Extensions; namespace Umbraco.Cms.Search.Core.Services.ContentIndexing; internal sealed class PublishedContentChangeStrategy : ContentChangeStrategyBase, IPublishedContentChangeStrategy { - private readonly IContentIndexingDataCollectionService _contentIndexingDataCollectionService; - private readonly IContentProtectionProvider _contentProtectionProvider; private readonly IContentService _contentService; - private readonly IEventAggregator _eventAggregator; private readonly ILogger _logger; + private readonly IIndexingService _indexingService; + private const string StrategyName = "PublishedContentChangeStrategy"; protected override bool SupportsTrashedContent => false; public PublishedContentChangeStrategy( - IContentIndexingDataCollectionService contentIndexingDataCollectionService, - IContentProtectionProvider contentProtectionProvider, IContentService contentService, - IEventAggregator eventAggregator, IUmbracoDatabaseFactory umbracoDatabaseFactory, IIdKeyMap idKeyMap, - ILogger logger) + ILogger logger, + IIndexingService indexingService) : base(umbracoDatabaseFactory, idKeyMap, logger) { - _contentIndexingDataCollectionService = contentIndexingDataCollectionService; - _contentProtectionProvider = contentProtectionProvider; _contentService = contentService; _logger = logger; - _eventAggregator = eventAggregator; + _indexingService = indexingService; } public async Task HandleAsync(IEnumerable indexInfos, IEnumerable changes, CancellationToken cancellationToken) @@ -72,18 +63,29 @@ change.ContentState is ContentState.Published await RemoveFromIndexAsync(indexInfosAsArray, pendingRemovals); pendingRemovals.Clear(); - await ReindexAsync(indexInfosAsArray, content, change.ChangeImpact is ChangeImpact.RefreshWithDescendants, cancellationToken); + await HandleContentChangeAsync(indexInfosAsArray, content, change.ChangeImpact is ChangeImpact.RefreshWithDescendants, cancellationToken); } } await RemoveFromIndexAsync(indexInfosAsArray, pendingRemovals); } - public async Task RebuildAsync(IndexInfo indexInfo, CancellationToken cancellationToken) + public async Task RebuildAsync(IndexInfo indexInfo, CancellationToken cancellationToken, bool useDatabase = false) { await indexInfo.Indexer.ResetAsync(indexInfo.IndexAlias); - IndexInfo[] indexInfos = [indexInfo]; + if (useDatabase) + { + await _indexingService.RebuildFromRepositoryAsync(indexInfo, StrategyName, cancellationToken); + } + else + { + await RebuildFromMemoryAsync(indexInfo, cancellationToken); + } + } + + private async Task RebuildFromMemoryAsync(IndexInfo indexInfo, CancellationToken cancellationToken) + { foreach (IContent content in _contentService.GetRootContent()) { if (cancellationToken.IsCancellationRequested) @@ -92,15 +94,18 @@ public async Task RebuildAsync(IndexInfo indexInfo, CancellationToken cancellati return; } - await ReindexAsync(indexInfos, content, true, cancellationToken); + await RebuildContentFromMemoryAsync(indexInfo, content, cancellationToken); } } - private async Task ReindexAsync(IndexInfo[] indexInfos, IContentBase content, bool forceReindexDescendants, CancellationToken cancellationToken) + /// + /// Used by HandleAsync: Calculates fields fresh, persists to DB, and adds to index. + /// + private async Task HandleContentChangeAsync(IndexInfo[] indexInfos, IContentBase content, bool forceReindexDescendants, CancellationToken cancellationToken) { // index the content - Variation[] indexedVariants = await UpdateIndexAsync(indexInfos, content, cancellationToken); - if (indexedVariants.Any() is false) + var result = await _indexingService.IndexContentAsync(indexInfos, content, StrategyName, true, cancellationToken); + if (result is false) { // we likely got here because a removal triggered a "refresh branch" notification, now we // need to delete every last culture of this content and all descendants @@ -110,11 +115,11 @@ private async Task ReindexAsync(IndexInfo[] indexInfos, IContentBase content, bo if (forceReindexDescendants) { - await ReindexDescendantsAsync(indexInfos, content, cancellationToken); + await HandleDescendantChangesAsync(indexInfos, content, cancellationToken); } } - private async Task ReindexDescendantsAsync(IndexInfo[] indexInfos, IContentBase content, CancellationToken cancellationToken) + private async Task HandleDescendantChangesAsync(IndexInfo[] indexInfos, IContentBase content, CancellationToken cancellationToken) { var removedDescendantIds = new List(); await EnumerateDescendantsByPath( @@ -133,8 +138,8 @@ await EnumerateDescendantsByPath( continue; } - Variation[] indexedVariants = await UpdateIndexAsync(indexInfos, descendant, cancellationToken); - if (indexedVariants.Any() is false) + var result = await _indexingService.IndexContentAsync(indexInfos, descendant, StrategyName, true, cancellationToken); + if (result is false) { // no variants to index, make sure this is removed from the index and skip any descendants moving forward // (the index implementation is responsible for deleting descendants at index level) @@ -145,39 +150,40 @@ await EnumerateDescendantsByPath( }); } - private async Task UpdateIndexAsync(IndexInfo[] indexInfos, IContentBase content, CancellationToken cancellationToken) + private async Task RebuildContentFromMemoryAsync(IndexInfo indexInfo, IContentBase content, CancellationToken cancellationToken) { - Variation[] variations = RoutablePublishedVariations(content); - if (variations.Length is 0) - { - return []; - } + var result = await _indexingService.IndexContentAsync([indexInfo], content, StrategyName, true, cancellationToken); - IEnumerable? fields = await _contentIndexingDataCollectionService.CollectAsync(content, true, cancellationToken); - if (fields is null) + if (result is false) { - return []; + return; } - // the fields collection is for all published variants of the content - but it's not certain that a published - // variant is also routable, because the published routing state can be broken at ancestor level. - fields = fields.Where(field => variations.Any(v => (field.Culture is null || v.Culture == field.Culture) && (field.Segment is null || v.Segment == field.Segment))).ToArray(); - - ContentProtection? contentProtection = await _contentProtectionProvider.GetContentProtectionAsync(content); - - foreach (IndexInfo indexInfo in indexInfos) - { - var notification = new IndexingNotification(indexInfo, content.Key, UmbracoObjectTypes.Document, variations, fields); - if (await _eventAggregator.PublishCancelableAsync(notification)) + // Rebuild all descendants + var removedDescendantIds = new List(); + await EnumerateDescendantsByPath( + UmbracoObjectTypes.Document, + content.Key, + (id, pageIndex, pageSize, query, ordering) => _contentService + .GetPagedDescendants(id, pageIndex, pageSize, out _, query, ordering) + .ToArray(), + async descendants => { - // the indexing operation was cancelled for this index; continue with the rest of the indexes - continue; - } + foreach (IContent descendant in descendants) + { + if (removedDescendantIds.Contains(descendant.ParentId)) + { + continue; + } - await indexInfo.Indexer.AddOrUpdateAsync(indexInfo.IndexAlias, content.Key, UmbracoObjectTypes.Document, variations, notification.Fields, contentProtection); - } + var descendantResult = await _indexingService.IndexContentAsync([indexInfo], content, StrategyName, true, cancellationToken); - return variations; + if (descendantResult is false) + { + removedDescendantIds.Add(descendant.Id); + } + } + }); } private async Task RemoveFromIndexAsync(IndexInfo[] indexInfos, Guid id) @@ -190,62 +196,6 @@ private async Task RemoveFromIndexAsync(IndexInfo[] indexInfos, IReadOnlyCollect return; } - foreach (IndexInfo indexInfo in indexInfos) - { - await indexInfo.Indexer.DeleteAsync(indexInfo.IndexAlias, ids); - } - } - - // NOTE: for the time being, segments are not individually publishable, but it will likely happen at some point, - // so this method deals with variations - not cultures. - private Variation[] RoutablePublishedVariations(IContentBase content) - { - if (content.IsPublished() is false) - { - return []; - } - - var variesByCulture = content.VariesByCulture(); - - // if the content varies by culture, the indexable cultures are the published - // cultures - otherwise "null" represents "no culture" - var cultures = content.PublishedCultures(); - - // now iterate all ancestors and make sure all cultures are published all the way up the tree - foreach (var ancestorId in content.AncestorIds()) - { - IContent? ancestor = _contentService.GetById(ancestorId); - if (ancestor is null || ancestor.Published is false) - { - // no published ancestor => don't index anything - cultures = []; - } - else if (variesByCulture && ancestor.VariesByCulture()) - { - // both the content and the ancestor are culture variant => only index the published cultures they have in common - cultures = cultures.Intersect(ancestor.PublishedCultures).ToArray(); - } - - // if we've already run out of cultures to index, there is no reason to iterate the ancestors any further - if (cultures.Any() == false) - { - break; - } - } - - // for now, segments are not individually routable, so we only need to deal with cultures and append all known segments - if (content.Properties.Any(p => p.PropertyType.VariesBySegment()) is false) - { - // no segment variant properties - just return the found cultures - return cultures.Select(c => new Variation(c, null)).ToArray(); - } - - // segments are not "known" - we can only determine segment variation by looking at the property values - return cultures.SelectMany(culture => content - .Properties - .SelectMany(property => property.Values.Where(value => value.Culture.InvariantEquals(culture))) - .DistinctBy(value => value.Segment).Select(value => value.Segment) - .Select(segment => new Variation(culture, segment))) - .ToArray(); + await _indexingService.RemoveAsync(indexInfos, StrategyName, ids.ToArray()); } } diff --git a/src/Umbraco.Cms.Search.Core/Umbraco.Cms.Search.Core.csproj b/src/Umbraco.Cms.Search.Core/Umbraco.Cms.Search.Core.csproj index 98df4c3..f43a60c 100644 --- a/src/Umbraco.Cms.Search.Core/Umbraco.Cms.Search.Core.csproj +++ b/src/Umbraco.Cms.Search.Core/Umbraco.Cms.Search.Core.csproj @@ -13,6 +13,7 @@ + diff --git a/src/Umbraco.Cms.Search.Provider.Examine/DependencyInjection/ServiceCollectionExtensions.cs b/src/Umbraco.Cms.Search.Provider.Examine/DependencyInjection/ServiceCollectionExtensions.cs index 0f4960f..2581ddd 100644 --- a/src/Umbraco.Cms.Search.Provider.Examine/DependencyInjection/ServiceCollectionExtensions.cs +++ b/src/Umbraco.Cms.Search.Provider.Examine/DependencyInjection/ServiceCollectionExtensions.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Search.Core.Configuration; +using Umbraco.Cms.Search.Core.Persistence; using Umbraco.Cms.Search.Core.Services; using Umbraco.Cms.Search.Core.Services.ContentIndexing; using Umbraco.Cms.Search.Provider.Examine.Configuration; @@ -19,6 +20,9 @@ public static void AddExamineSearchProviderServices(this IServiceCollection serv // register the in-memory searcher and indexer so they can be used explicitly for index registrations services.AddTransient(); services.AddTransient(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddTransient(); services.AddTransient(); diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantDocumentProtectionIndexTests.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantDocumentProtectionIndexTests.cs index 3c6341f..0a9e581 100644 --- a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantDocumentProtectionIndexTests.cs +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantDocumentProtectionIndexTests.cs @@ -84,6 +84,8 @@ public void DoesNotIndexContentProtectionIfNoneExists() [SetUp] public async Task CreateInvariantDocument() { + await PackageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); IContentType contentType = new ContentTypeBuilder() .WithAlias("invariant") .AddPropertyType() diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantDocumentTests.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantDocumentTests.cs index c0f2b05..54e3e7d 100644 --- a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantDocumentTests.cs +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantDocumentTests.cs @@ -1,11 +1,12 @@ using Examine; using Examine.Search; using NUnit.Framework; +using Umbraco.Cms.Core; using Umbraco.Cms.Core.Models; -using Umbraco.Cms.Search.Provider.Examine; using Umbraco.Cms.Search.Provider.Examine.Helpers; using Umbraco.Cms.Tests.Common.Builders; using Umbraco.Cms.Tests.Common.Builders.Extensions; +using Constants = Umbraco.Cms.Search.Provider.Examine.Constants; namespace Umbraco.Test.Search.Examine.Integration.Tests.ContentTests.IndexService; @@ -148,6 +149,9 @@ public void CanIndexAggregatedTexts(bool publish) [SetUp] public async Task CreateInvariantDocument() { + await PackageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); + DataType dataType = new DataTypeBuilder() .WithId(0) .WithoutIdentity() diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantDocumentTreeTests.Draft.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantDocumentTreeTests.Draft.cs index 1a0a6e2..e5261e6 100644 --- a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantDocumentTreeTests.Draft.cs +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantDocumentTreeTests.Draft.cs @@ -107,6 +107,9 @@ await WaitForIndexing(Cms.Search.Core.Constants.IndexAliases.DraftContent, () => private async Task CreateInvariantDocumentTree(bool publish) { + await PackageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); + DataType dataType = new DataTypeBuilder() .WithId(0) .WithoutIdentity() diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantFacetsIndexTests.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantFacetsIndexTests.cs index b00f1c9..425fd8d 100644 --- a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantFacetsIndexTests.cs +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantFacetsIndexTests.cs @@ -3,6 +3,7 @@ using Examine.Lucene; using Examine.Search; using NUnit.Framework; +using Umbraco.Cms.Core; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Search.Provider.Examine.Helpers; using Umbraco.Cms.Tests.Common.Builders; @@ -117,6 +118,12 @@ public async Task CanGetMultipleIntFacets(bool publish) }); } + [SetUp] + public async Task RunMigrations() + { + await PackageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); + } private async Task CreateCountDocType() { diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantSortableIndexTests.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantSortableIndexTests.cs index c40e4a0..5c4196e 100644 --- a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantSortableIndexTests.cs +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/InvariantSortableIndexTests.cs @@ -1,6 +1,7 @@ using Examine; using Examine.Search; using NUnit.Framework; +using Umbraco.Cms.Core; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Search.Provider.Examine.Helpers; using Umbraco.Cms.Tests.Common.Builders; @@ -36,6 +37,13 @@ public async Task CanGetSortedTitles(bool publish) }); } + [SetUp] + public async Task RunMigrations() + { + await PackageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); + } + private async Task CreateTitleDocType() { ContentType = new ContentTypeBuilder() diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/MediaIndexServiceTests.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/MediaIndexServiceTests.cs index feb307b..139d7e2 100644 --- a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/MediaIndexServiceTests.cs +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/MediaIndexServiceTests.cs @@ -21,6 +21,13 @@ public async Task CanIndexAnyMedia() Assert.That(results.TotalItemCount, Is.EqualTo(1)); } + [SetUp] + public async Task RunMigrations() + { + await PackageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); + } + private async Task CreateMediaAsync() { IMediaType mediaType = new MediaTypeBuilder() diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/MemberIndexServiceTests.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/MemberIndexServiceTests.cs index 46ade1a..38a6e9b 100644 --- a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/MemberIndexServiceTests.cs +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/MemberIndexServiceTests.cs @@ -20,6 +20,13 @@ public async Task CanIndexAnyMember() Assert.That(results.TotalItemCount, Is.EqualTo(1)); } + [SetUp] + public async Task RunMigrations() + { + await PackageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); + } + private async Task CreateMemberAsync() { IMemberType memberType = new MemberTypeBuilder() diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/VariantContentTreeTests.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/VariantContentTreeTests.cs index a9e8c68..7d5fca5 100644 --- a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/VariantContentTreeTests.cs +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/VariantContentTreeTests.cs @@ -219,6 +219,9 @@ private void VerifyVariance(IEnumerable expectedExistingCultures) [SetUp] public async Task CreateVariantDocumentTree() { + await PackageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); + ILanguage langDk = new LanguageBuilder() .WithCultureInfo("da-DK") .WithIsDefault(true) diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/VariantDocumentTests.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/VariantDocumentTests.cs index ff5f43a..8a00832 100644 --- a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/VariantDocumentTests.cs +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/IndexService/VariantDocumentTests.cs @@ -1,6 +1,7 @@ using Examine; using Examine.Search; using NUnit.Framework; +using Umbraco.Cms.Core; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Search.Provider.Examine.Helpers; using Umbraco.Cms.Tests.Common.Builders; @@ -156,6 +157,13 @@ public async Task CanIndexVariantTextBySegment(bool publish, string culture, str Assert.That(results.First().Values.First(x => x.Key == fieldName).Value, Is.EqualTo(expectedValue)); } + [SetUp] + public async Task RunMigrations() + { + await PackageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); + } + private async Task CreateVariantDocument() { DataType dataType = new DataTypeBuilder() diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/PersistenceTests/DocumentServiceTests.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/PersistenceTests/DocumentServiceTests.cs new file mode 100644 index 0000000..093bf68 --- /dev/null +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/PersistenceTests/DocumentServiceTests.cs @@ -0,0 +1,249 @@ +using System.Diagnostics; +using System.Reflection; +using Examine; +using Examine.Lucene.Providers; +using NUnit.Framework; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.HostedServices; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Models.ContentEditing; +using Umbraco.Cms.Core.Models.ContentTypeEditing; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Services.ContentTypeEditing; +using Umbraco.Cms.Core.Services.OperationStatus; +using Umbraco.Cms.Core.Sync; +using Umbraco.Cms.Infrastructure.Install; +using Umbraco.Cms.Infrastructure.Scoping; +using Umbraco.Cms.Search.Core.DependencyInjection; +using Umbraco.Cms.Search.Core.Models.Indexing; +using Umbraco.Cms.Search.Core.Models.Persistence; +using Umbraco.Cms.Search.Core.NotificationHandlers; +using Umbraco.Cms.Search.Core.Persistence; +using Umbraco.Cms.Tests.Common.Builders; +using Umbraco.Cms.Tests.Common.Testing; +using Umbraco.Cms.Tests.Integration.Testing; +using Umbraco.Test.Search.Examine.Integration.Attributes; +using Umbraco.Test.Search.Examine.Integration.Extensions; +using Umbraco.Test.Search.Examine.Integration.Tests.ContentTests.IndexService; +using Constants = Umbraco.Cms.Search.Core.Constants; + +namespace Umbraco.Test.Search.Examine.Integration.Tests.ContentTests.PersistenceTests; + +[TestFixture] +[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)] +public class DocumentServiceTests : UmbracoIntegrationTest +{ + private bool _indexingComplete; + + private PackageMigrationRunner _packageMigrationRunner => GetRequiredService(); + + private IRuntimeState RuntimeState => GetRequiredService(); + + private IContentTypeEditingService ContentTypeEditingService => GetRequiredService(); + + private IContentEditingService ContentEditingService => GetRequiredService(); + + private IContentService ContentService => GetRequiredService(); + + private Umbraco.Cms.Search.Core.Services.ContentIndexing.IDocumentService DocumentService => GetRequiredService(); + + private IDocumentRepository DocumentRepository => GetRequiredService(); + + private IContent _rootDocument = null!; + + protected override void CustomTestSetup(IUmbracoBuilder builder) + { + base.CustomTestSetup(builder); + + builder.AddExamineSearchProviderForTest(); + + builder.AddSearchCore(); + + builder.Services.AddUnique(); + builder.Services.AddUnique(); + builder.AddNotificationHandler(); + + // the core ConfigureBuilderAttribute won't execute from other assemblies at the moment, so this is a workaround + var testType = Type.GetType(TestContext.CurrentContext.Test.ClassName!); + if (testType is not null) + { + MethodInfo? methodInfo = testType.GetMethod(TestContext.CurrentContext.Test.Name); + if (methodInfo is not null) + { + foreach (ConfigureUmbracoBuilderAttribute attribute in methodInfo.GetCustomAttributes(typeof(ConfigureUmbracoBuilderAttribute), true).OfType()) + { + attribute.Execute(builder, testType); + } + } + } + } + + [Test] + public async Task CanRunMigration() + { + await TestSetup(false); + using IScope scope = ScopeProvider.CreateScope(autoComplete: true); + IEnumerable tables = scope.Database.SqlContext.SqlSyntax.GetTablesInSchema(scope.Database); + var result = tables.Any(x => x.InvariantEquals(Constants.Persistence.DocumentTableName)); + Assert.That(result, Is.True); + } + + [TestCase(true)] + [TestCase(false)] + public async Task AddsEntryToDatabaseAfterIndexing(bool publish) + { + await TestSetup(publish); + var changeStrategy = GetStrategy(publish); + using IScope scope = ScopeProvider.CreateScope(autoComplete: true); + Document? doc = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(doc, Is.Not.Null); + } + + [TestCase(true)] + [TestCase(false)] + public async Task UpdatesEntryInDatabaseAfterPropertyChange(bool publish) + { + await TestSetup(publish); + var indexAlias = GetIndexAlias(publish); + var changeStrategy = GetStrategy(publish); + + IndexField[] initialFields; + // Verify initial document exists + using (ScopeProvider.CreateScope(autoComplete: true)) + { + Document? initialDoc = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(initialDoc, Is.Not.Null); + initialFields = initialDoc!.Fields; + } + + + // Update the content name + _rootDocument.Name = "Updated Root Document"; + + await WaitForIndexing(indexAlias, () => + { + ContentService.Save(_rootDocument); + if (publish) + { + ContentService.Publish(_rootDocument, ["*"]); + } + + return Task.CompletedTask; + }); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify the document was updated + Document? updatedDoc = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(updatedDoc, Is.Not.Null); + Assert.That(updatedDoc!.Fields, Is.Not.EqualTo(initialFields)); + Assert.That(FieldsContainText(updatedDoc.Fields, "Updated Root Document"), Is.True); + } + } + + [TestCase(true)] + [TestCase(false)] + public async Task RemovesEntryFromDatabaseAfterDeletion(bool publish) + { + await TestSetup(publish); + var indexAlias = GetIndexAlias(publish); + var changeStrategy = GetStrategy(publish); + + // Verify initial document exists + using (ScopeProvider.CreateScope(autoComplete: true)) + { + Document? initialDoc = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(initialDoc, Is.Not.Null); + } + + // Delete the content + await WaitForIndexing(indexAlias, () => + { + ContentService.Delete(_rootDocument); + return Task.CompletedTask; + }); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify the document was removed + Document? deletedDoc = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(deletedDoc, Is.Null); + } + } + + private string GetIndexAlias(bool publish) => publish ? Constants.IndexAliases.PublishedContent : Constants.IndexAliases.DraftContent; + + private string GetStrategy(bool publish) => publish ? "PublishedContentChangeStrategy" : "DraftContentChangeStrategy"; + + public async Task TestSetup(bool publish) + { + await _packageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); + + ContentTypeCreateModel contentTypeCreateModel = ContentTypeEditingBuilder.CreateSimpleContentType( + "parentType", + "Parent Type"); + Attempt contentTypeAttempt = await ContentTypeEditingService.CreateAsync( + contentTypeCreateModel, + Umbraco.Cms.Core.Constants.Security.SuperUserKey); + Assert.That(contentTypeAttempt.Success, Is.True); + IContentType contentType = contentTypeAttempt.Result!; + ContentCreateModel rootCreateModel = ContentEditingBuilder.CreateSimpleContent(contentType.Key, "Root Document"); + + var indexAlias = GetIndexAlias(publish); + await WaitForIndexing(indexAlias, async () => + { + Attempt createRootResult = await ContentEditingService.CreateAsync(rootCreateModel, Umbraco.Cms.Core.Constants.Security.SuperUserKey); + Assert.That(createRootResult.Success, Is.True); + _rootDocument = createRootResult.Result.Content!; + + if (publish) + { + ContentService.Publish(_rootDocument, ["*"]); + } + }); + } + + protected async Task WaitForIndexing(string indexAlias, Func indexUpdatingAction) + { + var index = (LuceneIndex)GetRequiredService().GetIndex(indexAlias); + index.IndexCommitted += IndexCommited; + + var hasDoneAction = false; + + var stopWatch = Stopwatch.StartNew(); + + while (_indexingComplete is false) + { + if (hasDoneAction is false) + { + await indexUpdatingAction(); + hasDoneAction = true; + } + + if (stopWatch.ElapsedMilliseconds > 5000) + { + throw new TimeoutException("Indexing timed out"); + } + + await Task.Delay(250); + } + + _indexingComplete = false; + index.IndexCommitted -= IndexCommited; + } + + private void IndexCommited(object? sender, EventArgs e) + { + _indexingComplete = true; + } + + private static bool FieldsContainText(IndexField[] fields, string text) + => fields.Any(f => + (f.Value.Texts?.Any(t => t.Contains(text)) == true) || + (f.Value.TextsR1?.Any(t => t.Contains(text)) == true) || + (f.Value.TextsR2?.Any(t => t.Contains(text)) == true) || + (f.Value.TextsR3?.Any(t => t.Contains(text)) == true) || + (f.Value.Keywords?.Any(k => k.Contains(text)) == true)); +} diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/PersistenceTests/MediaServiceTests.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/PersistenceTests/MediaServiceTests.cs new file mode 100644 index 0000000..ecb45c4 --- /dev/null +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/PersistenceTests/MediaServiceTests.cs @@ -0,0 +1,214 @@ +using System.Diagnostics; +using System.Reflection; +using Examine; +using Examine.Lucene.Providers; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.HostedServices; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Sync; +using Umbraco.Cms.Infrastructure.Install; +using Umbraco.Cms.Infrastructure.Scoping; +using Umbraco.Cms.Search.Core.DependencyInjection; +using Umbraco.Cms.Search.Core.Models.Indexing; +using Umbraco.Cms.Search.Core.Models.Persistence; +using Umbraco.Cms.Search.Core.NotificationHandlers; +using Umbraco.Cms.Search.Core.Persistence; +using Umbraco.Cms.Tests.Common.Builders; +using Umbraco.Cms.Tests.Common.Builders.Extensions; +using Umbraco.Cms.Tests.Common.Testing; +using Umbraco.Cms.Tests.Integration.Testing; +using Umbraco.Test.Search.Examine.Integration.Attributes; +using Umbraco.Test.Search.Examine.Integration.Extensions; +using Umbraco.Test.Search.Examine.Integration.Tests.ContentTests.IndexService; +using Constants = Umbraco.Cms.Search.Core.Constants; + +namespace Umbraco.Test.Search.Examine.Integration.Tests.ContentTests.PersistenceTests; + +[TestFixture] +[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)] +public class MediaServiceTests : UmbracoIntegrationTest +{ + private bool _indexingComplete; + + private PackageMigrationRunner PackageMigrationRunner => GetRequiredService(); + + private IRuntimeState RuntimeState => Services.GetRequiredService(); + + private IMediaTypeService MediaTypeService => GetRequiredService(); + + private IMediaService MediaService => GetRequiredService(); + private IDocumentRepository DocumentRepository => GetRequiredService(); + + private IMedia _rootMedia = null!; + + protected override void CustomTestSetup(IUmbracoBuilder builder) + { + base.CustomTestSetup(builder); + + builder.AddExamineSearchProviderForTest(); + + builder.AddSearchCore(); + + builder.Services.AddUnique(); + builder.Services.AddUnique(); + builder.AddNotificationHandler(); + + // the core ConfigureBuilderAttribute won't execute from other assemblies at the moment, so this is a workaround + var testType = Type.GetType(TestContext.CurrentContext.Test.ClassName!); + if (testType is not null) + { + MethodInfo? methodInfo = testType.GetMethod(TestContext.CurrentContext.Test.Name); + if (methodInfo is not null) + { + foreach (ConfigureUmbracoBuilderAttribute attribute in methodInfo.GetCustomAttributes(typeof(ConfigureUmbracoBuilderAttribute), true).OfType()) + { + attribute.Execute(builder, testType); + } + } + } + } + + [Test] + public async Task AddsEntryToDatabaseAfterIndexing() + { + await TestSetup(); + using IScope scope = ScopeProvider.CreateScope(autoComplete: true); + Document? doc = await DocumentRepository.GetAsync(_rootMedia.Key, "DraftContentChangeStrategy"); + Assert.That(doc, Is.Not.Null); + } + + [Test] + public async Task UpdatesEntryInDatabaseAfterPropertyChange() + { + await TestSetup(); + + IndexField[] initialFields; + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify initial document exists + Document? initialDoc = await DocumentRepository.GetAsync(_rootMedia.Key, "DraftContentChangeStrategy"); + Assert.That(initialDoc, Is.Not.Null); + initialFields = initialDoc!.Fields; + } + + // Update the media name + _rootMedia.Name = "Updated Root Media"; + + await WaitForIndexing(Constants.IndexAliases.DraftMedia, () => + { + MediaService.Save(_rootMedia); + return Task.CompletedTask; + }); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify the document was updated + Document? updatedDoc = await DocumentRepository.GetAsync(_rootMedia.Key, "DraftContentChangeStrategy"); + Assert.That(updatedDoc, Is.Not.Null); + Assert.That(updatedDoc!.Fields, Is.Not.EqualTo(initialFields)); + Assert.That(FieldsContainText(updatedDoc.Fields, "Updated Root Media"), Is.True); + } + } + + [Test] + public async Task RemovesEntryFromDatabaseAfterDeletion() + { + await TestSetup(); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify initial document exists + Document? initialDoc = await DocumentRepository.GetAsync(_rootMedia.Key, "DraftContentChangeStrategy"); + Assert.That(initialDoc, Is.Not.Null); + } + + // Delete the media + await WaitForIndexing(Constants.IndexAliases.DraftMedia, () => + { + MediaService.Delete(_rootMedia); + return Task.CompletedTask; + }); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify the document was removed + Document? deletedDoc = await DocumentRepository.GetAsync(_rootMedia.Key, "DraftContentChangeStrategy"); + Assert.That(deletedDoc, Is.Null); + } + } + + private async Task TestSetup() + { + await PackageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); + + IMediaType mediaType = new MediaTypeBuilder() + .WithAlias("testMediaType") + .AddPropertyGroup() + .AddPropertyType() + .WithAlias("altText") + .WithDataTypeId(Umbraco.Cms.Core.Constants.DataTypes.Textbox) + .WithPropertyEditorAlias(Umbraco.Cms.Core.Constants.PropertyEditors.Aliases.TextBox) + .Done() + .Done() + .Build(); + await MediaTypeService.CreateAsync(mediaType, Umbraco.Cms.Core.Constants.Security.SuperUserKey); + + await WaitForIndexing(Constants.IndexAliases.DraftMedia, () => + { + _rootMedia = new MediaBuilder() + .WithMediaType(mediaType) + .WithName("Root Media") + .WithPropertyValues(new { altText = "The media alt text" }) + .Build(); + MediaService.Save(_rootMedia); + return Task.CompletedTask; + }); + } + + private async Task WaitForIndexing(string indexAlias, Func indexUpdatingAction) + { + var index = (LuceneIndex)GetRequiredService().GetIndex(indexAlias); + index.IndexCommitted += IndexCommited; + + var hasDoneAction = false; + + var stopWatch = Stopwatch.StartNew(); + + while (_indexingComplete is false) + { + if (hasDoneAction is false) + { + await indexUpdatingAction(); + hasDoneAction = true; + } + + if (stopWatch.ElapsedMilliseconds > 5000) + { + throw new TimeoutException("Indexing timed out"); + } + + await Task.Delay(250); + } + + _indexingComplete = false; + index.IndexCommitted -= IndexCommited; + } + + private void IndexCommited(object? sender, EventArgs e) + { + _indexingComplete = true; + } + + private static bool FieldsContainText(IndexField[] fields, string text) + => fields.Any(f => + (f.Value.Texts?.Any(t => t.Contains(text)) == true) || + (f.Value.TextsR1?.Any(t => t.Contains(text)) == true) || + (f.Value.TextsR2?.Any(t => t.Contains(text)) == true) || + (f.Value.TextsR3?.Any(t => t.Contains(text)) == true) || + (f.Value.Keywords?.Any(k => k.Contains(text)) == true)); +} diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/PersistenceTests/MemberServiceTests.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/PersistenceTests/MemberServiceTests.cs new file mode 100644 index 0000000..81250f8 --- /dev/null +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/PersistenceTests/MemberServiceTests.cs @@ -0,0 +1,217 @@ +using System.Diagnostics; +using System.Reflection; +using Examine; +using Examine.Lucene.Providers; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.HostedServices; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Sync; +using Umbraco.Cms.Infrastructure.Install; +using Umbraco.Cms.Infrastructure.Scoping; +using Umbraco.Cms.Search.Core.DependencyInjection; +using Umbraco.Cms.Search.Core.Models.Indexing; +using Umbraco.Cms.Search.Core.Models.Persistence; +using Umbraco.Cms.Search.Core.NotificationHandlers; +using Umbraco.Cms.Search.Core.Persistence; +using Umbraco.Cms.Tests.Common.Builders; +using Umbraco.Cms.Tests.Common.Builders.Extensions; +using Umbraco.Cms.Tests.Common.Testing; +using Umbraco.Cms.Tests.Integration.Testing; +using Umbraco.Test.Search.Examine.Integration.Attributes; +using Umbraco.Test.Search.Examine.Integration.Extensions; +using Umbraco.Test.Search.Examine.Integration.Tests.ContentTests.IndexService; +using Constants = Umbraco.Cms.Search.Core.Constants; + +namespace Umbraco.Test.Search.Examine.Integration.Tests.ContentTests.PersistenceTests; + +[TestFixture] +[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)] +public class MemberServiceTests : UmbracoIntegrationTest +{ + private bool _indexingComplete; + + private PackageMigrationRunner PackageMigrationRunner => GetRequiredService(); + + private IRuntimeState RuntimeState => Services.GetRequiredService(); + + private IMemberTypeService MemberTypeService => GetRequiredService(); + + private IMemberService MemberService => GetRequiredService(); + + private IDocumentRepository DocumentRepository => GetRequiredService(); + + private IMember _member = null!; + + protected override void CustomTestSetup(IUmbracoBuilder builder) + { + base.CustomTestSetup(builder); + + builder.AddExamineSearchProviderForTest(); + + builder.AddSearchCore(); + + builder.Services.AddUnique(); + builder.Services.AddUnique(); + builder.AddNotificationHandler(); + + // the core ConfigureBuilderAttribute won't execute from other assemblies at the moment, so this is a workaround + var testType = Type.GetType(TestContext.CurrentContext.Test.ClassName!); + if (testType is not null) + { + MethodInfo? methodInfo = testType.GetMethod(TestContext.CurrentContext.Test.Name); + if (methodInfo is not null) + { + foreach (ConfigureUmbracoBuilderAttribute attribute in methodInfo.GetCustomAttributes(typeof(ConfigureUmbracoBuilderAttribute), true).OfType()) + { + attribute.Execute(builder, testType); + } + } + } + } + + [Test] + public async Task AddsEntryToDatabaseAfterIndexing() + { + await TestSetup(); + using IScope scope = ScopeProvider.CreateScope(autoComplete: true); + Document? doc = await DocumentRepository.GetAsync(_member.Key, "DraftContentChangeStrategy"); + Assert.That(doc, Is.Not.Null); + } + + [Test] + public async Task UpdatesEntryInDatabaseAfterPropertyChange() + { + await TestSetup(); + + IndexField[] initialFields; + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify initial document exists + Document? initialDoc = await DocumentRepository.GetAsync(_member.Key, "DraftContentChangeStrategy"); + Assert.That(initialDoc, Is.Not.Null); + initialFields = initialDoc!.Fields; + } + + // Update the member name + _member.Name = "Updated Member Name"; + + await WaitForIndexing(Constants.IndexAliases.DraftMembers, () => + { + MemberService.Save(_member); + return Task.CompletedTask; + }); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify the document was updated + Document? updatedDoc = await DocumentRepository.GetAsync(_member.Key, "DraftContentChangeStrategy"); + Assert.That(updatedDoc, Is.Not.Null); + Assert.That(updatedDoc!.Fields, Is.Not.EqualTo(initialFields)); + Assert.That(FieldsContainText(updatedDoc.Fields, "Updated Member Name"), Is.True); + } + } + + [Test] + [Ignore("This does not work in 16, as the MemberCacheRefresher passes id, and the IdKeyMap is already cleared, this should work in 17 as it uses key.")] + public async Task RemovesEntryFromDatabaseAfterDeletion() + { + await TestSetup(); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify initial document exists + Document? initialDoc = await DocumentRepository.GetAsync(_member.Key, "DraftContentChangeStrategy"); + Assert.That(initialDoc, Is.Not.Null); + } + + // Delete the member + MemberService.Delete(_member); + await Task.Delay(4000); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify the document was removed + Document? deletedDoc = await DocumentRepository.GetAsync(_member.Key, "DraftContentChangeStrategy"); + Assert.That(deletedDoc, Is.Null); + } + } + + private async Task TestSetup() + { + await PackageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); + + IMemberType memberType = new MemberTypeBuilder() + .WithAlias("testMemberType") + .AddPropertyGroup() + .AddPropertyType() + .WithAlias("organization") + .WithDataTypeId(Umbraco.Cms.Core.Constants.DataTypes.Textbox) + .WithPropertyEditorAlias(Umbraco.Cms.Core.Constants.PropertyEditors.Aliases.TextBox) + .Done() + .Done() + .Build(); + await MemberTypeService.CreateAsync(memberType, Umbraco.Cms.Core.Constants.Security.SuperUserKey); + + await WaitForIndexing(Constants.IndexAliases.DraftMembers, () => + { + _member = new MemberBuilder() + .WithMemberType(memberType) + .WithName("Test Member") + .WithEmail("testmember@local") + .WithLogin("testmember@local", "Test123456") + .AddPropertyData() + .WithKeyValue("organization", "Test Organization") + .Done() + .Build(); + MemberService.Save(_member); + return Task.CompletedTask; + }); + } + + private async Task WaitForIndexing(string indexAlias, Func indexUpdatingAction) + { + var index = (LuceneIndex)GetRequiredService().GetIndex(indexAlias); + index.IndexCommitted += IndexCommited; + + var hasDoneAction = false; + + var stopWatch = Stopwatch.StartNew(); + + while (_indexingComplete is false) + { + if (hasDoneAction is false) + { + await indexUpdatingAction(); + hasDoneAction = true; + } + + if (stopWatch.ElapsedMilliseconds > 5000) + { + throw new TimeoutException("Indexing timed out"); + } + + await Task.Delay(250); + } + + _indexingComplete = false; + index.IndexCommitted -= IndexCommited; + } + + private void IndexCommited(object? sender, EventArgs e) + { + _indexingComplete = true; + } + + private static bool FieldsContainText(IndexField[] fields, string text) + => fields.Any(f => + (f.Value.Texts?.Any(t => t.Contains(text)) == true) || + (f.Value.TextsR1?.Any(t => t.Contains(text)) == true) || + (f.Value.TextsR2?.Any(t => t.Contains(text)) == true) || + (f.Value.TextsR3?.Any(t => t.Contains(text)) == true) || + (f.Value.Keywords?.Any(k => k.Contains(text)) == true)); +} diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/PersistenceTests/RebuildTests.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/PersistenceTests/RebuildTests.cs new file mode 100644 index 0000000..863417b --- /dev/null +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/PersistenceTests/RebuildTests.cs @@ -0,0 +1,331 @@ +using System.Diagnostics; +using System.Reflection; +using Examine; +using Examine.Lucene.Providers; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.HostedServices; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Models.ContentEditing; +using Umbraco.Cms.Core.Models.ContentTypeEditing; +using Umbraco.Cms.Core.Notifications; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Services.ContentTypeEditing; +using Umbraco.Cms.Core.Services.OperationStatus; +using Umbraco.Cms.Core.Sync; +using Umbraco.Cms.Infrastructure.Install; +using Umbraco.Cms.Search.Core.DependencyInjection; +using Umbraco.Cms.Search.Core.Models.Indexing; +using Umbraco.Cms.Search.Core.Models.Persistence; +using Umbraco.Cms.Search.Core.NotificationHandlers; +using Umbraco.Cms.Search.Core.Persistence; +using Umbraco.Cms.Search.Core.Services.ContentIndexing; +using Umbraco.Cms.Tests.Common.Builders; +using Umbraco.Cms.Tests.Common.Testing; +using Umbraco.Cms.Tests.Integration.Testing; +using Umbraco.Test.Search.Examine.Integration.Attributes; +using Umbraco.Test.Search.Examine.Integration.Extensions; +using Umbraco.Test.Search.Examine.Integration.Tests.ContentTests.IndexService; +using Constants = Umbraco.Cms.Search.Core.Constants; +using IScope = Umbraco.Cms.Infrastructure.Scoping.IScope; + +namespace Umbraco.Test.Search.Examine.Integration.Tests.ContentTests.PersistenceTests; + +[TestFixture] +[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)] +public class RebuildTests : UmbracoIntegrationTest +{ + private bool _indexingComplete; + + private PackageMigrationRunner PackageMigrationRunner => GetRequiredService(); + + private IRuntimeState RuntimeState => Services.GetRequiredService(); + + private IContentTypeEditingService ContentTypeEditingService => GetRequiredService(); + + private IContentEditingService ContentEditingService => GetRequiredService(); + + private IContentService ContentService => GetRequiredService(); + + private IDocumentRepository DocumentRepository => GetRequiredService(); + + private IContentIndexingService ContentIndexingService => GetRequiredService(); + + private IContent _rootDocument = null!; + + protected override void CustomTestSetup(IUmbracoBuilder builder) + { + base.CustomTestSetup(builder); + + builder.AddExamineSearchProviderForTest(); + + builder.AddSearchCore(); + + builder.Services.AddUnique(); + builder.Services.AddUnique(); + builder.AddNotificationHandler(); + + // the core ConfigureBuilderAttribute won't execute from other assemblies at the moment, so this is a workaround + var testType = Type.GetType(TestContext.CurrentContext.Test.ClassName!); + if (testType is not null) + { + MethodInfo? methodInfo = testType.GetMethod(TestContext.CurrentContext.Test.Name); + if (methodInfo is not null) + { + foreach (ConfigureUmbracoBuilderAttribute attribute in methodInfo.GetCustomAttributes(typeof(ConfigureUmbracoBuilderAttribute), true).OfType()) + { + attribute.Execute(builder, testType); + } + } + } + } + + [TestCase(true)] + [TestCase(false)] + public async Task RebuildWithoutDatabasePersistsDocumentsToDatabase(bool publish) + { + await CreateContentWithPersistence(publish); + var indexAlias = GetIndexAlias(publish); + var changeStrategy = GetStrategy(publish); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify document exists in database (from initial indexing) + Document? rootDocInitial = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(rootDocInitial, Is.Not.Null); + + // Delete the database entry to simulate a fresh state (e.g., after migration or database restore) + await DocumentRepository.DeleteAsync(_rootDocument.Key, changeStrategy); + + // Verify document no longer exists in database + Document? rootDocBefore = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(rootDocBefore, Is.Null); + } + + // Trigger rebuild + await WaitForIndexing(indexAlias, () => + { + ContentIndexingService.Rebuild(indexAlias); + return Task.CompletedTask; + }); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify document now exists in database again (rebuilt from content) + Document? rootDocAfter = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(rootDocAfter, Is.Not.Null); + Assert.That(FieldsContainText(rootDocAfter!.Fields, "Root Document"), Is.True); + } + } + + [TestCase(true)] + [TestCase(false)] + public async Task RebuildWithDatabaseDoesNotPersistDocumentsToDatabase(bool publish) + { + await CreateContentWithPersistence(publish); + var indexAlias = GetIndexAlias(publish); + var changeStrategy = GetStrategy(publish); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify document exists in database (from initial indexing) + Document? rootDocInitial = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(rootDocInitial, Is.Not.Null); + + // Delete the database entry to simulate a fresh state (e.g., after migration or database restore) + await DocumentRepository.DeleteAsync(_rootDocument.Key, changeStrategy); + + // Verify document no longer exists in database + Document? rootDocBefore = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(rootDocBefore, Is.Null); + } + + // Trigger rebuild with useDatabase=true (should NOT persist to database) + await WaitForIndexing(indexAlias, () => + { + ContentIndexingService.Rebuild(indexAlias, useDatabase: true); + return Task.CompletedTask; + }); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify document still does NOT exist in database (rebuild with useDatabase=true should not persist) + Document? rootDocAfter = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(rootDocAfter, Is.Null); + } + } + + [TestCase(true)] + [TestCase(false)] + public async Task RebuildUsesExistingDatabaseEntries(bool publish) + { + await CreateContentWithPersistence(publish); + var indexAlias = GetIndexAlias(publish); + var changeStrategy = GetStrategy(publish); + + IndexField[] rootFieldsBefore; + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify document exists in database + Document? rootDocBefore = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(rootDocBefore, Is.Not.Null); + + rootFieldsBefore = rootDocBefore!.Fields; + } + + // Trigger rebuild + await WaitForIndexing(indexAlias, () => + { + ContentIndexingService.Rebuild(indexAlias); + return Task.CompletedTask; + }); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify document still exists and fields are the same (fetched from DB, not recalculated) + Document? rootDocAfter = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + + Assert.That(rootDocAfter, Is.Not.Null); + Assert.That(rootDocAfter.Fields.Length, Is.GreaterThan(0)); + Assert.That(rootDocAfter.Fields.Length, Is.EqualTo(rootFieldsBefore.Length)); + } + } + + [TestCase(true)] + [TestCase(false)] + public async Task RebuildFromMemory_RecalculatesFields(bool publish) + { + await CreateContentWithPersistence(publish); + var indexAlias = GetIndexAlias(publish); + var changeStrategy = GetStrategy(publish); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify document exists in database with original name + Document? rootDocBefore = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(rootDocBefore, Is.Not.Null); + Assert.That(FieldsContainText(rootDocBefore!.Fields, "Root Document"), Is.True); + } + + // Update the content name directly (simulating a change) + _rootDocument.Name = "Updated Document Name"; + await WaitForIndexing(indexAlias, () => + { + ContentService.Save(_rootDocument); + if (publish) + { + ContentService.Publish(_rootDocument, ["*"]); + } + + return Task.CompletedTask; + }); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify the database now has the updated name + Document? rootDocUpdated = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(rootDocUpdated, Is.Not.Null); + Assert.That(FieldsContainText(rootDocUpdated!.Fields, "Updated Document Name"), Is.True); + + await DocumentRepository.DeleteAsync(_rootDocument.Key, changeStrategy); + } + + // Trigger rebuild with useDatabase=false (should recalculate, not use stale DB data) + await WaitForIndexing(indexAlias, () => + { + ContentIndexingService.Rebuild(indexAlias, useDatabase: false); + return Task.CompletedTask; + }); + + using (ScopeProvider.CreateScope(autoComplete: true)) + { + // Verify the database now has fresh recalculated fields with the actual content name + Document? rootDocAfter = await DocumentRepository.GetAsync(_rootDocument.Key, changeStrategy); + Assert.That(rootDocAfter, Is.Not.Null); + Assert.That(FieldsContainText(rootDocAfter!.Fields, "Updated Document Name"), Is.True); + Assert.That(FieldsContainText(rootDocAfter.Fields, "Root Document"), Is.False); + } + } + + private string GetIndexAlias(bool publish) => publish ? Constants.IndexAliases.PublishedContent : Constants.IndexAliases.DraftContent; + + private string GetStrategy(bool publish) => publish ? "PublishedContentChangeStrategy" : "DraftContentChangeStrategy"; + + /// + /// Creates content and waits for indexing (so database persistence happens). + /// + private async Task CreateContentWithPersistence(bool publish) + { + await PackageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); + + // Create content type + ContentTypeCreateModel contentTypeCreateModel = ContentTypeEditingBuilder.CreateSimpleContentType( + "testType", + "Test Type"); + Attempt contentTypeAttempt = await ContentTypeEditingService.CreateAsync( + contentTypeCreateModel, + Umbraco.Cms.Core.Constants.Security.SuperUserKey); + Assert.That(contentTypeAttempt.Success, Is.True); + IContentType contentType = contentTypeAttempt.Result!; + + var indexAlias = GetIndexAlias(publish); + + // Create root content with indexing + ContentCreateModel rootCreateModel = ContentEditingBuilder.CreateSimpleContent(contentType.Key, "Root Document"); + await WaitForIndexing(indexAlias, async () => + { + Attempt createRootResult = await ContentEditingService.CreateAsync(rootCreateModel, Umbraco.Cms.Core.Constants.Security.SuperUserKey); + Assert.That(createRootResult.Success, Is.True); + _rootDocument = createRootResult.Result.Content!; + + if (publish) + { + ContentService.Publish(_rootDocument, ["*"]); + } + }); + } + + protected async Task WaitForIndexing(string indexAlias, Func indexUpdatingAction) + { + var index = (LuceneIndex)GetRequiredService().GetIndex(indexAlias); + index.IndexCommitted += IndexCommited; + + var hasDoneAction = false; + + var stopWatch = Stopwatch.StartNew(); + + while (_indexingComplete is false) + { + if (hasDoneAction is false) + { + await indexUpdatingAction(); + hasDoneAction = true; + } + + if (stopWatch.ElapsedMilliseconds > 10000) + { + throw new TimeoutException("Indexing timed out"); + } + + await Task.Delay(250); + } + + _indexingComplete = false; + index.IndexCommitted -= IndexCommited; + } + + private void IndexCommited(object? sender, EventArgs e) + { + _indexingComplete = true; + } + + private static bool FieldsContainText(IndexField[] fields, string text) + => fields.Any(f => + (f.Value.Texts?.Any(t => t.Contains(text)) == true) || + (f.Value.TextsR1?.Any(t => t.Contains(text)) == true) || + (f.Value.TextsR2?.Any(t => t.Contains(text)) == true) || + (f.Value.TextsR3?.Any(t => t.Contains(text)) == true) || + (f.Value.Keywords?.Any(k => k.Contains(text)) == true)); +} diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/SearchService/InvariantDocumentTests.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/SearchService/InvariantDocumentTests.cs index 4293df9..25e16bf 100644 --- a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/SearchService/InvariantDocumentTests.cs +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/SearchService/InvariantDocumentTests.cs @@ -31,6 +31,23 @@ public async Task CanSearchName(bool publish) Assert.That(results.Documents.First().Id, Is.EqualTo(RootKey)); } + [TestCase(true)] + [TestCase(false)] + public async Task CanNotSearchDeletedName(bool publish) + { + await WaitForIndexing(GetIndexAlias(true), () => + { + IContent? content = ContentService.GetById(RootKey); + ContentService.Delete(content!); + return Task.CompletedTask; + }); + + var indexAlias = GetIndexAlias(publish); + + SearchResult results = await Searcher.SearchAsync(indexAlias, "Test", null, null, null, null, null, null, 0, 100); + Assert.That(results.Total, Is.EqualTo(0)); + } + [TestCase(true)] [TestCase(false)] public async Task CanSearchTextProperty(bool publish) diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/SearchService/SearcherTestBase.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/SearchService/SearcherTestBase.cs index 438c839..7f461e0 100644 --- a/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/SearchService/SearcherTestBase.cs +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/ContentTests/SearchService/SearcherTestBase.cs @@ -1,8 +1,17 @@ -using ISearcher = Umbraco.Cms.Search.Core.Services.ISearcher; +using NUnit.Framework; +using Umbraco.Cms.Core; +using ISearcher = Umbraco.Cms.Search.Core.Services.ISearcher; namespace Umbraco.Test.Search.Examine.Integration.Tests.ContentTests.SearchService; public abstract class SearcherTestBase : TestBase { protected ISearcher Searcher => GetRequiredService(); + + [SetUp] + public async Task RunMigrations() + { + await PackageMigrationRunner.RunPackageMigrationsIfPendingAsync("Umbraco CMS Search").ConfigureAwait(false); + Assert.That(RuntimeState.Level, Is.EqualTo(RuntimeLevel.Run)); + } } diff --git a/src/Umbraco.Test.Search.Examine.Integration/Tests/TestBase.cs b/src/Umbraco.Test.Search.Examine.Integration/Tests/TestBase.cs index 57d5890..4d770b6 100644 --- a/src/Umbraco.Test.Search.Examine.Integration/Tests/TestBase.cs +++ b/src/Umbraco.Test.Search.Examine.Integration/Tests/TestBase.cs @@ -8,6 +8,7 @@ using Umbraco.Cms.Core.Notifications; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Core.Sync; +using Umbraco.Cms.Infrastructure.Install; using Umbraco.Cms.Search.Core.DependencyInjection; using Umbraco.Cms.Search.Core.NotificationHandlers; using Umbraco.Cms.Tests.Common.Testing; @@ -41,6 +42,10 @@ public abstract class TestBase : UmbracoIntegrationTest protected ILanguageService LanguageService => GetRequiredService(); + protected PackageMigrationRunner PackageMigrationRunner => GetRequiredService(); + + protected IRuntimeState RuntimeState => GetRequiredService(); + protected void SaveAndPublish(IContent content) { diff --git a/src/Umbraco.Test.Search.Integration/Tests/ContentIndexingServiceTestsBase.cs b/src/Umbraco.Test.Search.Integration/Tests/ContentIndexingServiceTestsBase.cs index d0b4b3f..1b5b471 100644 --- a/src/Umbraco.Test.Search.Integration/Tests/ContentIndexingServiceTestsBase.cs +++ b/src/Umbraco.Test.Search.Integration/Tests/ContentIndexingServiceTestsBase.cs @@ -33,7 +33,7 @@ public Task HandleAsync(IEnumerable indexInfos, IEnumerable throw new NotImplementedException(); public List> HandledIndexInfos { get; } = new(); diff --git a/src/Umbraco.Test.Search.Integration/Tests/SearcherResolverTests.cs b/src/Umbraco.Test.Search.Integration/Tests/SearcherResolverTests.cs index 91b20d4..f3afacf 100644 --- a/src/Umbraco.Test.Search.Integration/Tests/SearcherResolverTests.cs +++ b/src/Umbraco.Test.Search.Integration/Tests/SearcherResolverTests.cs @@ -101,7 +101,7 @@ private class TestContentChangeStrategy : IContentChangeStrategy public Task HandleAsync(IEnumerable indexInfos, IEnumerable changes, CancellationToken cancellationToken) => throw new NotImplementedException(); - public Task RebuildAsync(IndexInfo indexInfo, CancellationToken cancellationToken) + public Task RebuildAsync(IndexInfo indexInfo, CancellationToken cancellationToken, bool useDatabase = false) => throw new NotImplementedException(); }