Condense progress tracking

Co-authored-by: Veselin Nikolov <veselin.nikolov@neotechnology.com>

Author: IoannisPanagiotas

algo/src/main/java/org/neo4j/gds/hdbscan/CondenseStep.java

@@ -23,12 +23,15 @@
 import org.neo4j.gds.collections.ha.HugeLongArray;
 import org.neo4j.gds.core.utils.paged.HugeAtomicBitSet;
 import org.neo4j.gds.core.utils.paged.HugeLongArrayQueue;
+import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker;
 
 class CondenseStep {
     private final long nodeCount;
+    private final ProgressTracker progressTracker;
 
-    CondenseStep(long nodeCount) {
+    CondenseStep(long nodeCount, ProgressTracker progressTracker) {
         this.nodeCount = nodeCount;
+        this.progressTracker = progressTracker;
     }
 
     CondensedTree condense(ClusterHierarchy clusterHierarchy, long minClusterSize) {
@@ -46,7 +49,7 @@ CondensedTree condense(ClusterHierarchy clusterHierarchy, long minClusterSize) {
         // After walking through the whole hierarchy and doing this we end up
         // with a much smaller tree with a small number of nodes,
         // each of which has data about how the size of the cluster at that node decreases over varying distance.
-
+        progressTracker.beginSubTask();
         var clusterHierarchyRoot = clusterHierarchy.root();
         var parent = HugeLongArray.newArray(clusterHierarchyRoot + 1);
         var lambda = HugeDoubleArray.newArray(clusterHierarchyRoot + 1);
@@ -100,7 +103,9 @@ CondensedTree condense(ClusterHierarchy clusterHierarchy, long minClusterSize) {
                 lambda.set(rightClusterId, fallingOutLambda);
                 size.set(rightClusterId - nodeCount, rightSize);
             }
+            progressTracker.logProgress();
         }
+        progressTracker.endSubTask();
 
         return new CondensedTree(currentCondensedRoot, parent, lambda, size, currentCondensedMaxClusterId, nodeCount);
     }

algo/src/main/java/org/neo4j/gds/hdbscan/HDBScan.java

@@ -68,7 +68,7 @@ public HugeLongArray compute() {
         var coreResult = computeCores(kdTree, nodeCount);
         var dualTreeMST = dualTreeMSTPhase(kdTree, coreResult);
         var clusterHierarchy = createClusterHierarchy(dualTreeMST);
-        var condenseStep = new CondenseStep(nodeCount);
+        var condenseStep = new CondenseStep(nodeCount,progressTracker);
         var condensedTree = condenseStep.condense(clusterHierarchy, minClusterSize);
         var labellingStep = new LabellingStep(condensedTree, nodeCount);
         return labellingStep.labels();

algo/src/main/java/org/neo4j/gds/hdbscan/HDBScanProgressTrackerCreator.java

@@ -29,7 +29,11 @@ static Task kdBuildingTask(String name, long nodeCount){
     }
 
     static Task hierarchyTask(String name, long nodeCount){
-        return Tasks.leaf(name,nodeCount-1);
+        return Tasks.leaf(name,nodeCount - 1);
+    }
+
+    static Task condenseTask(String name, long nodeCount){
+            return  Tasks.leaf(name,nodeCount - 1);
     }
 
 }

algo/src/test/java/org/neo4j/gds/hdbscan/ClusterHierarchyTest.java

@@ -154,11 +154,8 @@ void shouldLogProgress(){
         var progressTask = HDBScanProgressTrackerCreator.hierarchyTask("foo",3);
         var log = new GdsTestLog();
         var progressTracker = new TaskProgressTracker(progressTask, new LoggerForProgressTrackingAdapter(log), new Concurrency(1), EmptyTaskRegistryFactory.INSTANCE);
-        var clusterHierarchy = ClusterHierarchy.create(
-            3,
-            edges,
-            progressTracker
-        );
+
+        ClusterHierarchy.create(3, edges, progressTracker);
 
         Assertions.assertThat(log.getMessages(TestLog.INFO))
             .extracting(removingThreadId())

algo/src/test/java/org/neo4j/gds/hdbscan/CondenseStepTest.java

@@ -19,11 +19,21 @@
  */
 package org.neo4j.gds.hdbscan;
 
+import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.neo4j.gds.collections.ha.HugeDoubleArray;
 import org.neo4j.gds.collections.ha.HugeLongArray;
+import org.neo4j.gds.compat.TestLog;
+import org.neo4j.gds.core.concurrency.Concurrency;
+import org.neo4j.gds.core.utils.logging.LoggerForProgressTrackingAdapter;
+import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory;
+import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker;
+import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker;
+import org.neo4j.gds.logging.GdsTestLog;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.neo4j.gds.assertj.Extractors.removingThreadId;
+import static org.neo4j.gds.assertj.Extractors.replaceTimings;
 
 class CondenseStepTest {
 
@@ -38,7 +48,8 @@ void minClusterSizeTwo() {
 
         var clusterHierarchy = new ClusterHierarchy(root, left, right, lambda, size, nodeCount);
 
-        var condensedTree = new CondenseStep(nodeCount).condense(clusterHierarchy, 2L);
+        var condensedTree = new CondenseStep(nodeCount, ProgressTracker.NULL_TRACKER)
+            .condense(clusterHierarchy, 2L);
 
         assertThat(condensedTree.root()).isEqualTo(7L);
         assertThat(condensedTree.maximumClusterId()).isEqualTo(11L);
@@ -89,7 +100,8 @@ void minClusterSizeThree() {
 
         var clusterHierarchy = new ClusterHierarchy(root, left, right, lambda, size, nodeCount);
 
-        var condensedTree = new CondenseStep(nodeCount).condense(clusterHierarchy, 3L);
+        var condensedTree = new CondenseStep(nodeCount,ProgressTracker.NULL_TRACKER)
+            .condense(clusterHierarchy, 3L);
 
         assertThat(condensedTree.root()).isEqualTo(7L);
         assertThat(condensedTree.maximumClusterId()).isEqualTo(7L);
@@ -113,4 +125,34 @@ void minClusterSizeThree() {
         assertThat(condensedTree.fellOutOf(6L)).isEqualTo(7L);
         assertThat(condensedTree.lambda(6L)).isEqualTo(8d);
     }
+
+    @Test
+    void shouldLogProgress(){
+        var nodeCount = 7L;
+
+        var progressTask = HDBScanProgressTrackerCreator.condenseTask("condense",7);
+        var log = new GdsTestLog();
+        var progressTracker = new TaskProgressTracker(progressTask, new LoggerForProgressTrackingAdapter(log), new Concurrency(1), EmptyTaskRegistryFactory.INSTANCE);
+        var root = 12L;
+        var left = HugeLongArray.of(5, 4, 2, 9, 0, 11);
+        var right = HugeLongArray.of(6, 7, 3, 8, 1, 10);
+        var lambda = HugeDoubleArray.of(7d, 8d, 9d, 10d, 11d, 12d);
+        var size = HugeLongArray.of(2, 3, 2, 5, 2, 7);
+
+        var clusterHierarchy = new ClusterHierarchy(root, left, right, lambda, size, nodeCount);
+
+        new CondenseStep(nodeCount,progressTracker).condense(clusterHierarchy, 3L);
+
+        Assertions.assertThat(log.getMessages(TestLog.INFO))
+            .extracting(removingThreadId())
+            .extracting(replaceTimings())
+            .containsExactly(
+                "condense :: Start",
+                "condense 16%",
+                "condense 33%",
+                "condense 50%",
+                "condense 100%",
+                "condense :: Finished"
+            );
+    }
 }