Merge pull request #6587 from s1ck/idx-inc-scanning-relationships-importer

[IdxInc] Project inverse list in native projection

Author: s1ck

core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java

@@ -25,8 +25,8 @@
 import org.neo4j.gds.core.loading.CSRGraphStore;
 import org.neo4j.gds.core.loading.Capabilities;
 import org.neo4j.gds.core.loading.GraphStoreBuilder;
-import org.neo4j.gds.core.loading.IdMapAndProperties;
-import org.neo4j.gds.core.loading.RelationshipsAndProperties;
+import org.neo4j.gds.core.loading.NodeImportResult;
+import org.neo4j.gds.core.loading.RelationshipImportResult;
 import org.neo4j.gds.mem.MemoryUsage;
 
 import static org.neo4j.gds.utils.StringFormatting.formatWithLocale;
@@ -43,17 +43,17 @@ public CSRGraphStoreFactory(
     }
 
     protected CSRGraphStore createGraphStore(
-        IdMapAndProperties idMapAndProperties,
-        RelationshipsAndProperties relationshipsAndProperties
+        NodeImportResult nodeImportResult,
+        RelationshipImportResult relationshipImportResult
     ) {
         return new GraphStoreBuilder()
             .databaseId(DatabaseId.of(loadingContext.graphDatabaseService()))
             .capabilities(capabilities)
-            .schema(computeGraphSchema(idMapAndProperties, relationshipsAndProperties))
-            .nodes(idMapAndProperties.idMap())
-            .nodePropertyStore(idMapAndProperties.properties())
-            .relationships(relationshipsAndProperties.relationships())
-            .relationshipPropertyStores(relationshipsAndProperties.properties())
+            .schema(computeGraphSchema(nodeImportResult, relationshipImportResult))
+            .nodes(nodeImportResult.idMap())
+            .nodePropertyStore(nodeImportResult.properties())
+            .relationships(relationshipImportResult.relationships())
+            .relationshipPropertyStores(relationshipImportResult.properties())
             .concurrency(graphProjectConfig.readConcurrency())
             .build();
     }
@@ -64,5 +64,8 @@ protected void logLoadingSummary(GraphStore graphStore) {
         progressTracker.logInfo(formatWithLocale("Actual memory usage of the loaded graph: %s", memoryUsage));
     }
 
-    protected abstract GraphSchema computeGraphSchema(IdMapAndProperties idMapAndProperties, RelationshipsAndProperties relationshipsAndProperties);
+    protected abstract GraphSchema computeGraphSchema(
+        NodeImportResult nodeImportResult,
+        RelationshipImportResult relationshipImportResult
+    );
 }

core/src/main/java/org/neo4j/gds/api/Relationships.java

@@ -19,6 +19,7 @@
  */
 package org.neo4j.gds.api;
 
+import org.immutables.value.Value;
 import org.jetbrains.annotations.Nullable;
 import org.neo4j.gds.annotation.ValueClass;
 
@@ -52,7 +53,7 @@ static Relationships of(
         @Nullable AdjacencyProperties adjacencyProperties,
         double defaultPropertyValue
     ) {
-        Topology topology = ImmutableTopology.of(
+        var topology = ImmutableTopology.of(
             adjacencyList,
             relationshipCount,
             isMultiGraph
@@ -72,6 +73,9 @@ static Relationships of(
     interface Topology {
         AdjacencyList adjacencyList();
 
+        @Value.Parameter(false)
+        Optional<AdjacencyList> inverseAdjacencyList();
+
         long elementCount();
 
         boolean isMultiGraph();
@@ -82,6 +86,11 @@ public AdjacencyList adjacencyList() {
                 return AdjacencyList.EMPTY;
             }
 
+            @Override
+            public Optional<AdjacencyList> inverseAdjacencyList() {
+                return Optional.empty();
+            }
+
             @Override
             public long elementCount() {
                 return 0;

core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java

@@ -231,14 +231,14 @@ public static RelationshipPropertyStore buildRelationshipPropertyStore(
     }
 
     public static GraphSchema computeGraphSchema(
-        IdMapAndProperties idMapAndProperties,
+        NodeImportResult nodeImportResult,
         Function<NodeLabel, Collection<String>> propertiesByLabel,
-        RelationshipsAndProperties relationshipsAndProperties
+        RelationshipImportResult relationshipImportResult
     ) {
-        var properties = idMapAndProperties.properties().properties();
+        var properties = nodeImportResult.properties().properties();
 
         var nodeSchema = NodeSchema.empty();
-        for (var label : idMapAndProperties.idMap().availableNodeLabels()) {
+        for (var label : nodeImportResult.idMap().availableNodeLabels()) {
             var entry = nodeSchema.getOrCreateLabel(label);
             for (var propertyKey : propertiesByLabel.apply(label)) {
                 entry.addProperty(
@@ -247,15 +247,15 @@ public static GraphSchema computeGraphSchema(
                 );
             }
         }
-        idMapAndProperties.idMap().availableNodeLabels().forEach(nodeSchema::getOrCreateLabel);
+        nodeImportResult.idMap().availableNodeLabels().forEach(nodeSchema::getOrCreateLabel);
 
         var relationshipSchema = RelationshipSchema.empty();
-        relationshipsAndProperties
+        relationshipImportResult
             .properties()
             .forEach((relType, propertyStore) -> {
                 var entry = relationshipSchema.getOrCreateRelationshipType(
                     relType,
-                    relationshipsAndProperties.directions().get(relType)
+                    relationshipImportResult.directions().get(relType)
                 );
 
                 propertyStore
@@ -266,13 +266,13 @@ public static GraphSchema computeGraphSchema(
                     ));
             });
 
-        relationshipsAndProperties
+        relationshipImportResult
             .relationships()
             .keySet()
             .forEach(type -> {
                 relationshipSchema.getOrCreateRelationshipType(
                     type,
-                    relationshipsAndProperties.directions().get(type)
+                    relationshipImportResult.directions().get(type)
                 );
             });
 

core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java

@@ -120,12 +120,12 @@ public GraphDimensions estimationDimensions() {
 
     @Override
     protected GraphSchema computeGraphSchema(
-        IdMapAndProperties idMapAndProperties, RelationshipsAndProperties relationshipsAndProperties
+        NodeImportResult nodeImportResult, RelationshipImportResult relationshipImportResult
     ) {
         return CSRGraphStoreUtil.computeGraphSchema(
-            idMapAndProperties,
-            (__) -> idMapAndProperties.properties().keySet(),
-            relationshipsAndProperties
+            nodeImportResult,
+            (__) -> nodeImportResult.properties().keySet(),
+            relationshipImportResult
         );
     }
 

core/src/main/java/org/neo4j/gds/core/loading/CypherNodeLoader.java

@@ -34,7 +34,7 @@
 import java.util.stream.Collectors;
 
 @Value.Enclosing
-class CypherNodeLoader extends CypherRecordLoader<IdMapAndProperties> {
+class CypherNodeLoader extends CypherRecordLoader<NodeImportResult> {
 
     private final long nodeCount;
     private final ProgressTracker progressTracker;
@@ -97,13 +97,13 @@ void updateCounts(BatchLoadResult result) {
     }
 
     @Override
-    IdMapAndProperties result() {
+    NodeImportResult result() {
         var idMapAndProperties = nodesBuilder.build(highestNodeId);
         var idMap = idMapAndProperties.idMap();
         var nodeProperties = idMapAndProperties.nodeProperties().orElseGet(Map::of);
         var nodePropertiesWithPropertyMappings = propertiesWithPropertyMappings(nodeProperties);
 
-        return IdMapAndProperties.of(idMap, nodePropertiesWithPropertyMappings);
+        return NodeImportResult.of(idMap, nodePropertiesWithPropertyMappings);
     }
 
     @Override

core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java

@@ -45,7 +45,7 @@
 import java.util.stream.Collectors;
 
 @Value.Enclosing
-class CypherRelationshipLoader extends CypherRecordLoader<RelationshipsAndProperties> {
+class CypherRelationshipLoader extends CypherRecordLoader<RelationshipImportResult> {
 
     private final IdMap idMap;
     private final Context loaderContext;
@@ -133,16 +133,16 @@ BatchLoadResult loadSingleBatch(InternalTransaction tx, int bufferSize) {
     void updateCounts(BatchLoadResult result) {}
 
     @Override
-    RelationshipsAndProperties result() {
+    RelationshipImportResult result() {
         var relationshipsByType = loaderContext.relationshipBuildersByType.entrySet().stream().collect(Collectors.toMap(
             entry -> {
                 var projection = projectionBuilder.type(entry.getKey().name).build();
-                return RelationshipsAndProperties.RelationshipTypeAndProjection.of(entry.getKey(), projection);
+                return RelationshipImportResult.RelationshipTypeAndProjection.of(entry.getKey(), projection);
             },
             entry -> entry.getValue().buildAll()
         ));
 
-        return RelationshipsAndProperties.of(relationshipsByType);
+        return RelationshipImportResult.of(relationshipsByType);
     }
 
     @Override

core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java

@@ -227,12 +227,12 @@ protected ProgressTracker initProgressTracker() {
 
     @Override
     protected GraphSchema computeGraphSchema(
-        IdMapAndProperties idMapAndProperties, RelationshipsAndProperties relationshipsAndProperties
+        NodeImportResult nodeImportResult, RelationshipImportResult relationshipImportResult
     ) {
         return CSRGraphStoreUtil.computeGraphSchema(
-            idMapAndProperties,
+            nodeImportResult,
             (label) -> storeConfig.nodeProjections().projections().get(label).properties().propertyKeys(),
-            relationshipsAndProperties
+            relationshipImportResult
         );
     }
 
@@ -243,8 +243,8 @@ public CSRGraphStore build() {
         int concurrency = graphProjectConfig.readConcurrency();
         try {
             progressTracker.beginSubTask();
-            IdMapAndProperties nodes = loadNodes(concurrency);
-            RelationshipsAndProperties relationships = loadRelationships(nodes.idMap(), concurrency);
+            NodeImportResult nodes = loadNodes(concurrency);
+            RelationshipImportResult relationships = loadRelationships(nodes.idMap(), concurrency);
             CSRGraphStore graphStore = createGraphStore(nodes, relationships);
 
             logLoadingSummary(graphStore);
@@ -255,7 +255,7 @@ public CSRGraphStore build() {
         }
     }
 
-    private IdMapAndProperties loadNodes(int concurrency) {
+    private NodeImportResult loadNodes(int concurrency) {
         var scanningNodesImporter = new ScanningNodesImporterBuilder()
             .concurrency(concurrency)
             .graphProjectConfig(graphProjectConfig)
@@ -272,7 +272,7 @@ private IdMapAndProperties loadNodes(int concurrency) {
         }
     }
 
-    private RelationshipsAndProperties loadRelationships(IdMap idMap, int concurrency) {
+    private RelationshipImportResult loadRelationships(IdMap idMap, int concurrency) {
         var scanningRelationshipsImporter = new ScanningRelationshipsImporterBuilder()
             .idMap(idMap)
             .graphProjectConfig(graphProjectConfig)

…gds/core/loading/IdMapAndProperties.java …j/gds/core/loading/NodeImportResult.javacore/src/main/java/org/neo4j/gds/core/loading/IdMapAndProperties.java renamed to core/src/main/java/org/neo4j/gds/core/loading/NodeImportResult.java core/src/main/java/org/neo4j/gds/core/loading/IdMapAndProperties.java renamed to core/src/main/java/org/neo4j/gds/core/loading/NodeImportResult.java

@@ -30,13 +30,13 @@
 import java.util.Map;
 
 @ValueClass
-public interface IdMapAndProperties {
+public interface NodeImportResult {
 
     IdMap idMap();
 
     NodePropertyStore properties();
 
-    static IdMapAndProperties of(IdMap idMap, Map<PropertyMapping, NodePropertyValues> properties) {
+    static NodeImportResult of(IdMap idMap, Map<PropertyMapping, NodePropertyValues> properties) {
         NodePropertyStore.Builder builder = NodePropertyStore.builder();
         properties.forEach((mapping, nodeProperties) -> builder.putProperty(
             mapping.propertyKey(),
@@ -49,6 +49,6 @@ static IdMapAndProperties of(IdMap idMap, Map<PropertyMapping, NodePropertyValue
                     : nodeProperties.valueType().fallbackValue()
             )
         ));
-        return ImmutableIdMapAndProperties.of(idMap, builder.build());
+        return ImmutableNodeImportResult.of(idMap, builder.build());
     }
 }