Skip to content

Commit 81be931

Browse files
FlorentinDFlorentinD
authored
Merge pull request #10553 from soerenreichardt/export-command-queue
Export via batch descriptors
2 parents 75c2f4d + 2beefd7 commit 81be931

File tree

2 files changed

+178
-0
lines changed

2 files changed

+178
-0
lines changed

‎collections/src/main/java/org/neo4j/gds/collections/hsl/HugeSparseObjectList.java‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
/*
2+
* Copyright (c) "Neo4j"
3+
* Neo4j Sweden AB [http://neo4j.com]
4+
*
5+
* This file is part of Neo4j.
6+
*
7+
* Neo4j is free software: you can redistribute it and/or modify
8+
* it under the terms of the GNU General Public License as published by
9+
* the Free Software Foundation, either version 3 of the License, or
10+
* (at your option) any later version.
11+
*
12+
* This program is distributed in the hope that it will be useful,
13+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15+
* GNU General Public License for more details.
16+
*
17+
* You should have received a copy of the GNU General Public License
18+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
19+
*/
20+
package org.neo4j.gds.collections.hsl;
21+
22+
import org.neo4j.gds.collections.ArrayUtil;
23+
import org.neo4j.gds.collections.DrainingIterator;
24+
import org.neo4j.gds.collections.PageUtil;
25+
import org.neo4j.gds.mem.Estimate;
26+
27+
import java.lang.reflect.Array;
28+
import java.util.Arrays;
29+
import java.util.stream.Stream;
30+
31+
32+
public final class HugeSparseObjectList<E> {
33+
private static final int PAGE_SHIFT = 12;
34+
35+
private static final int PAGE_SIZE = 1 << PAGE_SHIFT;
36+
37+
private static final int PAGE_MASK = PAGE_SIZE - 1;
38+
39+
private static final long PAGE_SIZE_IN_BYTES = Estimate.sizeOfLongArray(PAGE_SIZE);
40+
private final Class<E> clazz;
41+
42+
private E[][] pages;
43+
44+
private final E defaultValue;
45+
46+
public HugeSparseObjectList(E defaultValue, long initialCapacity, Class<E> clazz) {
47+
this.clazz = clazz;
48+
int numPages = PageUtil.pageIndex(initialCapacity, PAGE_SHIFT);
49+
this.pages = (E[][]) Array.newInstance(clazz, numPages, PAGE_SIZE);
50+
this.defaultValue = defaultValue;
51+
}
52+
53+
public long capacity() {
54+
int numPages = pages.length;
55+
return ((long) numPages) << PAGE_SHIFT;
56+
}
57+
58+
public E get(long index) {
59+
int pageIndex = PageUtil.pageIndex(index, PAGE_SHIFT);
60+
int indexInPage = PageUtil.indexInPage(index, PAGE_MASK);
61+
if (pageIndex < pages.length) {
62+
E[] page = pages[pageIndex];
63+
if (page != null) {
64+
return page[indexInPage];
65+
}
66+
}
67+
return defaultValue;
68+
}
69+
70+
public boolean contains(long index) {
71+
int pageIndex = PageUtil.pageIndex(index, PAGE_SHIFT);
72+
if (pageIndex < pages.length) {
73+
E[] page = pages[pageIndex];
74+
if (page != null) {
75+
int indexInPage = PageUtil.indexInPage(index, PAGE_MASK);
76+
return !page[indexInPage].equals(defaultValue);
77+
}
78+
}
79+
return false;
80+
}
81+
82+
public DrainingIterator<E[]> drainingIterator() {
83+
return new DrainingIterator<>(pages, PAGE_SIZE);
84+
}
85+
86+
public void forAll(LongObjectConsumer<E> consumer) {
87+
E[][] pages = this.pages;
88+
for (int pageIndex = 0; pageIndex < pages.length; pageIndex++) {
89+
E[] page = pages[pageIndex];
90+
if (page == null) {
91+
continue;
92+
}
93+
for (int indexInPage = 0; indexInPage < page.length; indexInPage++) {
94+
E value = page[indexInPage];
95+
if (value.equals(defaultValue)) {
96+
continue;
97+
}
98+
long index = ((long) pageIndex << PAGE_SHIFT) | (long) indexInPage;
99+
consumer.consume(index, value);
100+
}
101+
}
102+
}
103+
104+
public void set(long index, E value) {
105+
int pageIndex = PageUtil.pageIndex(index, PAGE_SHIFT);
106+
int indexInPage = PageUtil.indexInPage(index, PAGE_MASK);
107+
getPage(pageIndex)[indexInPage] = value;
108+
}
109+
110+
public Stream<E> stream() {
111+
return Arrays.stream(this.pages).filter(obj -> !(obj == null)).flatMap(Arrays::stream);
112+
}
113+
114+
public boolean setIfAbsent(long index, E value) {
115+
int pageIndex = PageUtil.pageIndex(index, PAGE_SHIFT);
116+
int indexInPage = PageUtil.indexInPage(index, PAGE_MASK);
117+
E[] page = getPage(pageIndex);
118+
E currentValue = page[indexInPage];
119+
if (currentValue.equals(defaultValue)) {
120+
page[indexInPage] = value;
121+
return true;
122+
}
123+
return false;
124+
}
125+
126+
private E[] getPage(int pageIndex) {
127+
if (pageIndex >= pages.length) {
128+
grow(pageIndex + 1);
129+
}
130+
E[] page = pages[pageIndex];
131+
if (page == null) {
132+
page = allocateNewPage(pageIndex);
133+
}
134+
return page;
135+
}
136+
137+
private void grow(int minNewSize) {
138+
if (minNewSize <= pages.length) {
139+
return;
140+
}
141+
int newSize = ArrayUtil.oversize(minNewSize, Estimate.BYTES_OBJECT_REF);
142+
this.pages = Arrays.copyOf(this.pages, newSize);
143+
}
144+
145+
private E[] allocateNewPage(int pageIndex) {
146+
E[] page = (E[]) Array.newInstance(clazz, PAGE_SIZE);
147+
if (defaultValue != null) {
148+
Arrays.fill(page, defaultValue);
149+
}
150+
this.pages[pageIndex] = page;
151+
return page;
152+
}
153+
}

‎collections/src/main/java/org/neo4j/gds/collections/hsl/LongObjectConsumer.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Copyright (c) "Neo4j"
3+
* Neo4j Sweden AB [http://neo4j.com]
4+
*
5+
* This file is part of Neo4j.
6+
*
7+
* Neo4j is free software: you can redistribute it and/or modify
8+
* it under the terms of the GNU General Public License as published by
9+
* the Free Software Foundation, either version 3 of the License, or
10+
* (at your option) any later version.
11+
*
12+
* This program is distributed in the hope that it will be useful,
13+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15+
* GNU General Public License for more details.
16+
*
17+
* You should have received a copy of the GNU General Public License
18+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
19+
*/
20+
package org.neo4j.gds.collections.hsl;
21+
22+
@FunctionalInterface
23+
public interface LongObjectConsumer<E> {
24+
void consume(long index, E value);
25+
}

0 commit comments

Comments
 (0)