Skip to content

Commit 4956def

Browse files
vladakvladak
authored
make sure all IndexSearcher instances use an executor (#4008)
1 parent c3b0c92 commit 4956def

File tree

9 files changed

+120
-61
lines changed

9 files changed

+120
-61
lines changed

‎opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/IndexSearcherFactory.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
22+
*/
23+
package org.opengrok.indexer.configuration;
24+
25+
import org.apache.lucene.index.IndexReader;
26+
import org.apache.lucene.search.IndexSearcher;
27+
import org.apache.lucene.search.SearcherFactory;
28+
29+
/**
30+
* Factory for IndexSearcher objects with search executor.
31+
*/
32+
public class IndexSearcherFactory extends SearcherFactory {
33+
public IndexSearcher newSearcher(IndexReader reader) {
34+
return newSearcher(reader, null);
35+
}
36+
37+
@Override
38+
public IndexSearcher newSearcher(IndexReader reader, IndexReader prev) {
39+
// The previous IndexReader is not used here.
40+
return new IndexSearcher(reader, RuntimeEnvironment.getInstance().getSearchExecutor());
41+
}
42+
}

‎opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java‎

Lines changed: 34 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,9 @@ public final class RuntimeEnvironment {
104104
private final LazilyInstantiate<ExecutorService> lzRevisionExecutor;
105105
private static final RuntimeEnvironment instance = new RuntimeEnvironment();
106106

107+
private final LazilyInstantiate<SuperIndexSearcherFactory> lzSuperIndexSearcherFactory;
108+
private final LazilyInstantiate<IndexSearcherFactory> lzIndexSearcherFactory;
109+
107110
private final Map<Project, List<RepositoryInfo>> repository_map = new ConcurrentHashMap<>();
108111
private final Map<String, SearcherManager> searcherManagerMap = new ConcurrentHashMap<>();
109112

@@ -155,10 +158,11 @@ private RuntimeEnvironment() {
155158
configuration = new Configuration();
156159
configLock = new CloseableReentrantReadWriteLock();
157160
watchDog = new WatchDogService();
158-
lzIndexerParallelizer = LazilyInstantiate.using(() ->
159-
new IndexerParallelizer(this));
161+
lzIndexerParallelizer = LazilyInstantiate.using(() -> new IndexerParallelizer(this));
160162
lzSearchExecutor = LazilyInstantiate.using(this::newSearchExecutor);
161163
lzRevisionExecutor = LazilyInstantiate.using(this::newRevisionExecutor);
164+
lzSuperIndexSearcherFactory = LazilyInstantiate.using(this::newSuperIndexSearcherFactory);
165+
lzIndexSearcherFactory = LazilyInstantiate.using(this::newIndexSearcherFactory);
162166
}
163167

164168
// Instance of authorization framework and its lock.
@@ -211,6 +215,22 @@ public void shutdownRevisionExecutor() throws InterruptedException {
211215
getRevisionExecutor().awaitTermination(getIndexerCommandTimeout(), TimeUnit.SECONDS);
212216
}
213217

218+
private SuperIndexSearcherFactory newSuperIndexSearcherFactory() {
219+
return new SuperIndexSearcherFactory();
220+
}
221+
222+
public SuperIndexSearcherFactory getSuperIndexSearcherFactory() {
223+
return lzSuperIndexSearcherFactory.get();
224+
}
225+
226+
private IndexSearcherFactory newIndexSearcherFactory() {
227+
return new IndexSearcherFactory();
228+
}
229+
230+
public IndexSearcherFactory getIndexSearcherFactory() {
231+
return lzIndexSearcherFactory.get();
232+
}
233+
214234
/**
215235
* Get the one and only instance of the RuntimeEnvironment.
216236
*
@@ -1792,9 +1812,9 @@ public void maybeRefreshIndexSearchers() {
17921812
* Get IndexSearcher for given project.
17931813
* Each IndexSearcher is born from a SearcherManager object. There is one SearcherManager for every project.
17941814
* This schema makes it possible to reuse IndexSearcher/IndexReader objects so the heavy lifting
1795-
* (esp. system calls) performed in FSDirectory and DirectoryReader happens only once for a project.
1796-
* The caller has to make sure that the IndexSearcher is returned back
1797-
* to the SearcherManager. This is done with returnIndexSearcher().
1815+
* (esp. system calls) performed in {@code FSDirectory} and {@code DirectoryReader} happens only once for a project.
1816+
* The caller has to make sure that the IndexSearcher is returned to the SearcherManager.
1817+
* This is done with {@code searcherManagerInstance.release(indexSearcherInstance);}
17981818
* The return of the IndexSearcher should happen only after the search result data are read fully.
17991819
*
18001820
* @param projectName project
@@ -1803,24 +1823,24 @@ public void maybeRefreshIndexSearchers() {
18031823
*/
18041824
@SuppressWarnings("java:S2095")
18051825
public SuperIndexSearcher getIndexSearcher(String projectName) throws IOException {
1806-
SearcherManager mgr = searcherManagerMap.get(projectName);
1807-
SuperIndexSearcher searcher;
18081826

1827+
SearcherManager mgr = searcherManagerMap.get(projectName);
18091828
if (mgr == null) {
18101829
File indexDir = new File(getDataRootPath(), IndexDatabase.INDEX_DIR);
18111830
Directory dir = FSDirectory.open(new File(indexDir, projectName).toPath());
1812-
mgr = new SearcherManager(dir, new ThreadpoolSearcherFactory());
1831+
mgr = new SearcherManager(dir, getSuperIndexSearcherFactory());
18131832
searcherManagerMap.put(projectName, mgr);
18141833
}
1815-
searcher = (SuperIndexSearcher) mgr.acquire();
1834+
1835+
SuperIndexSearcher searcher = (SuperIndexSearcher) mgr.acquire();
18161836
searcher.setSearcherManager(mgr);
18171837

18181838
return searcher;
18191839
}
18201840

18211841
/**
1822-
* After new configuration is put into place, the set of projects might
1823-
* change so we go through the SearcherManager objects and close those where
1842+
* After new configuration is put into place, the set of projects might change,
1843+
* so we go through the SearcherManager objects and close those where
18241844
* the corresponding project is no longer present.
18251845
*/
18261846
public void refreshSearcherManagerMap() {
@@ -1847,17 +1867,14 @@ public void refreshSearcherManagerMap() {
18471867
}
18481868

18491869
/**
1850-
* Return collection of IndexReader objects as MultiReader object
1851-
* for given list of projects.
1852-
* The caller is responsible for releasing the IndexSearcher objects
1853-
* so we add them to the map.
1870+
* Return collection of IndexReader objects as MultiReader object for given list of projects.
1871+
* The caller is responsible for releasing the IndexSearcher objects.
18541872
*
18551873
* @param projects list of projects
18561874
* @param searcherList each SuperIndexSearcher produced will be put into this list
18571875
* @return MultiReader for the projects
18581876
*/
1859-
public MultiReader getMultiReader(SortedSet<String> projects,
1860-
ArrayList<SuperIndexSearcher> searcherList) {
1877+
public MultiReader getMultiReader(SortedSet<String> projects, List<SuperIndexSearcher> searcherList) {
18611878

18621879
IndexReader[] subreaders = new IndexReader[projects.size()];
18631880
int ii = 0;

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/ThreadpoolSearcherFactory.java renamed to opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/SuperIndexSearcherFactory.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
*/
1919

2020
/*
21-
* Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
21+
* Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
2222
*/
2323
package org.opengrok.indexer.configuration;
2424

@@ -31,12 +31,11 @@
3131
* to make sure the searcher threads are constrained to single thread pool.
3232
* @author vkotal
3333
*/
34-
class ThreadpoolSearcherFactory extends SearcherFactory {
34+
class SuperIndexSearcherFactory extends SearcherFactory {
3535

3636
@Override
3737
public SuperIndexSearcher newSearcher(IndexReader r, IndexReader prev) {
3838
// The previous IndexReader is not used here.
3939
return new SuperIndexSearcher(r, RuntimeEnvironment.getInstance().getSearchExecutor());
4040
}
41-
4241
}

‎opengrok-indexer/src/main/java/org/opengrok/indexer/history/DirectoryHistoryReader.java‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
*/
1919

2020
/*
21-
* Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved.
21+
* Copyright (c) 2005, 2022, Oracle and/or its affiliates. All rights reserved.
2222
*/
2323
package org.opengrok.indexer.history;
2424

@@ -93,16 +93,17 @@ public class DirectoryHistoryReader {
9393
* @throws IOException when index cannot be accessed
9494
*/
9595
public DirectoryHistoryReader(String path) throws IOException {
96-
IndexReader ireader = null;
96+
IndexReader indexReader = null;
9797
try {
9898
// Prepare for index search.
9999
String srcRoot = RuntimeEnvironment.getInstance().getSourceRootPath();
100-
ireader = IndexDatabase.getIndexReader(path);
101-
if (ireader == null) {
102-
throw new IOException("Could not locate index database");
100+
indexReader = IndexDatabase.getIndexReader(path);
101+
if (indexReader == null) {
102+
throw new IOException(String.format("Could not locate index database for '%s'", path));
103103
}
104104
// The search results will be sorted by date.
105-
IndexSearcher searcher = new IndexSearcher(ireader);
105+
IndexSearcher searcher = RuntimeEnvironment.getInstance().
106+
getIndexSearcherFactory().newSearcher(indexReader);
106107
SortField sfield = new SortField(QueryBuilder.DATE, SortField.Type.STRING, true);
107108
Sort sort = new Sort(sfield);
108109
QueryParser qparser = new QueryParser(QueryBuilder.PATH, new CompatibleAnalyser());
@@ -130,19 +131,20 @@ public DirectoryHistoryReader(String path) throws IOException {
130131
try {
131132
cdate = DateTools.stringToDate(doc.get(QueryBuilder.DATE));
132133
} catch (java.text.ParseException ex) {
133-
LOGGER.log(Level.WARNING, String.format("Could not get date for %s", path), ex);
134+
LOGGER.log(Level.WARNING, String.format("Could not get date for '%s'", path), ex);
134135
cdate = new Date();
135136
}
136137
int ls = rpath.lastIndexOf('/');
137138
if (ls != -1) {
138139
String rparent = rpath.substring(0, ls);
139140
String rbase = rpath.substring(ls + 1);
140141
History hist = null;
142+
File f = new File(srcRoot + rparent, rbase);
141143
try {
142-
File f = new File(srcRoot + rparent, rbase);
143144
hist = HistoryGuru.getInstance().getHistory(f);
144145
} catch (HistoryException e) {
145-
LOGGER.log(Level.WARNING, "An error occurred while getting history reader", e);
146+
LOGGER.log(Level.WARNING,
147+
String.format("An error occurred while getting history reader for '%s'", f), e);
146148
}
147149
if (hist == null) {
148150
put(cdate, "", "-", "", rpath);
@@ -164,11 +166,12 @@ public DirectoryHistoryReader(String path) throws IOException {
164166
// into history object.
165167
history = new History(entries);
166168
} finally {
167-
if (ireader != null) {
169+
if (indexReader != null) {
168170
try {
169-
ireader.close();
171+
indexReader.close();
170172
} catch (Exception ex) {
171-
LOGGER.log(Level.WARNING, "An error occurred while closing reader", ex);
173+
LOGGER.log(Level.WARNING,
174+
String.format("An error occurred while closing index reader for '%s'", path), ex);
172175
}
173176
}
174177
}

‎opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexAnalysisSettingsAccessor.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919

2020
/*
21+
* Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
2122
* Copyright (c) 2018, 2019, Chris Fraire <cfraire@me.com>.
2223
*/
2324
package org.opengrok.indexer.index;
@@ -37,6 +38,7 @@
3738
import org.apache.lucene.search.Query;
3839
import org.apache.lucene.search.TopDocs;
3940
import org.opengrok.indexer.analysis.CompatibleAnalyser;
41+
import org.opengrok.indexer.configuration.RuntimeEnvironment;
4042
import org.opengrok.indexer.search.QueryBuilder;
4143

4244
/**
@@ -73,9 +75,9 @@ public IndexAnalysisSettings3 read(IndexReader reader) throws IOException {
7375
* @return a defined instance, which is empty if none could be found
7476
* @throws IOException if I/O error occurs while searching Lucene
7577
*/
76-
public IndexAnalysisSettings3[] read(IndexReader reader, int n)
77-
throws IOException {
78-
IndexSearcher searcher = new IndexSearcher(reader);
78+
public IndexAnalysisSettings3[] read(IndexReader reader, int n) throws IOException {
79+
80+
IndexSearcher searcher = RuntimeEnvironment.getInstance().getIndexSearcherFactory().newSearcher(reader);
7981
Query q;
8082
try {
8183
q = new QueryParser(QueryBuilder.OBJUID, new CompatibleAnalyser()).

‎opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1852,10 +1852,10 @@ public int getNumFiles() throws IOException {
18521852
* Get an indexReader for the Index database where a given file.
18531853
*
18541854
* @param path the file to get the database for
1855-
* @return The index database where the file should be located or null if it
1856-
* cannot be located.
1855+
* @return The index database where the file should be located or {@code null} if it cannot be located.
18571856
*/
18581857
@SuppressWarnings("java:S2095")
1858+
@Nullable
18591859
public static IndexReader getIndexReader(String path) {
18601860
IndexReader ret = null;
18611861

@@ -1939,7 +1939,7 @@ private static Document getDocument(String path, IndexReader indexReader) throws
19391939

19401940
Document doc;
19411941
Query q = new QueryBuilder().setPath(path).build();
1942-
IndexSearcher searcher = new IndexSearcher(indexReader);
1942+
IndexSearcher searcher = RuntimeEnvironment.getInstance().getIndexSearcherFactory().newSearcher(indexReader);
19431943
Statistics stat = new Statistics();
19441944
TopDocs top = searcher.search(q, 1);
19451945
stat.report(LOGGER, Level.FINEST, "search via getDocument() done",

‎opengrok-indexer/src/main/java/org/opengrok/indexer/index/NumLinesLOCAccessor.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919

2020
/*
21+
* Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
2122
* Copyright (c) 2020, Chris Fraire <cfraire@me.com>.
2223
*/
2324
package org.opengrok.indexer.index;
@@ -41,6 +42,7 @@
4142
import org.opengrok.indexer.analysis.AccumulatedNumLinesLOC;
4243
import org.opengrok.indexer.analysis.NullableNumLinesLOC;
4344
import org.opengrok.indexer.analysis.NumLinesLOC;
45+
import org.opengrok.indexer.configuration.RuntimeEnvironment;
4446
import org.opengrok.indexer.search.QueryBuilder;
4547

4648
import java.io.File;
@@ -87,16 +89,15 @@ public void store(IndexWriter writer, IndexReader reader,
8789
* @return a value indicating whether any defined number-of-lines and
8890
* lines-of-code were found
8991
*/
90-
public boolean register(NumLinesLOCAggregator countsAggregator, IndexReader reader)
91-
throws IOException {
92+
public boolean register(NumLinesLOCAggregator countsAggregator, IndexReader reader) throws IOException {
9293

9394
/*
9495
* Search for existing documents with any value of PATH. Those are
9596
* documents representing source code files, as opposed to source code
9697
* directories or other object data (e.g. IndexAnalysisSettings3), which
9798
* have no stored PATH.
9899
*/
99-
IndexSearcher searcher = new IndexSearcher(reader);
100+
IndexSearcher searcher = RuntimeEnvironment.getInstance().getIndexSearcherFactory().newSearcher(reader);
100101

101102
Query query;
102103
try {
@@ -138,7 +139,7 @@ private void storeIterative(IndexWriter writer, IndexReader reader,
138139
List<AccumulatedNumLinesLOC> counts, boolean isAggregatingDeltas) throws IOException {
139140

140141
// Search for existing documents with QueryBuilder.D.
141-
IndexSearcher searcher = new IndexSearcher(reader);
142+
IndexSearcher searcher = RuntimeEnvironment.getInstance().getIndexSearcherFactory().newSearcher(reader);
142143

143144
for (AccumulatedNumLinesLOC entry : counts) {
144145
Query query = new TermQuery(new Term(QueryBuilder.D, entry.getPath()));
@@ -205,7 +206,7 @@ private boolean processFileCounts(NumLinesLOCAggregator countsAggregator,
205206

206207
private DSearchResult newDSearch(IndexReader reader, int n) throws IOException {
207208
// Search for existing documents with QueryBuilder.D.
208-
IndexSearcher searcher = new IndexSearcher(reader);
209+
IndexSearcher searcher = RuntimeEnvironment.getInstance().getIndexSearcherFactory().newSearcher(reader);
209210
Query query;
210211
try {
211212
QueryParser parser = new QueryParser(QueryBuilder.D, new CompatibleAnalyser());

0 commit comments

Comments
 (0)