diff --git a/parsing.api/apichanges.xml b/parsing.api/apichanges.xml --- a/parsing.api/apichanges.xml +++ b/parsing.api/apichanges.xml @@ -110,6 +110,21 @@ + + + Extended the QuerySupport to allow composite queries. + + + + + +

+ Extended the QuerySupport to allow composite queries with bool operators. +

+ + + + Added QuerySupport.findDependentRoots to find out source roots depending on given source root. diff --git a/parsing.api/nbproject/project.properties b/parsing.api/nbproject/project.properties --- a/parsing.api/nbproject/project.properties +++ b/parsing.api/nbproject/project.properties @@ -3,7 +3,7 @@ javac.source=1.6 javadoc.apichanges=${basedir}/apichanges.xml javadoc.arch=${basedir}/arch.xml -spec.version.base=1.65.0 +spec.version.base=1.66.0 test.config.stableBTD.includes=**/*Test.class test.config.stableBTD.excludes=\ diff --git a/parsing.api/nbproject/project.xml b/parsing.api/nbproject/project.xml --- a/parsing.api/nbproject/project.xml +++ b/parsing.api/nbproject/project.xml @@ -110,7 +110,7 @@ 2 - 2.22 + 2.24 diff --git a/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/lucene/DocumentBasedIndexManager.java b/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/lucene/DocumentBasedIndexManager.java --- a/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/lucene/DocumentBasedIndexManager.java +++ b/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/lucene/DocumentBasedIndexManager.java @@ -53,7 +53,7 @@ import org.netbeans.modules.parsing.impl.indexing.ClusteredIndexables; import org.netbeans.modules.parsing.impl.indexing.Pair; import org.netbeans.modules.parsing.impl.indexing.PathRegistry; -import org.netbeans.modules.parsing.lucene.support.DocumentIndex; +import org.netbeans.modules.parsing.lucene.support.DocumentIndex2; import org.netbeans.modules.parsing.lucene.support.DocumentIndexCache; import org.netbeans.modules.parsing.lucene.support.IndexManager; import org.openide.util.Exceptions; @@ -72,8 +72,8 @@ @org.netbeans.api.annotations.common.SuppressWarnings( value="DMI_COLLECTION_OF_URLS", justification="URLs have never host part") - private final Map> indexes = - new HashMap> (); + private final Map> indexes = + new HashMap> (); //@GuardedBy("this") private boolean closed; @@ -98,13 +98,13 @@ @org.netbeans.api.annotations.common.SuppressWarnings( value="DMI_COLLECTION_OF_URLS", justification="URLs have never host part") - public synchronized DocumentIndex.Transactional getIndex (final URL root, final Mode mode) throws IOException { + public synchronized DocumentIndex2.Transactional getIndex (final URL root, final Mode mode) throws IOException { assert root != null; assert PathRegistry.noHostPart(root) : root; if (closed) { return null; } - Pair li = indexes.get(root); + Pair li = indexes.get(root); if (li == null) { try { switch (mode) { @@ -113,8 +113,8 @@ final File file = Utilities.toFile(root.toURI()); file.mkdir(); final DocumentIndexCache cache = ClusteredIndexables.createDocumentIndexCache(); - final DocumentIndex.Transactional index = IndexManager.createTransactionalDocumentIndex(file, cache); - li = Pair.of(index, cache); + final DocumentIndex2.Transactional index = (DocumentIndex2.Transactional) IndexManager.createTransactionalDocumentIndex(file, cache); + li = Pair.of(index, cache); indexes.put(root,li); break; @@ -125,8 +125,8 @@ String[] children; if (file.isDirectory() && (children=file.list())!= null && children.length > 0) { final DocumentIndexCache cache = ClusteredIndexables.createDocumentIndexCache(); - final DocumentIndex.Transactional index = IndexManager.createTransactionalDocumentIndex(file, cache); - li = Pair.of(index, cache); + final DocumentIndex2.Transactional index = (DocumentIndex2.Transactional) IndexManager.createTransactionalDocumentIndex(file, cache); + li = Pair.of(index, cache); indexes.put(root,li); } break; @@ -141,14 +141,14 @@ @CheckForNull public synchronized DocumentIndexCache getCache(@NonNull final URL root) { - final Pair entry = indexes.get(root); + final Pair entry = indexes.get(root); return entry == null ? null : entry.second; } @CheckForNull - public synchronized DocumentIndex.Transactional getIndex(@NonNull final DocumentIndexCache cache) { + public synchronized DocumentIndex2.Transactional getIndex(@NonNull final DocumentIndexCache cache) { Parameters.notNull("cache", cache); //NOI18N - for (Pair e : indexes.values()) { + for (Pair e : indexes.values()) { if (cache.equals(e.second)) { return e.first; } @@ -161,7 +161,7 @@ return; } closed = true; - for (Pair index : indexes.values()) { + for (Pair index : indexes.values()) { try { index.first.close(); } catch (IOException ioe) { diff --git a/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/lucene/LayeredDocumentIndex.java b/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/lucene/LayeredDocumentIndex.java --- a/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/lucene/LayeredDocumentIndex.java +++ b/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/lucene/LayeredDocumentIndex.java @@ -49,29 +49,35 @@ import java.util.Iterator; import java.util.Set; import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.search.Query; +import org.netbeans.api.annotations.common.CheckForNull; import org.netbeans.api.annotations.common.NonNull; +import org.netbeans.api.annotations.common.NullAllowed; import org.netbeans.modules.parsing.impl.indexing.Pair; -import org.netbeans.modules.parsing.lucene.support.DocumentIndex; +import org.netbeans.modules.parsing.lucene.support.DocumentIndex2; import org.netbeans.modules.parsing.lucene.support.Index.Status; import org.netbeans.modules.parsing.lucene.support.IndexDocument; import org.netbeans.modules.parsing.lucene.support.IndexManager; import org.netbeans.modules.parsing.lucene.support.Queries.QueryKind; import org.openide.util.Exceptions; import static org.netbeans.modules.parsing.impl.indexing.TransientUpdateSupport.isTransientUpdate; +import org.netbeans.modules.parsing.lucene.support.Convertor; +import org.netbeans.modules.parsing.lucene.support.Convertors; +import org.netbeans.modules.parsing.lucene.support.Queries; /** * * @author Tomas Zezula */ -public final class LayeredDocumentIndex implements DocumentIndex.Transactional { +public final class LayeredDocumentIndex implements DocumentIndex2.Transactional { - private final DocumentIndex.Transactional base; + private final DocumentIndex2.Transactional base; private final Set filter = new HashSet(); //@GuardedBy("this") - private DocumentIndex overlay; + private DocumentIndex2 overlay; - LayeredDocumentIndex(@NonNull final DocumentIndex.Transactional base) { + LayeredDocumentIndex(@NonNull final DocumentIndex2.Transactional base) { assert base != null; this.base = base; } @@ -117,7 +123,7 @@ @Override public void store(boolean optimize) throws IOException { if (isTransientUpdate()) { - final Pair> ovl = getOverlayIfExists(); + final Pair> ovl = getOverlayIfExists(); if (ovl.first != null) { ovl.first.store(false); } @@ -165,31 +171,54 @@ @Override public Collection query(String fieldName, String value, QueryKind kind, String... fieldsToLoad) throws IOException, InterruptedException { - final Collection br = base.query(fieldName, value, kind, fieldsToLoad); - final Pair> ovl = getOverlayIfExists(); - if (ovl.first == null) { - return ovl.second == null ? br : filter(br,ovl.second); - } else { - return new ProxyCollection( - ovl.second == null ? br : filter(br,ovl.second), - ovl.first.query(fieldName, value, kind, fieldsToLoad)); - } + return query(Queries.createQuery(fieldName, fieldName, value, kind), Convertors.identity(), fieldsToLoad); } @Override public Collection findByPrimaryKey(String primaryKeyValue, QueryKind kind, String... fieldsToLoad) throws IOException, InterruptedException { final Collection br = base.findByPrimaryKey(primaryKeyValue, kind, fieldsToLoad); - final Pair> ovl = getOverlayIfExists(); + final Pair> ovl = getOverlayIfExists(); if (ovl.first == null) { - return ovl.second == null ? br : filter(br, ovl.second); + return ovl.second == null ? br : Filter.filter(br, ovl.second); } else { return new ProxyCollection( - ovl.second == null ? br : filter(br,ovl.second), + ovl.second == null ? br : Filter.filter(br,ovl.second), ovl.first.findByPrimaryKey(primaryKeyValue, kind, fieldsToLoad)); } } @Override + @NonNull + public Collection query( + @NonNull final Query query, + @NonNull final Convertor convertor, + @NullAllowed final String... fieldsToLoad) throws IOException, InterruptedException { + final Pair> ovl = getOverlayIfExists(); + Convertor filterConvertor; + if (ovl.second == null) { + filterConvertor = convertor; + } else { + filterConvertor = Convertors.compose( + new Filter(ovl.second), + convertor); + } + final Collection br = base.query( + query, + filterConvertor, + fieldsToLoad); + + if (ovl.first == null) { + return br; + } else { + return new ProxyCollection( + br, + ovl.first.query(query, convertor, fieldsToLoad)); + } + } + + + + @Override public void markKeyDirty(String primaryKey) { addToFilter(primaryKey); base.markKeyDirty(primaryKey); @@ -220,17 +249,17 @@ } @NonNull - private synchronized DocumentIndex getOverlay() throws IOException { + private synchronized DocumentIndex2 getOverlay() throws IOException { if (overlay == null) { - overlay = IndexManager.createDocumentIndex(IndexManager.createMemoryIndex(new KeywordAnalyzer())); + overlay = (DocumentIndex2) IndexManager.createDocumentIndex(IndexManager.createMemoryIndex(new KeywordAnalyzer())); } return overlay; } @NonNull - private synchronized Pair> getOverlayIfExists() throws IOException { + private synchronized Pair> getOverlayIfExists() throws IOException { final Set f = filter.isEmpty() ? null : new HashSet(filter); - return Pair.>of(overlay,f); + return Pair.>of(overlay,f); } private synchronized void addToFilter(@NonNull final String primaryKey) { @@ -249,19 +278,47 @@ } } } - - @NonNull - private static Collection filter ( - @NonNull final Collection base, - @NonNull final Set filter) { - assert !filter.isEmpty(); - final Collection res = new ArrayList(base.size()); - for (IndexDocument doc : base) { - if (!filter.contains(doc.getPrimaryKey())) { - res.add(doc); + + private static final class Filter implements Convertor { + + private final Set filter; + + Filter(@NonNull final Set filter) { + assert filter != null; + assert !filter.isEmpty(); + this.filter = filter; + } + + @Override + @CheckForNull + public IndexDocument convert(@NullAllowed final IndexDocument p) { + return filtered(p, filter) ? + null : + p; + } + + @NonNull + static Collection filter ( + @NonNull final Collection base, + @NonNull final Set filter) { + assert !filter.isEmpty(); + final Collection res = new ArrayList(base.size()); + for (IndexDocument doc : base) { + if (!filtered(doc, filter)) { + res.add(doc); + } } + return res; } - return res; + + private static boolean filtered( + @NullAllowed final IndexDocument doc, + @NonNull final Set filter) { + return doc == null ? + true : + filter.contains(doc.getPrimaryKey()); + } + } private static class ProxyCollection extends AbstractCollection { diff --git a/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/lucene/LuceneIndexFactory.java b/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/lucene/LuceneIndexFactory.java --- a/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/lucene/LuceneIndexFactory.java +++ b/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/lucene/LuceneIndexFactory.java @@ -52,7 +52,7 @@ import org.netbeans.modules.parsing.impl.indexing.IndexFactoryImpl; import org.netbeans.modules.parsing.impl.indexing.TransientUpdateSupport; import org.netbeans.modules.parsing.impl.indexing.Util; -import org.netbeans.modules.parsing.lucene.support.DocumentIndex; +import org.netbeans.modules.parsing.lucene.support.DocumentIndex2; import org.netbeans.modules.parsing.lucene.support.DocumentIndexCache; import org.netbeans.modules.parsing.lucene.support.IndexDocument; import org.netbeans.modules.parsing.lucene.support.IndexManager; @@ -130,7 +130,7 @@ } LayeredDocumentIndex res = indexes.get(luceneIndexFolder); if (res == null) { - final DocumentIndex.Transactional base = DocumentBasedIndexManager.getDefault().getIndex( + final DocumentIndex2.Transactional base = DocumentBasedIndexManager.getDefault().getIndex( luceneIndexFolder, mode); if (base != null) { diff --git a/parsing.api/src/org/netbeans/modules/parsing/spi/indexing/support/QuerySupport.java b/parsing.api/src/org/netbeans/modules/parsing/spi/indexing/support/QuerySupport.java --- a/parsing.api/src/org/netbeans/modules/parsing/spi/indexing/support/QuerySupport.java +++ b/parsing.api/src/org/netbeans/modules/parsing/spi/indexing/support/QuerySupport.java @@ -47,6 +47,7 @@ import java.lang.ref.SoftReference; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -56,12 +57,15 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.Set; import java.util.concurrent.Callable; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.lucene.search.BooleanClause; import org.netbeans.api.annotations.common.CheckForNull; import org.netbeans.api.annotations.common.NonNull; +import org.netbeans.api.annotations.common.NullAllowed; import org.netbeans.api.java.classpath.ClassPath; import org.netbeans.api.java.classpath.GlobalPathRegistry; import org.netbeans.api.java.queries.SourceForBinaryQuery; @@ -83,8 +87,10 @@ import org.netbeans.modules.parsing.impl.indexing.friendapi.IndexingController; import org.netbeans.modules.parsing.impl.indexing.lucene.LayeredDocumentIndex; import org.netbeans.modules.parsing.impl.indexing.lucene.LuceneIndexFactory; -import org.netbeans.modules.parsing.lucene.support.DocumentIndex; +import org.netbeans.modules.parsing.lucene.support.Convertor; +import org.netbeans.modules.parsing.lucene.support.DocumentIndex2; import org.netbeans.modules.parsing.lucene.support.Index; +import org.netbeans.modules.parsing.lucene.support.IndexDocument; import org.netbeans.modules.parsing.lucene.support.Queries; import org.openide.filesystems.FileObject; import org.openide.filesystems.FileStateInvalidException; @@ -319,79 +325,18 @@ Parameters.notNull("fieldName", fieldName); //NOI18N Parameters.notNull("fieldValue", fieldValue); //NOI18N Parameters.notNull("kind", kind); //NOI18N - try { - return Utilities.runPriorityIO(new Callable>() { + return getQueryFactory().field(fieldName, fieldValue, kind).execute(fieldsToLoad); + } - @Override - public Collection call() throws Exception { - Iterable> indices = indexerQuery.getIndices(roots); - // check if there are stale indices - for (Pair pair : indices) { - final LayeredDocumentIndex index = pair.second; - final Collection staleFiles = index.getDirtyKeys(); - final boolean scanningThread = RunWhenScanFinishedSupport.isScanningThread(); - LOG.log( - Level.FINE, - "Index: {0}, staleFiles: {1}, scanning thread: {2}", //NOI18N - new Object[]{ - index, - staleFiles, - scanningThread - }); - if (!staleFiles.isEmpty() && !scanningThread) { - final URL root = pair.first; - LinkedList list = new LinkedList(); - for (String staleFile : staleFiles) { - try { - list.add(Util.resolveUrl(root, staleFile, false)); - } catch (MalformedURLException ex) { - LOG.log(Level.WARNING, null, ex); - } - } - TransientUpdateSupport.setTransientUpdate(true); - try { - RepositoryUpdater.getDefault().enforcedFileListUpdate(root,list); - } finally { - TransientUpdateSupport.setTransientUpdate(false); - } - } - } - final List result = new LinkedList(); - for (Pair pair : indices) { - final DocumentIndex index = pair.second; - final URL root = pair.first; - final Collection pr = index.query( - fieldName, - fieldValue, - translateQueryKind(kind), - fieldsToLoad); - if (LOG.isLoggable(Level.FINE)) { - LOG.fine("query(\"" + fieldName + "\", \"" + fieldValue + "\", " + kind + ", " + printFiledToLoad(fieldsToLoad) + ") invoked at " + getClass().getSimpleName() + "@" + Integer.toHexString(System.identityHashCode(this)) + "[indexer=" + indexerQuery.getIndexerId() + "]:"); //NOI18N - for (org.netbeans.modules.parsing.lucene.support.IndexDocument idi : pr) { - LOG.fine(" " + idi); //NOI18N - } - LOG.fine("----"); //NOI18N - } - for (org.netbeans.modules.parsing.lucene.support.IndexDocument di : pr) { - result.add(new IndexResult(di, root)); - } - } - return result; - } - }); - } catch (Index.IndexClosedException ice) { - if (Installer.isClosed()) { - return Collections.emptySet(); - } else { - throw ice; - } - } catch (IOException ioe) { - throw ioe; - } catch (RuntimeException re) { - throw re; - } catch (Exception ex) { - throw new IOException(ex); - } + + /** + * Returns the factory for creating composite queries. + * @return the {@link QuerySupport.Query.Factory} + * @since 1.66 + */ + @NonNull + public Query.Factory getQueryFactory() { + return new Query.Factory(this); } /** @@ -433,6 +378,192 @@ CASE_INSENSITIVE_CAMEL_CASE; } + /** + * An index query. + * @since 1.66 + */ + public static final class Query { + + private final QuerySupport qs; + private final org.apache.lucene.search.Query queryImpl; + + private Query( + @NonNull final QuerySupport qs, + @NonNull final org.apache.lucene.search.Query queryImpl) { + assert qs != null; + assert queryImpl != null; + this.qs = qs; + this.queryImpl = queryImpl; + } + + /** + * Factory for an index queries. + */ + public static final class Factory { + + private final QuerySupport qs; + + private Factory(@NonNull final QuerySupport qs) { + assert qs != null; + this.qs = qs; + } + + /** + * Creates a query for required field value. + * @param fieldName the name of the tested field + * @param fieldValue the required value of the tested field + * @param kind the kind of the query + * @return the newly created query + */ + @NonNull + public Query field( + @NonNull final String fieldName, + @NonNull final String fieldValue, + @NonNull final Kind kind) { + Parameters.notNull("fieldName", fieldName); //NOI18N + Parameters.notNull("fieldValue", fieldValue); //NOI18N + Parameters.notNull("kind", kind); //NOI18N + return new Query( + qs, + Queries.createQuery( + fieldName, + fieldName, + fieldValue, + translateQueryKind(kind))); + } + + /** + * Creates a boolean AND query. + * @param queries the queries to compose into the AND query + * @return the newly created AND query + */ + @NonNull + public Query and(@NonNull final Query...queries) { + Parameters.notNull("queries", queries); //NOI18N + final org.apache.lucene.search.BooleanQuery bq = new org.apache.lucene.search.BooleanQuery(); + for (Query q : queries) { + bq.add(new BooleanClause(q.queryImpl, org.apache.lucene.search.BooleanClause.Occur.MUST)); + } + return new Query( + qs, + bq); + } + + /** + * Creates a boolean OR query. + * @param queries the queries to compose into the OR query + * @return the newly created OR query + */ + @NonNull + public Query or(@NonNull final Query...queries) { + Parameters.notNull("queries", queries); //NOI18N + final org.apache.lucene.search.BooleanQuery bq = new org.apache.lucene.search.BooleanQuery(); + for (Query q : queries) { + bq.add(new BooleanClause(q.queryImpl, org.apache.lucene.search.BooleanClause.Occur.SHOULD)); + } + return new Query( + qs, + bq); + } + } + + /** + * Executes the query. + * @param fieldsToLoad the filter for fields to be loaded into the {@link IndexResult}s + * @return the {@link Collection} of {@link IndexResult} matching the {@link QuerySupport.Query} + * @throws IOException in case of IO error. + */ + @NonNull + public Collection execute(@NullAllowed final String... fieldsToLoad) throws IOException { + try { + return Utilities.runPriorityIO(new Callable>() { + @Override + public Collection call() throws Exception { + Iterable> indices = qs.indexerQuery.getIndices(qs.roots); + // check if there are stale indices + for (Pair pair : indices) { + final LayeredDocumentIndex index = pair.second; + final Collection staleFiles = index.getDirtyKeys(); + final boolean scanningThread = RunWhenScanFinishedSupport.isScanningThread(); + LOG.log( + Level.FINE, + "Index: {0}, staleFiles: {1}, scanning thread: {2}", //NOI18N + new Object[]{ + index, + staleFiles, + scanningThread + }); + if (!staleFiles.isEmpty() && !scanningThread) { + final URL root = pair.first; + LinkedList list = new LinkedList(); + for (String staleFile : staleFiles) { + try { + list.add(Util.resolveUrl(root, staleFile, false)); + } catch (MalformedURLException ex) { + LOG.log(Level.WARNING, null, ex); + } + } + TransientUpdateSupport.setTransientUpdate(true); + try { + RepositoryUpdater.getDefault().enforcedFileListUpdate(root,list); + } finally { + TransientUpdateSupport.setTransientUpdate(false); + } + } + } + final Queue result = new ArrayDeque(); + for (Pair pair : indices) { + final DocumentIndex2 index = pair.second; + final URL root = pair.first; + final Collection pr = + index.query( + queryImpl, + new DocumentToResultConvertor(root), + fieldsToLoad); + result.addAll(pr); //TODO: Perf: Replace by ProxyCollection + if (LOG.isLoggable(Level.FINE)) { + LOG.log( + Level.FINE, "{0} (loading fields {1}) invoked at {2}@{3}[indexer={4}]:", //NOI18N + new Object[]{ + this, + printFiledToLoad(fieldsToLoad), + qs.getClass().getSimpleName(), + Integer.toHexString(System.identityHashCode(qs)), + qs.indexerQuery.getIndexerId()}); + for (IndexResult idi : pr) { + LOG.log(Level.FINE, " {0}", idi); //NOI18N + } + LOG.fine("----"); //NOI18N + } + } + return result; + } + }); + } catch (Index.IndexClosedException ice) { + if (Installer.isClosed()) { + return Collections.emptySet(); + } else { + throw ice; + } + } catch (IOException ioe) { + throw ioe; + } catch (RuntimeException re) { + throw re; + } catch (Exception ex) { + throw new IOException(ex); + } + + } + + @NonNull + @Override + public String toString() { + return String.format( + "QuerySupport.Query[%s]", //NOI18N + queryImpl); + } + } + // ------------------------------------------------------------------------ // Private implementation // ------------------------------------------------------------------------ @@ -720,4 +851,22 @@ return null; } } // End of IndexerQuery class + + private static final class DocumentToResultConvertor implements Convertor { + + private final URL root; + + DocumentToResultConvertor(@NonNull final URL root) { + this.root = root; + } + + @Override + @CheckForNull + public IndexResult convert(@NullAllowed final IndexDocument p) { + return p == null ? + null : + new IndexResult(p, root); + } + + } } diff --git a/parsing.api/test/unit/src/org/netbeans/modules/parsing/spi/indexing/support/IndexingSupportTest.java b/parsing.api/test/unit/src/org/netbeans/modules/parsing/spi/indexing/support/IndexingSupportTest.java --- a/parsing.api/test/unit/src/org/netbeans/modules/parsing/spi/indexing/support/IndexingSupportTest.java +++ b/parsing.api/test/unit/src/org/netbeans/modules/parsing/spi/indexing/support/IndexingSupportTest.java @@ -80,6 +80,8 @@ private FileObject cache; private FileObject f1; private FileObject f2; + private FileObject f3; + private FileObject f4; public IndexingSupportTest (final String name) { super (name); @@ -100,6 +102,10 @@ assert f1 != null; f2 = FileUtil.createData(root,"folder/b.foo"); assert f2 != null; + f3 = FileUtil.createData(root,"folder/c.foo"); + assert f3 != null; + f4 = FileUtil.createData(root,"folder/c.foo"); + assert f4 != null; FileUtil.setMIMEType("foo", MIME); //NOI18N } @@ -224,6 +230,180 @@ assertEquals("true", ir[1].getValue("flag")); } + + public void testIndexingQuerySupport2 () throws Exception { + // index + final Context ctx = SPIAccessor.getInstance().createContext( + CacheFolder.getDataFolder(root.toURL()), + root.toURL(), + "fooIndexer", + 1, + null, + false, + false, + false, + SuspendSupport.NOP, + null, + null); + assertNotNull(ctx); + final Indexable i1 = SPIAccessor.getInstance().create(new FileObjectIndexable(root, f1)); + final IndexingSupport is = IndexingSupport.getInstance(ctx); + assertNotNull(is); + IndexDocument doc1 = is.createDocument(i1); + assertNotNull(doc1); + doc1.addPair("class", "String", true, true); + doc1.addPair("package", "java.lang", true, true); + is.addDocument(doc1); + final Indexable i2 = SPIAccessor.getInstance().create(new FileObjectIndexable(root, f2)); + IndexDocument doc2 = is.createDocument(i2); + assertNotNull(doc2); + doc2.addPair("class", "Object", true, true); + doc2.addPair("package", "java.lang", true, true); + doc2.addPair("flag", "true", true, true); + is.addDocument(doc2); + SPIAccessor.getInstance().getIndexFactory(ctx).getIndex(ctx.getIndexFolder()).store(true); + + // query + QuerySupport qs = QuerySupport.forRoots("fooIndexer", 1, root); + Collection result = qs.getQueryFactory().field("class", "String", QuerySupport.Kind.EXACT).execute("class", "package"); + assertEquals(1, result.size()); + assertEquals("String", result.iterator().next().getValue("class")); + assertEquals("java.lang", result.iterator().next().getValue("package")); + assertEquals(f1, result.iterator().next().getFile()); + assertEquals(f1.getURL(), result.iterator().next().getUrl()); + result = qs.getQueryFactory().field("class", "Str", QuerySupport.Kind.PREFIX).execute("class", "package"); + assertEquals(1, result.size()); + assertEquals("String", result.iterator().next().getValue("class")); + assertEquals("java.lang", result.iterator().next().getValue("package")); + result = qs.getQueryFactory().field("class", "S.*g", QuerySupport.Kind.REGEXP).execute("class", "package"); + assertEquals(1, result.size()); + assertEquals("String", result.iterator().next().getValue("class")); + assertEquals("java.lang", result.iterator().next().getValue("package")); + result = qs.getQueryFactory().field("class", "S", QuerySupport.Kind.CAMEL_CASE).execute("class", "package"); + assertEquals(1, result.size()); + assertEquals("String", result.iterator().next().getValue("class")); + assertEquals("java.lang", result.iterator().next().getValue("package")); + result = qs.getQueryFactory().field("class", "", QuerySupport.Kind.PREFIX).execute("class", "package"); + assertEquals(2, result.size()); + IndexResult[] ir = new IndexResult[2]; + ir = result.toArray(ir); + assertEquals("String", ir[0].getValue("class")); + assertEquals("java.lang", ir[0].getValue("package")); + assertEquals("Object", ir[1].getValue("class")); + assertEquals("java.lang", ir[1].getValue("package")); + result = qs.getQueryFactory().field("class", "F", QuerySupport.Kind.PREFIX).execute("class", "package"); + assertEquals(0, result.size()); + + // search for documents that contain field called 'flag' + result = qs.getQueryFactory().field("flag", "", QuerySupport.Kind.PREFIX).execute(); + assertEquals(1, result.size()); + assertEquals("Object", result.iterator().next().getValue("class")); + assertEquals("java.lang", result.iterator().next().getValue("package")); + assertEquals("true", result.iterator().next().getValue("flag")); + + // search for all documents + result = qs.getQueryFactory().field("", "", QuerySupport.Kind.PREFIX).execute(); + assertEquals(2, result.size()); + ir = new IndexResult[2]; + ir = result.toArray(ir); + assertEquals("String", ir[0].getValue("class")); + assertEquals("java.lang", ir[0].getValue("package")); + assertNull(ir[0].getValue("flag")); + assertEquals("Object", ir[1].getValue("class")); + assertEquals("java.lang", ir[1].getValue("package")); + assertEquals("true", ir[1].getValue("flag")); + } + + public void testIndexingQuerySupport3 () throws Exception { + // index + final Context ctx = SPIAccessor.getInstance().createContext( + CacheFolder.getDataFolder(root.toURL()), + root.toURL(), + "fooIndexer", + 1, + null, + false, + false, + false, + SuspendSupport.NOP, + null, + null); + assertNotNull(ctx); + final Indexable i1 = SPIAccessor.getInstance().create(new FileObjectIndexable(root, f1)); + final IndexingSupport is = IndexingSupport.getInstance(ctx); + assertNotNull(is); + IndexDocument doc1 = is.createDocument(i1); + assertNotNull(doc1); + doc1.addPair("class", "String", true, true); + doc1.addPair("package", "java.lang", true, true); + is.addDocument(doc1); + final Indexable i2 = SPIAccessor.getInstance().create(new FileObjectIndexable(root, f2)); + IndexDocument doc2 = is.createDocument(i2); + assertNotNull(doc2); + doc2.addPair("class", "Object", true, true); + doc2.addPair("package", "java.lang", true, true); + is.addDocument(doc2); + final Indexable i3 = SPIAccessor.getInstance().create(new FileObjectIndexable(root, f3)); + IndexDocument doc3 = is.createDocument(i3); + assertNotNull(doc3); + doc3.addPair("class", "Object", true, true); + doc3.addPair("package", "org.omg.CORBA", true, true); + is.addDocument(doc3); + final Indexable i4 = SPIAccessor.getInstance().create(new FileObjectIndexable(root, f4)); + IndexDocument doc4 = is.createDocument(i3); + assertNotNull(doc4); + doc4.addPair("class", "Integer", true, true); + doc4.addPair("package", "java.lang", true, true); + is.addDocument(doc4); + SPIAccessor.getInstance().getIndexFactory(ctx).getIndex(ctx.getIndexFolder()).store(true); + + // query + QuerySupport qs = QuerySupport.forRoots("fooIndexer", 1, root); + Collection result = qs.getQueryFactory().field("class", "Object", QuerySupport.Kind.EXACT).execute("class", "package"); + assertEquals(2, result.size()); + IndexResult[] ir = result.toArray(new IndexResult[2]); + assertEquals("Object", ir[0].getValue("class")); + assertEquals("java.lang", ir[0].getValue("package")); + assertEquals("Object", ir[1].getValue("class")); + assertEquals("org.omg.CORBA", ir[1].getValue("package")); + result = qs.getQueryFactory(). + and( + qs.getQueryFactory().field("class", "Object", QuerySupport.Kind.EXACT), + qs.getQueryFactory().field("package", "org.omg.CORBA", QuerySupport.Kind.EXACT)).execute("class", "package"); + assertEquals(1, result.size()); + ir = result.toArray(new IndexResult[1]); + assertEquals("Object", ir[0].getValue("class")); + assertEquals("org.omg.CORBA", ir[0].getValue("package")); + result = qs.getQueryFactory(). + and( + qs.getQueryFactory().field("package", "java.lang", QuerySupport.Kind.EXACT), + qs.getQueryFactory().or( + qs.getQueryFactory().field("class", "String", QuerySupport.Kind.EXACT), + qs.getQueryFactory().field("class", "Integer", QuerySupport.Kind.EXACT)) + ).execute("class", "package"); + assertEquals(2, result.size()); + ir = result.toArray(new IndexResult[2]); + assertEquals("String", ir[0].getValue("class")); + assertEquals("java.lang", ir[0].getValue("package")); + assertEquals("Integer", ir[1].getValue("class")); + assertEquals("java.lang", ir[1].getValue("package")); + result = qs.getQueryFactory(). + or( + qs.getQueryFactory().and( + qs.getQueryFactory().field("package", "java.lang", QuerySupport.Kind.EXACT), + qs.getQueryFactory().field("class", "String", QuerySupport.Kind.EXACT)), + qs.getQueryFactory().and( + qs.getQueryFactory().field("package", "java.lang", QuerySupport.Kind.EXACT), + qs.getQueryFactory().field("class", "Integer", QuerySupport.Kind.EXACT)) + ).execute("class", "package"); + assertEquals(2, result.size()); + ir = result.toArray(new IndexResult[2]); + assertEquals("String", ir[0].getValue("class")); + assertEquals("java.lang", ir[0].getValue("package")); + assertEquals("Integer", ir[1].getValue("class")); + assertEquals("java.lang", ir[1].getValue("package")); + } + public void testQuerySupportCaching() throws Exception { // index final Context ctx = SPIAccessor.getInstance().createContext( diff --git a/parsing.lucene/apichanges.xml b/parsing.lucene/apichanges.xml --- a/parsing.lucene/apichanges.xml +++ b/parsing.lucene/apichanges.xml @@ -110,6 +110,21 @@ + + + Added DocumentIndex2 allowing execution of arbitrary Lucene query. + + + + + +

+ Added DocumentIndex2 extending the DocumentIndex by execution of arbitrary Lucene query. +

+ + + + Added possibility to use the default DocumentIndex with custom IndexDocuments. diff --git a/parsing.lucene/nbproject/project.properties b/parsing.lucene/nbproject/project.properties --- a/parsing.lucene/nbproject/project.properties +++ b/parsing.lucene/nbproject/project.properties @@ -3,7 +3,7 @@ javadoc.apichanges=${basedir}/apichanges.xml javac.compilerargs=-Xlint -Xlint:-serial -spec.version.base=2.23.0 +spec.version.base=2.24.0 test.config.stableBTD.includes=**/*Test.class test.config.stableBTD.excludes=\ **/LuceneIndexTest.class diff --git a/parsing.lucene/src/org/netbeans/modules/parsing/lucene/DocumentIndexImpl.java b/parsing.lucene/src/org/netbeans/modules/parsing/lucene/DocumentIndexImpl.java --- a/parsing.lucene/src/org/netbeans/modules/parsing/lucene/DocumentIndexImpl.java +++ b/parsing.lucene/src/org/netbeans/modules/parsing/lucene/DocumentIndexImpl.java @@ -43,10 +43,11 @@ package org.netbeans.modules.parsing.lucene; import java.io.IOException; -import java.util.ArrayList; +import java.util.ArrayDeque; import java.util.Collection; import java.util.HashSet; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; @@ -56,19 +57,21 @@ import org.apache.lucene.document.FieldSelector; import org.apache.lucene.search.Query; import org.netbeans.api.annotations.common.NonNull; +import org.netbeans.api.annotations.common.NullAllowed; import org.netbeans.modules.parsing.lucene.support.Convertor; -import org.netbeans.modules.parsing.lucene.support.DocumentIndex; +import org.netbeans.modules.parsing.lucene.support.DocumentIndex2; import org.netbeans.modules.parsing.lucene.support.DocumentIndexCache; import org.netbeans.modules.parsing.lucene.support.Index; import org.netbeans.modules.parsing.lucene.support.IndexDocument; import org.netbeans.modules.parsing.lucene.support.Queries; import org.netbeans.modules.parsing.lucene.support.Queries.QueryKind; +import org.openide.util.Parameters; /** * * @author Tomas Zezula */ -public class DocumentIndexImpl implements DocumentIndex, Runnable { +public class DocumentIndexImpl implements DocumentIndex2, Runnable { private static final Convertor DEFAULT_ADD_CONVERTOR = Convertors.newIndexDocumentToDocumentConvertor(); private static final Convertor DEFAULT_QUERY_CONVERTOR = Convertors.newDocumentToIndexDocumentConvertor(); @@ -248,17 +251,11 @@ assert fieldName != null; assert value != null; assert kind != null; - final List result = new ArrayList(); final Query query = Queries.createQuery(fieldName, fieldName, value, kind); - FieldSelector selector = null; - if (fieldsToLoad != null && fieldsToLoad.length > 0) { - final String[] fieldsWithSource = new String[fieldsToLoad.length+1]; - System.arraycopy(fieldsToLoad, 0, fieldsWithSource, 0, fieldsToLoad.length); - fieldsWithSource[fieldsToLoad.length] = IndexDocumentImpl.FIELD_PRIMARY_KEY; - selector = Queries.createFieldSelector(fieldsWithSource); - } - luceneIndex.query(result, queryConvertor, selector, null, query); - return result; + return query( + query, + org.netbeans.modules.parsing.lucene.support.Convertors.identity(), + fieldsToLoad); } @Override @@ -270,6 +267,30 @@ } @Override + @NonNull + public Collection query( + @NonNull final Query query, + @NonNull final Convertor convertor, + @NullAllowed final String... fieldsToLoad) throws IOException, InterruptedException { + Parameters.notNull("query", query); //NOI18N + Parameters.notNull("convertor", convertor); //NOI18N + final Collection result = new ArrayDeque(); + FieldSelector selector = null; + if (fieldsToLoad != null && fieldsToLoad.length > 0) { + final String[] fieldsWithSource = Arrays.copyOf(fieldsToLoad, fieldsToLoad.length+1); + fieldsWithSource[fieldsToLoad.length] = IndexDocumentImpl.FIELD_PRIMARY_KEY; + selector = Queries.createFieldSelector(fieldsWithSource); + } + luceneIndex.query( + result, + org.netbeans.modules.parsing.lucene.support.Convertors.compose(queryConvertor, convertor), + selector, + null, + query); + return result; + } + + @Override public void markKeyDirty(final String primaryKey) { synchronized (this) { if (LOGGER.isLoggable(Level.FINE)) { @@ -302,24 +323,26 @@ @Override public String toString () { - return "DocumentIndex["+luceneIndex.toString()+"]"; //NOI18N + return String.format( + "DocumentIndexImpl[%s]", //NOI18N + luceneIndex.toString()); } @NonNull - public static DocumentIndex create( + public static DocumentIndex2 create( @NonNull final Index index, @NonNull final DocumentIndexCache cache) { return new DocumentIndexImpl(index, cache); } @NonNull - public static DocumentIndex.Transactional createTransactional( + public static DocumentIndex2.Transactional createTransactional( @NonNull final Index.Transactional index, @NonNull final DocumentIndexCache cache) { return new DocumentIndexImpl.Transactional(index, cache); } - private final static class Transactional extends DocumentIndexImpl implements DocumentIndex.Transactional { + private final static class Transactional extends DocumentIndexImpl implements DocumentIndex2.Transactional { private Transactional( @NonNull final Index.Transactional index, diff --git a/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/Convertors.java b/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/Convertors.java new file mode 100644 --- /dev/null +++ b/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/Convertors.java @@ -0,0 +1,119 @@ +/* + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright 2013 Oracle and/or its affiliates. All rights reserved. + * + * Oracle and Java are registered trademarks of Oracle and/or its affiliates. + * Other names may be trademarks of their respective owners. + * + * The contents of this file are subject to the terms of either the GNU + * General Public License Version 2 only ("GPL") or the Common + * Development and Distribution License("CDDL") (collectively, the + * "License"). You may not use this file except in compliance with the + * License. You can obtain a copy of the License at + * http://www.netbeans.org/cddl-gplv2.html + * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the + * specific language governing permissions and limitations under the + * License. When distributing the software, include this License Header + * Notice in each file and include the License file at + * nbbuild/licenses/CDDL-GPL-2-CP. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the GPL Version 2 section of the License file that + * accompanied this code. If applicable, add the following below the + * License Header, with the fields enclosed by brackets [] replaced by + * your own identifying information: + * "Portions Copyrighted [year] [name of copyright owner]" + * + * If you wish your version of this file to be governed by only the CDDL + * or only the GPL Version 2, indicate your decision by adding + * "[Contributor] elects to include this software in this distribution + * under the [CDDL or GPL Version 2] license." If you do not indicate a + * single choice of license, a recipient has the option to distribute + * your version of this file under either the CDDL, the GPL Version 2 or + * to extend the choice of license to its licensees as provided above. + * However, if you add GPL Version 2 code and therefore, elected the GPL + * Version 2 license, then the option applies only if the new code is + * made subject to such option by the copyright holder. + * + * Contributor(s): + * + * Portions Copyrighted 2013 Sun Microsystems, Inc. + */ +package org.netbeans.modules.parsing.lucene.support; + +import org.netbeans.api.annotations.common.CheckForNull; +import org.netbeans.api.annotations.common.NonNull; +import org.netbeans.api.annotations.common.NullAllowed; +import org.openide.util.Parameters; + +/** + * {@link Convertor} utilities. + * @author Tomas Zezula + * @since 2.24 + */ +public final class Convertors { + + private static final Convertor IDENTITY = + new Convertor() { + @Override + @CheckForNull + public Object convert(@NullAllowed final Object p) { + return p; + } + }; + + private Convertors() {} + + /** + * Identity convertor. + * Returns the identity function, function returning its parameter. + * @param type of the both parameter and result. + * @return returns identity convertor. + */ + @SuppressWarnings("unchecked") + public static Convertor identity() { + return (Convertor) IDENTITY; + } + + /** + * Composite convertor. + * Returns the composition of two functions F(p)->second(first(p)). + * @param

the parameter type of the first {@link Convertor} + * @param the result type of the first {@link Convertor} + * @param the result type of the second {@link Convertor} + * @param first the first {@link Convertor} + * @param second the second {@link Convertor} + * @return the composite {@link Convertor} + */ + public static Convertor compose( + @NonNull final Convertor first, + @NonNull final Convertor second) { + Parameters.notNull("first", first); //NOI18N + Parameters.notNull("second", second); //NOI18N + return new CompositeConvertor (first, second); + } + + + + private static final class CompositeConvertor implements Convertor { + + private final Convertor first; + private final Convertor second; + + CompositeConvertor( + @NonNull final Convertor first, + @NonNull final Convertor second) { + this.first = first; + this.second = second; + } + + + @Override + @CheckForNull + public R convert(@NullAllowed P p) { + return second.convert(first.convert(p)); + } + + } + +} diff --git a/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/DocumentIndex2.java b/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/DocumentIndex2.java new file mode 100644 --- /dev/null +++ b/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/DocumentIndex2.java @@ -0,0 +1,76 @@ +/* + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright 2013 Oracle and/or its affiliates. All rights reserved. + * + * Oracle and Java are registered trademarks of Oracle and/or its affiliates. + * Other names may be trademarks of their respective owners. + * + * The contents of this file are subject to the terms of either the GNU + * General Public License Version 2 only ("GPL") or the Common + * Development and Distribution License("CDDL") (collectively, the + * "License"). You may not use this file except in compliance with the + * License. You can obtain a copy of the License at + * http://www.netbeans.org/cddl-gplv2.html + * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the + * specific language governing permissions and limitations under the + * License. When distributing the software, include this License Header + * Notice in each file and include the License file at + * nbbuild/licenses/CDDL-GPL-2-CP. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the GPL Version 2 section of the License file that + * accompanied this code. If applicable, add the following below the + * License Header, with the fields enclosed by brackets [] replaced by + * your own identifying information: + * "Portions Copyrighted [year] [name of copyright owner]" + * + * If you wish your version of this file to be governed by only the CDDL + * or only the GPL Version 2, indicate your decision by adding + * "[Contributor] elects to include this software in this distribution + * under the [CDDL or GPL Version 2] license." If you do not indicate a + * single choice of license, a recipient has the option to distribute + * your version of this file under either the CDDL, the GPL Version 2 or + * to extend the choice of license to its licensees as provided above. + * However, if you add GPL Version 2 code and therefore, elected the GPL + * Version 2 license, then the option applies only if the new code is + * made subject to such option by the copyright holder. + * + * Contributor(s): + * + * Portions Copyrighted 2013 Sun Microsystems, Inc. + */ +package org.netbeans.modules.parsing.lucene.support; + +import java.io.IOException; +import java.util.Collection; +import org.apache.lucene.search.Query; +import org.netbeans.api.annotations.common.NonNull; +import org.netbeans.api.annotations.common.NullAllowed; + +/** + * Document based index allowing arbitrary Lucene Query. + * @author Tomas Zezula + * @since 2.24 + */ +public interface DocumentIndex2 extends DocumentIndex { + + /** + * Performs the Lucene query on the index. + * @param query to perform + * @param fieldsToLoad fields to load into returned document + * @return the Collection of {@link IndexDocument} matching the query. + * @throws IOException in case of IO error. + * @throws InterruptedException if the search is interrupted. + */ + @NonNull + public Collection query ( + @NonNull Query query, + @NonNull Convertor convertor, + @NullAllowed String... fieldsToLoad) throws IOException, InterruptedException; + + /** + * Transactional {@link DocumentIndex2}. + */ + public interface Transactional extends DocumentIndex2, DocumentIndex.Transactional { + } +}