[hg] main-silver: Reusing Lucene documents to help GC.

  • From: Tomas Zezula < >
  • To:
  • Subject: [hg] main-silver: Reusing Lucene documents to help GC.
  • Date: Wed, 06 Feb 2013 13:58:32 -0800

changeset 001624a69a9c in main-silver ((none))
details: http://hg.netbeans.org/main-silver/rev/001624a69a9c
description:
        Reusing Lucene documents to help GC.

diffstat:

 parsing.api/nbproject/project.xml                                            
          |    2 +-
 
parsing.api/src/org/netbeans/modules/parsing/impl/indexing/ClusteredIndexables.java
    |  195 ++++++---
 parsing.lucene/apichanges.xml                                                
          |   15 +
 parsing.lucene/nbproject/project.properties                                  
          |    2 +-
 
parsing.lucene/src/org/netbeans/modules/parsing/lucene/DocumentIndexImpl.java 
         |   38 +-
 
parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/DocumentIndexCache.java
 |   29 +
 6 files changed, 194 insertions(+), 87 deletions(-)

diffs (477 lines):

diff --git a/parsing.api/nbproject/project.xml 
b/parsing.api/nbproject/project.xml
--- a/parsing.api/nbproject/project.xml
+++ b/parsing.api/nbproject/project.xml
@@ -110,7 +110,7 @@
                     <compile-dependency/>
                     <run-dependency>
                         <release-version>2</release-version>
-                        <specification-version>2.20</specification-version>
+                        <specification-version>2.22</specification-version>
                     </run-dependency>
                 </dependency>
                 <dependency>
diff --git 
a/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/ClusteredIndexables.java
 
b/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/ClusteredIndexables.java
--- 
a/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/ClusteredIndexables.java
+++ 
b/parsing.api/src/org/netbeans/modules/parsing/impl/indexing/ClusteredIndexables.java
@@ -61,15 +61,16 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.netbeans.api.annotations.common.NonNull;
 import org.netbeans.api.annotations.common.NullAllowed;
 import 
org.netbeans.modules.parsing.impl.indexing.lucene.DocumentBasedIndexManager;
+import org.netbeans.modules.parsing.lucene.support.Convertor;
 import org.netbeans.modules.parsing.lucene.support.DocumentIndex;
 import org.netbeans.modules.parsing.lucene.support.DocumentIndexCache;
 import org.netbeans.modules.parsing.lucene.support.IndexDocument;
-import org.netbeans.modules.parsing.lucene.support.IndexManager;
 import org.netbeans.modules.parsing.spi.indexing.Indexable;
 import org.openide.util.Parameters;
 import org.openide.util.Utilities;
@@ -136,10 +137,10 @@
     @NonNull
     public static IndexDocument createDocument(@NonNull final String 
primaryKey) {
         Parameters.notNull("primaryKey", primaryKey);   //NOI18N
-        return new DocumentStore.MemoryIndexDocument(primaryKey);
+        return new MemIndexDocument(primaryKey);
     }
 
-    public static interface AttachableDocumentIndexCache extends 
DocumentIndexCache {
+    public static interface AttachableDocumentIndexCache extends 
DocumentIndexCache.WithCustomIndexDocument {
         void attach(@NonNull final String mode, @NonNull final 
ClusteredIndexables ci);
         void detach();
     }
@@ -268,6 +269,16 @@
     //<editor-fold defaultstate="collapsed" desc="DocumentIndexCache 
Implementation">
     private static final class DocumentIndexCacheImpl implements 
AttachableDocumentIndexCache {
       
+        private static final Convertor<IndexDocument, Document> 
ADD_CONVERTOR =
+                new Convertor<IndexDocument, Document>() {
+                    @NonNull
+                    @Override
+                    public Document convert(@NonNull final IndexDocument 
doc) {
+                        final ReusableIndexDocument rdoc = 
(ReusableIndexDocument) doc;
+                        return rdoc.doc;
+                    }
+                };
+      
         private ClusteredIndexables deleteIndexables;
         private ClusteredIndexables indexIndexables;
         private BitSet deleteFromDeleted;
@@ -327,7 +338,7 @@
 
         @Override
         public boolean addDocument(IndexDocument document) {
-            if (!(document instanceof DocumentStore.MemoryIndexDocument)) {
+            if (!(document instanceof MemIndexDocument)) {
                 throw new 
IllegalArgumentException(document.getClass().getName());
             }
             boolean shouldFlush = init();
@@ -381,6 +392,16 @@
             return toAdd != null ? toAdd : 
Collections.<IndexDocument>emptySet();
         }
 
+        @Override
+        public Convertor<IndexDocument, Document> createAddConvertor() {
+            return ADD_CONVERTOR;
+        }
+
+        @Override
+        public Convertor<Document, IndexDocument> createQueryConvertor() {
+            return null;
+        }
+
         private static void ensureNotReBound(
                 @NullAllowed final ClusteredIndexables oldCi,
                 @NonNull final ClusteredIndexables newCi) {
@@ -689,6 +710,99 @@
     }
     //</editor-fold>
 
+    //<editor-fold defaultstate="collapsed" desc="Input IndexDocument 
(inserted into cache)">
+    private static final class MemIndexDocument implements IndexDocument {
+
+        private static final String[] EMPTY = new String[0];
+        static final String FIELD_PRIMARY_KEY = "_sn";  //NOI18N
+
+        private final List<Fieldable> fields = new ArrayList<Fieldable>();
+
+        MemIndexDocument(@NonNull final String primaryKey) {
+            Parameters.notNull("primaryKey", primaryKey);   //NOI18N
+            fields.add(sourceNameField(primaryKey));
+        }
+
+        public List<Fieldable> getFields() {
+            return fields;
+        }
+
+        @Override
+        public String getPrimaryKey() {
+            return getValue(FIELD_PRIMARY_KEY);
+        }
+
+        @Override
+        public void addPair(String key, String value, boolean searchable, 
boolean stored) {
+            final Field field = new Field (key, value,
+                    stored ? Field.Store.YES : Field.Store.NO,
+                    searchable ? Field.Index.NOT_ANALYZED_NO_NORMS : 
Field.Index.NO);
+            fields.add (field);
+        }
+
+        @Override
+        public String getValue(String key) {
+            for (Fieldable field : fields) {
+                if (field.name().equals(key)) {
+                    return field.stringValue();
+                }
+            }
+            return null;
+        }
+
+        @Override
+        public String[] getValues(String key) {
+            final List<String> result = new ArrayList<String>();
+            for (Fieldable field : fields) {
+                if (field.name().equals(key)) {
+                    result.add(field.stringValue());
+                }
+            }
+            return result.toArray(result.isEmpty() ? EMPTY : new 
String[result.size()]);
+        }
+
+        private Fieldable sourceNameField(@NonNull String primaryKey) {
+            return new Field(FIELD_PRIMARY_KEY, primaryKey, Field.Store.YES, 
Field.Index.NOT_ANALYZED_NO_NORMS);
+        }
+    }
+    //</editor-fold>
+
+    //<editor-fold defaultstate="collapsed" desc="Output IndexDocument 
(output of the cache)">
+    //@NotThreadSafe
+    private static final class ReusableIndexDocument implements 
IndexDocument {
+
+        private final Document doc = new Document();
+
+        @Override
+        public String getPrimaryKey() {
+            return doc.get(MemIndexDocument.FIELD_PRIMARY_KEY);
+        }
+
+        @Override
+        public String getValue(String key) {
+            return doc.get(key);
+        }
+
+        @Override
+        public String[] getValues(String key) {
+            return doc.getValues(key);
+        }
+
+        @Override
+        public void addPair(String key, String value, boolean searchable, 
boolean stored) {
+            doc.add(new Field (
+                key,
+                value,
+                stored ? Field.Store.YES : Field.Store.NO,
+                searchable ? Field.Index.NOT_ANALYZED_NO_NORMS : 
Field.Index.NO));
+        }
+
+        void clear() {
+            doc.getFields().clear();
+        }
+    }
+    //</editor-fold>
+
     //<editor-fold defaultstate="collapsed" desc="Added IndexDocuments 
Collection (optimized for high number of fields).">
     /*test*/ static final class DocumentStore extends 
AbstractCollection<IndexDocument>{
 
@@ -720,10 +834,10 @@
         
         boolean addDocument(@NonNull final IndexDocument doc) {
             boolean res = false;
-            if (!(doc instanceof MemoryIndexDocument)) {
+            if (!(doc instanceof MemIndexDocument)) {
                 throw new IllegalArgumentException();
             }
-            for (Fieldable fld : ((MemoryIndexDocument)doc).getFields()) {
+            for (Fieldable fld : ((MemIndexDocument)doc).getFields()) {
                 final String fldName = fld.name();
                 final boolean stored = fld.isStored();
                 final boolean indexed = fld.isIndexed();
@@ -806,9 +920,11 @@
 
             private int cur = 0;
             private final List<String> names;
+            private final ReusableIndexDocument doc;
 
             It() {
                 names = new ArrayList<String>(fieldNames.keySet());
+                doc = new ReusableIndexDocument();
             }
 
             @Override
@@ -816,15 +932,12 @@
                 return cur<docsPointer;
             }
 
-            /**
-             * Todo: Perf: non needed creation of IndexDocument, single can 
be used.
-             */
             @Override
             public IndexDocument next() {
                 if (cur>=docsPointer) {
                     throw new NoSuchElementException();
                 }
-                IndexDocument doc = null;
+                doc.clear();
                 int nameIndex;
                 while ((nameIndex=docs[cur++]) != 0) {
                     final boolean stored = (nameIndex & 4) == 4;
@@ -837,10 +950,6 @@
                                 dataPointer :
                                 docs[cur+2];
                     final String value = new String (data,dataStart, dataEnd 
- dataStart);
-                    if (doc == null) {
-                        assert "_sn".equals(names.get(nameIndex));  //NOI18N
-                        doc = IndexManager.createDocument(value);
-                    }
                     doc.addPair(
                             names.get(nameIndex),
                             value,
@@ -855,65 +964,7 @@
             }
         }
         //</editor-fold>
-
-        //<editor-fold defaultstate="collapsed" desc="In Memory 
IndexDocument (in to cache)">
-        private static final class MemoryIndexDocument implements 
IndexDocument {
-
-            private static final String FIELD_PRIMARY_KEY = "_sn";  //NOI18N
-            private static final String[] EMPTY = new String[0];
-
-            private final List<Fieldable> fields = new 
ArrayList<Fieldable>();
-
-            MemoryIndexDocument(@NonNull final String primaryKey) {
-                Parameters.notNull("primaryKey", primaryKey);   //NOI18N
-                fields.add(sourceNameField(primaryKey));
-            }
-
-            public List<Fieldable> getFields() {
-                return fields;
-            }
-
-            @Override
-            public String getPrimaryKey() {
-                return getValue(FIELD_PRIMARY_KEY);
-            }
-
-            @Override
-            public void addPair(String key, String value, boolean 
searchable, boolean stored) {
-                final Field field = new Field (key, value,
-                        stored ? Field.Store.YES : Field.Store.NO,
-                        searchable ? Field.Index.NOT_ANALYZED_NO_NORMS : 
Field.Index.NO);
-                fields.add (field);
-            }
-
-            @Override
-            public String getValue(String key) {
-                for (Fieldable field : fields) {
-                    if (field.name().equals(key)) {
-                        return field.stringValue();
-                    }
-                }
-                return null;
-            }
-
-            @Override
-            public String[] getValues(String key) {
-                final List<String> result = new ArrayList<String>();
-                for (Fieldable field : fields) {
-                    if (field.name().equals(key)) {
-                        result.add(field.stringValue());
-                    }
-                }
-                return result.toArray(result.isEmpty() ? EMPTY : new 
String[result.size()]);
-            }
-
-            private Fieldable sourceNameField(@NonNull String primaryKey) {
-                return new Field(FIELD_PRIMARY_KEY, primaryKey, 
Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS);
-            }
         }
         //</editor-fold>
 
     }
-    //</editor-fold>
-
-}
diff --git a/parsing.lucene/apichanges.xml b/parsing.lucene/apichanges.xml
--- a/parsing.lucene/apichanges.xml
+++ b/parsing.lucene/apichanges.xml
@@ -110,6 +110,21 @@
 <!-- ACTUAL CHANGES BEGIN HERE: -->
 
   <changes>
+  <change id="DocumentIndexCache-WithCustomIndexDocument">
+    <api name="LuceneSupport"/>
+    <summary>Added possibility to use the default <code>DocumentIndex</code> 
with custom <code>IndexDocument</code>s.</summary>
+    <version major="2" minor="22"/>
+    <date day="6" month="2" year="2013"/>
+    <author login="tzezula"/>
+    <compatibility source="compatible" binary="compatible" addition="yes"/>
+    <description>
+        <p>
+            Added a possibility to use the default 
<code>DocumentIndex</code> created by <code>IndexManager</code>
+            with a custom <code>IndexDocument</code> implementation.
+        </p>
+    </description>
+    <class package="org.netbeans.modules.parsing.lucene.support" 
name="DocumentIndexCache"/>
+  </change>
   <change id="TransactionalDocumentIndex-clear">
     <api name="LuceneSupport"/>
     <summary>Added <code>DocumentIndex.Transactional.clear</code> to allow 
complete index clean up.</summary>
diff --git a/parsing.lucene/nbproject/project.properties 
b/parsing.lucene/nbproject/project.properties
--- a/parsing.lucene/nbproject/project.properties
+++ b/parsing.lucene/nbproject/project.properties
@@ -3,7 +3,7 @@
 javadoc.apichanges=${basedir}/apichanges.xml
 javac.compilerargs=-Xlint -Xlint:-serial
 
-spec.version.base=2.21.0
+spec.version.base=2.22.0
 test.config.stableBTD.includes=**/*Test.class
 test.config.stableBTD.excludes=\
     **/LuceneIndexTest.class
diff --git 
a/parsing.lucene/src/org/netbeans/modules/parsing/lucene/DocumentIndexImpl.java
 
b/parsing.lucene/src/org/netbeans/modules/parsing/lucene/DocumentIndexImpl.java
--- 
a/parsing.lucene/src/org/netbeans/modules/parsing/lucene/DocumentIndexImpl.java
+++ 
b/parsing.lucene/src/org/netbeans/modules/parsing/lucene/DocumentIndexImpl.java
@@ -70,21 +70,23 @@
  */
 public class DocumentIndexImpl implements DocumentIndex, Runnable {
     
+    private static final Convertor<IndexDocument,Document> 
DEFAULT_ADD_CONVERTOR = Convertors.newIndexDocumentToDocumentConvertor();    
+    private static final Convertor<Document,IndexDocumentImpl> 
DEFAULT_QUERY_CONVERTOR = Convertors.newDocumentToIndexDocumentConvertor();
+    private static final Convertor<String,Query> REMOVE_CONVERTOR = 
Convertors.newSourceNameToQueryConvertor();
+    private static final Logger LOGGER = 
Logger.getLogger(DocumentIndexImpl.class.getName());
+    
+    private final Set</*@GuardedBy("this")*/String> dirtyKeys = new 
HashSet<String>();
+    //@GuardedBy ("this")
+    private final DocumentIndexCache cache;
     private final Index luceneIndex;
-    //@GuardedBy (this)
-    private final DocumentIndexCache cache;
+    private final Convertor<? super IndexDocument, ? extends Document> 
addConvertor;
+    private final Convertor<? super Document, ? extends IndexDocument> 
queryConvertor;
     /**
      * Transactional extension to the index
      */
     final Index.Transactional txLuceneIndex;
+    final AtomicBoolean requiresRollBack = new AtomicBoolean();
             
-    private static final Convertor<IndexDocument,Document> ADD_CONVERTOR = 
Convertors.newIndexDocumentToDocumentConvertor();
-    private static final Convertor<String,Query> REMOVE_CONVERTOR = 
Convertors.newSourceNameToQueryConvertor();
-    private static final Convertor<Document,IndexDocumentImpl> 
QUERY_CONVERTOR = Convertors.newDocumentToIndexDocumentConvertor();
-    private static final Logger LOGGER = 
Logger.getLogger(DocumentIndexImpl.class.getName());
-    
-    private final Set</*@GuardedBy("this")*/String> dirtyKeys = new 
HashSet<String>();
-    final AtomicBoolean requiresRollBack = new AtomicBoolean();
 
     private DocumentIndexImpl (
             @NonNull final Index index,
@@ -93,6 +95,16 @@
         assert cache != null;
         this.luceneIndex = index;
         this.cache = cache;
+        Convertor<IndexDocument,Document> _addConvertor = null;
+        Convertor<Document,IndexDocument> _queryConvertor = null;
+        if (cache instanceof DocumentIndexCache.WithCustomIndexDocument) {
+            final DocumentIndexCache.WithCustomIndexDocument 
cacheWithCustomDoc =
+                    (DocumentIndexCache.WithCustomIndexDocument) cache;
+            _addConvertor = cacheWithCustomDoc.createAddConvertor();
+            _queryConvertor = cacheWithCustomDoc.createQueryConvertor();
+        }
+        addConvertor = _addConvertor != null ? _addConvertor : 
DEFAULT_ADD_CONVERTOR;
+        queryConvertor = _queryConvertor != null ? _queryConvertor : 
DEFAULT_QUERY_CONVERTOR;
         if (index instanceof Index.Transactional) {
             this.txLuceneIndex = (Index.Transactional)index;
         } else {
@@ -204,14 +216,14 @@
                 txLuceneIndex.txStore(
                         _toAdd,
                         _toRemove,
-                        ADD_CONVERTOR,
+                        addConvertor,
                         REMOVE_CONVERTOR
                 );
             } else {
                 luceneIndex.store(
                         _toAdd,
                         _toRemove,
-                        ADD_CONVERTOR,
+                        addConvertor,
                         REMOVE_CONVERTOR,
                         optimize);
             }
@@ -236,7 +248,7 @@
         assert fieldName != null;
         assert value != null;
         assert kind != null;
-        final List<IndexDocumentImpl> result = new 
LinkedList<IndexDocumentImpl>();
+        final List<IndexDocument> result = new LinkedList<IndexDocument>();
         final Query query = Queries.createQuery(fieldName, fieldName, value, 
kind);
         FieldSelector selector = null;
         if (fieldsToLoad != null && fieldsToLoad.length > 0) {
@@ -245,7 +257,7 @@
             fieldsWithSource[fieldsToLoad.length] = 
IndexDocumentImpl.FIELD_PRIMARY_KEY;
             selector = Queries.createFieldSelector(fieldsWithSource);
         }        
-        luceneIndex.query(result, QUERY_CONVERTOR, selector, null, query);
+        luceneIndex.query(result, queryConvertor, selector, null, query);
         return result;
     }
     
diff --git 
a/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/DocumentIndexCache.java
 
b/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/DocumentIndexCache.java
--- 
a/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/DocumentIndexCache.java
+++ 
b/parsing.lucene/src/org/netbeans/modules/parsing/lucene/support/DocumentIndexCache.java
@@ -42,6 +42,8 @@
 package org.netbeans.modules.parsing.lucene.support;
 
 import java.util.Collection;
+import org.apache.lucene.document.Document;
+import org.netbeans.api.annotations.common.CheckForNull;
 import org.netbeans.api.annotations.common.NonNull;
 
 /**
@@ -89,4 +91,31 @@
     @NonNull
     Collection<? extends IndexDocument> getAddedDocuments();
 
+
+    /**
+     * Cache which allows custom {@link IndexDocument}s implementations.
+     * @since 2.22
+     */
+    interface WithCustomIndexDocument extends DocumentIndexCache {
+        /**
+         * Creates a {@link Convertor} from custom {@link IndexDocument} 
implementation.
+         * The returned {@link Convertor} translates the custom {@link 
IndexDocument}s
+         * returned by the {@link DocumentIndexCache#getAddedDocuments()} to 
{@link Document}s.
+         * @return the {@link Convertor} or null if a default convertor, 
converting from
+         * {@link IndexDocument}s created by {@link 
IndexManager#createDocument}, should be used.
+         */
+        @CheckForNull
+        Convertor<IndexDocument, Document> createAddConvertor();
+
+        /**
+         * Creates a {@link Convertor} to custom {@link IndexDocument} 
implementation.
+         * The returned {@link Convertor} translates the {@link Document}s
+         * created by the {@link Index#query} to custom {@link 
IndexDocument}s.
+         * @return the {@link Convertor} or null if a default convertor, 
converting to
+         * {@link IndexDocument}s created by {@link 
IndexManager#createDocument}, should be used.
+         */
+        @CheckForNull
+        Convertor<Document, IndexDocument> createQueryConvertor();
 }
+
+}

[hg] main-silver: Reusing Lucene documents to help GC.

Tomas Zezula 02/06/2013

Project Features

About this Project

Editor was started in November 2009, is owned by Martin Ryzl, and has 147 members.
By use of this website, you agree to the NetBeans Policies and Terms of Use (revision 20140418.2d69abc). © 2013, Oracle Corporation and/or its affiliates. Sponsored by Oracle logo
 
 
Close
loading
Please Confirm
Close