Attachment #62062 for bug #117450

This Bugzilla instance is a read-only archive of historic NetBeans bug reports. To report a bug in NetBeans please follow the project's instructions for reporting issues.

View | Details | Raw Unified | Return to bug 117450
Collapse All | Expand All

Lines 1-5 Link Here

(-)a/editor.util/manifest.mf (-1 / +1 lines)
1	Manifest-Version: 1.0	1	Manifest-Version: 1.0
2	OpenIDE-Module: org.netbeans.modules.editor.util/1	2	OpenIDE-Module: org.netbeans.modules.editor.util/1
3	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/editor/util/Bundle.properties	3	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/editor/util/Bundle.properties
4	OpenIDE-Module-Specification-Version: 1.23	4	OpenIDE-Module-Specification-Version: 1.24
5	AutoUpdate-Show-In-Client: false	5	AutoUpdate-Show-In-Client: false




    /**
     * Ensure that the given start and end parameters are valid indices
     * of the given text.
     * @param start must be &gt;=0 and &lt;=end.
     * @param end must be &gt;=start and &lt;=textLength.
     * @param length total length of a charsequence.
     * @throws IndexOutOfBoundsException if the start or end are not within bounds.
     */
    public static void checkIndexesValid(int start, int end, int length) {
        if (start < 0) {
            throw new IndexOutOfBoundsException("start=" + start + " < 0"); // NOI18N
        }
        if (end < start) {
            throw new IndexOutOfBoundsException("end=" + end + " < start=" + start); // NOI18N
        }
        if (end > length) {
            throw new IndexOutOfBoundsException("end=" + end + " > length()=" + length); // NOI18N
            + " > text.length()=" + text.length()); // NOI18N
        }
    }

    /**
     * Ensure that the given start and end parameters are valid indices
     * of the given text.
     * @param text non-null char sequence.
     * @param start must be &gt;=0 and &lt;=end.
     * @param end must be &gt;=start and &lt;=<code>text.length()</code>.
     * @throws IndexOutOfBoundsException if the start or end are not within bounds
     *  of the given text.
     */
    public static void checkIndexesValid(CharSequence text, int start, int end) {
        checkIndexesValid(start, end, text.length());
    }

}

Lines 60-67 Link Here

(-)a/editor.util/src/org/netbeans/lib/editor/util/FlyOffsetGapList.java (-1 / +6 lines)
60	private int offsetGapLength = Integer.MAX_VALUE / 2; // 32 bytes	60	private int offsetGapLength = Integer.MAX_VALUE / 2; // 32 bytes
61		61
62	public FlyOffsetGapList() {	62	public FlyOffsetGapList() {
		63	this(10);
63	}	64	}
64		65
		66	public FlyOffsetGapList(int initialCapacity) {
		67	super(initialCapacity);
		68	}
		69
65	/**	70	/**
66	* Get the raw offset of the given element currently stored in the list.	71	* Get the raw offset of the given element currently stored in the list.
67	*	72	*

Lines 82-89 Link Here

(-)a/editor.util/src/org/netbeans/lib/editor/util/OffsetGapList.java (-1 / +6 lines)
82	private int offsetGapLength = Integer.MAX_VALUE / 2; // 32 bytes	82	private int offsetGapLength = Integer.MAX_VALUE / 2; // 32 bytes
83		83
84	public OffsetGapList() {	84	public OffsetGapList() {
		85	this(10);
85	}	86	}
86		87
		88	public OffsetGapList(int initialCapacity) {
		89	super(initialCapacity);
		90	}
		91
87	/**	92	/**
88	* Get the raw offset of the given element currently stored in the list.	93	* Get the raw offset of the given element currently stored in the list.
89	*	94	*

Lines 138-143 Link Here

(-)a/editor.util/test/unit/src/org/netbeans/lib/editor/util/CharSequenceUtilitiesTest.java (+9 lines)
138		138
139	// endsWith	139	// endsWith
140	assertTrue(CharSequenceUtilities.endsWith(string, string.substring(CHARS_LENGTH - SUBSTR_LENGTH)));	140	assertTrue(CharSequenceUtilities.endsWith(string, string.substring(CHARS_LENGTH - SUBSTR_LENGTH)));
		141
		142	CharSequenceUtilities.checkIndexesValid(0, 3, 3); // start,end,length
		143	CharSequenceUtilities.checkIndexesValid(1, 3, 3);
		144	try {
		145	CharSequenceUtilities.checkIndexesValid(1, 4, 3);
		146	TestCase.fail("IndexOutOfBoundsException was expected.");
		147	} catch (IndexOutOfBoundsException e) {
		148	// Expected
		149	}
141	}	150	}
142		151
143	public void generateChars(char[] chars) {	152	public void generateChars(char[] chars) {




<!-- ACTUAL CHANGES BEGIN HERE: -->

  <changes>
  <change id="Joined-sections-Lexing">
      <api name="api"/>
      <summary>Joined Sections Lexing</summary>
      <version major="1" minor="28"/>
      <date day="28" month="5" year="2008"/>
      <author login="mmetelka"/>
      <compatibility source="compatible" binary="compatible" semantic="compatible" deletion="no" addition="yes" modification="no"/>
      <description>
        <p>
            Embeddings that request input sections to be joined before lexing
            are now lexed as a single section.
            <br/>
            <code>Token.isRemoved()</code> was added to check whether a particular token
            is still present in token hierarchy or whether it was removed as part of a modification.
        </p>
      </description>
      <issue number="117450"/>
  </change>

  <change id="Lexer-API-Cleanup">
      <api name="api"/>
      <summary>Lexer API Cleanup</summary>

Lines 43-46 Link Here

(-)a/lexer/nbproject/project.properties (-1 / +1 lines)
43	javadoc.arch=${basedir}/arch.xml	43	javadoc.arch=${basedir}/arch.xml
44	javadoc.apichanges=${basedir}/apichanges.xml	44	javadoc.apichanges=${basedir}/apichanges.xml
45	javadoc.docfiles=${basedir}/api/doc	45	javadoc.docfiles=${basedir}/api/doc
46	spec.version.base=1.27.0	46	spec.version.base=1.28.0

Lines 52-58 Link Here

(-)a/lexer/nbproject/project.xml (-1 / +1 lines)
52	<compile-dependency/>	52	<compile-dependency/>
53	<run-dependency>	53	<run-dependency>
54	<release-version>1</release-version>	54	<release-version>1</release-version>
55	<specification-version>1.15</specification-version>	55	<specification-version>1.24</specification-version>
56	</run-dependency>	56	</run-dependency>
57	</dependency>	57	</dependency>
58	<dependency>	58	<dependency>

Lines 40-45 Link Here

(-)a/lexer/src/org/netbeans/api/lexer/Token.java (-1 / +37 lines)
40	*/	40	*/
41		41
42	package org.netbeans.api.lexer;	42	package org.netbeans.api.lexer;
		43
		44	import java.util.List;
43		45
44	/**	46	/**
45	* Token describes a lexical element of input text.	47	* Token describes a lexical element of input text.
Lines 207-218 Link Here
207	* @return true if the token is flyweight or false otherwise.	209	* @return true if the token is flyweight or false otherwise.
208	*/	210	*/
209	public abstract boolean isFlyweight();	211	public abstract boolean isFlyweight();
		212
		213	/**
		214	* Check whether this token is no longer part of the token hierarchy.
		215	*
		216	* @return true if the token was removed from the token hierarchy
		217	* or false if it's still present in the hierarchy.
		218	*/
		219	public abstract boolean isRemoved();
210		220
211	/**	221	/**
212	* Check whether this token represents a complete token	222	* Check whether this token represents a complete token
213	* or whether it's a part of a complete token.	223	* or whether it's a particular part of a complete token.
		224	* <br/>
		225	* Some lexers may also use this information to express an incomplete token.
		226	* For example an unclosed block comment at the end of java source
		227	* is represented as a BLOCK_COMMENT token id and {@link PartType#START}.
		228	*
		229	* @return {@link PartType#COMPLETE} for regular token or other part types
		230	* for particular token parts.
214	*/	231	*/
215	public abstract PartType partType();	232	public abstract PartType partType();
		233
		234	/**
		235	* Get a complete token that is joined from multiple parts (this token is one of those parts).
		236	*
		237	* @return complete token or null if this token is not a part of any token.
		238	*/
		239	public abstract Token<T> joinToken();
		240
		241	/**
		242	* Get all token parts comprising this token ordered from lowest to highest part's offset.
		243	* <br/>
		244	* It's guaranteed that each token part is continuous in the input text
		245	* (there are no gaps inside the token part's text).
		246	* <br/>
		247	* On the other hand there may be textual gaps between two adajcent token parts.
		248	*
		249	* @return list of token parts or null if the token is continuous.
		250	*/
		251	public abstract List<? extends Token<T>> joinedParts();
216		252
217	/**	253	/**
218	* Quickly determine whether this token has any extra properties.	254	* Quickly determine whether this token has any extra properties.

Lines 123-129 Link Here

(-)a/lexer/src/org/netbeans/api/lexer/TokenHierarchyEvent.java (-1 / +1 lines)
123	* if this event's type is not {@link TokenHierarchyEventType#MODIFICATION}.	123	* if this event's type is not {@link TokenHierarchyEventType#MODIFICATION}.
124	*/	124	*/
125	public int modificationOffset() {	125	public int modificationOffset() {
126	return info.modificationOffset();	126	return info.modOffset();
127	}	127	}
128		128
129	/**	129	/**




package org.netbeans.api.lexer;

import java.util.ConcurrentModificationException;
import org.netbeans.lib.lexer.EmbeddedTokenList;
import org.netbeans.lib.lexer.EmbeddingContainer;
import org.netbeans.lib.lexer.JoinTokenList;
import org.netbeans.lib.lexer.SubSequenceTokenList;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.lib.lexer.TokenList;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.TokenOrEmbedding;

/**
 * Token sequence allows to iterate between tokens

    public int offset() {
        checkTokenNotNull();
        if (tokenOffset == -1) {
            tokenOffset = tokenList.tokenOffsetByIndex(tokenIndex);
        }
        return tokenOffset;
    }

     */
    public TokenSequence<?> embedded() {
        checkTokenNotNull();
        return embeddedImpl(null, false);
    }
    
    private <ET extends TokenId> TokenSequence<ET> embeddedImpl(Language<ET> embeddedLanguage) {
        if (token.isFlyweight())
            return null;
        TokenList<ET> embeddedTokenList = LexerUtilsConstants.embeddedTokenList(
                tokenList, tokenIndex, embeddedLanguage);
        return (embeddedTokenList != null)
                ? new TokenSequence<ET>(embeddedTokenList)
                : null;
    }

    /**

     */
    public <ET extends TokenId> TokenSequence<ET> embedded(Language<ET> embeddedLanguage) {
        checkTokenNotNull();
        return embeddedImpl(embeddedLanguage, false);
    }

    /**
     * Get embedded token sequence that possibly joins multiple embeddings
     * with the same language paths (if the embeddings allow it - see
     * {@link LanguageEmbedding#joinSections()}) into a single input text
     * which is then lexed as a single continuous text.
     * <br/>
     * If any of the resulting tokens crosses embedding's boundaries then the token
     * is split into multiple part tokens.
     * <br/>
     * If the embedding does not join sections then this method behaves
     * like {@link #embedded()}.
     * 
     * @return embedded sequence or null if no embedding exists for this token.
     *  The token sequence will be positioned before first token of this embedding
     *  or to a join token in case the first token of this embedding is part of the join token.
     */
    public TokenSequence<?> embeddedJoined() {
        checkTokenNotNull();
        return embeddedImpl(null, true);
    }

    /**
     * Get embedded token sequence if the token
     * to which this token sequence is currently positioned
     * has a language embedding.
     * 
     * @throws IllegalStateException if {@link #token()} returns null.
     */
    public <ET extends TokenId> TokenSequence<ET> embeddedJoined(Language<ET> embeddedLanguage) {
        checkTokenNotNull();
        return embeddedImpl(embeddedLanguage, true);
    }

    private <ET extends TokenId> TokenSequence<ET> embeddedImpl(Language<ET> embeddedLanguage, boolean joined) {
        if (token.isFlyweight())
            return null;

        EmbeddedTokenList<ET> embeddedTokenList
                = EmbeddingContainer.embeddedTokenList(tokenList, tokenIndex, embeddedLanguage, true);
        if (embeddedTokenList != null) {
            embeddedTokenList.embeddingContainer().updateStatus();
            TokenSequence<ET> tse;
            JoinTokenList<ET> joinTokenList;
            if (joined && (joinTokenList = embeddedTokenList.joinTokenList()) != null) {
                tse = new TokenSequence<ET>(joinTokenList);
                // Position to this etl's index
                tse.moveIndex(joinTokenList.activeStartJoinIndex());
            } else { // Request regular TS or no joining available
                tse = new TokenSequence<ET>(embeddedTokenList);
            }
            return tse;
        }
        return null;
    }

    /**

        checkModCount();
        if (token != null) // Token already fetched
            tokenIndex++;
        TokenOrEmbedding<T> tokenOrEmbedding = tokenList.tokenOrEmbedding(tokenIndex);
        if (tokenOrEmbedding != null) { // Might be null if no more tokens available
            AbstractToken<T> origToken = token;
            token = tokenOrEmbedding.token();
            // If origToken == null then the right offset might already be pre-computed from move()
            if (tokenOffset != -1) {
                if (origToken != null) {

    public boolean movePrevious() {
        checkModCount();
        if (tokenIndex > 0) {
            AbstractToken<T> origToken = token;
            tokenIndex--;
            token = tokenList.tokenOrEmbedding(tokenIndex).token();
            if (tokenOffset != -1) {
                // If the token list is continuous or the original token
                // is flyweight (there cannot be a gap before flyweight token)

    public int moveIndex(int index) {
        checkModCount();
        if (index >= 0) {
            TokenOrEmbedding<T> tokenOrEmbedding = tokenList.tokenOrEmbedding(index);
            if (tokenOrEmbedding != null) { // enough tokens
                resetTokenIndex(index, -1);
            } else {// Token at the requested index does not exist - leave orig. index
                resetTokenIndex(tokenCount(), -1);
            }
        } else {// index < 0
            resetTokenIndex(0, -1);
        }
        return index - tokenIndex;
    }
    

     * <p>
     * If token filtering is used there may be gaps that are not covered
     * by any tokens and if the offset is contained in such gap then
     * the token sequence will be positioned before the token that precedes the gap.
     * </p>
     *
     *

     * @return difference between the reqeuested offset
     *  and the start offset of the token
     *  before which the the token sequence gets positioned.
     *  <br/>
     *  If positioned right after the last token then (offset - last-token-end-offset)
     *  is returned.
     * 
     * @throws ConcurrentModificationException if this token sequence
     *  is no longer valid because of an underlying mutable input source modification.
     */
    public int move(int offset) {
        checkModCount();
        int[] indexAndTokenOffset = tokenList.tokenIndex(offset);
        if (indexAndTokenOffset[0] != -1) { // Valid index and token-offset
            resetTokenIndex(indexAndTokenOffset[0], indexAndTokenOffset[1]);
        } else { // No tokens in token list (indexAndOffset[1] == 0)
            resetTokenIndex(0, -1); // Set Index to zero and offset to invalid
            if (tokenList.tokenOrEmbeddingContainer(0) == null) { // really no tokens at all
                // In this case the token sequence could not be positioned yet
                // so no need to reset "index" or other vars
                resetTokenIndex(0);
                return offset;
            }
            // Re-get the present token count (could be created a chunk of tokens at once)
            tokenCount = tokenList.tokenCountCurrent();
        }
        return offset - indexAndTokenOffset[1];
        // tokenCount surely >0
        int prevTokenOffset = tokenList.tokenOffset(tokenCount - 1);
        if (offset > prevTokenOffset) { // may need to create further tokens if they do not exist
            // Force token list to create subsequent tokens
            // Cannot subtract offset by each token's length because
            // there may be gaps between tokens due to token id filter use.
            int tokenLength = LexerUtilsConstants.token(tokenList, tokenCount - 1).length();
            while (offset >= prevTokenOffset + tokenLength) { // above present token
                Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(tokenCount);
                if (tokenOrEmbeddingContainer != null) {
                    AbstractToken t = LexerUtilsConstants.token(tokenOrEmbeddingContainer);
                    if (t.isFlyweight()) { // need to use previous tokenLength
                        prevTokenOffset += tokenLength;
                    } else { // non-flyweight token - retrieve offset
                        prevTokenOffset = tokenList.tokenOffset(tokenCount);
                    }
                    tokenLength = t.length();
                    tokenCount++;

                } else { // no more tokens => position behind last token
                    resetTokenIndex(tokenCount);
                    tokenOffset = prevTokenOffset + tokenLength; // May assign the token's offset in advance
                    return offset - tokenOffset;
                }
            }
            resetTokenIndex(tokenCount - 1);
            tokenOffset = prevTokenOffset; // May assign the token's offset in advance
            return offset - prevTokenOffset;
        }
        
        // The offset is within the currently recognized tokens
        // Use binary search
        int low = 0;
        int high = tokenCount - 1;
        
        while (low <= high) {
            int mid = (low + high) / 2;
            int midStartOffset = tokenList.tokenOffset(mid);
            
            if (midStartOffset < offset) {
                low = mid + 1;
            } else if (midStartOffset > offset) {
                high = mid - 1;
            } else {
                // Token starting exactly at offset found
                resetTokenIndex(mid);
                tokenOffset = midStartOffset;
                return 0; // right at the token begining
            }
        }
        
        // Not found exactly and high + 1 == low => high < low
        // BTW there may be gaps between tokens; if offset is in gap then position to higher token
        if (high >= 0) { // could be -1
            AbstractToken t = LexerUtilsConstants.token(tokenList, high);
            prevTokenOffset = tokenList.tokenOffset(high);
            // If gaps allowed check whether the token at "high" contains the offset
            if (!tokenList.isContinuous() && offset > prevTokenOffset + t.length()) {
                // Offset in the gap above the "high" token
                high++;
                prevTokenOffset += t.length();
            }
        } else { // at least one token exists => use token at index 0
            high = 0;
            prevTokenOffset = tokenList.tokenOffset(0); // result may differ from 0
        }
        resetTokenIndex(high);
        tokenOffset = prevTokenOffset;
        return offset - prevTokenOffset;
    }
    
    /**

     * @see #tokenCount()
     */
    public boolean isEmpty() {
        return (tokenIndex == 0 && tokenList.tokenOrEmbedding(0) == null);
    }

    /**

            tl = stl.delegate();
            startOffset = Math.max(startOffset, stl.limitStartOffset());
            endOffset = Math.min(endOffset, stl.limitEndOffset());
        } else {// Regular token list
            tl = tokenList;
        }
        return new TokenSequence<T>(new SubSequenceTokenList<T>(tl, startOffset, endOffset));
    }
    

    @Override
    public String toString() {
        return LexerUtilsConstants.appendTokenList(null, tokenList,
                tokenIndex, 0, Integer.MAX_VALUE, true, 0, true).toString();
    }
    
    private void resetTokenIndex(int index, int offset) {
        // Position to the given index e.g. by move() and moveIndex()
        tokenIndex = index;
        token = null;
        tokenOffset = offset;
    }

    private void checkTokenNotNull() {




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer;

import java.util.ArrayList;
import java.util.Set;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.InputAttributes;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.token.TextToken;


/**
 * Token list used for immutable inputs.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class BatchTokenList<T extends TokenId>
extends ArrayList<TokenOrEmbedding<T>> implements TokenList<T> {
    
    /** Flag for additional correctness checks (may degrade performance). */
    private static final boolean testing = Boolean.getBoolean("netbeans.debug.lexer.test");
    
    private static boolean maintainLAState;
    
    /**
     * Check whether lookaheads and states are stored for testing purposes.
     */
    public static boolean isMaintainLAState() {
        return maintainLAState;
    }
    
    public static void setMaintainLAState(boolean maintainLAState) {
        BatchTokenList.maintainLAState = maintainLAState;
    }
    
    private final TokenHierarchyOperation<?,T> tokenHierarchyOperation;
    
    private final CharSequence inputSourceText;

    private final LanguagePath languagePath;
    
    private final Set<T> skipTokenIds;
    
    private final InputAttributes inputAttributes;
    
    /**
     * Lexer input used for lexing of the input.
     */
    private LexerInputOperation<T> lexerInputOperation;

    private LAState laState;
    
    
    public BatchTokenList(TokenHierarchyOperation<?,T> tokenHierarchyOperation, CharSequence inputText,
    Language<T> language, Set<T> skipTokenIds, InputAttributes inputAttributes) {
        this.tokenHierarchyOperation = tokenHierarchyOperation;
        this.inputSourceText = inputText;
        this.languagePath = LanguagePath.get(language);
        this.skipTokenIds = skipTokenIds;
        this.inputAttributes = inputAttributes;
        if (testing) { // Maintain lookaheads and states when in test environment
            laState = LAState.empty();
        }
        this.lexerInputOperation = createLexerInputOperation();
    }

    protected LexerInputOperation<T> createLexerInputOperation() {
        return new TextLexerInputOperation<T>(this);
    }

    public TokenList<?> rootTokenList() {
        return this; // this list should always be the root list of the token hierarchy
    }
    
    public CharSequence inputSourceText() {
        return inputSourceText;
    }

    public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {
        return tokenHierarchyOperation;
    }
    
    public LanguagePath languagePath() {
        return languagePath;
    }
    
    public synchronized int tokenCount() {
        if (lexerInputOperation != null) { // still lexing
            tokenOrEmbeddingImpl(Integer.MAX_VALUE);
        }
        return size();
    }
    
    public int tokenCountCurrent() {
        return size();
    }

    public int tokenOffset(AbstractToken<T> token) {
        int rawOffset = token.rawOffset();
        // Children offsets should be absolute
        return rawOffset;
    }

    public int tokenOffsetByIndex(int index) {
        AbstractToken<T> token = existingToken(index);
        int offset;
        if (token.isFlyweight()) {
            offset = 0;
            while (--index >= 0) {
                token = existingToken(index);
                offset += token.length();
                if (!token.isFlyweight()) {
                    offset += token.offset(null);
                    break;
                }
            }
        } else { // non-flyweight offset
            offset = token.offset(null);
        }
        return offset;
    }

    public int[] tokenIndex(int offset) {
        return LexerUtilsConstants.tokenIndexLazyTokenCreation(this, offset);
    }

    public synchronized TokenOrEmbedding<T> tokenOrEmbedding(int index) {
        return tokenOrEmbeddingImpl(index);
    }
    
    private TokenOrEmbedding<T> tokenOrEmbeddingImpl(int index) {
        while (lexerInputOperation != null && index >= size()) {
            AbstractToken<T> token = lexerInputOperation.nextToken();
            if (token != null) { // lexer returned valid token
                add(token);
                if (laState != null) { // maintaining lookaheads and states
                    laState = laState.add(lexerInputOperation.lookahead(),
                            lexerInputOperation.lexerState());
                }
            } else { // no more tokens from lexer
                lexerInputOperation.release();
                lexerInputOperation = null;
                trimToSize();
            }
        }
        return (index < size()) ? get(index) : null;
    }
    
    private AbstractToken<T> existingToken(int index) {
        return get(index).token();
    }

    public int lookahead(int index) {
        return (laState != null) ? laState.lookahead(index) : -1;
    }

    public Object state(int index) {
        return (laState != null) ? laState.state(index) : null;
    }

    public int startOffset() {
        return 0;
    }

    public int endOffset() {
        int cntM1 = tokenCount() - 1;
        if (cntM1 >= 0)
            return tokenOffsetByIndex(cntM1) + tokenOrEmbeddingImpl(cntM1).token().length();
        return 0;
    }

    public boolean isRemoved() {
        return false;
    }

    public int modCount() {
        return LexerUtilsConstants.MOD_COUNT_IMMUTABLE_INPUT; // immutable input
    }
    
    public synchronized AbstractToken<T> replaceFlyToken(
    int index, AbstractToken<T> flyToken, int offset) {
        TextToken<T> nonFlyToken = ((TextToken<T>)flyToken).createCopy(this, offset);
        set(index, nonFlyToken);
        return nonFlyToken;
    }

    public void wrapToken(int index, EmbeddingContainer<T> embeddingContainer) {
        set(index, embeddingContainer);
    }

    public InputAttributes inputAttributes() {
        return inputAttributes;
    }
    
    public boolean isContinuous() {
        return (skipTokenIds == null);
    }
    
    public Set<T> skipTokenIds() {
        return skipTokenIds;
    }

}




    public void notifyError(String errorMessage) {
        if (lexerInputOperation != null) {
            int parentIndex = parent.readIndex(); // Get the 
//            lexerInputOperation.notifyPreprocessorError(
//                new CharPreprocessorError(errorMessage, parent.deepRawLength(parentIndex)));
        }
    }


        return tokenLength;
    }
    
    public void assignTokenLength(int tokenLength, boolean skipToken) {
        this.tokenLength = tokenLength;
        // Modify tokenLength for preprocessed characters
        parent.assignTokenLength(parentLength(tokenLength), skipToken);
    }
    
    public PreprocessedTextStorage createPreprocessedTextStorage(CharSequence rawText,

     * This method is called after the token has been recognized
     * to clear internal data related to processing of token's characters.
     */
    public void consumeTokenLength() {
        if (prepStartIndex != lookaheadIndex) { // some prep chars (may be after token length)
            if (prepStartIndex < tokenLength) { // prep chars before token end
                if (prepEndIndex <= tokenLength) { // no preprocessed chars past token end


        readIndex -= tokenLength;
        lookaheadIndex -= tokenLength;
        parent.consumeTokenLength();

        if (testing)
            consistencyCheck();




     * token length in the root lexer input operation due to character
     * preprocessing.
     * <br/>
     * The tokenLength at a particular level should be cached by the corresponding provider.
     *
     * @param skipToken whether the token will be skipped due to filtering of its id.
     * @return true if the token is preprocessed or false otherwise.
     */
    void assignTokenLength(int tokenLength, boolean skipToken);
    
    /**
     * Notify this provider that the token was created and
     * that the tokenLength number of characters should be consumed
     * (tokenLength should continue to be held by the provider).
     */
    void consumeTokenLength();
    
    /**
     * Collect extra preprocessed characters from the parent providers.




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer;

/**
 * Class that wraps a each embedded token list contained in join token list.
 * 
 * @author Miloslav Metelka
 */

public final class EmbeddedJoinInfo {
    
    public EmbeddedJoinInfo(JoinTokenListBase base, int rawJoinTokenIndex, int rawTokenListIndex) {
        assert (base != null);
        this.base = base;
        this.rawJoinTokenIndex = rawJoinTokenIndex;
        this.rawTokenListIndex = rawTokenListIndex;
    }
    
    /**
     * Reference to join token list base as a join-related extension
     * of this ETL.
     * In fact this is the only field through which the join token list base instance
     * is referenced.
     */
    public final JoinTokenListBase base; // 12 bytes (8-super + 4)

    /**
     * Index in terms of join token list
     * that corresponds to first token of wrapped ETL.
     * <br/>
     * The index must be gap-preprocessed.
     */
    int rawJoinTokenIndex; // 16 bytes

    /**
     * Index of related ETL in a join token list (base).
     * <br/>
     * The index must be gap-preprocessed.
     */
    int rawTokenListIndex; // 20 bytes

    /**
     * Number of items to go forward to reach last part of a join token.
     * Zero otherwise.
     */
    private int joinTokenLastPartShift; // 24 bytes

    public int joinTokenIndex() {
        return base.joinTokenIndex(rawJoinTokenIndex);
    }

    public void setRawJoinTokenIndex(int rawJoinTokenIndex) {
        this.rawJoinTokenIndex = rawJoinTokenIndex;
    }

    public int tokenListIndex() {
        return base.tokenListIndex(rawTokenListIndex);
    }

    public int joinTokenLastPartShift() {
        return joinTokenLastPartShift;
    }
    
    public void setJoinTokenLastPartShift(int joinTokenLastPartShift) {
        this.joinTokenLastPartShift = joinTokenLastPartShift;
    }

    public StringBuilder dumpInfo(StringBuilder sb) {
        if (sb == null)
            sb = new StringBuilder(70);
        sb.append("jti=").append(joinTokenIndex());
        sb.append(", tli=").append(tokenListIndex());
        sb.append(", lps=").append(joinTokenLastPartShift());
        return sb;
    }

    @Override
    public String toString() {
        return dumpInfo(null).toString();
    }

}




import org.netbeans.lib.editor.util.FlyOffsetGapList;
import org.netbeans.lib.lexer.inc.MutableTokenList;
import org.netbeans.api.lexer.InputAttributes;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.inc.TokenListChange;
import org.netbeans.spi.lexer.LanguageEmbedding;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.token.JoinToken;
import org.netbeans.lib.lexer.token.TextToken;



 */

public final class EmbeddedTokenList<T extends TokenId>
extends FlyOffsetGapList<TokenOrEmbedding<T>> implements MutableTokenList<T> {
    
    /** Flag for additional correctness checks (may degrade performance). */
    private static final boolean testing = Boolean.getBoolean("netbeans.debug.lexer.test");

     * made but was unsuccessful.
     */
    public static final EmbeddedTokenList<TokenId> NO_DEFAULT_EMBEDDING
            = new EmbeddedTokenList<TokenId>(null, null, null);
    
    /**
     * Embedding container carries info about the token into which this

     */
    private EmbeddedTokenList<?> nextEmbeddedTokenList; // 52 bytes
    
    /**
     * Additional information in case this ETL is contained in a JoinTokenList.
     * <br/>
     * Through this info a reference to the JoinTokenList is held. There is no other
     * indexed structure so the EmbeddedTokenList members of TokenListList
     * must be binary-searched.
     */
    public EmbeddedJoinInfo joinInfo; // 56 bytes

    
    public EmbeddedTokenList(EmbeddingContainer<?> embeddingContainer,
            LanguagePath languagePath, LanguageEmbedding<T> embedding
    ) {
        super(1); // Suitable for adding join-token parts
        this.embeddingContainer = embeddingContainer;
        this.languagePath = languagePath;
        this.embedding = embedding;
        this.nextEmbeddedTokenList = nextEmbedding;

        if (embeddingContainer != null) { // ec may be null for NO_DEFAULT_EMBEDDING only
            initLAState();
            embeddingContainer.updateStatusImpl(); // Ensure startOffset() is up-to-date
        }
    }

    public void initAllTokens() {
        assert (!embedding.joinSections()); // Joined token creation must be used instead
//        initLAState();
            root().tokenHierarchyOperation().tokenListList(languagePath);
        } else { // not joining => can lex individually
            init(null);
        }
    }
    
    public void init(Object relexState) {
        laState = (modCount() != -1 || testing) ? LAState.empty() : null;

        // Lex the whole input represented by token at once
        LexerInputOperation<T> lexerInputOperation = createLexerInputOperation(
                0, startOffset(), null);
        AbstractToken<T> token = lexerInputOperation.nextToken();
        while (token != null) {
            addToken(token, lexerInputOperation);
            add(token);
            if (laState != null) {
                laState = laState.add(lexerInputOperation.lookahead(),
                        lexerInputOperation.lexerState());
            }
            token = lexerInputOperation.nextToken();
        }
        lexerInputOperation.release();
        lexerInputOperation = null;
        trimStorageToSize();
    }
    
    private void initLAState() {
        this.laState = (modCount() != LexerUtilsConstants.MOD_COUNT_IMMUTABLE_INPUT || testing)
                ? LAState.empty() // Will collect LAState
                : null;
    }

    /**
     * Return join token list with active token list positioned to this ETL
     * or return null if this.joinInfo == null.
     */
    public JoinTokenList<T> joinTokenList() {
        if (joinInfo != null) {
            TokenListList<T> tokenListList = rootTokenList().tokenHierarchyOperation().existingTokenListList(languagePath);
            int etlIndex = tokenListList.findIndex(startOffset());
            int tokenListStartIndex = etlIndex - joinInfo.tokenListIndex();
            JoinTokenList<T> jtl = new JoinTokenList<T>(tokenListList, joinInfo.base, tokenListStartIndex);
            // Position to this etl's join index
            jtl.setActiveTokenListIndex(etlIndex - tokenListStartIndex);
            return jtl;
        }
        return null;
    }

    /**
     * Add token without touching laState - suitable for JoinToken's handling.
     *
     * @param token non-null token
     */
    public void addToken(AbstractToken<T> token) {
        updateElementOffsetAdd(token); // must subtract startOffset()
        add(token);
    }

    public void addToken(AbstractToken<T> token, LexerInputOperation<T> lexerInputOperation) {
        addToken(token);
        if (laState != null) { // maintaining lookaheads and states
            // Only get LA and state when necessary (especially lexerState() may be costly)
            laState = laState.add(lexerInputOperation.lookahead(), lexerInputOperation.lexerState());
        }
    }

    /**
     * Used when dealing with PartToken instances.
     */
    public void addToken(AbstractToken<T> token, int lookahead, Object state) {
        addToken(token);
        if (laState != null) { // maintaining lookaheads and states
            laState = laState.add(lookahead, state);
        }
    }

    public void trimStorageToSize() {
        trimToSize(); // Compact storage
        if (laState != null)
            laState.trimToSize();
    }
    
    public EmbeddedTokenList<?> nextEmbeddedTokenList() {
     * Check whether this embedded token list is initialized.
     * <br/>
     * If not then the updating process should not touch it unless
     * the token list list exists for this particular language path.
     */
    public boolean isInited() {
        return (laState != LAState.initState());
    }
    
    EmbeddedTokenList<?> nextEmbeddedTokenList() {
        return nextEmbeddedTokenList;
    }
    

    }

    public int tokenCount() {
        return tokenCountCurrent();
            if (laState == LAState.initState())
                init();
            return size();
        }
    }
    
    public int tokenCountCurrent() {
        return size();
    }

    public int joinTokenCount() {
        int tokenCount = tokenCountCurrent();
        if (tokenCount > 0 && joinInfo.joinTokenLastPartShift() > 0)
            tokenCount--;
        return tokenCount;
    }

    public TokenOrEmbedding<T> tokenOrEmbedding(int index) {
        synchronized (rootTokenList()) {
            return (index < size()) ? get(index) : null;
        }
    }
    
    private Token existingToken(int index) {
        return LexerUtilsConstants.token(tokenOrEmbeddingContainer(index));
    }
    
    public int lookahead(int index) {

     * For token hierarchy snapshots the returned value is corrected
     * in the TokenSequence explicitly by adding TokenSequence.tokenOffsetDiff.
     */
    public int tokenOffsetByIndex(int index) {
//        embeddingContainer().checkStatusUpdated();
        return elementOffset(index);
    }

    public int tokenOffset(AbstractToken<T> token) {
        if (token.getClass() == JoinToken.class) {
            return token.offset(null);
        }
        int rawOffset = token.rawOffset();
    
    public int childTokenOffsetNoUpdate(int rawOffset) {
//        embeddingContainer().checkStatusUpdated();
        int relOffset = (rawOffset < offsetGapStart())
                ? rawOffset
                : rawOffset - offsetGapLength();
        return startOffset() + relOffset;
    }

    public int[] tokenIndex(int offset) {
        return LexerUtilsConstants.tokenIndexBinSearch(this, offset, tokenCountCurrent());
     * against start offset of the root token.
     */
    public int childTokenOffsetShift(int rawOffset) {
        // Need to make sure that the startOffsetShift is up-to-date
        embeddingContainer.updateStatus();
        return embeddingContainer.rootTokenOffsetShift() + childTokenRelOffset(rawOffset);
    }

    /**
     * Get child token's real offset which is always a relative value
     * to startOffset value.
     */
    private int childTokenRelOffset(int rawOffset) {
        return (rawOffset < offsetGapStart())
                ? rawOffset
                : rawOffset - offsetGapLength();
    }

    public char childTokenCharAt(int rawOffset, int index) {
//        embeddingContainer().checkStatusUpdated();
        // Do not update the start offset shift - the token.text()
        // did it before returning its result and its contract
        // specifies that.
        // Return chars by delegating to rootToken
        return embeddingContainer.charAt(
                embedding.startSkipLength() + childTokenRelOffset(rawOffset) + index);
    }

    public int modCount() {
        // Mod count of EC must be returned to allow custom removed embeddings to work
        //  - they set LexerUtilsConstants.MOD_COUNT_REMOVED as cachedModCount.
        // calls EmbeddingContainer.embeddedTokenList()
        // which calls which contains the synchronization and calls updateStatusImpl().
        return embeddingContainer.cachedModCount();
    }
    
    @Override
    public int startOffset() { // used by FlyOffsetGapList
//        embeddingContainer.checkStatusUpdated();
        return embeddingContainer.branchTokenStartOffset() + embedding.startSkipLength();
    }
    
    public int endOffset() {
//        embeddingContainer.checkStatusUpdated();
        return embeddingContainer.branchTokenStartOffset() + embeddingContainer.token().length()
                - embedding.endSkipLength();
    }
    
    public int textLength() {
        return embeddingContainer.token().length() - embedding.startSkipLength() - embedding.endSkipLength();
    }
    
    public boolean isRemoved() {
        embeddingContainer.updateStatusImpl();
        return embeddingContainer.isRemoved();
    }

    public TokenList<?> rootTokenList() {
        return embeddingContainer.rootTokenList();
    }

    public CharSequence inputSourceText() {
        return rootTokenList().inputSourceText();
    }

    public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {
        return rootTokenList().tokenHierarchyOperation();
    }
    
    protected int elementRawOffset(TokenOrEmbedding<T> elem) {
        return elem.token().rawOffset();
    }

    protected void setElementRawOffset(TokenOrEmbedding<T> elem, int rawOffset) {
        elem.token().setRawOffset(rawOffset);
            ? ((EmbeddingContainer)elem).token().rawOffset()
            : ((AbstractToken<?>)elem).rawOffset();
    }

    protected void setElementRawOffset(Object elem, int rawOffset) {
        if (elem.getClass() == EmbeddingContainer.class)
            ((EmbeddingContainer)elem).token().setRawOffset(rawOffset);
        else
            ((AbstractToken<?>)elem).setRawOffset(rawOffset);
    }
    
    protected boolean isElementFlyweight(TokenOrEmbedding<T> elem) {
        return elem.token().isFlyweight();
        return (elem.getClass() != EmbeddingContainer.class)
            && ((AbstractToken<?>)elem).isFlyweight();
    }
    
    protected int elementLength(TokenOrEmbedding<T> elem) {
        return elem.token().length();
    }
    
    public AbstractToken<T> replaceFlyToken(
    int index, AbstractToken<T> flyToken, int offset) {
        synchronized (rootTokenList()) {
            TextToken<T> nonFlyToken = ((TextToken<T>)flyToken).createCopy(this, offset2Raw(offset));
            set(index, nonFlyToken);
            return nonFlyToken;
        }
    }

    public void wrapToken(int index, EmbeddingContainer<T> embeddingContainer) {
        synchronized (rootTokenList()) {
            set(index, embeddingContainer);
        }
    }

    public InputAttributes inputAttributes() {
        return rootTokenList().inputAttributes();
    }

    // MutableTokenList extra methods
    public TokenOrEmbedding<T> tokenOrEmbeddingUnsync(int index) {
        return get(index);
    }

    public int tokenCountCurrent() {
        return size();
    }

    public LexerInputOperation<T> createLexerInputOperation(
    int tokenIndex, int relexOffset, Object relexState) {
//        embeddingContainer.checkStatusUpdated();
//        AbstractToken<?> branchToken = embeddingContainer.token();
        int endOffset = endOffset();
//        assert (!branchToken.isRemoved()) : "No lexing when token is removed";
//        assert (relexOffset >= startOffset()) : "Invalid relexOffset=" + relexOffset + " < startOffset()=" + startOffset();
        assert (relexOffset <= endOffset) : "Invalid relexOffset=" + relexOffset + " > endOffset()=" + endOffset;
        return new TextLexerInputOperation<T>(this, tokenIndex, relexState, relexOffset, endOffset);
            );
        }
        int endOffset = tokenStartOffset + tokenText.length()
            - embedding.endSkipLength();
        return new TextLexerInputOperation<T>(this, tokenIndex, relexState, tokenText,
                tokenStartOffset, relexOffset, endOffset);
    }

    public boolean isFullyLexed() {
        return true;
    }

    public void replaceTokens(TokenListChange<T> change, int diffLength) {
        int index = change.index();
        // Remove obsolete tokens (original offsets are retained)
        int removedTokenCount = change.removedTokenCount();
        int rootModCount = rootTokenList().modCount();
        AbstractToken<T> firstRemovedToken = null;
        if (removedTokenCount > 0) {
            @SuppressWarnings("unchecked")
            TokenOrEmbedding<T>[] removedTokensOrEmbeddings = new TokenOrEmbedding[removedTokenCount];
            copyElements(index, index + removedTokenCount, removedTokensOrEmbeddings, 0);
            firstRemovedToken = removedTokensOrEmbeddings[0].token();
            for (int i = 0; i < removedTokenCount; i++) {
                TokenOrEmbedding<T> tokenOrEmbedding = removedTokensOrEmbeddings[i];
                // It's necessary to update-status of all removed tokens' contained embeddings
                // since otherwise (if they would not be up-to-date) they could not be updated later
                // as they lose their parent token list which the update-status relies on.
                EmbeddingContainer<T> ec = tokenOrEmbedding.embedding();
                if (ec != null) {
                    ec.updateStatusUnsyncAndMarkRemoved();
                    assert (ec.cachedModCount() != rootModCount) : "ModCount already updated"; // NOI18N
                }
                AbstractToken<T> token = tokenOrEmbedding.token();
                if (!token.isFlyweight()) {
                    updateElementOffsetRemove(token);
                    token.setTokenList(null);
                }
            }
            remove(index, removedTokenCount); // Retain original offsets
            laState.remove(index, removedTokenCount); // Remove lookaheads and states
            change.setRemovedTokens(removedTokensOrEmbeddings);
        }

        if (diffLength != 0) { // JoinTokenList may pass 0 to not do any offset updates
            // Move and fix the gap according to the performed modification.
            // Instead of modOffset the gap is located at first relexed token's start
            // because then the already precomputed index corresponding to the given offset
            // can be reused. Otherwise there would have to be another binary search for index.
            int startOffset = startOffset(); // updateStatus() should already be called
            if (offsetGapStart() != change.offset() - startOffset) {
                // Minimum of the index of the first removed index and original computed index
                moveOffsetGap(change.offset() - startOffset, change.index());
            }
            updateOffsetGapLength(-diffLength);
        }
        remove(index, removeTokenCount); // Retain original offsets
        laState.remove(index, removeTokenCount); // Remove lookaheads and states
        change.setRemovedTokens(removedTokensOrEmbeddingContainers);
        change.setRemovedEndOffset(offset);

        // Move and fix the gap according to the performed modification.
        int startOffset = startOffset(); // updateStatus() should already be called
        if (offsetGapStart() != change.offset() - startOffset) {
            // Minimum of the index of the first removed index and original computed index
            moveOffsetGap(change.offset() - startOffset, Math.min(index, change.offsetGapIndex()));
        }
        updateOffsetGapLength(-diffLength);

        // Add created tokens.
        // This should be called early when all the members are true tokens
        List<TokenOrEmbedding<T>> addedTokenOrEmbeddings = change.addedTokenOrEmbeddings();
        if (addedTokenOrEmbeddings != null) {
            for (TokenOrEmbedding<T> tokenOrEmbedding : addedTokenOrEmbeddings) {
                updateElementOffsetAdd(tokenOrEmbedding.token());
                AbstractToken<T> token = (AbstractToken<T>)tokenOrBranch;
                updateElementOffsetAdd(token);
            }
            addAll(index, addedTokenOrEmbeddings);
            laState = laState.addAll(index, change.laState());
            change.syncAddedTokenCount();
            // Check for bounds change only
            if (removedTokenCount == 1 && addedTokenOrEmbeddings.size() == 1) {
                // Compare removed and added token ids and part types
                AbstractToken<T> removedToken = LexerUtilsConstants.token(removedTokensOrEmbeddingContainers[0]);
                AbstractToken<T> addedToken = change.addedToken(0);
                if (firstRemovedToken.id() == addedToken.id()
                    && firstRemovedToken.partType() == addedToken.partType()
                ) {
                    change.markBoundsChange();
                }

        this.embeddingContainer = embeddingContainer;
    }
    
    public StringBuilder dumpInfo(StringBuilder sb) {
        if (sb == null) {
            sb = new StringBuilder(50);
        }
        sb.append("ETL<").append(startOffset());
        sb.append(",").append(endOffset());
        sb.append("> TC=").append(tokenCountCurrent());
        sb.append("(").append(joinTokenCount()).append(')');
        if (joinInfo != null) {
            sb.append(" JI:");
            joinInfo.dumpInfo(sb);
        }
        sb.append(", IHC=").append(System.identityHashCode(this));
        sb.append('\n');
        return sb;
    }

    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder(256);
        dumpInfo(sb);
        LexerUtilsConstants.appendTokenList(sb, this);
        return sb.toString();
    }

Lines 41-46 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/EmbeddingContainer.java (-170 / +169 lines)
41		41
42	package org.netbeans.lib.lexer;	42	package org.netbeans.lib.lexer;
43		43
		44	import org.netbeans.lib.lexer.inc.TokenHierarchyUpdate;
44	import org.netbeans.api.lexer.Language;	45	import org.netbeans.api.lexer.Language;
45	import org.netbeans.api.lexer.LanguagePath;	46	import org.netbeans.api.lexer.LanguagePath;
46	import org.netbeans.api.lexer.TokenHierarchyEventType;	47	import org.netbeans.api.lexer.TokenHierarchyEventType;
Lines 68-75 Link Here
68	* @version 1.00	69	* @version 1.00
69	*/	70	*/
70		71
71	public final class EmbeddingContainer<T extends TokenId> {	72	public final class EmbeddingContainer<T extends TokenId> implements TokenOrEmbedding<T> {
72		73
73	/**	74	/**
74	* Get embedded token list.	75	* Get embedded token list.
75	*	76	*
Lines 79-106 Link Here
79	* should be obtained.	80	* should be obtained.
80	* @param language whether only language embeddding of the particular language	81	* @param language whether only language embeddding of the particular language
81	* was requested. It may be null if any embedding should be returned.	82	* was requested. It may be null if any embedding should be returned.
		83	* @param initTokensInNew true if tokens should be created when a new ETL gets created.
		84	* False in case this is called from TokenListList to grab ETLs for sections joining.
82	*/	85	*/
83	public static <T extends TokenId, ET extends TokenId> EmbeddedTokenList<ET> embeddedTokenList(	86	public static <T extends TokenId, ET extends TokenId> EmbeddedTokenList<ET> embeddedTokenList(
84	TokenList<T> tokenList, int index, Language<ET> embeddedLanguage) {	87	TokenList<T> tokenList, int index, Language<ET> embeddedLanguage, boolean initTokensInNew) {
85	EmbeddingContainer<T> ec;	88	TokenList<?> rootTokenList = tokenList.rootTokenList();
86	AbstractToken<T> token;
87	EmbeddingPresence ep;
88	TokenList<?> rootTokenList = tokenList.root();
89	synchronized (rootTokenList) {	89	synchronized (rootTokenList) {
90	Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index);	90	TokenOrEmbedding<T> tokenOrEmbedding = tokenList.tokenOrEmbedding(index);
91	if (tokenOrEmbeddingContainer.getClass() == EmbeddingContainer.class) {	91	EmbeddingContainer<T> ec = tokenOrEmbedding.embedding();
92	// Embedding container exists	92	AbstractToken<T> token = tokenOrEmbedding.token();
93	@SuppressWarnings("unchecked")	93	EmbeddingPresence ep;
94	EmbeddingContainer<T> ecUC = (EmbeddingContainer<T>)tokenOrEmbeddingContainer;	94	if (ec != null) {
95	ec = ecUC;
96	token = ec.token();
97	ep = null;	95	ep = null;
98
99	} else { // No embedding was created yet	96	} else { // No embedding was created yet
100	ec = null;
101	@SuppressWarnings("unchecked")
102	AbstractToken<T> t = (AbstractToken<T>)tokenOrEmbeddingContainer;
103	token = t;
104	// Check embedding presence	97	// Check embedding presence
105	ep = LexerUtilsConstants.innerLanguageOperation(tokenList.languagePath()).embeddingPresence(token.id());	98	ep = LexerUtilsConstants.innerLanguageOperation(tokenList.languagePath()).embeddingPresence(token.id());
106	if (ep == EmbeddingPresence.NONE) {	99	if (ep == EmbeddingPresence.NONE) {
Lines 112-118 Link Here
112	// need to be processed to find the embedded token list for requested language.	105	// need to be processed to find the embedded token list for requested language.
113	EmbeddedTokenList<?> prevEtl;	106	EmbeddedTokenList<?> prevEtl;
114	if (ec != null) {	107	if (ec != null) {
115	ec.updateStatusImpl();	108	ec.updateStatusUnsync();
116	EmbeddedTokenList<?> etl = ec.firstEmbeddedTokenList();	109	EmbeddedTokenList<?> etl = ec.firstEmbeddedTokenList();
117	prevEtl = null;	110	prevEtl = null;
118	while (etl != null) {	111	while (etl != null) {
Lines 154-161 Link Here
154	setEmbeddingPresence(token.id(), EmbeddingPresence.ALWAYS_QUERY);	147	setEmbeddingPresence(token.id(), EmbeddingPresence.ALWAYS_QUERY);
155	}	148	}
156	// Check whether the token contains enough text to satisfy embedding's start and end skip lengths	149	// Check whether the token contains enough text to satisfy embedding's start and end skip lengths
157	CharSequence text = token.text(); // Should not be null here but rather check	150	if (token.isRemoved() \|\| embedding.startSkipLength() + embedding.endSkipLength() > token.length()) {
158	if (text == null \|\| embedding.startSkipLength() + embedding.endSkipLength() > text.length()) {
159	return null;	151	return null;
160	}	152	}
161	if (ec == null) {	153	if (ec == null) {
Lines 165-174 Link Here
165	LanguagePath embeddedLanguagePath = LanguagePath.get(languagePath,	157	LanguagePath embeddedLanguagePath = LanguagePath.get(languagePath,
166	embedding.language());	158	embedding.language());
167	EmbeddedTokenList<ET> etl = new EmbeddedTokenList<ET>(ec,	159	EmbeddedTokenList<ET> etl = new EmbeddedTokenList<ET>(ec,
168	embeddedLanguagePath, embedding, null);	160	embeddedLanguagePath, embedding);
169	// Preceding code should ensure that (prevEtl.nextEmbeddedTokenList == null)	161	// Preceding code should ensure that (prevEtl.nextEmbeddedTokenList == null)
170	// so no need to call etl.setNextEmbeddedTokenList(prevEtl.nextEmbeddedTokenList())	162	// so no need to call etl.setNextEmbeddedTokenList(prevEtl.nextEmbeddedTokenList())
171	ec.addEmbeddedTokenList(prevEtl, etl, true);	163	ec.addEmbeddedTokenList(prevEtl, etl, true);
		164
		165	if (initTokensInNew) {
		166	if (embedding.joinSections()) {
		167	// Init corresponding TokenListList
		168	rootTokenList.tokenHierarchyOperation().tokenListList(embeddedLanguagePath);
		169	} else { // sections not joined
		170	etl.initAllTokens();
		171	}
		172	}
172	return (embeddedLanguage == null \|\| embeddedLanguage == embedding.language()) ? etl : null;	173	return (embeddedLanguage == null \|\| embeddedLanguage == embedding.language()) ? etl : null;
173	}	174	}
174	// Update embedding presence to NONE	175	// Update embedding presence to NONE
Lines 200-206 Link Here
200	public static <T extends TokenId, ET extends TokenId> boolean createEmbedding(	201	public static <T extends TokenId, ET extends TokenId> boolean createEmbedding(
201	TokenList<T> tokenList, int index, Language<ET> embeddedLanguage,	202	TokenList<T> tokenList, int index, Language<ET> embeddedLanguage,
202	int startSkipLength, int endSkipLength, boolean joinSections) {	203	int startSkipLength, int endSkipLength, boolean joinSections) {
203	TokenList<?> rootTokenList = tokenList.root();	204	TokenList<?> rootTokenList = tokenList.rootTokenList();
204	// Only create embedddings for valid operations so not e.g. for removed token list	205	// Only create embedddings for valid operations so not e.g. for removed token list
205	AbstractToken<T> token;	206	AbstractToken<T> token;
206	EmbeddingContainer<T> ec;	207	EmbeddingContainer<T> ec;
Lines 217-228 Link Here
217	tokenHierarchyOperation = tokenList.tokenHierarchyOperation();	218	tokenHierarchyOperation = tokenList.tokenHierarchyOperation();
218	tokenHierarchyOperation.ensureWriteLocked();	219	tokenHierarchyOperation.ensureWriteLocked();
219		220
220	Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index);	221	TokenOrEmbedding<T> tokenOrEmbedding = tokenList.tokenOrEmbedding(index);
221	if (tokenOrEmbeddingContainer.getClass() == EmbeddingContainer.class) {	222	ec = tokenOrEmbedding.embedding();
222	// Embedding container exists	223	token = tokenOrEmbedding.token();
223	@SuppressWarnings("unchecked")	224	if (ec != null) {
224	EmbeddingContainer<T> ecUC = (EmbeddingContainer<T>)tokenOrEmbeddingContainer;
225	ec = ecUC;
226	EmbeddedTokenList etl2 = ec.firstEmbeddedTokenList();	225	EmbeddedTokenList etl2 = ec.firstEmbeddedTokenList();
227	while (etl2 != null) {	226	while (etl2 != null) {
228	if (embeddedLanguage == etl2.languagePath().innerLanguage()) {	227	if (embeddedLanguage == etl2.languagePath().innerLanguage()) {
Lines 230-240 Link Here
230	}	229	}
231	etl2 = etl2.nextEmbeddedTokenList();	230	etl2 = etl2.nextEmbeddedTokenList();
232	}	231	}
233	token = ec.token();
234	} else {	232	} else {
235	@SuppressWarnings("unchecked")
236	AbstractToken<T> t = (AbstractToken<T>)tokenOrEmbeddingContainer;
237	token = t;
238	if (token.isFlyweight()) { // embedding cannot exist for this flyweight token	233	if (token.isFlyweight()) { // embedding cannot exist for this flyweight token
239	return false;	234	return false;
240	}	235	}
Lines 253-264 Link Here
253	tokenHierarchyOperation.addLanguagePath(embeddedLanguagePath);	248	tokenHierarchyOperation.addLanguagePath(embeddedLanguagePath);
254	// Make the embedded token list to be the first in the list	249	// Make the embedded token list to be the first in the list
255	etl = new EmbeddedTokenList<ET>(	250	etl = new EmbeddedTokenList<ET>(
256	ec, embeddedLanguagePath, embedding, ec.firstEmbeddedTokenList());	251	ec, embeddedLanguagePath, embedding);
257	ec.addEmbeddedTokenList(null, etl, false);	252	ec.addEmbeddedTokenList(null, etl, false);
258		253
259	// Fire the embedding creation to the clients	254	// Fire the embedding creation to the clients
260	// Threading model may need to be changed if necessary	255	// Threading model may need to be changed if necessary
261	tokenStartOffset = ec.tokenStartOffset();	256	tokenStartOffset = ec.branchTokenStartOffset();
262	eventInfo = new TokenHierarchyEventInfo(	257	eventInfo = new TokenHierarchyEventInfo(
263	tokenHierarchyOperation,	258	tokenHierarchyOperation,
264	TokenHierarchyEventType.EMBEDDING_CREATED,	259	TokenHierarchyEventType.EMBEDDING_CREATED,
Lines 269-285 Link Here
269	// When joining sections ensure that the token list list gets created	264	// When joining sections ensure that the token list list gets created
270	// and the embedded tokens get created because they must exist	265	// and the embedded tokens get created because they must exist
271	// before possible next updating of the token list.	266	// before possible next updating of the token list.
272	TokenListList tll = tokenHierarchyOperation.existingTokenListList(etl.languagePath());	267	TokenListList<ET> tll = tokenHierarchyOperation.existingTokenListList(etl.languagePath());
		268	if (!embedding.joinSections()) {
		269	etl.initAllTokens();
		270	}
273	if (tll != null) {	271	if (tll != null) {
274	// Update tll by embedding creation	272	// Update tll by embedding creation
275	new TokenHierarchyUpdate(eventInfo).updateCreateEmbedding(etl);	273	new TokenHierarchyUpdate(eventInfo).updateCreateOrRemoveEmbedding(etl, true);
276	} else { // tll == null	274	} else { // tll == null
277	if (embedding.joinSections()) {	275	if (embedding.joinSections()) {
278	// Force token list list creation only when joining sections	276	// Force token list list creation only when joining sections
279	tll = tokenHierarchyOperation.tokenListList(etl.languagePath());	277	tll = tokenHierarchyOperation.tokenListList(etl.languagePath());
280	}	278	}
281	}	279	}
282
283	}	280	}
284		281
285	// Construct outer token change info	282	// Construct outer token change info
Lines 304-327 Link Here
304		301
305	public static <T extends TokenId, ET extends TokenId> boolean removeEmbedding(	302	public static <T extends TokenId, ET extends TokenId> boolean removeEmbedding(
306	TokenList<T> tokenList, int index, Language<ET> embeddedLanguage) {	303	TokenList<T> tokenList, int index, Language<ET> embeddedLanguage) {
307	TokenList<?> rootTokenList = tokenList.root();	304	TokenList<?> rootTokenList = tokenList.rootTokenList();
308	// Only create embedddings for valid operations so not e.g. for removed token list	305	// Only create embedddings for valid operations so not e.g. for removed token list
309	EmbeddingContainer<T> ec;	306	EmbeddingContainer<T> ec;
310	synchronized (rootTokenList) {	307	synchronized (rootTokenList) {
311	// Check TL.isRemoved() under syncing of rootTokenList	308	// Check TL.isRemoved() under syncing of rootTokenList
312	if (tokenList.isRemoved()) // Do not create embedding for removed TLs	309	if (tokenList.isRemoved()) // Do not remove embedding for removed TLs
313	return false;	310	return false;
314	// If TL is not removed then THO should be non-null	311	// If TL is not removed then THO should be non-null
315	TokenHierarchyOperation<?,?> tokenHierarchyOperation = tokenList.tokenHierarchyOperation();	312	TokenHierarchyOperation<?,?> tokenHierarchyOperation = tokenList.tokenHierarchyOperation();
316	tokenHierarchyOperation.ensureWriteLocked();	313	tokenHierarchyOperation.ensureWriteLocked();
317		314
318	Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index);	315	TokenOrEmbedding<T> tokenOrEmbedding = tokenList.tokenOrEmbedding(index);
319	if (tokenOrEmbeddingContainer.getClass() == EmbeddingContainer.class) {	316	ec = tokenOrEmbedding.embedding();
320	// Embedding container exists	317	if (ec != null) {
321	@SuppressWarnings("unchecked")	318	ec.updateStatusUnsync();
322	EmbeddingContainer<T> ecUC = (EmbeddingContainer<T>)tokenOrEmbeddingContainer;
323	ec = ecUC;
324	ec.updateStatusImpl();
325	EmbeddedTokenList<?> etl = ec.firstEmbeddedTokenList();	319	EmbeddedTokenList<?> etl = ec.firstEmbeddedTokenList();
326	EmbeddedTokenList<?> prevEtl = null;	320	EmbeddedTokenList<?> prevEtl = null;
327	while (etl != null) {	321	while (etl != null) {
Lines 337-346 Link Here
337	// State that the removed embedding was not default - should not matter anyway	331	// State that the removed embedding was not default - should not matter anyway
338	ec.addEmbeddedTokenList(null, etl, false);	332	ec.addEmbeddedTokenList(null, etl, false);
339	etl.setEmbeddingContainer(ec);	333	etl.setEmbeddingContainer(ec);
340	ec.invalidateChildren();	334	ec.markChildrenRemovedDeep();
341		335
342	// Fire the embedding creation to the clients	336	// Fire the embedding creation to the clients
343	int startOffset = ec.tokenStartOffset();	337	int startOffset = ec.branchTokenStartOffset();
344	TokenHierarchyEventInfo eventInfo = new TokenHierarchyEventInfo(	338	TokenHierarchyEventInfo eventInfo = new TokenHierarchyEventInfo(
345	tokenHierarchyOperation,	339	tokenHierarchyOperation,
346	TokenHierarchyEventType.EMBEDDING_REMOVED,	340	TokenHierarchyEventType.EMBEDDING_REMOVED,
Lines 369-375 Link Here
369	TokenListList tll = tokenHierarchyOperation.existingTokenListList(etl.languagePath());	363	TokenListList tll = tokenHierarchyOperation.existingTokenListList(etl.languagePath());
370	if (tll != null) {	364	if (tll != null) {
371	// update-status already called	365	// update-status already called
372	new TokenHierarchyUpdate(eventInfo).updateRemoveEmbedding(etl);	366	new TokenHierarchyUpdate(eventInfo).updateCreateOrRemoveEmbedding(etl, false);
373	}	367	}
374		368
375	// Fire the change	369	// Fire the change
Lines 383-446 Link Here
383	return false;	377	return false;
384	}	378	}
385		379
386	private AbstractToken<T> token; // 12 bytes (8-super + 4)	380	/**
		381	* Token wrapped by this EC.
		382	*/
		383	private AbstractToken<T> branchToken; // 12 bytes (8-super + 4)
		384
		385	/**
		386	* Root token list of the hierarchy should never be null and is final.
		387	*
		388	*/
		389	private final TokenList<?> rootTokenList; // 16 bytes
387		390
388	/**	391	/**
389	* Cached modification count allows to determine whether the start offset	392	* Cached modification count allows to determine whether the start offset
390	* needs to be recomputed.	393	* needs to be recomputed.
391	*/	394	*/
392	private int cachedModCount; // 16 bytes	395	private int cachedModCount; // 20 bytes
393		396
394	/**
395	* Root token list of the hierarchy.
396	*
397	*/
398	private final TokenList<?> rootTokenList; // 20 bytes
399
400	/**
401	* The root embedding container to which this embedding container relates.
402	* <br/>
403	* It's used for getting of the start offset of the contained tokens
404	* and for getting of their text.
405	*/
406	private AbstractToken<?> rootToken; // 24 bytes
407
408	/**	397	/**
409	* Cached start offset of the token for which this embedding container	398	* Cached start offset of the token for which this embedding container
410	* was created.	399	* was created.
		400	* <br/>
		401	* Its value may be shared by multiple embedded token lists.
411	*/	402	*/
412	private int tokenStartOffset; // 28 bytes	403	private int branchTokenStartOffset; // 24 bytes
413		404
414	/**	405	/**
415	* First embedded token list in the single-linked list.	406	* First embedded token list in the single-linked list.
416	*/	407	*/
417	private EmbeddedTokenList<?> firstEmbeddedTokenList; // 32 bytes	408	private EmbeddedTokenList<?> firstEmbeddedTokenList; // 28 bytes
418		409
419	/**
420	* Difference between start offset of the first token in this token list
421	* against the start offset of the root token.
422	* <br>
423	* The offset gets refreshed upon <code>updateStartOffset()</code>.
424	*/
425	private int offsetShiftFromRootToken; // 36 bytes
426
427	/**	410	/**
428	* Embedded token list that represents the default embedding.	411	* Embedded token list that represents the default embedding.
429	* It may be <code>EmbeddedTokenList.NO_DEFAULT_EMBEDDING</code>	412	* It may be <code>EmbeddedTokenList.NO_DEFAULT_EMBEDDING</code>
430	* for failed attempt to create a default embedding.	413	* for failed attempt to create a default embedding.
431	*/	414	*/
432	private EmbeddedTokenList<?> defaultEmbeddedTokenList; // 40 bytes	415	private EmbeddedTokenList<?> defaultEmbeddedTokenList; // 32 bytes
433		416
434	EmbeddingContainer(AbstractToken<T> token, TokenList<?> rootTokenList) {	417	EmbeddingContainer(AbstractToken<T> branchToken, TokenList<?> rootTokenList) {
435	this.token = token;	418	if (branchToken == null)
		419	throw new IllegalArgumentException("branchToken cannot be null");
		420	if (rootTokenList == null)
		421	throw new IllegalArgumentException("rootTokenList cannot be null");
		422	this.branchToken = branchToken;
436	this.rootTokenList = rootTokenList;	423	this.rootTokenList = rootTokenList;
437	this.rootToken = token; // Has to be non-null since updateStatusImpl() would not update null rootToken
438	// cachedModCount must differ from root's one to sync offsets	424	// cachedModCount must differ from root's one to sync offsets
439	// Root mod count can be >= 0 or -1 for non-incremental token lists	425	// Root mod count can be >= 0 or -1 for non-incremental token lists
440	this.cachedModCount = -2;	426	this.cachedModCount = LexerUtilsConstants.MOD_COUNT_EMBEDDED_INITIAL;
441	// Update the tokenStartOffset etc. - this assumes that the token	427	// Update the tokenStartOffset etc. - this assumes that the token
442	// is already parented till the root token list.	428	// is already parented till the root token list.
443	updateStatusImpl();	429	updateStatusUnsync();
444	}	430	}
445		431
446	/**	432	/**
Lines 456-479 Link Here
456	* @param ec non-null existing embedding container.	442	* @param ec non-null existing embedding container.
457	*/	443	*/
458	EmbeddingContainer(EmbeddingContainer<T> ec) {	444	EmbeddingContainer(EmbeddingContainer<T> ec) {
459	this(ec.token(), ec.rootTokenList()); // Force init of tokenStartOffset and rootTokenOffsetShift	445	this(ec.token(), ec.rootTokenList()); // Force init of tokenStartOffset
460	invalidate();	446	markRemoved();
461	}	447	}
462		448
463	private void invalidate() {	449	private void markRemoved() {
464	this.rootToken = null;	450	// Set cachedModCount to LexerUtilsConstants.MOD_COUNT_REMOVED which should not occur
465	// Set cachedModCount to -2 which should not occur for regular cases	451	// for regular cases which should force existing token sequences to be invalidated.
466	// which should force existing token sequences to be invalidated.	452	this.cachedModCount = LexerUtilsConstants.MOD_COUNT_REMOVED;
467	this.cachedModCount = -2;
468	}	453	}
469		454
470	void invalidateChildren() {	455	void markChildrenRemovedDeep() { // Used by custom embedding removal
471	EmbeddedTokenList etl = firstEmbeddedTokenList;	456	EmbeddedTokenList etl = firstEmbeddedTokenList;
472	while (etl != null && etl != EmbeddedTokenList.NO_DEFAULT_EMBEDDING) {	457	while (etl != null && etl != EmbeddedTokenList.NO_DEFAULT_EMBEDDING) {
473	for (int i = etl.tokenCountCurrent() - 1; i >= 0; i--) {	458	for (int i = etl.tokenCountCurrent() - 1; i >= 0; i--) {
474	Object tokenOrEC = etl.tokenOrEmbeddingContainerUnsync(i);	459	EmbeddingContainer ec = etl.tokenOrEmbeddingUnsync(i).embedding();
475	if (tokenOrEC.getClass() == EmbeddingContainer.class) {	460	if (ec != null) {
476	((EmbeddingContainer)tokenOrEC).invalidateChildren();	461	ec.updateStatusUnsync(); // First update the status
		462	ec.markChildrenRemovedDeep();
		463	ec.markRemoved(); // Mark removed with the correctly updated offsets
477	}	464	}
478	}	465	}
479	etl = etl.nextEmbeddedTokenList();	466	etl = etl.nextEmbeddedTokenList();
Lines 484-509 Link Here
484	return cachedModCount;	471	return cachedModCount;
485	}	472	}
486		473
487	/**	474	public final AbstractToken<T> token() {
488	* Check if this embedding container is up-to-date (updateStatusImpl() was called on it)	475	return branchToken;
489	* which is useful for missing-update-status checks.
490	*/
491	public void checkStatusUpdated() {
492	if (cachedModCount != -2 && cachedModCount != rootTokenList.modCount()
493	&& !checkStatusUpdatedThrowingException
494	) {
495	// Prevent OOME because of nested throwing of exc
496	checkStatusUpdatedThrowingException = true;
497	String excMsg = "!!!INTERNAL ERROR!!! Status not updated on " +
498	this + "\nin token hierarchy\n" + rootTokenList.tokenHierarchyOperation();
499	checkStatusUpdatedThrowingException = false;
500	throw new IllegalStateException(excMsg);
501	}
502	}	476	}
503	private static boolean checkStatusUpdatedThrowingException;	477
504		478	public final EmbeddingContainer<T> embedding() {
505	public AbstractToken<T> token() {	479	return this;
506	return token;
507	}	480	}
508		481
509	/**	482	/**
Lines 511-548 Link Here
511	* The updateStatusImpl() should be called afterwards to update tokenStartOffset etc.	484	* The updateStatusImpl() should be called afterwards to update tokenStartOffset etc.
512	*/	485	*/
513	public void reinit(AbstractToken<T> token) {	486	public void reinit(AbstractToken<T> token) {
514	this.token = token;	487	this.branchToken = token;
515	TokenList<?> parentTokenList = token.tokenList();	488	cachedModCount = LexerUtilsConstants.MOD_COUNT_EMBEDDED_INITIAL;
516	assert (parentTokenList != null);	489	updateStatusUnsync();
517	if (parentTokenList.getClass() == EmbeddedTokenList.class) {
518	rootToken = ((EmbeddedTokenList<?>)parentTokenList).rootToken();
519	} else { // parent is a root token list: rootToken == token
520	rootToken = token;
521	}
522	updateStatusImpl();
523	}	490	}
524		491
525	public TokenList<?> rootTokenList() {	492	public TokenList<?> rootTokenList() {
526	return rootTokenList;	493	return rootTokenList;
527	}	494	}
528		495
529	public AbstractToken<?> rootToken() {	496	public int branchTokenStartOffset() {
530	return rootToken;
531	}
532
533	public int tokenStartOffset() {
534	// checkStatusUpdated();	497	// checkStatusUpdated();
535	return tokenStartOffset;	498	return branchTokenStartOffset;
536	}
537
538	public int rootTokenOffsetShift() {
539	// checkStatusUpdated();
540	return offsetShiftFromRootToken;
541	}
542
543	public char charAt(int tokenRelOffset) {
544	// checkStatusUpdated();
545	return rootToken.charAt(offsetShiftFromRootToken + tokenRelOffset);
546	}	499	}
547		500
548	public EmbeddedTokenList<?> firstEmbeddedTokenList() {	501	public EmbeddedTokenList<?> firstEmbeddedTokenList() {
Lines 606-651 Link Here
606	*/	559	*/
607	public boolean isRemoved() {	560	public boolean isRemoved() {
608	// checkStatusUpdated();	561	// checkStatusUpdated();
609	return (rootToken == null);	562	return (cachedModCount == LexerUtilsConstants.MOD_COUNT_REMOVED);
610	}	563	}
611		564
612	public void updateStatusAndInvalidate() {	565	public void updateStatusUnsyncAndMarkRemoved() {
613	updateStatusImpl();	566	updateStatusUnsync();
614	invalidate();	567	markRemoved();
615	}
616
617	public boolean updateStatus() {
618	synchronized (rootTokenList) {
619	return (updateStatusImpl() != null);
620	}
621	}	568	}
622		569
623	/**	570	/**
624	* Update and return root token corresponding to this embedding container.	571	* Update status of this container in a synchronized way
		572	* ensuring that no other thread will interfere - this is suitable
		573	* for cases when there may be multiple concurrent readers
		574	* using a token hierarchy and calling Token.offset() for example.
		575	* <br/>
		576	* Status updating fixes value of cached start offset of wrapped branch token
		577	* (calling branch token(s) on upper level(s) for multiple embeddings' nesting).
		578	*
		579	* @return true if token is still part of token hierarchy or false
		580	* if it was removed.
625	*/	581	*/
626	public AbstractToken<?> updateStatusImpl() {	582	public boolean updateStatus() {
627	if (rootToken == null)	583	synchronized (rootTokenList) {
628	return null; // Removed from hierarchy	584	return updateStatusUnsync();
629	int rootModCount;	585	}
630	if (cachedModCount != (rootModCount = rootTokenList.modCount())) {	586	}
631	cachedModCount = rootModCount;	587
632	TokenList<?> parentTokenList = token.tokenList();	588	/**
633	if (parentTokenList == null) {	589	* Unsynced synchronization of container - this method should only be used
634	rootToken = null;	590	* when there may be only a single thread accessing token hierarchy i.e. during
635	} else if (parentTokenList.getClass() == EmbeddedTokenList.class) {	591	* token hierarchy modifications upon mutable input source modifications.
636	EmbeddedTokenList<?> parentEtl = (EmbeddedTokenList<?>)parentTokenList;	592	*
637	rootToken = parentEtl.embeddingContainer().updateStatusImpl();	593	* @return true if token is still part of token hierarchy or false
638	tokenStartOffset = parentEtl.childTokenOffsetNoUpdate(token.rawOffset());	594	* if it was removed.
639	EmbeddingContainer parentEC = parentEtl.embeddingContainer();	595	* @see #updateStatus()
640	offsetShiftFromRootToken = tokenStartOffset - parentEC.tokenStartOffset()	596	*/
641	+ parentEC.rootTokenOffsetShift();	597	public boolean updateStatusUnsync() {
642	} else { // parent is a root token list: rootToken == token	598	return (updateStatusImpl(rootTokenList.modCount()) != LexerUtilsConstants.MOD_COUNT_REMOVED);
643	rootToken = token;	599	}
644	tokenStartOffset = token.offset(null);	600
645	offsetShiftFromRootToken = 0;	601	/**
		602	* Update the status of this embedding container when current mod count
		603	* of a root token list is given.
		604	*
		605	* @param rootModCount modCount of a root token list. The token list either
		606	* updates to it or to LexerUtilsConstants.MOD_COUNT_REMOVED if it's removed
		607	* from a token hierarchy. If called by nested embeddings they should finally
		608	* update to the same value.
		609	* @return current modCount of this container.
		610	*/
		611	protected int updateStatusImpl(int rootModCount) {
		612	if (cachedModCount != LexerUtilsConstants.MOD_COUNT_REMOVED &&
		613	cachedModCount != rootModCount
		614	) {
		615	TokenList<T> parentTokenList = branchToken.tokenList();
		616	if (parentTokenList == null) { // branch token removed from its parent token list
		617	cachedModCount = LexerUtilsConstants.MOD_COUNT_REMOVED;
		618	} else if (parentTokenList.getClass() == EmbeddedTokenList.class) { // deeper level embedding
		619	EmbeddedTokenList<T> parentEtl = (EmbeddedTokenList<T>)parentTokenList;
		620	cachedModCount = parentEtl.embeddingContainer().updateStatusImpl(rootModCount);
		621	// After status of parent(s) was updated get the current branch token's offset
		622	branchTokenStartOffset = parentEtl.tokenOffset(branchToken);
		623	} else { // parent of branch token is a non-null root token list.
		624	cachedModCount = rootModCount;
		625	branchTokenStartOffset = parentTokenList.tokenOffset(branchToken);
646	}	626	}
647	}	627	}
648	return rootToken;	628	return cachedModCount;
649	}	629	}
650		630
		631	/**
		632	* Check if this embedding container is up-to-date (updateStatusImpl() was called on it)
		633	* which is useful for missing-update-status checks.
		634	*/
		635	public void checkStatusUpdated() {
		636	if (cachedModCount != LexerUtilsConstants.MOD_COUNT_REMOVED
		637	&& cachedModCount != rootTokenList.modCount()
		638	&& !checkStatusUpdatedThrowingException
		639	) {
		640	// Prevent OOME because of nested throwing of exc
		641	checkStatusUpdatedThrowingException = true;
		642	String excMsg = "!!!INTERNAL ERROR!!! Status not updated on " +
		643	this + "\nin token hierarchy\n" + rootTokenList.tokenHierarchyOperation();
		644	checkStatusUpdatedThrowingException = false;
		645	throw new IllegalStateException(excMsg);
		646	}
		647	}
		648	private static boolean checkStatusUpdatedThrowingException;
		649
651	}	650	}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer;

import java.util.List;
import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.ArrayUtilities;
import org.netbeans.spi.lexer.LexerInput;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.token.JoinToken;
import org.netbeans.lib.lexer.token.PartToken;
import org.netbeans.spi.lexer.TokenPropertyProvider;

/**
 * Lexer input operation over multiple joined sections (embedded token lists).
 * <br/>
 * It produces regular tokens (to be added directly into ETL represented by
 * {@link #activeTokenList()} and also special {@link #JoinToken} instances
 * in case a token spans boundaries of multiple ETLs.
 * <br/>
 * It can either work over JoinTokenList directly or, during a modification,
 * it simulates that certain token lists are already removed/added to underlying token list.
 * <br/>
 * 
 * {@link #recognizedTokenLastInTokenList()} gives information whether the lastly
 * produced token ends right at boundary of the activeTokenList.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public class JoinLexerInputOperation<T extends TokenId> extends LexerInputOperation<T> {
    
    CharSequence inputSourceText;

    private TokenListText readText; // For servicing read()
    
    private TokenListText readExistingText;
    
    /**
     * Token list in which the last recognized token started.
     */
    private EmbeddedTokenList<T> activeTokenList;
    
    /**
     * Index of activeTokenList in JTL.
     */
    private int activeTokenListIndex;
    
    /**
     * End offset of the active token list.
     */
    private int activeTokenListEndOffset;
    
    /**
     * Real token's start offset used to derive the token's offset in ETL.
     * Since tokenStartOffset is affected by TokenListList.readOffsetShift
     * it cannot be used for this purpose.
     */
    private int realTokenStartOffset;
    
    private boolean recognizedTokenJoined; // Whether recognized token will consist of parts
    

    public JoinLexerInputOperation(JoinTokenList<T> joinTokenList, int relexJoinIndex, Object lexerRestartState,
            int activeTokenListIndex, int relexOffset
    ) {
        super(joinTokenList, relexJoinIndex, lexerRestartState);
        this.inputSourceText = joinTokenList.inputSourceText();
        this.activeTokenListIndex = activeTokenListIndex;
        tokenStartOffset = relexOffset;
        readOffset = relexOffset;
        // Assign realTokenStartOffset after fetchActiveTokenList() since it would overwrite it
        realTokenStartOffset = relexOffset;
    }

    public final void init() {
        // Following code uses tokenList() method overriden in MutableJoinLexerInputOperation
        // so the following code would fail when placed in constructor since the constructor of MJLIO would not yet run.
        fetchActiveTokenList();
        readText = new TokenListText();
        readText.init();
    }

    /**
     * Get active ETL into which the last produced token should be added.
     * For join tokens there is an ETL into which a last part of JT should be added.
     */
    public EmbeddedTokenList<T> activeTokenList() {
        return activeTokenList;
    }
    
    /**
     * Get index of active ETL into which the last produced token should be added.
     * For join tokens there is an index of the last ETL into which a last part of JT should be added.
     */
    public int activeTokenListIndex() {
        return activeTokenListIndex;
    }
    
    /**
     * True if the last returned token is last in {@link #activeTokenList()}.
     * For join tokens this applies to the last part of join token.
     */
    public boolean recognizedTokenLastInTokenList() {
        // realTokenStartOffset is set to the end of last recognized token
        return (realTokenStartOffset == activeTokenListEndOffset);
    }

    @Override
    public int lastTokenEndOffset() {
        return realTokenStartOffset;
    }

    public int read(int offset) { // index >= 0 is guaranteed by contract
        return readText.read(offset);
    }

    public char readExisting(int offset) {
        if (readText.isInBounds(offset)) {
            return readText.inBoundsChar(offset);
        }
        if (readExistingText == null) {
            readExistingText = new TokenListText();
            readExistingText.initFrom(readText);
        }
        return readExistingText.existingChar(offset);
    }

    @Override
    public void assignTokenLength(int tokenLength) {
        super.assignTokenLength(tokenLength);
        // Check whether activeTokenList needs to be changed due to various flags
        if (recognizedTokenLastInTokenList()) { // Advance to next token list
            // Since this is done when recognizing a next token it should be ok when recognizing
            // last token in the last ETL (it should not go beyond last ETL).
            do {
                activeTokenListIndex++;
                fetchActiveTokenList();
            } while (realTokenStartOffset == activeTokenListEndOffset); // Skip empty ETLs
        }
        // Advance to end of currently recognized token
        realTokenStartOffset += tokenLength;
        // Joined token past ETL's boundary
        recognizedTokenJoined = (realTokenStartOffset > activeTokenListEndOffset);
    }
    
    private void fetchActiveTokenList() {
        activeTokenList = tokenList(activeTokenListIndex);
        realTokenStartOffset = activeTokenList.startOffset();
        activeTokenListEndOffset = activeTokenList.endOffset();
    }
    
    public EmbeddedTokenList<T> tokenList(int tokenListIndex) { // Also used by JoinTokenListChange
        return ((JoinTokenList<T>) tokenList).tokenList(tokenListIndex);
    }

    protected int tokenListCount() {
        return ((JoinTokenList<T>) tokenList).tokenListCount();
    }

    protected void fillTokenData(AbstractToken<T> token) {
        if (!recognizedTokenJoined) {
            token.setTokenList(activeTokenList);
            // Subtract tokenLength since this is already advanced to end of token
            token.setRawOffset(realTokenStartOffset - tokenLength);
        }
    }
    
    @Override
    protected boolean isFlyTokenAllowed() {
        return super.isFlyTokenAllowed() && !recognizedTokenJoined;
    }
    
    @Override
    protected AbstractToken<T> createDefaultTokenInstance(T id) {
        if (recognizedTokenJoined) {
            return createJoinToken(id, null, PartType.COMPLETE);
        } else { // Regular case
            return super.createDefaultTokenInstance(id);
        }
    }

    @Override
    protected AbstractToken<T> createPropertyTokenInstance(T id,
    TokenPropertyProvider<T> propertyProvider, PartType partType) {
        if (recognizedTokenJoined) {
            return createJoinToken(id, null, partType);
        } else { // Regular case
            return super.createPropertyTokenInstance(id, propertyProvider, partType);
        }
    }
    
    private AbstractToken<T> createJoinToken(T id,
    TokenPropertyProvider<T> propertyProvider, PartType partType) {
        // Create join token
        // realTokenStartOffset is already advanced by tokenLength so first decrease it
        realTokenStartOffset -= tokenLength;
        JoinToken<T> joinToken = new JoinToken<T>(id, tokenLength, propertyProvider, partType);
        int joinPartCountEstimate = readText.tokenListIndex - activeTokenListIndex + 1;
        @SuppressWarnings("unchecked")
        PartToken<T>[] parts = new PartToken[joinPartCountEstimate];
        int partLength = activeTokenListEndOffset - realTokenStartOffset;
        PartToken<T> partToken = new PartToken<T>(id, partLength, propertyProvider, PartType.START, joinToken, 0, 0);
        partToken.setTokenList(activeTokenList);
        partToken.setRawOffset(realTokenStartOffset); // realTokenStartOffset already decreased by tokenLength
        parts[0] = partToken;
        int partIndex = 1;
        int partTextOffset = partLength; // Length of created parts so far
        int firstPartTokenListIndex = activeTokenListIndex;
        do {
            activeTokenListIndex++;
            fetchActiveTokenList();
            // realTokenStartOffset set to start activeTokenList
            PartType partPartType;
            // Attempt total ETL's length as partLength
            partLength = activeTokenListEndOffset - realTokenStartOffset;
            if (partLength == 0) {
                continue;
            }
            if (partTextOffset + partLength >= tokenLength) { // Last part
                partLength = tokenLength - partTextOffset;
                // If the partType of the join token is not complete then this will be PartType.MIDDLE
                partPartType = (partType == PartType.START) ? PartType.MIDDLE : PartType.END;
            } else { // Non-last part
                partPartType = PartType.MIDDLE;
            }

            partToken = new PartToken<T>(id, partLength, propertyProvider, partPartType, joinToken, partIndex, partTextOffset);
            // realTokenStartOffset still points to start of activeTokenList
            partToken.setRawOffset(realTokenStartOffset); // ETL.startOffset() will be subtracted upon addition to ETL
            partToken.setTokenList(activeTokenList);
            partTextOffset += partLength;
            parts[partIndex++] = partToken;
        } while (partTextOffset < tokenLength);
        // Update realTokenStartOffset which pointed to start of activeTokenList
        realTokenStartOffset += partLength;
        // Check that the array does not have any extra items
        if (partIndex < parts.length) {
            @SuppressWarnings("unchecked")
            PartToken<T>[] tmp = new PartToken[partIndex];
            System.arraycopy(parts, 0, tmp, 0, partIndex);
            parts = tmp;
        }
        List<PartToken<T>> partList = ArrayUtilities.unmodifiableList(parts);
        joinToken.setJoinedParts(partList, activeTokenListIndex - firstPartTokenListIndex);
        // joinToken.setTokenList() makes no sense - JoinTokenList instances are temporary
        // joinToken.setRawOffset() makes no sense - offset taken from initial part
        return joinToken;
    }
    
    /**
     * Class for reading of text of subsequent ETLs - it allows to see their text
     * as a consecutive character sequence (inputSourceText is used as a backing char sequence)
     * with an increasing readIndex (it's not decremented after token's recognition).
     */
    final class TokenListText {

        int tokenListIndex;

        int tokenListStartOffset;

        int tokenListEndOffset;

        /**
         * A constant added to readOffset to allow a smoothly increasing reading offset
         * when reading through multiple ETLs with gaps among them.
         */
        int readOffsetShift;

        void init() {
            EmbeddedTokenList<T> etl = tokenList(activeTokenListIndex);
            tokenListStartOffset = etl.startOffset();
            tokenListEndOffset = etl.endOffset();
            // No extra shift for first token
        }

        void initFrom(TokenListText text) {
            this.tokenListIndex = text.tokenListIndex;
            this.tokenListStartOffset = text.tokenListStartOffset;
            this.tokenListEndOffset = text.tokenListEndOffset;
            this.readOffsetShift = text.readOffsetShift;
        }

        /**
         * Read next char or return EOF.
         */
        int read(int offset) {
            offset += readOffsetShift;
            if (offset < tokenListEndOffset) {
                return inputSourceText.charAt(offset);
            } else {
                while (++tokenListIndex < tokenListCount()) {
                    EmbeddedTokenList etl = tokenList(tokenListIndex);
                    tokenListStartOffset = etl.startOffset();
                    // Increase offset shift by the size of gap between ETLs
                    readOffsetShift += tokenListStartOffset - tokenListEndOffset;
                    // Also shift given offset value
                    offset += tokenListStartOffset - tokenListEndOffset;
                    tokenListEndOffset = etl.endOffset();
                    if (readOffset < tokenListEndOffset) { // ETL might be empty
                        return inputSourceText.charAt(offset);
                    }
                }
                tokenListIndex--; // Return to (tokenListCount() - 1)
                return LexerInput.EOF;
            }
        }

        /**
         * Check whether currently set text covers the given relative index.
         * 
         * @param index index in the same metrics as readIndex.
         * @return whether the given index is within current bounds.
         */
        boolean isInBounds(int offset) {
            offset += readOffsetShift;
            return offset >= tokenListStartOffset && offset < tokenListEndOffset;
        }
        
        /**
         * Get char that was previously verified to be within bounds.
         */
        char inBoundsChar(int offset) {
            offset += readOffsetShift;
            return inputSourceText.charAt(offset);
        }
        
        char existingChar(int offset) {
            offset += readOffsetShift;
            if (offset < tokenListStartOffset) {
                while (true) { // Char should exist
                    tokenListIndex--;
                    EmbeddedTokenList etl = tokenList(tokenListIndex);
                    tokenListEndOffset = etl.endOffset();
                    // Decrease offset shift by the size of gap between ETLs
                    readOffsetShift -= tokenListStartOffset - tokenListEndOffset;
                    // Also shift given offset value
                    offset -= tokenListStartOffset - tokenListEndOffset;
                    tokenListStartOffset = etl.startOffset();
                    if (readOffset >= tokenListStartOffset) { // ETL might be empty
                        return inputSourceText.charAt(offset);
                    }
                }
                
            } else if (offset >= tokenListEndOffset) {
                while (true) { // Char should exist
                    tokenListIndex++;
                    EmbeddedTokenList etl = tokenList(tokenListIndex);
                    tokenListStartOffset = etl.startOffset();
                    // Increase offset shift by the size of gap between ETLs
                    readOffsetShift += tokenListStartOffset - tokenListEndOffset;
                    // Also shift given offset value
                    offset += tokenListStartOffset - tokenListEndOffset;
                    tokenListEndOffset = etl.endOffset();
                    if (readOffset < tokenListEndOffset) { // ETL might be empty
                        return inputSourceText.charAt(offset);
                    }
                }
                
            }
            // Index within current bounds
            return inputSourceText.charAt(offset);
        }
        
    }
    
    @Override
    public String toString() {
        return super.toString() + ", realTokenStartOffset=" + realTokenStartOffset + // NOI18N
                ", activeTokenListIndex=" + activeTokenListIndex + // NOI18N
                ", activeTokenListEndOffset=" + activeTokenListEndOffset; // NOI18N
    }

}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer;

import java.util.List;
import java.util.Set;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.InputAttributes;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.ArrayUtilities;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.token.JoinToken;
import org.netbeans.lib.lexer.token.PartToken;


/**
 * Join token list over certain range of ETLs of a TokenListList.
 * <br/>
 * There must always be at least one ETL in a JTL since otherwise there would be nobody
 * holding EmbeddedJoinInfo that holds JoinTokenListBase which is crucial for JTL.
 * <br/>
 * It does not have any physical storage for tokens. Regular tokens
 * are stored in individual ETLs. Tokens split across multiple ETLs
 * are represented as PartToken in each ETL referencing a JoinToken.
 * The only "countable" part is a last part of a JoinToken.
 * <br/>
 * Lookaheads and states are assigned to a last part of the JoinToken
 * and it's stored normally in ETL like for regular tokens.
 * 
 * @author Miloslav Metelka
 */

public class JoinTokenList<T extends TokenId> implements TokenList<T> {
    
    /**
     * Create join token list over an uninitialized set of embedded token lists
     * and this method will perform initial lexing of the contained embedded token lists.
     * 
     * @param tokenListList non-null tokenListList
     * @param tokenListStartIndex index of first ETL contained in the desired JTL.
     * @param tokenListCount total number of ETLs contained in the created JTL.
     * @return non-null JTL.
     */
    public static <T extends TokenId> JoinTokenList<T> init(
            TokenListList<T> tokenListList, int tokenListStartIndex, int tokenListCount
    ) {
        assert (tokenListCount > 0) : "tokenListCount must be >0";
        JoinTokenListBase base = new JoinTokenListBase(tokenListCount, tokenListList.rootTokenList().modCount());
        // Create join token list - just init first ETL's join info (read by JTL's constructor)
        JoinTokenList<T> jtl = new JoinTokenList<T>(tokenListList, base, tokenListStartIndex);
        // Notify will initialize all ETL.joinInfo except the first one (already inited)
        JoinLexerInputOperation<T> lexerInputOperation = new JoinLexerInputOperation<T>(jtl, 0, null, 0,
                tokenListList.get(tokenListStartIndex).startOffset());
        lexerInputOperation.init();
        AbstractToken<T> token;
        int joinTokenCount = 0;
        int activeTokenListIndex = -1;
        EmbeddedTokenList<T> activeTokenList = null;
        while ((token = lexerInputOperation.nextToken()) != null) {
            if (activeTokenList != lexerInputOperation.activeTokenList()) {
                do {
                    // There may be empty ETLs and ETLs that will contain part tokens
                    activeTokenListIndex++;
                    activeTokenList = jtl.tokenList(activeTokenListIndex);
                    activeTokenList.joinInfo = new EmbeddedJoinInfo(base, joinTokenCount, activeTokenListIndex);
                } while (activeTokenListIndex < lexerInputOperation.activeTokenListIndex());
            }
            if (token.getClass() == JoinToken.class) {
                // ETL for last part
                JoinToken<T> joinToken = (JoinToken<T>) token;
                List<PartToken<T>> joinedParts = joinToken.joinedParts();
                // Index for active list of addition
                // There may be ETLs so token list count may differ from part count
                int extraTokenListSpanCount = joinToken.extraTokenListSpanCount();
                int tokenListIndex = activeTokenListIndex - extraTokenListSpanCount;
                int joinedPartIndex = 0;
                // Only add without the last part (will be added normally outside the loop)
                // The last ETL can not be empty (must contain the last non-empty token part)
                for (int i = 0; i < extraTokenListSpanCount; i++) {
                    EmbeddedTokenList<T> etl = tokenListList.get(tokenListStartIndex + tokenListIndex + i);
                    // Check whether token list is non-empty by checking a text length that it covers.
                    // Do not use elt.tokenCount() since the tokens are just being added into ETL.
                    if (etl.textLength() > 0) {
                        etl.addToken(joinedParts.get(joinedPartIndex++), 0, null);
                    }
                    etl.joinInfo.setJoinTokenLastPartShift(extraTokenListSpanCount - i);
                    // Increase total number of tokens - use just the first part's ETL since others are ignored.
                    // There's a single token part in ETL only
                }
                // Last part will be added normally by subsequent code
                token = joinedParts.get(joinedPartIndex); // Should be (joinedParts.size()-1)
            }
            activeTokenList.addToken(token, lexerInputOperation);
            joinTokenCount++;
        }
        // Init possible empty ETL at the end
        while (activeTokenListIndex < lexerInputOperation.activeTokenListIndex()) {
            EmbeddedTokenList<T> etl = jtl.tokenList(activeTokenListIndex);
            assert (etl.joinInfo == null);
            etl.joinInfo = new EmbeddedJoinInfo(base, joinTokenCount, activeTokenListIndex);
            activeTokenListIndex++;
        }
        // Trim storage of all ETLs to their current size
        for (int i = tokenListCount - 1; i >= 0; i--) {
            tokenListList.get(tokenListStartIndex + i).trimStorageToSize();
        }
        base.joinTokenCount = joinTokenCount;
        // Could possibly subtract gap lengths but should not be necessary
        return jtl;
    }
    
    /** Backing token list list that holds ETLs. */
    protected final TokenListList<T> tokenListList; // 16 bytes
    
    /** Info about token list count and join token index gap. */
    protected final JoinTokenListBase base; // 12 bytes (8-super + 4)
    
    /** Start index of ETLs in TLL used by this JTL. */
    protected final int tokenListStartIndex; // 20 bytes
    
    /** Index of active token list. */
    protected int activeTokenListIndex; // 24 bytes

    /** Token list currently servicing requests. */
    protected  EmbeddedTokenList<T> activeTokenList; // 28 bytes
    
    /** Start join index of activeTokenList */
    protected int activeStartJoinIndex; // 32 bytes
    
    /** End join index of activeTokenList */
    protected int activeEndJoinIndex; // 36 bytes

    public JoinTokenList(TokenListList<T> tokenListList, JoinTokenListBase base, int tokenListStartIndex) {
        this.tokenListList = tokenListList;
        this.base = base;
        this.tokenListStartIndex = tokenListStartIndex;
        this.activeTokenListIndex = -1; // Signal invalid value
    }

    public LanguagePath languagePath() {
        return tokenListList.languagePath();
    }

    public TokenListList<T> tokenListList() {
        return tokenListList;
    }

    public JoinTokenListBase base() {
        return base;
    }

    public int tokenListStartIndex() {
        return tokenListStartIndex;
    }

    /**
     * Get token list contained in this join token list.
     * 
     * @param index >=0 index of the token list in this joined token list.
     * @return non-null embedded token list at the given index.
     */
    public EmbeddedTokenList<T> tokenList(int index) {
        if (index < 0)
            throw new IndexOutOfBoundsException("index=" + index + " < 0"); // NOI18N
        if (index >= base.tokenListCount)
            throw new IndexOutOfBoundsException("index=" + index + " >= size()=" + base.tokenListCount); // NOI18N
        return tokenListList.get(tokenListStartIndex + index);
    }
    
    public int tokenListCount() {
        return base.tokenListCount;
    }
    
    
    public int tokenCountCurrent() {
        return base.joinTokenCount;
    }

    public int tokenCount() {
        return tokenCountCurrent();
    }
    
    public int activeStartJoinIndex() { // Use by TS.embeddedImpl()
        return activeStartJoinIndex;
    }

    public int activeEndJoinIndex() { // Use by TokenListUpdater
        return activeEndJoinIndex;
    }

    public int activeTokenListIndex() {
        return activeTokenListIndex;
    }

    public void setActiveTokenListIndex(int activeTokenListIndex) { // Used by ETL.joinTokenList()
        if (this.activeTokenListIndex != activeTokenListIndex) {
            this.activeTokenListIndex = activeTokenListIndex;
            fetchActiveTokenListData();
        }
    }

    public EmbeddedTokenList<T> activeTokenList() {
        return activeTokenList;
    }

    public TokenOrEmbedding<T> tokenOrEmbedding(int index) {
        locateTokenListByIndex(index);
        TokenOrEmbedding<T> tokenOrEmbedding = activeTokenList.tokenOrEmbedding(index - activeStartJoinIndex);
        // Need to return complete token in case a token part was retrieved
        AbstractToken<T> token;
        if (index == activeStartJoinIndex && // token part can only be the first in ETL
            tokenOrEmbedding != null && // could be beyond end?
            (token = tokenOrEmbedding.token()).getClass() == PartToken.class
        ) {
            tokenOrEmbedding = ((PartToken<T>)token).joinTokenOrEmbedding();
        }
        return tokenOrEmbedding;
    }

    public int tokenOffset(AbstractToken<T> token) {
        // Should never be called for any token instances
        throw new IllegalStateException("Internal error - should never be called");
    }

    public int tokenOffsetByIndex(int index) {
        locateTokenListByIndex(index);
        // Need to treat specially token parts - return offset of complete token
        AbstractToken<T> token;
        if (index == activeStartJoinIndex && // token part can only be the first in ETL
            (token = activeTokenList.tokenOrEmbedding(index - activeStartJoinIndex).token()).getClass() == PartToken.class
        ) {
            return ((PartToken<T>)token).joinToken().offset(null);
        }
        return activeTokenList.tokenOffsetByIndex(index - activeStartJoinIndex);
    }

    public int tokenListIndex(int offset, int startIndex, int endIndex) {
        // First find the right ETL for the given offset and store it in activeTokenListIndex
        // Use binary search
        int low = startIndex;
        int high = endIndex - 1;
        while (low <= high) {
            int mid = (low + high) >>> 1;
            int midStartOffset = tokenList(mid).startOffset();
            
            if (midStartOffset < offset) {
                low = mid + 1;
            } else if (midStartOffset > offset) {
                high = mid - 1;
            } else {
                // Token starting exactly at ETL.startOffset()
                high = mid;
                break;
            }
        }
        // Use lower index => high
        return high; // May return -1
    }

    public int[] tokenIndex(int offset) {
        // Check if the current active token list covers the given offset.
        // If not covered then only search below/above the current active ETL.
        // It not only improves performance but it is NECESSARY for proper functionality
        // of TokenListUpdater.updateJoined() since it may skip removed ETLs
        // by manually using setActiveTokenListIndex() in the area below/above the removed ETLs.
        boolean activeStartsBelowOffset = ((offset >= activeTokenList.startOffset()) || activeTokenListIndex == 0);
        if (activeStartsBelowOffset) {
            if (offset < activeTokenList.endOffset() ||
                (activeTokenListIndex + 1 == tokenListCount() ||
                    offset < tokenList(activeTokenListIndex + 1).startOffset())
            ) {
                // Current active ETL covers the area
            } else if (activeTokenListIndex + 1 < tokenListCount()) { // Search above
                activeTokenListIndex = tokenListIndex(offset, activeTokenListIndex + 1, tokenListCount());
                fetchActiveTokenListData();
            }
        } else if (activeTokenListIndex > 0) { // Search below
            activeTokenListIndex = tokenListIndex(offset, 0, activeTokenListIndex);
            if (activeTokenListIndex < 0) {
                activeTokenListIndex = 0;
            }
            fetchActiveTokenListData();
        }

        // Now search within a single ETL by binary search
        EmbeddedJoinInfo joinInfo = activeTokenList.joinInfo;
        int joinTokenLastPartShift = joinInfo.joinTokenLastPartShift();
        int searchETLTokenCount = activeTokenList.joinTokenCount();
        int[] indexAndTokenOffset = LexerUtilsConstants.tokenIndexBinSearch(activeTokenList, offset, searchETLTokenCount);
        int etlIndex = indexAndTokenOffset[0]; // Index inside etl
        indexAndTokenOffset[0] += joinInfo.joinTokenIndex(); // Make the index joinIndex
        if (etlIndex == searchETLTokenCount && joinTokenLastPartShift > 0) { // Must move activeTokenList to last part
            // Get last part and find out how much forward is the last part
            activeTokenListIndex += joinTokenLastPartShift;
            fetchActiveTokenListData();
            PartToken<T> lastPartToken = (PartToken<T>) activeTokenList.tokenOrEmbeddingUnsync(0).token();
            indexAndTokenOffset[1] = lastPartToken.joinToken().offset(null);
            
        } else if (etlIndex == 0) { // Possibly last part of a join token
            AbstractToken<T> token = activeTokenList.tokenOrEmbedding(0).token();
            if (token.getClass() == PartToken.class) {
                // indexAndTokenOffset[0] is already ok - just fix token's offset
                indexAndTokenOffset[1] = ((PartToken<T>)token).joinToken().offset(null);
            }
        }
        return indexAndTokenOffset;
    }

    public AbstractToken<T> replaceFlyToken(int index, AbstractToken<T> flyToken, int offset) {
        locateTokenListByIndex(index);
        return activeTokenList.replaceFlyToken(index - activeStartJoinIndex, flyToken, offset);
    }

    public void wrapToken(int index, EmbeddingContainer<T> embeddingContainer) {
        locateTokenListByIndex(index);
        // !!! TBD - must not wrap complete tokens of join token list.
        // Instead wrap all part tokens with another join token list
        activeTokenList.wrapToken(index - activeStartJoinIndex, embeddingContainer);
    }

    public final int modCount() {
        return rootTokenList().modCount();
    }
    
    public InputAttributes inputAttributes() {
        return rootTokenList().inputAttributes();
    }
    
    public int lookahead(int index) {
        // Locate embedded token list for the last token part (only that one stores the LA)
        locateTokenListByIndex(index);
        return activeTokenList.lookahead(index - activeStartJoinIndex);
    }

    public Object state(int index) {
        // Locate embedded token list for the last token part (only that one stores the state)
        locateTokenListByIndex(index);
        return activeTokenList.state(index - activeStartJoinIndex);
    }

    public final TokenList<?> rootTokenList() {
        return tokenListList.rootTokenList();
    }

    public CharSequence inputSourceText() {
        return rootTokenList().inputSourceText();
    }

    public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {
        return rootTokenList().tokenHierarchyOperation();
    }
    
    public boolean isContinuous() {
        return false; // TBD can be partially continuous - could be improved
    }

    public Set<T> skipTokenIds() {
        return null; // Not a top-level list -> no skip token ids
    }

    public int startOffset() {
        return tokenList(0).startOffset();
    }

    public int endOffset() {
        return tokenList(base.tokenListCount - 1).endOffset();
    }
    
    public void updateStatus() {
        synchronized (rootTokenList()) {
            updateStatusUnsync();
        }
    }

    /**
     * Unsynced synchronization of join token list - this method should only be used
     * when there may be only a single thread accessing token hierarchy i.e. during
     * token hierarchy modifications upon mutable input source modifications.
     * 
     * @see #updateStatus()
     */
    public void updateStatusUnsync() {
        int rootModCount;
        if (base.lastModCount != (rootModCount = rootTokenList().modCount())) {
            base.lastModCount = rootModCount;
            // Update status of all the contained ETLs
            for (int i = 0; i < base.tokenListCount; i++) {
                tokenList(i).embeddingContainer().updateStatus();
            }
        }
    }

    public boolean isRemoved() {
        return false; // Should never be parented
    }

    /**
     * Get index of token list where a token for a particular joinInde starts
     * and index where it's located.
     *
     * @param index index in JTL.
     * @return [0] contains token-list-index and [1] index of token-start info.
     */
    public int tokenStartLocalIndex(int index) {
        if (index == tokenCount()) {
            if (activeTokenListIndex < tokenListCount() - 1) {
                activeTokenListIndex = tokenListCount() - 1;
                fetchActiveTokenListData();
            }
            return activeTokenList.tokenCountCurrent(); // Index at end (can't be join tokens)
        }

        locateTokenListByIndex(index);
        AbstractToken<T> token = activeTokenList.tokenOrEmbeddingUnsync(index - activeStartJoinIndex).token();
        if (token.getClass() == PartToken.class) { // Last part of join token
            PartToken<T> partToken = (PartToken<T>) token;
            activeTokenListIndex -= partToken.joinToken().extraTokenListSpanCount();
            fetchActiveTokenListData();
            // The first part of join token is last in the active ETL
            return activeTokenList.tokenCountCurrent() - 1;
        }
        return index - activeStartJoinIndex;
    }

    /**
     * Locate the right activeTokenList to service the requested join index.
     *
     * @param joinIndex index in a join token list.
     * @throws IndexOutOfBoundsException for joinIndex below zero.
     */
    protected final void locateTokenListByIndex(int joinIndex) {
        if (joinIndex < activeStartJoinIndex) {
            if (joinIndex < 0)
                throw new IndexOutOfBoundsException("index=" + joinIndex + " < 0");
            // Must be lower segment - first try the one below
            activeTokenListIndex--;
            fetchActiveTokenListData();
            if (joinIndex < activeStartJoinIndex) { // Still not covered
                // Do binary search on <0, activeTokenListIndex - 1>
                positionToJoinIndex(joinIndex, 0, activeTokenListIndex - 1);
            }

        } else if (joinIndex >= activeEndJoinIndex) {
            if (activeTokenListIndex + 1 < tokenListCount()) {
                activeTokenListIndex++;
                fetchActiveTokenListData();
                if (joinIndex >= activeEndJoinIndex) { // Still too high
                    // Do binary search on <activeTokenListIndex + 1, tokenListCount-1>
                    positionToJoinIndex(joinIndex, activeTokenListIndex + 1, base.tokenListCount - 1);
                }
            }
        }
        // The index is within bounds of activeTokenList
    }

    private void positionToJoinIndex(int joinIndex, int low, int high) {
        while (low <= high) {
            activeTokenListIndex = (low + high) >>> 1;
            fetchActiveTokenListData();
            if (activeStartJoinIndex < joinIndex) {
                low = activeTokenListIndex + 1;
            } else if (activeStartJoinIndex > joinIndex) {
                high = activeTokenListIndex - 1;
            } else { // first token of the active token list
                return;
            }
        }
        // low == high + 1
        if (activeTokenListIndex != high) {
            activeTokenListIndex = high;
            fetchActiveTokenListData();
        }
    }

    protected final void fetchActiveTokenListData() {
        activeTokenList = tokenList(activeTokenListIndex);
        activeStartJoinIndex = activeTokenList.joinInfo.joinTokenIndex();
        activeEndJoinIndex = activeStartJoinIndex + activeTokenList.joinTokenCount();
    }

    public String checkConsistency() {
        // Check regular consistency without checking embeddings
        String error = LexerUtilsConstants.checkConsistencyTokenList(this, false);
        if (error == null) {
            // Check individual ETLs and their join infos
            int joinTokenCount = 0;
            JoinToken<T> activeJoinToken = null;
            int joinedPartCount = 0;
            int nextCheckPartIndex = 0;
            for (int tokenListIndex = 0; tokenListIndex < tokenListCount(); tokenListIndex++) {
                EmbeddedTokenList<T> etl = tokenList(tokenListIndex);
                error = LexerUtilsConstants.checkConsistencyTokenList(etl, false);
                if (error != null)
                    return error;

                if (etl.joinInfo == null) {
                    return "Null joinInfo for ETL at token-list-index " + tokenListIndex; // NOI18N
                }
                if (joinTokenCount != etl.joinInfo.joinTokenIndex()) {
                    return "joinTokenIndex=" + joinTokenCount + " != etl.joinInfo.joinTokenIndex()=" + // NOI18N
                            etl.joinInfo.joinTokenIndex() + " at token-list-index " + tokenListIndex; // NOI18N
                }
                if (tokenListIndex != etl.joinInfo.tokenListIndex()) {
                    return "token-list-index=" + tokenListIndex + " != etl.joinInfo.tokenListIndex()=" + // NOI18N
                            etl.joinInfo.tokenListIndex();
                }

                int etlTokenCount = etl.tokenCount();
                int etlJoinTokenCount = etlTokenCount;
                if (etlTokenCount > 0) {
                    AbstractToken<T> token = etl.tokenOrEmbeddingUnsync(0).token();
                    int startCheckIndex = 0;
                    // Check first token (may also be the last token)
                    if (activeJoinToken != null) {
                        if (token.getClass() != PartToken.class) {
                            return "Unfinished joinToken at token-list-index=" + tokenListIndex; // NOI18N
                        }
                        error = checkConsistencyJoinToken(activeJoinToken, token, nextCheckPartIndex++, tokenListIndex);
                        if (error != null) {
                            return error;
                        }
                        if (nextCheckPartIndex == joinedPartCount) {
                            activeJoinToken = null; // activeJoinToken ended
                        } else { // For non-last there must be no other tokens in the list
                            if (etlTokenCount > 1) {
                                return "More than one token and non-last part of unfinished join token" +  // NOI18N
                                        " at token-list-index " + tokenListIndex; // NOI18N
                            }
                            // etlTokenCount so the first token is last too
                            // and this is an ETL with single token part that continues activeJoinToken
                            etlJoinTokenCount--;
                        }
                        startCheckIndex = 1;
                    }
                    // Check last token
                    if (etlTokenCount > startCheckIndex) {
                        assert (activeJoinToken == null);
                        token = etl.tokenOrEmbeddingUnsync(etlTokenCount - 1).token();
                        if (token.getClass() == PartToken.class) {
                            etlJoinTokenCount--;
                            activeJoinToken = ((PartToken<T>) token).joinToken();
                            joinedPartCount = activeJoinToken.joinedParts().size();
                            nextCheckPartIndex = 0;
                            if (joinedPartCount < 2) {
                                return "joinedPartCount=" + joinedPartCount + " < 2";
                            }
                            error = checkConsistencyJoinToken(activeJoinToken, token, nextCheckPartIndex++, tokenListIndex);
                            if (error != null)
                                return error;
                        }
                    }
                    // Check that no other token are part tokens than the relevant ones
                    for (int j = startCheckIndex; j < etlJoinTokenCount; j++) {
                        if (etl.tokenOrEmbeddingUnsync(j).token().getClass() == PartToken.class) {
                            return "Inside PartToken at index " + j + "; joinTokenCount=" + etlJoinTokenCount; // NOI18N
                        }
                    }
                }
                if (etlJoinTokenCount != etl.joinTokenCount()) {
                    return "joinTokenCount=" + etlJoinTokenCount + " != etl.joinTokenCount()=" + // NOI18N
                            etl.joinTokenCount() + " at token-list-index " + tokenListIndex; // NOI18N
                }
                joinTokenCount += etlJoinTokenCount;
            } // end-of-for over ETLs
            if (activeJoinToken != null) {
                return "Unfinished join token at end";
            }
            if (joinTokenCount != base.joinTokenCount) {
                return "joinTokenCount=" + joinTokenCount + " != base.joinTokenCount=" + base.joinTokenCount; // NOI18N
            }
        }
        // Check placement of index gap
        return error;
    }

    private String checkConsistencyJoinToken(JoinToken<T> joinToken, AbstractToken<T> token, int partIndex, int tokenListIndex) {
        PartToken<T> partToken = (PartToken<T>) token;
        if (joinToken.joinedParts().get(partIndex) != token) {
            return "activeJoinToken.joinedParts().get(" + partIndex + // NOI18N
                    ") != token at token-list-index" + tokenListIndex; // NOI18N
        }
        if (partToken.joinToken() != joinToken) {
            return "Invalid join token of part at partIndex " + partIndex + // NOI18N
                    " at token-list-index " + tokenListIndex; // NOI18N
        }
        EmbeddedTokenList<T> etl = tokenList(tokenListIndex);
        int lps = etl.joinInfo.joinTokenLastPartShift();
        if (lps < 0) {
            return "lps=" + lps + " < 0";
        }

        if (tokenListIndex + lps >= tokenListCount()) {
            return "Invalid lps=" + lps + // NOI18N
                    " at token-list-index " + tokenListIndex + // NOI18N
                    "; tokenListCount=" + tokenListCount(); // NOI18N
        }
        AbstractToken<T> lastPart = tokenList(tokenListIndex + lps).tokenOrEmbeddingUnsync(0).token();
        if (lastPart.getClass() != PartToken.class) {
            return "Invalid lps: lastPart not PartToken " + lastPart.dumpInfo(null, null, true, 0) + // NOI18N
                    " at token-list-index " + tokenListIndex; // NOI18N
        }
        if (((PartToken<T>)lastPart).joinToken().lastPart() != lastPart) {
            return "Invalid lps: Not last part " + lastPart.dumpInfo(null, null, true, 0) + // NOI18N
                    " at token-list-index " + tokenListIndex; // NOI18N
        }
        return null;
    }

    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder(256);
        int tokenListCount = tokenListCount();
        int digitCount = String.valueOf(tokenListCount - 1).length();
        for (int i = 0; i < tokenListCount; i++) {
            ArrayUtilities.appendBracketedIndex(sb, i, digitCount);
            tokenList(i).dumpInfo(sb); // includes '\n'
        }
        return LexerUtilsConstants.appendTokenList(sb, this).toString();
    }

}





/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer;

/**
 * Structure holding a token list count that bases a JoinTokenList.
 * <br/>
 * There is also an information regarding a join token index gap inside the series
 * of contained embedded token lists.
 * <br/>
 * Join token list can be created as a wrapper around this class with a knowledge
 * of token list list and an token list index at which the join token list would start.
 * 
 * @author Miloslav Metelka
 */

public final class JoinTokenListBase {
    
    private static final int INDEX_GAP_LENGTH_INITIAL_SIZE = (Integer.MAX_VALUE >> 1);
    
    /**
     * Number of embedded token lists contained in join token list.
     */
    int tokenListCount; // 12 bytes (8-super + 4)
    
    /**
     * Total count of tokens contained in JoinTokenList.
     */
    int joinTokenCount; // 16 bytes
    
    /**
     * Last modCount for which the JoinTokenList was updated.
     * <br/>
     * JoinTokenList.updateStatus() may be used to update.
     */
    int lastModCount; // 20 bytes
    
    /**
     * Index among contained embedded token lists where both index-related gaps are located.
     */
    int indexGapsIndex; // 24 bytes
    
    /**
     * Length of an index gap for computation of indexes in a JoinTokenList
     * based on ETLs.
     * <br/>
     * The above gap checking is done by checking whether the index is above gap length
     * since the initial gap length is so high that the indexes should never reach
     * its size (even decreased by added items).
     */
    int joinTokenIndexGapLength = INDEX_GAP_LENGTH_INITIAL_SIZE; // 28 bytes
    
    /**
     * Length of an index gap for computation of index of ETL in a JoinTokenList
     * which is useful for finding of a start-token-list-index of the join token list.
     * <br/>
     * The above gap checking is done by checking whether the index is above gap length
     * since the initial gap length is so high that the indexes should never reach
     * its size (even decreased by added items).
     */
    int tokenListIndexGapLength = INDEX_GAP_LENGTH_INITIAL_SIZE; // 32 bytes
    
    JoinTokenListBase(int tokenListCount, int lastModCount) {
        this.tokenListCount = tokenListCount;
        this.lastModCount = lastModCount;
        // Move index gap to be at the end of all contained token lists
        this.indexGapsIndex = tokenListCount;
    }
    
    int joinTokenIndex(int rawJoinTokenIndex) {
        return (rawJoinTokenIndex < joinTokenIndexGapLength)
                ? rawJoinTokenIndex
                : rawJoinTokenIndex - joinTokenIndexGapLength;
    }
    
    int tokenListIndex(int rawTokenListIndex) {
        return (rawTokenListIndex < tokenListIndexGapLength)
                ? rawTokenListIndex
                : rawTokenListIndex - tokenListIndexGapLength;
    }
    
    /**
     * Move both gaps in sync so that ETL.JoinInfo in an ETL at "index" is above both gaps.
     * 
     * @param tokenListList non-null TLL.
     * @param tokenListStartIndex points to first list belonging to a JTL.
     * @param index index to which the gaps should be moved.
     */
    
    public void moveIndexGap(TokenListList<?> tokenListList, int tokenListStartIndex, int index) {
        if (index < indexGapsIndex) {
            // Items above index should be moved to be above gap
            do {
                EmbeddedJoinInfo joinInfo = tokenListList.get(tokenListStartIndex + index).joinInfo;
                joinInfo.rawTokenListIndex += tokenListIndexGapLength;
                joinInfo.rawJoinTokenIndex += joinTokenIndexGapLength;
            } while (++index < indexGapsIndex);
        } else if (index > indexGapsIndex) {
            // Items above index should be moved to be above gap
            do {
                index--;
                EmbeddedJoinInfo joinInfo = tokenListList.get(tokenListStartIndex + index).joinInfo;
                joinInfo.rawTokenListIndex += tokenListIndexGapLength;
                joinInfo.rawJoinTokenIndex += joinTokenIndexGapLength;
            } while (index > indexGapsIndex);
        }
    }

    public void tokenListModNotify(int tokenListCountDiff) {
        // Gap assumed to be above last added token list or above
        indexGapsIndex += tokenListCountDiff; // Move gap above added 
        tokenListCount += tokenListCountDiff;
        tokenListIndexGapLength -= tokenListCountDiff;
    }

    public int joinTokenCount() {
        return joinTokenCount;
    }

    public void updateJoinTokenCount(int joinTokenCountDiff) {
        joinTokenCount += joinTokenCountDiff;
        joinTokenIndexGapLength -= joinTokenCountDiff;
    }

    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder(70);
        sb.append("tokenListCount=").append(tokenListCount);
        sb.append(", tokenCount=").append(joinTokenCount);
        sb.append(", lastModCount=").append(lastModCount);
        sb.append(", indexGapsIndex=").append(indexGapsIndex);
        return sb.toString();
    }

}





    private static final LAState EMPTY = new NoState(0);
    
    private static final LAState INIT_STATE = new NoState(0);

    public static LAState empty() {
        return EMPTY;
    }
    
    /**
     * Special state for marking that an embedded token list was not inited yet.
     */
    public static LAState initState() {
        return INIT_STATE;
    }

    static int withExtraCapacity(int capacity) {
        return capacity * 3 / 2 + 4;
    }

        return capacity() - gapLength;
    }

    /**
     * Add a particular lookahead and state.
     *
     * @param lookahead
     * @param state
     * @return either same or a new LAState containing the given lookahead and state.
     */
    public final LAState add(int lookahead, Object state) {
        LAState ret;
        if (gapLength > 0) { // enough space


    protected abstract LAState upgrade(int capacity, Class laStateClass);

    /**
     * Whether an upgrade is necessary when the given laStateClass needs to be used.
     *
     * @param laStateClass non-null requested laStateClass
     * @return true if upgrade is necessary.
     */
    protected abstract boolean isUpgrade(Class laStateClass);

    protected abstract Class addToGapStart(int lookahead, Object state);




        if (!existingLanguagePaths.contains(lp)) {
            newLanguagePaths.add(lp);
        }
        Language<T> language = (Language<T>)LexerUtilsConstants.<T>innerLanguage(lp);
        Language<T> language = (Language<T>)lp.innerLanguage();
        if (!exploredLanguages.contains(language)) {
            exploredLanguages.add(language);
            Set<T> ids = language.tokenIds();

        WeakListeners.create(PropertyChangeListener.class, this, LanguageManager.getInstance()));
    }
    
    public Language<T> language() {
        return language;
    }
    
    public synchronized TokenValidator<T> tokenValidator(T id) {
        if (tokenValidators == null) {
            tokenValidators = allocateTokenValidatorArray(language.maxOrdinal() + 1);

        }
        FlyItem<T> item = flyItems[id.ordinal()];
        if (item == null) {
            item = new FlyItem<T>(id, text);
            flyItems[id.ordinal()] = item;
            flyItems[id.ordinal()] = new FlyItem<T>(token);
        } else { // already a valid item
            token = item.token();
            if (token.text() != text) {
                token = item.token2();
                if (token == null || token.text() != text) {
                    token = item.token();
                    if (!CharSequenceUtilities.textEquals(token.text(), text)) {
                        token = item.token2();
                        if (token == null || !CharSequenceUtilities.textEquals(token.text(), text)) {
                            // Create new token
                            token = new TextToken<T>(id, text);
                            token.makeFlyweight();
                        }
                        item.pushToken(token);
                    }
                } else { // found token2
                    item.pushToken(token);
                }
            }
        }
        return item.flyToken(id, text);
        return token;
    }
    
    public synchronized EmbeddingPresence embeddingPresence(T id) {

        return (TokenValidator<T>[]) new TokenValidator[length];
    }

    
    private static final class FlyItem<T extends TokenId> {
        
        private TextToken<T> token; // Most used (first candidate)
        
        private TextToken<T> token2; // Second most used
        
        FlyItem(T id, String text) {
            newToken(id, text);
            token2 = token; // Make both item non-null
        }

        TextToken<T> flyToken(T id, String text) {
            // First do a quick check for equality only in both items
            if (token.text() != text) {
                if (token2.text() == text) {
                    // Swap token and token2 => token will contain the right value
                    swap();
                } else { // token.text() != text && token2.text() != text
                    // Now deep-compare of text of both tokens
                    if (!CharSequenceUtilities.textEquals(token.text(), text)) {
                        if (!CharSequenceUtilities.textEquals(token2.text(), text)) {
                            token2 = token;
                            newToken(id, text);
                        } else { // swap
                            swap();
                        }
                    }
                }
            }
            return token;
        }
        
        void newToken(T id, String text) {
            // Create new token
            token = new TextToken<T>(id, text);
            token.makeFlyweight();
        }
        
        private void swap() {
            TextToken<T> tmp = token;
            token = token2;
            token2 = tmp;
        }
        
    }





package org.netbeans.lib.lexer;

import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.CharSequenceUtilities;
import org.netbeans.lib.lexer.token.ComplexToken;
import org.netbeans.spi.lexer.Lexer;
import org.netbeans.spi.lexer.LexerInput;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.token.CustomTextToken;
import org.netbeans.lib.lexer.token.DefaultToken;
import org.netbeans.lib.lexer.token.PropertyToken;
import org.netbeans.spi.lexer.LanguageHierarchy;
import org.netbeans.spi.lexer.LexerRestartInfo;
import org.netbeans.spi.lexer.TokenFactory;
import org.netbeans.spi.lexer.TokenPropertyProvider;

/**
 * Implementation of the functionality related to lexer input.

 * @version 1.00
 */

public abstract class LexerInputOperation<T extends TokenId> {

    // -J-Dorg.netbeans.lib.lexer.LexerInputOperation.level=FINE
    static final Logger LOG = Logger.getLogger(LexerInputOperation.class.getName());
    
    protected final TokenList<T> tokenList;
    private static final boolean testing = Boolean.getBoolean("netbeans.debug.lexer.test");
    
    /**
     * Current reading index which usually corresponds to real offset.
     * <br/>
     * It should be set to its initial value in the constructor by descendants.
     */
    protected int readOffset;
    
    /**
     * A value that designates a start of a token being currently recognized.
     */
    protected int tokenStartOffset;

    /**
     * Maximum index from which the char was fetched for current
     * (or previous) tokens recognition.

     * The index is updated lazily - only when EOF is reached
     * and when backup() is called.
     */
    private int lookaheadOffset;
    
    /**
     * Token length computed by assignTokenLength().
     */
    protected int tokenLength;
    
    protected Lexer<T> lexer;
     * Computed and cached token length.
     */
    private int tokenLength;
    
    protected final LanguageOperation<T> innerLanguageOperation;
    
    private final boolean mutableInput;
    
    private final Lexer<T> lexer;
    
    /**
     * Start of the token being currently recognized.
     */
    private int tokenStartIndex;

    private boolean lexerFinished;
    
    /**
     * How many flyweight tokens were created in a row.
     */
    private int flyTokenSequenceLength;
    
    private List<CharPreprocessorError> preprocessErrorList;
    
    /**
     * Total count of preprocessors used during lexing.
     * It's used to determine whether extra preprocessed chars need to be used.
     */
    protected int preprocessingLevelCount;

    private CharProvider.ExtraPreprocessedChars extraPreprocessedChars;
    
    private Language<T> language;

    public LexerInputOperation(TokenList<T> tokenList, int tokenIndex, Object lexerRestartState) {
        this.tokenList = tokenList;
        LanguagePath languagePath = tokenList.languagePath();
        this.innerLanguageOperation = LexerUtilsConstants.innerLanguageOperation(languagePath);
        
        // Determine flyTokenSequenceLength setting
        while (--tokenIndex >= 0 && tokenList.tokenOrEmbedding(tokenIndex).token().isFlyweight()) {
            flyTokenSequenceLength++;
        }

        LanguageHierarchy<T> languageHierarchy = LexerApiPackageAccessor.get().languageHierarchy(
                LexerUtilsConstants.<T>innerLanguage(languagePath));
        LanguageHierarchy<T> languageHierarchy = LexerApiPackageAccessor.get().languageHierarchy(language);
        TokenFactory<T> tokenFactory = LexerSpiPackageAccessor.get().createTokenFactory(this);
        LexerInput lexerInput = LexerSpiPackageAccessor.get().createLexerInput(this);
        // Check whether character preprocessing is necessary
//        CharPreprocessor p = LexerSpiPackageAccessor.get().createCharPreprocessor(languageHierarchy);
//        if (p != null) {
//            preprocessingLevelCount++;
//            preprocessorOperation = new CharPreprocessorOperation(
//                    ((preprocessorOperation != null)
//                        ? (CharProvider)preprocessorOperation
//                        : this),
//                    p,
//                    this
//            );
//        }
        
        LexerInput lexerInput = LexerSpiPackageAccessor.get().createLexerInput(
                (preprocessorOperation != null) ? preprocessorOperation : this);

        LexerRestartInfo<T> info = LexerSpiPackageAccessor.get().createLexerRestartInfo(
                lexerInput, tokenFactory, lexerRestartState,
                languagePath, tokenList.inputAttributes());
        lexer = LexerSpiPackageAccessor.get().createLexer(languageHierarchy, info);
    }

    public abstract int read(int offset);

    public abstract char readExisting(int offset);

    /**
     * Fill appropriate data like token list and offset into a non-flyweight token.
     * <br/>
     * This method should also move over the token's characters by increasing
     * starting offset of the token and possibly other related variables.
     * 
     * @param token non-null non-flyweight token.
     */
    protected abstract void fillTokenData(AbstractToken<T> token);
    
    public final int read() {
        int c = read(readOffset++);
        if (c == LexerInput.EOF) {
            lookaheadOffset = readOffset; // count EOF char into lookahead
            readOffset--; // readIndex must not include EOF
        }
        return c;
    }
    
    public final int readLength() {
        return readOffset - tokenStartOffset;
        return length;
    }
    
    public final char readExistingAtIndex(int index) {
        return readExisting(tokenStartOffset + index);
        return index;
    }
    
    public final int readIndex() {
        return readIndex;
    }
    
    public final void backup(int count) {
        if (lookaheadOffset < readOffset) {
            lookaheadOffset = readOffset;
        }
        readOffset -= count;
    }
    
    public final int lookahead() {
        return (lookaheadOffset > readOffset) ? (lookaheadOffset - readOffset) : 0;
    }

    public AbstractToken<T> nextToken() {
        while (true) {
            AbstractToken<T> token = (AbstractToken<T>)lexer.nextToken();
            if (token == null) {
                checkLexerInputFinished();
                return null;
            }
            // Check if token id of the new token belongs to the language
            Language<T> language = innerLanguageOperation.language();
            // Check that the id belongs to the language
            if (!isSkipToken(token) && !language.tokenIds().contains(token.id())) {
                String msgPrefix = "Invalid TokenId=" + token.id()
                        + " returned from lexer="
                        + lexer + " for language=" + language + ":\n";
                if (token.id().ordinal() > language.maxOrdinal()) {
                    throw new IllegalStateException(msgPrefix +
                            "Language.maxOrdinal()=" + language.maxOrdinal() + " < " + token.id().ordinal());
                } else { // Ordinal ok but different id with that ordinal contained in language
                    throw new IllegalStateException(msgPrefix +
                            "Language contains no or different tokenId with ordinal="
                            + token.id().ordinal() + ": " + language.tokenId(token.id().ordinal()));
                }
            }
            // Skip token's chars
            tokenStartOffset += tokenLength;
            if (!isSkipToken(token))
                return token;
        } // Continue to fetch non-skip token
    }

    /**
     * Used by token list updater after nextToken() to determine start offset of a token 
     * to be recognized next. Overriden for join token lists since join tokens
     * may span multiple ETLs.
     * 
     * @return start offset of a next token that would be recognized.
     *
     * @return &gt;=0 number of characters between the rightmost reading index reached
     *   and the present read position.
     *   <br/>
     *   The EOF (when reached by reading) is treated as a single character
     *   in lookahead.
     *   <br/>
     *   If there is an active character preprocessor the returned value
     *   is a raw length of the lookahead.
     */
    public int lastTokenEndOffset() {
        return tokenStartOffset;
    }

    public AbstractToken<T> getFlyweightToken(T id, String text) {
        assert (text.length() <= readLength());
        // Compare each recognized char with the corresponding char in text
        if (LOG.isLoggable(Level.FINE)) {
            for (int i = 0; i < text.length(); i++) {
                if (text.charAt(i) != readExisting(i)) {
                    throw new IllegalArgumentException("Flyweight text in " + // NOI18N
                            "TokenFactory.getFlyweightToken(" + id + ", \"" + // NOI18N
                            CharSequenceUtilities.debugText(text) + "\") " + // NOI18N
                            "differs from recognized text: '" + // NOI18N
                            CharSequenceUtilities.debugChar(readExisting(i)) +
                            "' != '" + CharSequenceUtilities.debugChar(text.charAt(i)) + // NOI18N
                            "' at index=" + i // NOI18N
                    );
                }
            }
        }

        assignTokenLength(text.length());
        AbstractToken<T> token;
        if ((token = checkSkipToken(id)) == null) {
            if (isFlyTokenAllowed()) {
                token = innerLanguageOperation.getFlyweightToken(id, text);
                flyTokenSequenceLength++;
            } else { // Create regular token
                token = createDefaultTokenInstance(id);
                fillTokenData(token);
                flyTokenSequenceLength = 0;
            }
        }
        return token;
    }
    
    private AbstractToken<T> checkSkipToken(T id) {
        if (isSkipTokenId(id)) {
            // Prevent fly token occurrence after skip token to have a valid offset
            flyTokenSequenceLength = LexerUtilsConstants.MAX_FLY_SEQUENCE_LENGTH;
            return skipToken();
        }
        return null;
    }
    
    public AbstractToken<T> createToken(T id, int length) {
        assignTokenLength(length);
        AbstractToken<T> token;
        if ((token = checkSkipToken(id)) == null) {
            token = createDefaultTokenInstance(id);
            fillTokenData(token);
            flyTokenSequenceLength = 0;
        }
        return token;
    }

    protected AbstractToken<T> createDefaultTokenInstance(T id) {
        return new DefaultToken<T>(id, tokenLength);
    }

    public AbstractToken<T> createToken(T id, int length, PartType partType) {
        if (partType == null)
            throw new IllegalArgumentException("partType must be non-null");
        if (partType == PartType.COMPLETE)
            return createToken(id, length);

        return createPropertyToken(id, length, null, partType);
    }

    public AbstractToken<T> createPropertyToken(T id, int length,
    TokenPropertyProvider<T> propertyProvider, PartType partType) {
        if (partType == null)
            partType = PartType.COMPLETE;
        
        assignTokenLength(length);
        AbstractToken<T> token;
        if ((token = checkSkipToken(id)) == null) {
            token = createPropertyTokenInstance(id, propertyProvider, partType);
            fillTokenData(token);
            flyTokenSequenceLength = 0;
        }
        return token;
    }

    protected AbstractToken<T> createPropertyTokenInstance(T id,
    TokenPropertyProvider<T> propertyProvider, PartType partType) {
        return new PropertyToken<T>(id, tokenLength, propertyProvider, partType);
    }

    public AbstractToken<T> createCustomTextToken(T id, int length, CharSequence customText) {
        assignTokenLength(length);
        AbstractToken<T> token;
        if ((token = checkSkipToken(id)) == null) {
            token = createCustomTextTokenInstance(id, customText);
            fillTokenData(token);
            flyTokenSequenceLength = 0;
        }
        return token;
    }
    
    protected AbstractToken<T> createCustomTextTokenInstance(T id, CharSequence customText) {
        return new CustomTextToken<T>(id, customText, tokenLength);
    }

    public boolean isSkipTokenId(T id) {
        Set<T> skipTokenIds = tokenList.skipTokenIds();
        return (skipTokenIds != null && skipTokenIds.contains(id));
    }

    protected final int tokenLength() {
        return tokenLength;
    }

    public void assignTokenLength(int tokenLength) {
        if (tokenLength > readLength()) {
            throw new IndexOutOfBoundsException("tokenLength=" + tokenLength // NOI18N
                    + " >" + readLength());
        }
        this.tokenLength = tokenLength;
    }
    
    public void tokenApproved() {
        tokenStartIndex += tokenLength;
        readIndex -= tokenLength;
        lookaheadIndex -= tokenLength;
    }
    
    protected final TokenList<T> tokenList() {
        return tokenList;
    }
    
    protected final int tokenStartIndex() {
        return tokenStartIndex;
    }

    public final void setTokenStartIndex(int tokenStartIndex) {
        this.tokenStartIndex = tokenStartIndex;
    }

    protected final CharPreprocessorOperation preprocessor() {
        return preprocessorOperation;
    }
    
    public final boolean isMutableInput() {
        return mutableInput;
    }
    
    public final boolean isStoreLookaheadAndState() {
        return isMutableInput() || testing;
    }
    
    public AbstractToken<T> nextToken() {
        assert (!lexerFinished);
        while (true) {
            @SuppressWarnings("unchecked")
            AbstractToken<T> token = (AbstractToken<T>)lexer().nextToken();
            if (token == null) {
                LexerUtilsConstants.checkLexerInputFinished(
                        (preprocessorOperation != null) ? (CharProvider)preprocessorOperation : this, this);
                lexerFinished = true;
                return null;
            } else {
                // Check that the id belongs to the language
                if (token != TokenFactory.SKIP_TOKEN && !language.tokenIds().contains(token.id())) {
                    String msgPrefix = "Invalid TokenId=" + token.id()
                            + " returned from lexer="
                            + lexer() + " for language=" + language + ":\n";
                    if (token.id().ordinal() > language.maxOrdinal()) {
                        throw new IllegalStateException(msgPrefix +
                                "Language.maxOrdinal()=" + language.maxOrdinal() + " < " + token.id().ordinal());
                    } else { // Ordinal ok but different id with that ordinal contained in language
                        throw new IllegalStateException(msgPrefix +
                                "Language contains no or different tokenId with ordinal="
                                + token.id().ordinal() + ": " + language.tokenId(token.id().ordinal()));
                    }
                }
                approveToken(token);
            }
            if (token == TokenFactory.SKIP_TOKEN)
                continue; // Fetch next token
            return token;
        }
    }
    
    /**
     * Notification that the token was recognized.
     * @param tokenLength length of the recognized token.
     * @param skip whether the token should be skipped
     * @return true if the token holding preprocessed text should be created.
     *  If skip is true then false is returned.
     */
    public final boolean tokenRecognized(int tokenLength, boolean skip) {
        if (preprocessorOperation != null) {
            preprocessorOperation.tokenRecognized(tokenLength);
        } else { // no preprocessor
            tokenRecognized(tokenLength);
        }

        // If the token is not skipped check whether preprocessed token
        // should be created instead of the regular token.
        if (!skip && tokenLength != this.tokenLength
                || (preprocessErrorList != null 
                    && preprocessErrorList.get(0).index() < this.tokenLength)
        ) {
            if (extraPreprocessedChars == null && preprocessingLevelCount > 1) {
                // For more than one preprocessing level need to handle
                // extra preprocessed chars before and after the main ones
                // on the parent levels.
                extraPreprocessedChars = new CharProvider.ExtraPreprocessedChars();
            }
            return true;
        }
        return false;
    }
    
    public void notifyPreprocessorError(CharPreprocessorError error) {
        if (preprocessErrorList == null) {
            preprocessErrorList = new GapList<CharPreprocessorError>();
        }
        preprocessErrorList.add(error);
    }

//    public final void initPreprocessedToken(AbstractToken<T> token) {
//        CharPreprocessorError error = null;
//        if (preprocessErrorList != null && preprocessErrorList.size() > 0) {
//            for (int i = preprocessErrorList.size() - 1; i >= 0; i--) {
//                error = preprocessErrorList.get(i);
//                if (error.index() < tokenLength) {
//                    preprocessErrorList.remove(i);
//                } else {// Above errors for this token
//                    // Relocate - subtract token length
//                    error.updateIndex(-tokenLength);
//                    error = null;
//                }
//            }
//        }
//        
//        PreprocessedTextStorage storage = preprocessorOperation.createPreprocessedTextStorage(
//                token.text(), extraPreprocessedChars);
//        
//        if (token.getClass() == ComplexToken.class) {
//            ((ComplexToken)token).initPrep(storage, error);
//        } else {
//            ((PreprocessedTextToken)token).initPrep(storage, error);
//        }
//    }
    
    public void collectExtraPreprocessedChars(CharProvider.ExtraPreprocessedChars epc,
    int prepStartIndex, int prepEndIndex, int topPrepEndIndex) {
        // No extra preprocessed characters
    }
    
    public final LanguageOperation<T> languageOperation() {
        return LexerUtilsConstants.innerLanguageOperation(tokenList.languagePath());
    }
    
    public final Object lexerState() {
        return lexer.state();
    }

    protected boolean isFlyTokenAllowed() {
        return (flyTokenSequenceLength < LexerUtilsConstants.MAX_FLY_SEQUENCE_LENGTH);
    }
    
    public final boolean isSkipToken(AbstractToken<T> token) {
        return (token == LexerUtilsConstants.SKIP_TOKEN);
    }
    
    @SuppressWarnings("unchecked")
    public final AbstractToken<T> skipToken() {
        return (AbstractToken<T>)LexerUtilsConstants.SKIP_TOKEN;
    }

    /**
     * Release the underlying lexer. This method can be called multiple times.
     */
    public final void release() {
        if (lexer != null) {
            lexer.release();
            lexer = null;
        }
    }
    
    /**
     * Check that there are no more characters to be read from the given
     * lexer input operation.
     */
    private void checkLexerInputFinished() {
        if (read() != LexerInput.EOF) {
            throw new IllegalStateException(
                "Lexer " + lexer + // NOI18N
                " returned null token" + // NOI18N
                " but EOF was not read from lexer input yet." + // NOI18N
                " Fix the lexer."// NOI18N
            );
        }
        if (readLength() > 0) {
            throw new IllegalStateException(
                "Lexer " + lexer + // NOI18N
                " returned null token but lexerInput.readLength()=" + // NOI18N
                readLength() +
                " - these characters need to be tokenized." + // NOI18N
                " Fix the lexer." // NOI18N
            );
        }
    }

    @Override
    public String toString() {
        return "tokenStartOffset=" + tokenStartOffset + ", readOffset=" + readOffset + // NOI18N
                ", lookaheadOffset=" + lookaheadOffset;
    }

    public final Lexer lexer() {
        return lexer;
    }
    
    public final InputAttributes inputAttributes() {
        return tokenList.inputAttributes();
    }
    
    public final void release() {
        lexer.release();
    }
    
}

Lines 107-113 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/LexerSpiPackageAccessor.java (-1 / +1 lines)
107		107
108	public abstract <T extends TokenId> boolean isRetainTokenText(LanguageHierarchy<T> languageHierarchy, T id);	108	public abstract <T extends TokenId> boolean isRetainTokenText(LanguageHierarchy<T> languageHierarchy, T id);
109		109
110	public abstract LexerInput createLexerInput(CharProvider charProvider);	110	public abstract LexerInput createLexerInput(LexerInputOperation<?> operation);
111		111
112	public abstract Language<?> language(MutableTextInput<?> mti);	112	public abstract Language<?> language(MutableTextInput<?> mti);
113		113




import org.netbeans.api.lexer.InputAttributes;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.ArrayUtilities;
import org.netbeans.lib.lexer.inc.FilterSnapshotTokenList;
import org.netbeans.lib.lexer.inc.SnapshotTokenList;
import org.netbeans.lib.lexer.inc.TokenHierarchyEventInfo;
import org.netbeans.spi.lexer.LanguageHierarchy;
import org.netbeans.spi.lexer.LexerInput;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.token.TextToken;
import org.netbeans.spi.lexer.LanguageEmbedding;

/**

    public static final int MAX_FLY_SEQUENCE_LENGTH = 5;
    
    /**
     * Token list's modCount for the case when the source input is unmodifiable.
     * at once in a mutable input setup.
     * <br>
     * The created tokens will be notified in one token change event.
     * <br>
     * This should roughly cover a single page with text
     * (so that an initial page of text is lexed at once)
     * but it's not strictly necessary.
     */
    public static final int MOD_COUNT_IMMUTABLE_INPUT = -1;
    
    /**
     * ModCount when the particular token list was removed from the token hierarchy.
     * <br>
     * This should avoid notifying of token creations too many times
     * for large inputs.
     */
    public static final int MOD_COUNT_REMOVED = -2;
    
    /**
     * Maximum token length that has the TokenLength objects cached by TokenLength.CACHE.
     */
    public static final int MAX_CACHED_TOKEN_LENGTH = 200;
    
    /**
     * Threshold (used by TokenLength) above which the DefaultToken implementations will
     * start to cache the Token.text().toString() result in itself.
     */
    public static final short CACHE_TOKEN_TO_STRING_THRESHOLD = 900;
    
    /**
     * Threshold similar to TOKEN_TEXT_STRING_THRESHOLD but for a case when a root token list's text
     * is a String instance. In that case a String.substring(start, end) will be used
     * which is considerably cheaper than a regular case because the character data
     * will be shared with the root text and there will be no character copying.
     */
    public static final short INPUT_TEXT_STRING_THRESHOLD = 300;
    
    /**
     * Used by TokenLength as a measure of a String instance production.
     */
    public static final short TOKEN_LENGTH_STRING_CREATION_FACTOR = 50;
    
    /**
     * Initial size of a buffer for copying a text of a Reader.
     */
    public static final int READER_TEXT_BUFFER_SIZE = 4096;
    
    static {
        // Require the following to only use THRESHOLD in certain checks
        assert (CACHE_TOKEN_TO_STRING_THRESHOLD >= INPUT_TEXT_STRING_THRESHOLD);
    }

    public static final AbstractToken<?> SKIP_TOKEN
        = new TextToken<TokenId>(
            new TokenIdImpl("skip-token-id; special id of TokenFactory.SKIP_TOKEN; " + // NOI18N
                    " It should never be part of token sequence", 0, null), // NOI18N
            "" // empty skip token text NOI18N
        );
    
    /**
     * Initial embedded token list's modCount prior it was synced
     * with the root token list's modCount.
     */
    public static final int MOD_COUNT_EMBEDDED_INITIAL = -3;

    public static void tokenLengthZeroOrNegative(int tokenLength) {
        if (tokenLength == 0) {
            throw new IllegalArgumentException(

     * and if no embedding is found then the <code>LanguageProvider.findLanguageEmbedding()</code>.
     */
    public static <T extends TokenId> LanguageEmbedding<?>
    findEmbedding(LanguageHierarchy<T> languageHierarchy, AbstractToken<T> token,
    LanguagePath languagePath, InputAttributes inputAttributes) {
        LanguageEmbedding<?> embedding =
                LexerSpiPackageAccessor.get().embedding(

        return embedding;
    }
    
    /**
     * Returns token from the given object which is either the token
     * or an embedding container.
     * <br/>
     * The method casts the resulting token to the generic type requested by the caller.
     */
    public static <T extends TokenId> AbstractToken<T> token(Object tokenOrEmbeddingContainer) {
        @SuppressWarnings("unchecked")
        AbstractToken<T> token = (AbstractToken<T>)
            ((tokenOrEmbeddingContainer.getClass() == EmbeddingContainer.class)
                ? ((EmbeddingContainer)tokenOrEmbeddingContainer).token()
                : (AbstractToken<?>)tokenOrEmbeddingContainer);
        return token;
    }

    public static <T extends TokenId> AbstractToken<T> token(TokenList<T> tokenList, int index) {
        return token(tokenList.tokenOrEmbeddingContainer(index));
    }
    
    public static int maxLanguagePathSize(Set<LanguagePath> paths) {
        int maxPathSize = 0;
        for (LanguagePath lp : paths) {

        }
        return maxPathSize;
    }
    
    /**
     * Get index of the token that "contains" the given offset.
     * If the offset is beyond the existing tokens the method asks
     * for next tokens by <code>tokenList.tokenOrEmbedding()</code>.
     * 
     * @param offset offset for which the token index should be found.
     * @return array of two items where the [0] is token's index and [1] is its offset.
     *  <br/>
     *  If offset &gt;= last-token-end-offset then [0] contains token-count and
     *  [1] conains last-token-end-offset.
     *  <br/>
     *  [0] may contain -1 to indicate that there are no tokens in the token list
     *  ([1] then contains zero).
     */
    public static int[] tokenIndexLazyTokenCreation(TokenList<?> tokenList, int offset) {
        // Token count in the list may change as possibly other threads
        // keep asking for tokens. Root token list impls create tokens lazily
        // when asked by clients.
        // The intent is to not force creation of all token (because of using a binary search)
        // so first a last token is checked whether it covers the requested offset.
        int tokenCount = tokenList.tokenCountCurrent(); // presently created token count
        if (tokenCount == 0) { // no tokens yet -> attempt to create at least one
            if (tokenList.tokenOrEmbedding(0) == null) { // really no tokens at all
                return new int[] { -1, 0 };
            }
            // Re-get the present token count (could be created a chunk of tokens at once)
            tokenCount = tokenList.tokenCountCurrent();
        }

        // tokenCount surely >0
        int prevTokenOffset = tokenList.tokenOffsetByIndex(tokenCount - 1);
        if (offset > prevTokenOffset) { // may need to create further tokens if they do not exist
            // Force token list to create subsequent tokens
            // Cannot subtract offset by each token's length because
            // there may be gaps between tokens due to token id filter use.
            int tokenLength = tokenList.tokenOrEmbedding(tokenCount - 1).token().length();
            while (offset >= prevTokenOffset + tokenLength) { // above present token
                TokenOrEmbedding<?> tokenOrEmbedding = tokenList.tokenOrEmbedding(tokenCount);
                if (tokenOrEmbedding != null) {
                    AbstractToken<?> t = tokenOrEmbedding.token();
                    if (t.isFlyweight()) { // need to use previous tokenLength
                        prevTokenOffset += tokenLength;
                    } else { // non-flyweight token - retrieve offset
                        prevTokenOffset = tokenList.tokenOffsetByIndex(tokenCount);
                    }
                    tokenLength = t.length();
                    tokenCount++;

                } else { // no more tokens => position behind last token
                    return new int[] { tokenCount, prevTokenOffset + tokenLength };
                }
            }
            return new int[] { tokenCount - 1, prevTokenOffset };
        }
        // Now do a regular binary search
        return tokenIndexBinSearch(tokenList, offset, tokenCount);
    }
    
    /**
     * Get index of the token that "contains" the given offset by using binary search
     * in existing tokens.
     * 
     * @param offset offset for which the token index should be found.
     * @return array of two items where the [0] is token's index and [1] is its offset.
     *  <br/>
     *  If offset &gt;= last-token-end-offset then [0] contains token-count and
     *  [1] conains last-token-end-offset.
     *  <br/>
     *  [0] may contain -1 to indicate that there are no tokens in the token list
     *  ([1] then contains zero).
     */
    public static int[] tokenIndexBinSearch(TokenList<?> tokenList, int offset, int tokenCount) {
        // The offset is within the currently recognized tokens
        // Use binary search
        int low = 0;
        int high = tokenCount - 1;
        int mid = -1;
        int midStartOffset = -1;
        while (low <= high) {
            mid = (low + high) >>> 1;
            midStartOffset = tokenList.tokenOffsetByIndex(mid);
            
            if (midStartOffset < offset) {
                low = mid + 1;
            } else if (midStartOffset > offset) {
                high = mid - 1;
            } else {
                // Token starting exactly at offset found
                return new int[] { mid, midStartOffset}; // right at the token begining
            }
        }
        
        // Not found exactly and high + 1 == low => high < low
        // BTW there may be gaps between tokens; if offset is in gap then position to lower token
        if (high >= 0) { // could be -1
            if (low == tokenCount) { // Could be beyond end of last token
                AbstractToken<?> t = tokenList.tokenOrEmbedding(high).token();
                // Use current midStartOffset
                if (offset >= midStartOffset + t.length()) { // beyond end of last token
                    // Offset in the gap above the "high" token
                    high++;
                    midStartOffset += t.length();
                } else if (mid != high) {
                    midStartOffset = tokenList.tokenOffsetByIndex(high);
                }
            } else if (mid != high) {
                midStartOffset = tokenList.tokenOffsetByIndex(high);
            }
        } else { // high == -1 => mid == 0
            if (tokenCount == 0) { // Need to return -1
                return new int[] { -1, 0 };
            }
            high = 0;
            // Use current midStartOffset
        }
        return new int[] { high, midStartOffset };
    }

    public static int updatedStartOffset(EmbeddedTokenList<?> etl, TokenHierarchyEventInfo eventInfo) {
        etl.embeddingContainer().updateStatusUnsync();
        int startOffset = etl.startOffset();
        return (etl.isRemoved() && startOffset > eventInfo.modOffset())
                ? Math.max(startOffset - eventInfo.removedLength(), eventInfo.modOffset())
                : startOffset;
    }

    public static <T extends TokenId> StringBuilder appendTokenList(StringBuilder sb, TokenList<T> tokenList) {
        return appendTokenList(sb, tokenList, -1, 0, Integer.MAX_VALUE, true, 0, true);
    }

    public static <T extends TokenId> StringBuilder appendTokenListIndented(
        StringBuilder sb, TokenList<T> tokenList, int indent
    ) {
        return appendTokenList(sb, tokenList, -1, 0, Integer.MAX_VALUE, false, indent, true);
    }

    public static <T extends TokenId> StringBuilder appendTokenList(StringBuilder sb,
            TokenList<T> tokenList, int currentIndex, int startIndex, int endIndex,
            boolean appendEmbedded, int indent, boolean dumpTokenText
    ) {
        if (sb == null) {
            sb = new StringBuilder(200);
        }
        TokenHierarchy<?> tokenHierarchy;
        if (tokenList instanceof SnapshotTokenList) {
            tokenHierarchy = ((SnapshotTokenList<T>)tokenList).snapshot().tokenHierarchy();
        } else {
            tokenHierarchy = null;
        }

        endIndex = Math.min(tokenList.tokenCountCurrent(), endIndex);

            sb.append((i == currentIndex) ? '*' : 'T');
            ArrayUtilities.appendBracketedIndex(sb, i, digitCount);
            appendTokenInfo(sb, tokenList, i, tokenHierarchy,
                    appendEmbedded, indent, dumpTokenText);
            sb.append('\n');
        }
        return sb;

            || (state1 != null && state1.equals(state2));
    }
    
    /**
     * Get end state of the given token list that may be used for relexing
     * of the next section.
     * <br/>
     * If the section is empty but it does not join the sections then null state
     * is returned.
     * 
     * @param tokenList non-null token list.
     * @return end state or {@link #INVALID_STATE} if previous token list must be queried.
     */
    public static Object endState(EmbeddedTokenList<?> tokenList) {
        int tokenCount = tokenList.tokenCount();
        return (tokenCount > 0)
                ? tokenList.state(tokenList.tokenCount() - 1)
                : tokenList.embedding().joinSections() ? INVALID_STATE : null;
    }
    
    /**
     * Get end state of the given token list that may be used for relexing
     * of the next section.
     * <br/>
     * If the section is empty but it does not join the sections then null state
     * is returned.
     * 
     * @param tokenList non-null token list.
     * @param lastEndState current state that will be overriden in case this section
     *  is not empty while joining the sections.
     * @return end state or {@link #INVALID_STATE} if previous token list must be queried.
     */
    public static Object endState(EmbeddedTokenList<?> tokenList, Object lastEndState) {
        int tokenCount = tokenList.tokenCount();
        return (tokenCount > 0)
                ? tokenList.state(tokenList.tokenCount() - 1)
                : tokenList.embedding().joinSections() ? lastEndState : null;
    }
    
    public static String idToString(TokenId id) {
        return id.name() + '[' + id.ordinal() + ']'; // NOI18N;
    }
    
    public static <T extends TokenId> void appendTokenInfo(StringBuilder sb,
            TokenList<T> tokenList, int index,
            TokenHierarchy tokenHierarchy, boolean appendEmbedded, int indent,
            boolean dumpTokenText
        if (embeddedTokenList != null) {
            ((EmbeddedTokenList)embeddedTokenList).embeddingContainer().updateStatus();
            TokenList<T> tl = tokenList;
            if (tokenList.getClass() == SubSequenceTokenList.class) {
                tl = ((SubSequenceTokenList<T>)tokenList).delegate();
            }

            if (tl.getClass() == FilterSnapshotTokenList.class) {
                embeddedTokenList = new FilterSnapshotTokenList<ET>(embeddedTokenList,
                        ((FilterSnapshotTokenList<T>)tl).tokenOffsetDiff());

            } else if (tl.getClass() == SnapshotTokenList.class) {
                Token<T> token = token(tokenList, tokenIndex);
                embeddedTokenList = new FilterSnapshotTokenList<ET>(embeddedTokenList,
                        tokenList.tokenOffset(tokenIndex) - token.offset(null));
            }
            return embeddedTokenList;
        }
        return null;
    }
    
    public static void appendTokenInfo(StringBuilder sb, TokenList tokenList, int index,
            TokenHierarchy tokenHierarchy, boolean appendEmbedded, int indent
    ) {
        appendTokenInfo(sb, tokenList.tokenOrEmbedding(index),
                tokenList.lookahead(index), tokenList.state(index),
                tokenHierarchy, appendEmbedded, indent, dumpTokenText);
    }

    public static <T extends TokenId> void appendTokenInfo(StringBuilder sb,
            TokenOrEmbedding<T> tokenOrEmbedding, int lookahead, Object state,
            TokenHierarchy<?> tokenHierarchy, boolean appendEmbedded, int indent,
            boolean dumpTokenText
    ) {
        if (tokenOrEmbedding == null) {
            sb.append("<NULL-TOKEN>");
        } else { // regular token
            EmbeddingContainer<T> ec = tokenOrEmbedding.embedding();
            AbstractToken<T> token = tokenOrEmbedding.token();
            token.dumpInfo(sb, tokenHierarchy, dumpTokenText, indent);
                ec = (EmbeddingContainer<?>)tokenOrEmbeddingContainer;
                token = ec.token();
            } else {
                ec = null;
                token = (Token<?>)tokenOrEmbeddingContainer;
            }
            sb.append(((AbstractToken<?>)token).dumpInfo(tokenHierarchy));
            appendLAState(sb, lookahead, state);
            sb.append(", ");
            appendIdentityHashCode(sb, token);

                    ArrayUtilities.appendSpaces(sb, indent);
                    sb.append("Embedding[").append(index).append("]: \"").append(etl.languagePath().mimePath()).append("\"\n");
                    if (appendEmbedded) {
                        appendTokenList(sb, etl, -1, 0, Integer.MAX_VALUE, appendEmbedded, indent, true);
                    }
                    etl = etl.nextEmbeddedTokenList();
                    index++;

        }
    }
    
    public static String checkConsistencyTokenList(TokenList<?> tokenList, boolean checkEmbedded) {
        return checkConsistencyTokenList(tokenList, checkEmbedded, ArrayUtilities.emptyIntArray(), tokenList.startOffset());
    }

    private static String checkConsistencyTokenList(TokenList<?> tokenList, boolean checkEmbedded,
    int[] parentIndexes, int firstTokenOffset) {
        int tokenCountCurrent = tokenList.tokenCountCurrent();
        int[] indexes = ArrayUtilities.intArray(parentIndexes, parentIndexes.length + 1);
        boolean continuous = tokenList.isContinuous();
        int lastOffset = firstTokenOffset;
        for (int i = 0; i < tokenCountCurrent; i++) {
            TokenOrEmbedding<?> tokenOrEmbedding = tokenList.tokenOrEmbedding(i);
            if (tokenOrEmbedding == null) {
                return dumpContext("Null token", tokenList, i, parentIndexes); // NOI18N
            }
            AbstractToken<?> token = tokenOrEmbedding.token();
            if (token.isRemoved()) {
                return dumpContext("Token is removed", tokenList, i, parentIndexes);
            }
            // Check whether tokenList.startOffset() corresponds to the start of first token
            if (i == 0 && continuous && tokenCountCurrent > 0 && !token.isFlyweight()) {
                if (token.offset(null) != tokenList.startOffset()) {
                    return dumpContext("firstToken.offset()=" + token.offset(null) +
                            " != tokenList.startOffset()=" + tokenList.startOffset(),
                            tokenList, i, parentIndexes);
                }
            }
            if (!token.isFlyweight() && token.tokenList() != tokenList && !(tokenList instanceof JoinTokenList)) {
                return dumpContext("Invalid token.tokenList()=" + token.tokenList(),
                        tokenList, i, parentIndexes);
            }
            if (token.text() == null) {
                return dumpContext("Null token.text()=" + token.tokenList(),
                        tokenList, i, parentIndexes);
            }
            int offset = (token.isFlyweight()) ? lastOffset : token.offset(null);
            if (offset < 0) {
                return dumpContext("Token offset=" + offset + " < 0", tokenList, i, parentIndexes); // NOI18N
            }
            if (offset < lastOffset) {
                return dumpContext("Token offset=" + offset + " < lastOffset=" + lastOffset,
                        tokenList, i, parentIndexes);
            }
            if (offset > lastOffset && continuous) {
                return dumpContext("Gap between tokens; offset=" + offset + ", lastOffset=" + lastOffset,
                        tokenList, i, parentIndexes);
            }
            lastOffset = offset + token.length();
            EmbeddingContainer<?> ec = tokenOrEmbedding.embedding();
            if (ec != null && checkEmbedded) {
                EmbeddedTokenList<?> etl = ec.firstEmbeddedTokenList();
                while (etl != null) {
                    String error = checkConsistencyTokenList(etl, checkEmbedded, indexes,
                            offset + etl.embedding().startSkipLength());
                    if (error != null)
                        return error;
                    etl = etl.nextEmbeddedTokenList();
                }
            }
        }
        return null;
    }
    
    private static String dumpContext(String msg, TokenList<?> tokenList, int index, int[] parentIndexes) {
        StringBuilder sb = new StringBuilder();
        sb.append(msg);
        sb.append(" at index="); // NOI18N
        sb.append(index);
        sb.append(" of tokens of language "); // NOI18N
        sb.append(tokenList.languagePath().innerLanguage().mimeType());
        sb.append('\n');
        LexerUtilsConstants.appendTokenList(sb, tokenList, index, index - 2, index + 3, false, 0, true);
        sb.append("\nParents:\n"); // NOI18N
        sb.append(tracePath(parentIndexes, tokenList));
        return sb.toString();
    }
    
    public static String findTokenContext(AbstractToken<?> token) {
        return findTokenContext(token, token.tokenList().rootTokenList(), ArrayUtilities.emptyIntArray());
    }

    private static String findTokenContext(AbstractToken<?> token, TokenList<?> tokenList, int[] parentIndexes) {
        int tokenCountCurrent = tokenList.tokenCountCurrent();
        int[] indexes = ArrayUtilities.intArray(parentIndexes, parentIndexes.length + 1);
        for (int i = 0; i < tokenCountCurrent; i++) {
            TokenOrEmbedding<?> tokenOrEmbedding = tokenList.tokenOrEmbedding(i);
            if (tokenOrEmbedding == null) {
                continue;
            }
            EmbeddingContainer<?> ec = tokenOrEmbedding.embedding();
            if (ec != null) {
                if (ec.token() == token) {
                    return dumpContext("Token found.", tokenList, i, indexes);
                }
                EmbeddedTokenList<?> etl = ec.firstEmbeddedTokenList();
                while (etl != null) {
                    String context = findTokenContext(token, etl, indexes);
                    if (context != null)
                        return context;
                    etl = etl.nextEmbeddedTokenList();
                }

            } else if (tokenOrEmbedding == token) {
                return dumpContext("Token found.", tokenList, i, indexes);
            }
        }
        return null;
    }

    private static String tracePath(int[] indexes, TokenList<?> tokenList) {
        StringBuilder sb  = new StringBuilder();
        LanguagePath languagePath = tokenList.languagePath();
        TokenList<?> rootTokenList = tokenList.rootTokenList();
        tokenList = rootTokenList;
        for (int i = 0; i < indexes.length; i++) {
            appendTokenInfo(sb, tokenList, i, rootTokenList.tokenHierarchyOperation().tokenHierarchy(), false, 0, true);
            // Assign language to variable to get rid of javac bug for incremental compilation on 1.5 
            Language<?> language = languagePath.language(i);
            tokenList = EmbeddingContainer.embeddedTokenList(tokenList, indexes[i], language, true);
        }
        return sb.toString();
    }
    
    private LexerUtilsConstants() {
        // no instances
    }




    private int lastRawLengthShift;
    
    private int tokenEndRawLengthShift;
    
    private int tokenStartIndex; // Extra added to compile
        this(tokenList, 0, null, prepText, 0, 0, prepText.length());
    }

    public PreprocessedTextLexerInputOperation(TokenList<T> tokenList, int tokenIndex,
    Object lexerRestartState, PreprocessedTextStorage prepText, int prepTextStartOffset,
    int startOffset, int endOffset) {
        super(tokenList, tokenIndex, lexerRestartState, startOffset, endOffset);
                prepTextStartOffset, startOffset, endOffset);
        this.preprocessedText = prepText;
        int index = startOffset - prepTextStartOffset;
        if (index > 0) {
            tokenStartRawLengthShift = preprocessedText.rawLengthShift(index);
            lastRawLengthShift = tokenStartRawLengthShift;
        }
        preprocessingLevelCount++; // extra level of preprocessing
    }

    public int deepRawLength(int length) {
        return length + preprocessedText.rawLengthShift(tokenStartIndex() + length - 1)
                - tokenStartRawLengthShift;
    }
    
    public int deepRawLengthShift(int index) {
        return preprocessedText.rawLengthShift(tokenStartIndex() + index)
                - tokenStartRawLengthShift;
    }

    public int read(int index) { // index >= 0 is guaranteed by contract
        index += tokenStartIndex;
        if (index < readEndIndex()) {
            // Check whether the char is preprocessed
            int rls = preprocessedText.rawLengthShift(index);

        }
    }

    public void assignTokenLength(int tokenLength) {
        super.tokenRecognized(tokenLength);
        tokenEndRawLengthShift = preprocessedText.rawLengthShift(
                tokenStartIndex + tokenLength - 1);
    }
    
    protected void tokenApproved() {
        // Increase base raw length shift by the token's last-char shift
        tokenStartRawLengthShift += tokenEndRawLengthShift;


                prepEndIndex -= tokenLength();
            }
        }
        super.tokenApproved();
    }
    
    public void collectExtraPreprocessedChars(CharProvider.ExtraPreprocessedChars epc,

                    // for the present token and the ending chars could possibly
                    // be non-preprocessed (prepEndIndex > tokenLength)
                    while (--i >= prepStartIndex && postCount > 0
                            && preprocessedText.rawLengthShift(i + tokenStartIndex) == tokenEndRawLengthShift
                    ) { // not preprocessed
                        postCount--;
                    }

            assert (preCount >= 0 && postCount >= 0);
            epc.ensureExtraLength(preCount + postCount);
            while (--preCount >= 0) {
//                epc.insert(readExisting(prepStartIndex - 1), deepRawLength(prepStartIndex) - prepStartIndex);
                prepStartIndex--;
            }
            while (--postCount >= 0) {
//                epc.append(readExisting(prepEndIndex), deepRawLength(prepEndIndex) - topPrepEndIndex);
                prepEndIndex++;
                topPrepEndIndex++;
            }




            // No upper bound for end index so use tokenCount() (can be improved if desired)
            limitEndIndex = tokenList.tokenCount();
        } else { // Valid limit end offset
            int[] indexAndTokenOffset = tokenList.tokenIndex(limitEndOffset);
            limitEndIndex = indexAndTokenOffset[0];
            if (limitEndIndex != -1) {
                // If the limitStartOffset is "inside" a token and it's not at or beyond end of TL
                if (limitEndOffset != indexAndTokenOffset[1] && limitEndIndex < tokenList.tokenCountCurrent()) {
                    limitEndIndex++; // Include the token that contains the offset
                }
            } else { // No tokens at all
                limitEndIndex = 0;
            if (limitEndIndex > 0) {
                // tokenCount surely >0
                int tokenOffset = tokenList.tokenOffset(limitEndIndex - 1);
                if (limitEndOffset > tokenOffset) { // may need to create further tokens if they do not exist
                    // Force token list to create subsequent tokens
                    // Cannot subtract offset by each token's length because
                    // there may be gaps between tokens due to token id filter use.
                    AbstractToken<?> token = token(limitEndIndex - 1);
                    int tokenLength = token.length();
                    while (limitEndOffset > tokenOffset + tokenLength) { // above present token
                        Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(limitEndIndex);
                        if (tokenOrEmbeddingContainer != null) {
                            token = LexerUtilsConstants.token(tokenOrEmbeddingContainer);
                            if (tokenList.isContinuous() || token.isFlyweight()) {
                                tokenOffset += tokenLength;
                            } else { // retrieve offset
                                tokenOffset = tokenList.tokenOffset(limitEndIndex);
                            }
                            tokenLength = token.length();
                            limitEndIndex++;
                        } else { // no more tokens => break
                            break;
                        }
                    }

                } else { // end index within existing tokens
                    // The offset is within the currently recognized tokens
                    // Use binary search
                    int low = 0;
                    limitEndIndex--;

                    while (low <= limitEndIndex) {
                        int mid = (low + limitEndIndex) / 2;
                        int midStartOffset = tokenList.tokenOffset(mid);

                        if (midStartOffset < limitEndOffset) {
                            low = mid + 1;
                        } else if (midStartOffset > limitEndOffset) {
                            limitEndIndex = mid - 1;
                        } else { // Token starting exactly at offset found
                            limitEndIndex = mid - 1;
                            break;
                        }
                    }
                    limitEndIndex++; // Increase from 'high' to end index
                }
            }
        }
            
        // Compute limitStartIndex (currently == 0)
        if (limitEndIndex > 0 && limitStartOffset > 0) {
            // Although the binary search could only be in <0,limitEndIndex> bounds
            // use regular TL.tokenIndex() because it has substantially better performance
            // e.g. in JoinTokenList.
            int[] indexAndTokenOffset = tokenList.tokenIndex(limitStartOffset);
            limitStartIndex = indexAndTokenOffset[0];
            // Check if the limitStartOffset is not in gap after end of token at limitStartIndex
            if (limitStartIndex < tokenList.tokenCountCurrent() &&
                indexAndTokenOffset[1] + tokenList.tokenOrEmbedding(limitStartIndex).token().length() <= limitStartOffset
                    high = mid - 1;
                } else { // Token starting exactly at offset found
                    limitStartIndex = mid + 1;
                    break;
                }
            }
            // Include previous token if it "includes" limitStartOffset (also handles gaps between tokens properly)
            if (limitStartIndex > 0 &&
                    tokenList.tokenOffset(limitStartIndex - 1) + token(limitStartIndex - 1).length() > limitStartOffset
            ) {
                limitStartIndex++;
            }
        }
    }

        return limitEndOffset;
    }
    
    public TokenOrEmbedding<T> tokenOrEmbedding(int index) {
        index += limitStartIndex;
        return (index < limitEndIndex)
            ? tokenList.tokenOrEmbedding(index)
            : null;
    }

    public int tokenOffsetByIndex(int index) {
        index += limitStartIndex;
        if (index >= limitEndIndex)
            throw new IndexOutOfBoundsException("index=" + index + " >= limitEndIndex=" + limitEndIndex);
        return tokenList.tokenOffsetByIndex(index);
    }

    public int[] tokenIndex(int offset) {
        return LexerUtilsConstants.tokenIndexBinSearch(this, offset, tokenCountCurrent());
    }

    public int tokenCount() {
        return tokenCountCurrent();
    }

    public int tokenCountCurrent() {

        return tokenList.languagePath();
    }

    public int tokenOffset(AbstractToken<T> token) {
        return tokenList.tokenOffset(token);
    }

    public char childTokenCharAt(int rawOffset, int index) {
        throw new IllegalStateException("Unexpected call.");
    }

    public void wrapToken(int index, EmbeddingContainer<T> embeddingContainer) {
        tokenList.wrapToken(limitStartIndex + index, embeddingContainer);
    }

    public TokenList<?> rootTokenList() {
        return tokenList.rootTokenList();
    }

    public CharSequence inputSourceText() {
        return rootTokenList().inputSourceText();
    }

    public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {

    
    public int startOffset() {
        if (tokenCountCurrent() > 0 || tokenCount() > 0)
            return tokenOffsetByIndex(0);
        return limitStartOffset;
    }

    public int endOffset() {
        int cntM1 = tokenCount() - 1;
        if (cntM1 >= 0)
            return tokenOffsetByIndex(cntM1) + tokenList.tokenOrEmbedding(cntM1).token().length();
        return limitStartOffset;
    }
    

        return tokenList.isRemoved();
    }

    private AbstractToken<T> token(int index) {
        return LexerUtilsConstants.token(tokenList, index);
    }
    
}




 * @author Miloslav Metelka
 * @version 1.00
 */

public class TextLexerInputOperation<T extends TokenId> extends LexerInputOperation<T> {
    
    /**
     * Input text from which the reading of characters is done.
     */
    private final CharSequence inputSourceText;

    private final int inputTextStartOffset;
    
    /**
     * Point beyond which the reading cannot go.
     */
    private int readEndOffset;
    


    public TextLexerInputOperation(TokenList<T> tokenList) {
        this(tokenList, 0, null, 0, -1);
    }

    public TextLexerInputOperation(TokenList<T> tokenList, int tokenIndex,
    Object lexerRestartState, int startOffset, int endOffset) {
    int startOffset, int endOffset) {
        super(tokenList, tokenIndex, lexerRestartState);
        this.inputSourceText = tokenList.inputSourceText();
        if (endOffset == -1) {
            endOffset = inputSourceText.length();
        }
        startOffset -= inputTextStartOffset;
        endOffset -= inputTextStartOffset;
        assert (0 <= startOffset) && (startOffset <= endOffset)
            && (endOffset <= inputSourceText.length())
            : "startOffset=" + startOffset + ", endOffset=" + endOffset
                + ", inputSourceText.length()=" + inputSourceText.length();
        tokenStartOffset = startOffset;
        readOffset = tokenStartOffset;
        readEndOffset = endOffset;
    }
    
    public int read(int offset) {
        if (offset < readEndOffset) {
            return inputSourceText.charAt(offset);
            return inputText.charAt(index);
        } else { // must read next or return EOF
            return LexerInput.EOF;
        }
    }

    public char readExisting(int offset) {
        return inputSourceText.charAt(offset);
    }

    protected void fillTokenData(AbstractToken<T> token) {
        token.setTokenList(tokenList);
        token.setRawOffset(tokenStartOffset);

        } else if (token.isFlyweight()) {
            assert isFlyTokenAllowed();
            flyTokenAdded();

        } else { // non-flyweight token
            token.setTokenList(tokenList());
            token.setRawOffset(inputTextStartOffset + tokenStartIndex());
            clearFlySequence();
        }

        tokenApproved();
    }
    
    protected final int readEndIndex() {
        return readEndOffset;
    }

}





package org.netbeans.lib.lexer;

import java.io.IOException;
import java.io.Reader;
import java.util.Collections;
import java.util.HashMap;

import org.netbeans.api.lexer.TokenHierarchyEvent;
import org.netbeans.api.lexer.TokenHierarchyListener;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.lib.lexer.batch.CopyTextTokenList;
import org.netbeans.lib.lexer.batch.TextTokenList;
import org.netbeans.lib.lexer.inc.IncTokenList;
import org.netbeans.lib.lexer.inc.TokenHierarchyEventInfo;
import org.netbeans.spi.lexer.MutableTextInput;

import org.netbeans.api.lexer.TokenId;
import org.netbeans.api.lexer.TokenSequence;
import org.netbeans.lib.editor.util.ArrayUtilities;
import org.netbeans.lib.lexer.inc.TokenHierarchyUpdate;
import org.netbeans.lib.lexer.inc.TokenListChange;
import org.netbeans.lib.lexer.token.AbstractToken;

/**
 * Token hierarchy operation services tasks of its associated token hierarchy.

 */

public final class TokenHierarchyOperation<I, T extends TokenId> { // "I" stands for input

    // -J-Dorg.netbeans.lib.lexer.TokenHierarchyOperation.level=FINE
    static final Logger LOG = Logger.getLogger(TokenHierarchyOperation.class.getName());
    
    // -J-Dorg.netbeans.spi.lexer.MutableTextInput.level=FINE
    private static final Logger LOG_LOCK = Logger.getLogger(MutableTextInput.class.getName()); // Logger for read/write-lock

     * Mutable text input for mutable token hierarchy or null otherwise.
     */
    private MutableTextInput<I> mutableTextInput;

    /**
     * Root token list of this hierarchy. It is created in constructor and never changed
     * during the whole lifetime of the token hierarchy.
     */
    private final TokenList<T> rootTokenList;
    
    /**
     * The hierarchy can be made inactive to release the tokens

     * If a token list list is contained then all its parents
     * with the shorter language path are also mandatorily maintained.
     */
    private Map<LanguagePath,TokenListList<?>> path2tokenListList;
    
    private int maxTokenListListPathSize;
    

        @SuppressWarnings("unchecked")
        I input = (I)inputReader;
        this.inputSource = input;

        // Instead of using an original CopyTextTokenList that allowed to skip
        // individual characters of all flyweight tokens do just a copy of all chars
        // from the Reader. TBD - do a lazy reading instead of pre-reading.
        char[] chars = new char[LexerUtilsConstants.READER_TEXT_BUFFER_SIZE];
        int offset = 0;
        try {
            while (true) {
                int readLen = inputReader.read(chars, offset, chars.length - offset);
                if (readLen == -1) // End of stream
                    break;
                offset += readLen;
                if (offset == chars.length) { // Full buffer
                    chars = ArrayUtilities.charArray(chars); // Double array size
                }
            }
        } catch (IOException e) {
            // Ignored silently - there should be a wrapping reader catching and properly handling
            // this IOException.
        } finally {
            // Attempt to close the Reader
            try {
                inputReader.close();
            } catch (IOException e) {
                // Ignored silently - there should be a wrapping reader catching and properly handling
                // this IOException.
            }
        }
        String inputText = new String(chars, 0, offset); // Copy of reader's whole text

        this.rootTokenList = new BatchTokenList<T>(this, inputText,
                language, skipTokenIds, inputAttributes);
        init();
        activity = Activity.ACTIVE;

        @SuppressWarnings("unchecked")
        I input = (I)inputText;
        this.inputSource = input;
        if (copyInputText) {
            // Instead of using an original CopyTextTokenList (that allowed to skip
            // individual characters of all flyweight tokens) do just a copy of the full text
            // and use regular BatchTokenList.
            inputText = inputText.toString();
        }
        this.rootTokenList = new BatchTokenList<T>(this, inputText,
                        language, skipTokenIds, inputAttributes);
        init();
        activity = Activity.ACTIVE;

                    return;
                }
            } else { // Wishing to be inactive
                change = TokenListChange.createRebuildChange(incTokenList);
                incTokenList.replaceTokens(change, 0);
//                change.setOffset(0);
                incTokenList.replaceTokens(change, incTokenList.tokenCountCurrent(), 0);
                incTokenList.setLanguagePath(null);
                incTokenList.reinit();
            }

        ensureReadLocked();
        synchronized (rootTokenList) {
            return isActiveImpl()
                ? new TokenSequenceList(rootTokenList, languagePath, startOffset, endOffset)
                : null;
        }
    }

     * <br/>
     * If the list needs to be created or it was non-mandatory.
     */
    public <ET extends TokenId> TokenListList<ET> tokenListList(LanguagePath languagePath) {
        assert isActiveNoInit() : "Token hierarchy expected to be active.";
        @SuppressWarnings("unchecked")
        TokenListList<ET> tll = (TokenListList<ET>) path2tokenListList().get(languagePath);
        if (tll == null) {
            tll = new TokenListList<ET>(rootTokenList, languagePath);
            path2tokenListList.put(languagePath, tll);
            maxTokenListListPathSize = Math.max(languagePath.size(), maxTokenListListPathSize);
            // Also create parent token list lists if they don't exist yet

        return tll;
    }
    
    /**
     * Get existing token list list or null if the TLL does not exist yet.
     */
    public <ET extends TokenId> TokenListList<ET> existingTokenListList(LanguagePath languagePath) {
        synchronized (rootTokenList()) {
            @SuppressWarnings("unchecked")
            TokenListList<ET> tll = (path2tokenListList != null) 
                    ? (TokenListList<ET>) path2tokenListList.get(languagePath)
                    : null;
            return tll;
        }
    }

    private Map<LanguagePath,TokenListList<?>> path2tokenListList() {
        if (path2tokenListList == null) {
            path2tokenListList = new HashMap<LanguagePath,TokenListList<?>>(4, 0.5f);
        }
        return path2tokenListList;
    }
    
    public int maxTokenListListPathSize() {
     * Get existing token list list or null if the TLL does not exist yet.
     */
    public TokenListList existingTokenListList(LanguagePath languagePath) {
        synchronized (rootTokenList()) {
            return (path2tokenListList != null) ? path2tokenListList.get(languagePath) : null;
        }
    }

    int maxTokenListListPathSize() {
        return maxTokenListListPathSize;
    }


            if (isActiveNoInit()) {
                IncTokenList<T> incTokenList = (IncTokenList<T>)rootTokenList;
                incTokenList.incrementModCount();
                TokenListChange<T> change = new TokenListChange<T>(incTokenList);
                CharSequence text = LexerSpiPackageAccessor.get().text(mutableTextInput);
                TokenHierarchyEventInfo eventInfo = new TokenHierarchyEventInfo(
                    this, TokenHierarchyEventType.REBUILD, 0, 0, "", 0);
                TokenListChange<T> change = TokenListChange.createRebuildChange(incTokenList);
                incTokenList.replaceTokens(change, 0);
                change.setAddedEndOffset(0); // Tokens will be recreated lazily

                incTokenList.replaceTokens(change, incTokenList.tokenCountCurrent(), 0);
                incTokenList.reinit(); // Will relex tokens lazily

                eventInfo.setTokenChangeInfo(change.tokenChangeInfo());

            TokenHierarchyEventInfo eventInfo = new TokenHierarchyEventInfo(
                    this, TokenHierarchyEventType.MODIFICATION,
                    offset, removedLength, removedText, insertedLength);
            new TokenHierarchyUpdate(eventInfo).update();
            IncTokenList<T> incTokenList = (IncTokenList<T>)rootTokenList;

            if (LOG.isLoggable(Level.FINEST)) {
                // Display current state of the hierarchy by faking its text
                // through original text
                CharSequence text = incTokenList.text();
                assert (text != null);
                incTokenList.setText(eventInfo.originalText());
                // Dump all contents
                LOG.finest(toString());
                // Return the original text
                incTokenList.setText(text);
            }
            
            if (LOG.isLoggable(Level.FINE)) {
                StringBuilder sb = new StringBuilder(150);
                sb.append("<<<<<<<<<<<<<<<<<< LEXER CHANGE START ------------------\n"); // NOI18N
                sb.append(eventInfo.modificationDescription(false));
                TokenHierarchyUpdate.LOG.fine(sb.toString());
            }

            new TokenHierarchyUpdate(eventInfo).update(incTokenList);
            
            if (LOG.isLoggable(Level.FINE)) {
                LOG.fine("AFFECTED: " + eventInfo.dumpAffected() + "\n"); // NOI18N
                String extraMsg = "";
                if (LOG.isLoggable(Level.FINER)) {
                    // Check consistency of the whole token hierarchy
                    String error = checkConsistency();
                    if (error != null) {
                        String msg = "!!!CONSISTENCY-ERROR!!!: " + error + "\n";
                        if (LOG.isLoggable(Level.FINEST)) {
                            throw new IllegalStateException(msg);
                        } else {
                            LOG.finer(msg);
                        }
                    } else {
                        extraMsg = "(TokenHierarchy Check OK) ";
                    }
                }
                LOG.fine(">>>>>>>>>>>>>>>>>> LEXER CHANGE END " + extraMsg + "------------------\n"); // NOI18N
            }

            if (LOG.isLoggable(Level.FINEST)) {
                LOG.finest("AFTER UPDATE:\n");
                LOG.finest(toString());
            }

            fireTokenHierarchyChanged(eventInfo);
        }
    }

//
//    private void checkIsSnapshot() {
//        if (!isSnapshot()) {
//            throw new IllegalStateException("Not a snapshot");    
//        }
//    }
//

     */
    public String checkConsistency() {
        // Check root token list first
        String error = LexerUtilsConstants.checkConsistencyTokenList(rootTokenList(), true);
        // Check token-list lists
        if (error == null && path2tokenListList != null) {
            for (TokenListList<?> tll : path2tokenListList.values()) {
                // Check token-list list consistency
                error = tll.checkConsistency();
                if (error != null)
                    return error;
                // Check each individual token list in token-list list
                for (TokenList<?> tl : tll) {
                    error = LexerUtilsConstants.checkConsistencyTokenList(tl, false);
                    if (error != null) {
                        return error;
                    }

            }
        }
        return error;
    }
    
    private String checkConsistencyTokenList(TokenList<?> tokenList,
    int[] parentIndexes, int firstTokenOffset) {
        int tokenCountCurrent = tokenList.tokenCountCurrent();
        int[] indexes = ArrayUtilities.intArray(parentIndexes, parentIndexes.length + 1);
        boolean continuous = tokenList.isContinuous();
        int lastOffset = firstTokenOffset;
        for (int i = 0; i < tokenCountCurrent; i++) {
            Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(i);
            if (tokenOrEmbeddingContainer == null) {
                return dumpContext("Null token", tokenList, i, parentIndexes); // NOI18N
            }
            AbstractToken<?> token = LexerUtilsConstants.token(tokenOrEmbeddingContainer);
            // Check whether tokenList.startOffset() corresponds to the start of first token
            if (i == 0 && continuous && tokenCountCurrent > 0 && !token.isFlyweight()) {
                if (token.offset(null) != tokenList.startOffset()) {
                    return dumpContext("firstToken.offset()=" + token.offset(null) +
                            " != tokenList.startOffset()=" + tokenList.startOffset(),
                            tokenList, i, parentIndexes);
                }
            }
            if (!token.isFlyweight() && token.tokenList() != tokenList) {
                return dumpContext("Invalid token.tokenList()=" + token.tokenList(),
                        tokenList, i, parentIndexes);
            }
            if (token.text() == null) {
                return dumpContext("Null token.text()=" + token.tokenList(),
                        tokenList, i, parentIndexes);
            }
            int offset = (token.isFlyweight()) ? lastOffset : token.offset(null);
            if (offset < 0) {
                return dumpContext("Token offset=" + offset + " < 0", tokenList, i, parentIndexes); // NOI18N
            }
            if (offset < lastOffset) {
                return dumpContext("Token offset=" + offset + " < lastOffset=" + lastOffset,
                        tokenList, i, parentIndexes);
            }
            if (offset > lastOffset && continuous) {
                return dumpContext("Gap between tokens; offset=" + offset + ", lastOffset=" + lastOffset,
                        tokenList, i, parentIndexes);
            }
            lastOffset = offset + token.length();
            if (tokenOrEmbeddingContainer.getClass() == EmbeddingContainer.class) {
                EmbeddingContainer<?> ec = (EmbeddingContainer<?>)tokenOrEmbeddingContainer;
                EmbeddedTokenList<?> etl = ec.firstEmbeddedTokenList();
                while (etl != null) {
                    String error = checkConsistencyTokenList(etl, indexes, offset + etl.embedding().startSkipLength());
                    if (error != null)
                        return error;
                    etl = etl.nextEmbeddedTokenList();
                }
            }
        }
        return null;
    }
    
    private String dumpContext(String msg, TokenList<?> tokenList, int index, int[] parentIndexes) {
        StringBuilder sb = new StringBuilder();
        sb.append(msg);
        sb.append(" at index="); // NOI18N
        sb.append(index);
        sb.append(" of tokens of language "); // NOI18N
        sb.append(tokenList.languagePath().innerLanguage().mimeType());
        sb.append('\n');
        LexerUtilsConstants.appendTokenList(sb, tokenList, index, index - 2, index + 3, false, 0);
        sb.append("\nParents:\n"); // NOI18N
        sb.append(tracePath(parentIndexes, tokenList.languagePath()));
        return sb.toString();
    }
    
    public String findTokenContext(AbstractToken<?> token) {
        return findTokenContext(token, rootTokenList(), ArrayUtilities.emptyIntArray());
    }

    private String findTokenContext(AbstractToken<?> token, TokenList<?> tokenList, int[] parentIndexes) {
        int tokenCountCurrent = tokenList.tokenCountCurrent();
        int[] indexes = ArrayUtilities.intArray(parentIndexes, parentIndexes.length + 1);
        for (int i = 0; i < tokenCountCurrent; i++) {
            Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(i);
            if (tokenOrEmbeddingContainer == null) {
                continue;
            }
            if (tokenOrEmbeddingContainer.getClass() == EmbeddingContainer.class) {
                EmbeddingContainer<?> ec = (EmbeddingContainer<?>)tokenOrEmbeddingContainer;
                if (ec.token() == token) {
                    return dumpContext("Token found.", tokenList, i, indexes);
                }
                EmbeddedTokenList<?> etl = ec.firstEmbeddedTokenList();
                while (etl != null) {
                    String context = findTokenContext(token, etl, indexes);
                    if (context != null)
                        return context;
                    etl = etl.nextEmbeddedTokenList();
                }

            } else if (tokenOrEmbeddingContainer == token) {
                return dumpContext("Token found.", tokenList, i, indexes);
            }
        }
        return null;
    }

    private String tracePath(int[] indexes, LanguagePath languagePath) {
        StringBuilder sb  = new StringBuilder();
        TokenList<?> tokenList = rootTokenList();
        for (int i = 0; i < indexes.length; i++) {
            LexerUtilsConstants.appendTokenInfo(sb, tokenList, i,
                    tokenHierarchy(), false, 0);
            // Assign language to variable to get rid of javac bug for incremental compilation on 1.5 
            Language<?> language = languagePath.language(i);
            tokenList = EmbeddingContainer.embeddedTokenList(tokenList, indexes[i], language);
        }
        return sb.toString();
    }
    
//    private final class SnapshotRef extends WeakReference<TokenHierarchyOperation<I,T>> implements Runnable {




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.inc.IncTokenList;
import org.netbeans.lib.lexer.inc.MutableTokenList;
import org.netbeans.lib.lexer.inc.TokenHierarchyEventInfo;
import org.netbeans.lib.lexer.inc.TokenListChange;
import org.netbeans.lib.lexer.inc.TokenListUpdater;
import static org.netbeans.lib.lexer.LexerUtilsConstants.INVALID_STATE;

/**
 * Request for updating of token hierarchy after text modification
 * or custom embedding creation/removal.
 * <br/>
 * This class contains all the data and methods related to updating.
 *
 * @author Miloslav Metelka
 */

public final class TokenHierarchyUpdate {
    
    // -J-Dorg.netbeans.lib.lexer.TokenHierarchyUpdate.level=FINE
    static final Logger LOG = Logger.getLogger(TokenHierarchyUpdate.class.getName());

    final TokenHierarchyEventInfo eventInfo;
    
    private Map<LanguagePath,TLLInfo> path2info;
    
    /**
     * Infos ordered from higher top levels of the hierarchy to lower levels.
     * Useful for top-down updating at the end.
     */
    private List<List<TLLInfo>> levelInfos;
    
    TokenListChange<?> rootChange;

    public TokenHierarchyUpdate(TokenHierarchyEventInfo eventInfo) {
        this.eventInfo = eventInfo;
    }

    public void update(IncTokenList<?> incTokenList) {
        incTokenList.incrementModCount();
        // Update top-level token list first
        // It does not need any updateStatusImpl() since it's only for embedded token lists
        rootChange = updateTokenListByModification(incTokenList, null);
        eventInfo.setTokenChangeInfo(rootChange.tokenChangeInfo());

        if (LOG.isLoggable(Level.FINE)) {
            LOG.fine("ROOT CHANGE: " + rootChange.toString(0) + "\n"); // NOI18N
        }

        // If there is an active lexer input operation (for not-yet finished
        // top-level token list lexing) refresh it because it would be damaged
        // by the performed token list update
        if (!incTokenList.isFullyLexed()) {
            incTokenList.refreshLexerInputOperation();
        }

        // Now the update goes to possibly embedded token list lists
        // based on the top-level change. If there are embeddings that join sections
        // this becomes a fairly complex thing.
        // 1. The updating must always go from upper levels to lower levels of the token hierarchy
        //    to ensure that the tokens of the possible joined embeddings get updated properly
        //    as the tokens created/removed at upper levels may contain embeddings that will
        //    need to be added/removed from token list lists on lower level.
        // 2. A single insert/remove may produce token updates at several
        //    places in the document. A top-level change of token with embedding
        //    will request the embedded token list update and that token list
        //    may be connected with another joined token list(s) with the same language path
        //    and the update may continue into these joined token lists.

        // 3. The algorithm must collect both removed and added token lists
        //    in the TLLInfo.
        // 4. For a removed token list the updating must check nested embedded token lists
        //    because some embedded tokens of the removed embedded token list might contain
        //    another embedding that might also be maintained as token list list
        //    and need to be updated.
        // 5. The parent token list lists
        //    are always maintained too which simplifies the updating algorithm
        //    and speeds it up because the token list list marks whether it has any children
        //    or not and so the deep traversing only occurs if there are any children present.
        // 6. Additions may produce nested additions too so they need to be makred
        //    similarly to removals.
        if (rootChange.isBoundsChange()) {
            processBoundsChangeEmbeddings(rootChange, null);
        } else {
            // Mark changed area based on start of first mod.token and end of last mod.token
            // of the root-level change
            eventInfo.setMinAffectedStartOffset(rootChange.offset());
            eventInfo.setMaxAffectedEndOffset(rootChange.addedEndOffset());
            processNonBoundsChange(rootChange);
        }

        processLevelInfos();
    }
    
    public void updateCreateEmbedding(EmbeddedTokenList<?> addedTokenList) {
        TLLInfo info = info(addedTokenList.languagePath());
        if (info != NO_INFO) {
            if (LOG.isLoggable(Level.FINE)) {
                LOG.fine("THU.updateCreateEmbedding(): " + addedTokenList.toStringHeader());
            }
            info.markAdded(addedTokenList);
            processLevelInfos();
        }
    }
    
    /**
     * update-status must be called by the caller.
     * @param removedTokenList token list removed by TS.removeEmbedding().
     */
    public void updateRemoveEmbedding(EmbeddedTokenList<?> removedTokenList) {
        TLLInfo info = info(removedTokenList.languagePath());
        if (info != NO_INFO) {
            if (LOG.isLoggable(Level.FINE)) {
                LOG.fine("THU.updateRemoveEmbedding(): " + removedTokenList.toStringHeader());
            }
            // update-status called by caller.
            info.markRemoved(removedTokenList);
            processLevelInfos();
        }
    }
    
    void processBoundsChangeEmbeddings(TokenListChange<?> change, TokenListChange<?> parentChange) {
        // Add an embedded change to the parent change (if exists)
        if (parentChange != null) {
            parentChange.tokenChangeInfo().addEmbeddedChange(change.tokenChangeInfo());
        }
        Object tokenOrEC = change.tokenChangeInfo().removedTokenList().tokenOrEmbeddingContainer(0);
        if (tokenOrEC.getClass() == EmbeddingContainer.class) {
            TLLInfo info;
            boolean hasChildren;
            if (change.languagePath().size() > 1) {
                info = info(change.languagePath());
                hasChildren = (info != NO_INFO) ? info.tokenListList().hasChildren() : false;
            } else { // root-level
                info = NO_INFO;
                hasChildren = (eventInfo.tokenHierarchyOperation().maxTokenListListPathSize() > 0);
            }
            EmbeddingContainer<?> ec = (EmbeddingContainer<?>)tokenOrEC;
            rewrapECToken(ec, change); // Includes updateStatusImpl()
            EmbeddedTokenList<?> etl = ec.firstEmbeddedTokenList();
            if (etl != null && etl != EmbeddedTokenList.NO_DEFAULT_EMBEDDING) {
                // Check the text length beyond modification => end skip length must not be affected
                int modRelOffset = eventInfo.modificationOffset() - change.offset();
                int beyondModLength = change.addedEndOffset() - (eventInfo.modificationOffset() + eventInfo.diffLengthOrZero());
                EmbeddedTokenList<?> prevEtl = null;
                do {
                    TLLInfo childInfo = hasChildren ? info(etl.languagePath()) : NO_INFO;
                    // Check whether the change was not in the start or end skip lengths
                    // and if so then remove the embedding
                    if (modRelOffset >= etl.embedding().startSkipLength()
                            && beyondModLength >= etl.embedding().endSkipLength()
                    ) { // Modification within embedding's bounds => embedding can stay
                        // Mark that the embedding should be updated
                        if (childInfo != NO_INFO) {
                            // update-status called by rewrap-ec-token above
                            childInfo.markBoundsChange(etl);
                        } else { // No child but want to update nested possible bounds changes
                            if (etl.isInited()) {
                                parentChange = change;
                                // Perform change in child - it surely does not join the sections
                                // since otherwise the childInfo could not be null
                                // update-status done above for the embedding container
                                change = updateTokenListByModification(etl, null);
                                if (change.isBoundsChange()) {
                                    processBoundsChangeEmbeddings(change, parentChange);
                                } else {
                                    eventInfo.setMinAffectedStartOffset(change.offset());
                                    eventInfo.setMaxAffectedEndOffset(change.addedEndOffset());
                                }
                            }
                        }
                        prevEtl = etl;
                        etl = etl.nextEmbeddedTokenList();

                    } else { // Mod in skip lengths => Remove the etl from chain
                        if (childInfo != NO_INFO) {
                            // update-status already done as part of rewrap-token
                            childInfo.markRemoved(etl);
                        }
                        // Remove embedding and get the next embedded token list (prevEtl stays the same)
                        etl = ec.removeEmbeddedTokenList(prevEtl, etl);
                    }
                } while (etl != null && etl != EmbeddedTokenList.NO_DEFAULT_EMBEDDING);
            }
        } 
    }
    
    void processNonBoundsChange(TokenListChange<?> change) {
        TLLInfo info;
        boolean hasChildren;
        if (change.languagePath().size() >= 2) {
            info = info(change.languagePath());
            hasChildren = (info != NO_INFO && info.tokenListList().hasChildren());
        } else { // root change
            info = NO_INFO;
            hasChildren = (eventInfo.tokenHierarchyOperation().maxTokenListListPathSize() > 0);
        }
        if (hasChildren) {
            // First mark the removed embeddings
            TokenList<?> removedTokenList = change.tokenChangeInfo().removedTokenList();
            if (removedTokenList != null) {
                markRemovedEmbeddings(removedTokenList);
            }

            // Now mark added embeddings
            TokenList<?> currentTokenList = change.tokenChangeInfo().currentTokenList();
            markAddedEmbeddings(currentTokenList, change.index(), change.addedTokensOrBranchesCount());
        }
    }
    
    /**
     * Collect removed embeddings for the given token list recursively
     * and nest deep enough for all maintained children
     * token list lists.
     */
    private void markRemovedEmbeddings(TokenList<?> removedTokenList) {
        int tokenCount = removedTokenList.tokenCountCurrent();
        for (int i = 0; i < tokenCount; i++) {
            Object tokenOrEC = removedTokenList.tokenOrEmbeddingContainer(i);
            if (tokenOrEC.getClass() == EmbeddingContainer.class) {
                EmbeddingContainer<?> ec = (EmbeddingContainer<?>)tokenOrEC;
                ec.updateStatusImpl(); // Update status since markRemoved() will need it
                EmbeddedTokenList<?> etl = ec.firstEmbeddedTokenList();
                while (etl != null && etl != EmbeddedTokenList.NO_DEFAULT_EMBEDDING) {
                    TLLInfo info = info(etl.languagePath());
                    if (info != NO_INFO) {
                        // update-status called above
                        info.markRemoved(etl);
                    }
                    etl = etl.nextEmbeddedTokenList();
                }
            }
        }
    }
    
    private void markAddedEmbeddings(TokenList<?> tokenList, int index, int addedCount) {
        for (int i = 0; i < addedCount; i++) {
            // Ensure that the default embedding gets possibly created
            EmbeddedTokenList<?> etl = EmbeddingContainer.embeddedTokenList(tokenList, index + i, null);
            if (etl != null) {
                TLLInfo info = info(etl.languagePath());
                if (info != NO_INFO) {
                    // Mark that there was a new embedded token list added
                    // There should be no updateStatusImpl() necessary since the token lists are new
                    // and the parent embedding container was surely updated by the updating process.
                    info.markAdded(etl);
                }
            }
        }
    }

    private void processLevelInfos() {
        // Now relex the changes in affected token list lists
        // i.e. fix the tokens after the token lists removals/additions.
        // The higher-level updates
        if (levelInfos != null) {
            // The list can be extended by additional items dynamically during iteration
            for (int i = 0; i < levelInfos.size(); i++) {
                List<TLLInfo> infos = levelInfos.get(i);
                // The "infos" list should not be extended by additional items dynamically during iteration
                // However an extra items can be added at the deeper levels.
                for (int j = 0; j < infos.size(); j++) {
                    infos.get(j).update();
                }
            }
        }
        
        // Assert that update was called on all infos
        if (LOG.isLoggable(Level.FINER) && levelInfos != null) {
            for (List<TLLInfo> infos : levelInfos) {
                for (TLLInfo info : infos) {
                    if (!info.updateCalled) {
                        throw new IllegalStateException("Update not called on tokenListList\n" + // NOI18N
                                info.tokenListList);
                    }
                }
            }
        }
    }
    
    <T extends TokenId> TokenListChange<T> updateTokenListByModification(
    MutableTokenList<T> tokenList, Object zeroIndexRelexState) {
        TokenListChange<T> change = new TokenListChange<T>(tokenList);
//        if (tokenList instanceof EmbeddedTokenList) {
//            ((EmbeddedTokenList)tokenList).embeddingContainer().checkStatusUpdated();
//        }
        TokenListUpdater.update(tokenList, eventInfo.modificationOffset(),
                eventInfo.insertedLength(), eventInfo.removedLength(), change, zeroIndexRelexState);
        return change;
    }

    /**
     * Return tll info or NO_INFO if the token list list is not maintained
     * for the given language path.
     */
    private TLLInfo info(LanguagePath languagePath) {
        if (path2info == null) { // Init since it will contain NO_INFO
            path2info = new HashMap<LanguagePath,TLLInfo>(4, 0.5f);
        }
        TLLInfo info = path2info.get(languagePath);
        if (info == null) {
            TokenListList tll = eventInfo.tokenHierarchyOperation().existingTokenListList(languagePath);
            if (tll != null) {
                info = new TLLInfo(this, tll);
                int index = languagePath.size() - 2;
                if (levelInfos == null) {
                    levelInfos = new ArrayList<List<TLLInfo>>(index + 1);
                }
                while (levelInfos.size() <= index) {
                    levelInfos.add(new ArrayList<TLLInfo>(2));
                }
                levelInfos.get(index).add(info);
            } else { // No token list list for the given language path
                info = NO_INFO;
            }
            path2info.put(languagePath, info);
        }
        return info;
    }
    
    private <T extends TokenId> void rewrapECToken(EmbeddingContainer<T> ec, TokenListChange<?> change) {
        @SuppressWarnings("unchecked")
        TokenListChange<T> tChange = (TokenListChange<T>)change;
        ec.reinit(tChange.addedToken(0));
        ec.updateStatusImpl();
        tChange.tokenList().wrapToken(tChange.index(), ec);
    }

    /**
     * Special constant value to avoid double map search for token list lists updating.
     */
    static final TLLInfo NO_INFO = new TLLInfo(null, null);
        
    /**
     * Information about update in a single token list list.
     */
    static final class TLLInfo {
        
        final TokenHierarchyUpdate update;
        
        final TokenListList tokenListList;

        int index;

        int removeCount;

        List<TokenList<?>> added;
        
        TokenListChange<?> change;
        
        boolean updateCalled;
        
        public TLLInfo(TokenHierarchyUpdate update, TokenListList tokenListList) {
            this.update = update;
            this.tokenListList = tokenListList;
            this.index = -1;
            this.added = Collections.emptyList();
        }
        
        public TokenListList tokenListList() {
            return tokenListList;
        }
        
        /**
         * Mark the given token list as removed in this token list list.
         * All removed token lists should be marked by their increasing offset
         * so it should be necessary to search for the index just once.
         * <br/>
         * It's expected that updateStatusImpl() was already called
         * on the corresponding embedding container.
         */
        public void markRemoved(EmbeddedTokenList<?> removedTokenList) {
            boolean indexWasMinusOne; // Used for possible exception cause debugging
//            removedTokenList.embeddingContainer().checkStatusUpdated();
            if (index == -1) {
                checkUpdateNotCalledYet();
                indexWasMinusOne = true;
                index = tokenListList.findIndexDuringUpdate(removedTokenList,
                        update.eventInfo.modificationOffset(), update.eventInfo.removedLength());
                assert (index >= 0) : "index=" + index + " < 0"; // NOI18N
            } else { // Index already initialized
                indexWasMinusOne = false;
            }
            TokenList<?> markedForRemoveTokenList = tokenListList.getOrNull(index + removeCount);
            if (markedForRemoveTokenList != removedTokenList) {
                int realIndex = tokenListList.indexOf(removedTokenList);
                throw new IllegalStateException("Removing at index=" + index + // NOI18N
                        " but real index is " + realIndex + // NOI18N
                        " (indexWasMinusOne=" + indexWasMinusOne + ").\n" + // NOI18N
                        "Wishing to remove tokenList\n" + removedTokenList + // NOI18N
                        "\nbut marked-for-remove tokenList is \n" + markedForRemoveTokenList + // NOI18N
                        "\nfrom tokenListList\n" + tokenListList + // NOI18N
                        "\n\nModification description:\n" + update.eventInfo.modificationDescription(true) // NOI18N
                );
            }
            removeCount++;
        }

        /**
         * Mark the given token list to be added to this list of token lists.
         * At the end first the token lists marked for removal will be removed
         * and then the token lists marked for addition will be added.
         * <br/>
         * It's expected that updateStatusImpl() was already called
         * on the corresponding embedding container.
         */
        public void markAdded(EmbeddedTokenList<?> addedTokenList) {
//            addedTokenList.embeddingContainer().checkStatusUpdated();
            if (added.size() == 0) {
                checkUpdateNotCalledYet();
                if (index == -1) {
                    index = tokenListList.findIndex(addedTokenList.startOffset());
                    assert (index >= 0) : "index=" + index + " < 0"; // NOI18N
                }
                added = new ArrayList<TokenList<?>>(4);
            }
            added.add(addedTokenList);
        }
        
        /**
         * Mark that a parent's token list's bounds change need to be propagated
         * into the given (child) token list.
         * <br/>
         * It's expected that updateStatusImpl() was already called
         * on the corresponding embedding container.
         */
        public void markBoundsChange(EmbeddedTokenList<?> etl) {
            assert (index == -1) : "index=" + index + " != -1"; // Should be the first one
//            etl.embeddingContainer().checkStatusUpdated();
            checkUpdateNotCalledYet();
            index = tokenListList.findIndex(etl.startOffset());
        }
        
        public void update() {
            checkUpdateNotCalledYet();
            updateCalled = true;
            // Update this level (and language path).
            // All the removed and added sections resulting from parent change(s)
            // are already marked.
            if (index == -1)
                return; // Nothing to do

            if (removeCount == 0 && added.size() == 0) { // Bounds change only
                if (LOG.isLoggable(Level.FINE)) {
                    LOG.fine("TLLInfo.update(): BOUNDS-CHANGE: " + tokenListList.languagePath().mimePath() + // NOI18N
                            " index=" + index + // NOI18N
                            '\n'
                    );
                }

                EmbeddedTokenList<?> etl = tokenListList.get(index);
                etl.embeddingContainer().updateStatusImpl();
                Object matchState = LexerUtilsConstants.endState(etl);
                Object relexState = tokenListList.relexState(index);
                // update-status called above
                TokenListChange<?> chng = update.updateTokenListByModification(etl, relexState);
                relexState = LexerUtilsConstants.endState(etl, relexState);
                // Prevent bounds change in case the states at the end of the section would not match
                // which leads to relexing of the next section.
                if (chng.isBoundsChange() && LexerUtilsConstants.statesEqual(relexState, matchState)) {
                    TokenListChange<?> parentChange = (tokenListList.languagePath().size() == 2)
                            ? update.rootChange
                            : update.info(tokenListList.languagePath().parent()).change;
                    update.processBoundsChangeEmbeddings(chng, parentChange);
                } else { // Regular change
                    update.processNonBoundsChange(chng);
                }
                relexAfterLastModifiedSection(index + 1, relexState, matchState);

            } else { // Non-bounds change
                if (LOG.isLoggable(Level.FINE)) {
                    LOG.fine("TLLInfo.update(): REPLACE: " + tokenListList.languagePath().mimePath() + // NOI18N
                            " index=" + index + // NOI18N
                            ", removeCount=" + removeCount + // NOI18N
                            ", added.size()=" + added.size() + // NOI18N
                            '\n'
                    );
                }

                TokenList<?>[] removed = tokenListList.replace(index, removeCount, added);
                // Mark embeddings of removed token lists as removed
                if (tokenListList.hasChildren()) {
                    for (int i = 0; i < removed.length; i++) {
                        TokenList<?> removedTokenList = removed[i];
                        update.markRemovedEmbeddings(removedTokenList);
                    }
                }

                Object relexState; // State from which the relexing will start
                Object matchState = INVALID_STATE; // State that needs to be reached by relexing
                if (tokenListList.joinSections()) { // Need to find the right relexState
                    // Must update the token list by incremental algorithm
                    // Find non-empty token list and take last token's state
                    relexState = tokenListList.relexState(index);
                    for (int i = removed.length - 1; i >= 0 && matchState == INVALID_STATE; i--) {
                        matchState = LexerUtilsConstants.endState((EmbeddedTokenList<?>)removed[i]);
                    }
                    // Find the start state as the previous non-empty section's last token's state
                    // for case there would be no token lists added or all the added sections
                    // would be empty.
                    if (matchState == INVALID_STATE) // None or just empty sections were removed
                        matchState = relexState;

                } else { // Not joining the sections
                    relexState = null;
                }

                // Relex all the added token lists (just by asking for tokenCount - init() will be done)
                for (int i = 0; i < added.size(); i++) {
                    EmbeddedTokenList<?> tokenList = (EmbeddedTokenList<?>)added.get(i);
                    assert (!tokenList.isInited());
                    tokenList.init(relexState);
                    if (tokenList.embedding().joinSections()) {
                        tokenListList.setJoinSections(true);
                    }
                    relexState = LexerUtilsConstants.endState((EmbeddedTokenList<?>)tokenList, relexState);
                    if (tokenListList.hasChildren()) {
                        update.markAddedEmbeddings(tokenList, 0, tokenList.tokenCount());
                    }
                    // Added token lists should not require updateStatus()
                    update.eventInfo.setMaxAffectedEndOffset(tokenList.endOffset());
                }

                if (tokenListList.joinSections()) {
                    index += added.size();
                    relexAfterLastModifiedSection(index, relexState, matchState);
                }
            }
            
//            for (EmbeddedTokenList<?> etl : tokenListList) {
//                etl.embeddingContainer().updateStatusImpl();
//                if (etl.embeddingContainer().isRemoved())
//                    throw new IllegalStateException();
//            }
            // Set index to -1 to simplify correctness checking
            index = -1;
        }
        
        void checkUpdateNotCalledYet() {
            if (updateCalled) {
                throw new IllegalStateException("Update already called on \n" + tokenListList);
            }
        }
        
        private void relexAfterLastModifiedSection(int index, Object relexState, Object matchState) {
            // Must continue relexing existing section(s) (from a different start state)
            // until the relexing will stop before the last token of the given section.
            EmbeddedTokenList<?> etl;
            while (!LexerUtilsConstants.statesEqual(relexState, matchState)
                && (etl = tokenListList.getOrNull(index)) != null
            ) {
                etl.embeddingContainer().updateStatusImpl();
                if (etl.tokenCount() > 0) {
                    // Remember state after the last token of the given section
                    matchState = etl.state(etl.tokenCount() - 1);
                    // updateStatusImpl() just called
                    TokenListChange<?> chng = updateTokenListAtStart(etl, etl.startOffset(), relexState);
                    update.processNonBoundsChange(chng);
                    // Since the section is non-empty (checked above) there should be >0 tokens
                    relexState = etl.state(etl.tokenCount() - 1);
                }
                index++;
            }
        }
        
        private <T extends TokenId> TokenListChange<T> updateTokenListAtStart(
        EmbeddedTokenList<T> etl, int offset, Object zeroIndexRelexState) {
            TokenListChange<T> chng = new TokenListChange<T>(etl);
//            etl.embeddingContainer().checkStatusUpdated();
            TokenListUpdater.update(etl, offset, 0, 0, chng, zeroIndexRelexState);
            update.eventInfo.setMaxAffectedEndOffset(chng.addedEndOffset());
            return chng;
        }
        
    }
    
}

Lines 92-98 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/TokenList.java (-31 / +49 lines)
92	* @param >=0 index of the token in this list.	92	* @param >=0 index of the token in this list.
93	* @return valid token or null if the index is too high.	93	* @return valid token or null if the index is too high.
94	*/	94	*/
95	Object tokenOrEmbeddingContainer(int index);	95	TokenOrEmbedding<T> tokenOrEmbedding(int index);
96		96
97	/**	97	/**
98	* Replace flyweight token at the given index with its non-flyweight copy.	98	* Replace flyweight token at the given index with its non-flyweight copy.
Lines 120-130 Link Here
120	* Get absolute offset of the token at the given index in the token list.	120	* Get absolute offset of the token at the given index in the token list.
121	* <br>	121	* <br>
122	* This method can only be called if the token at the given index	122	* This method can only be called if the token at the given index
123	* was already fetched by {@link tokenOrEmbeddingContainer(int)}.	123	* was already fetched by {@link tokenOrEmbedding(int)}.
		124	* <br/>
		125	* For EmbeddedTokenList a EmbeddingContainer.updateStatus() must be called
		126	* prior this method to obtain up-to-date results.
124	*/	127	*/
125	int tokenOffset(int index);	128	int tokenOffsetByIndex(int index);
126		129
127	/**	130	/**
		131	* Get absolute offset of a token contained in this token list.
		132	* <br/>
		133	* For EmbeddedTokenList a EmbeddingContainer.updateStatus() must be called
		134	* prior this method to obtain up-to-date results.
		135	*
		136	* @param token non-null child token of this token list.
		137	* @return absolute offset in the input.
		138	*/
		139	int tokenOffset(AbstractToken<T> token);
		140
		141	/**
		142	* Get index of the token that "contains" the given offset.
		143	* <br/>
		144	* The result is in sync with TokenSequence.moveOffset().
		145	*
		146	* @param offset offset for which the token index should be found.
		147	* @return array of two items where the [0] is token's index and [1] is its offset.
		148	* <br/>
		149	* If offset >= last-token-end-offset then [0] contains token-count and
		150	* [1] conains last-token-end-offset.
		151	* <br/>
		152	* [0] may contain -1 to indicate that there are no tokens in the token list
		153	* ([1] then contains zero).
		154	*/
		155	int[] tokenIndex(int offset);
		156
		157	/**
128	* Get total count of tokens in the list.	158	* Get total count of tokens in the list.
129	* <br/>	159	* <br/>
130	* For token lists that create the tokens lazily	160	* For token lists that create the tokens lazily
Lines 154-164 Link Here
154	* <p>	184	* <p>
155	* This is also used to check whether this token list corresponds to mutable input	185	* This is also used to check whether this token list corresponds to mutable input
156	* or not because unmodifiable lists return -1 from this method.	186	* or not because unmodifiable lists return -1 from this method.
		187	* </p>
157	*	188	*
158	* <p>	189	* <p>
159	* For branch token lists the {@link #updateStartOffsetShift()} ensures	190	* For embedded token lists this value should be update to root's one
160	* that the value returned by this method is most up-to-date	191	* prior constructing child token sequence.
161	* (equals to the root list's one).	192	* </p>
162	*	193	*
163	* @return number of modifications performed to the list.	194	* @return number of modifications performed to the list.
164	* <br/>	195	* <br/>
Lines 167-199 Link Here
167	int modCount();	198	int modCount();
168		199
169	/**	200	/**
170	* Get absolute offset of the child token with the given raw offset	201	* Get the root token list of the token list hierarchy.
171	* in the underlying input.
172	*
173	* @param rawOffset raw offset of the child token.
174	* @return absolute offset in the input.
175	*/	202	*/
176	int childTokenOffset(int rawOffset);	203	TokenList<?> rootTokenList();
177		204
178	/**	205	/**
179	* Get character of a token from the character sequence represented	206	* Get text of the whole input source.
180	* by this support.
181	*
182	* @param rawOffset raw offset of the child token.
183	* The given offset value may need to be preprocessed before using (it depends
184	* on a nature of the token list).
185	* @param index index inside the token's text that should be returned.
186	* This value cannot be simply added to the previous parameter
187	* for mutable token lists as the value could errorneously point
188	* into a middle of the offset gap then.
189	* @return appropriate character that the token has requested.
190	*/	207	*/
191	char childTokenCharAt(int rawOffset, int index);	208	CharSequence inputSourceText();
192
193	/**
194	* Get the root token list of the token list hierarchy.
195	*/
196	TokenList<?> root();
197		209
198	/**	210	/**
199	* Get token hierarchy operation for this token list or null	211	* Get token hierarchy operation for this token list or null
Lines 271-276 Link Here
271	* If token filtering is used then the first token may start at higher offset.	283	* If token filtering is used then the first token may start at higher offset.
272	* <br/>	284	* <br/>
273	* It's guaranteed that there will be no token starting below this offset.	285	* It's guaranteed that there will be no token starting below this offset.
		286	* <br/>
		287	* For EmbeddedTokenList a EmbeddingContainer.updateStatus() must be called
		288	* prior this method to obtain up-to-date results.
274	*/	289	*/
275	int startOffset();	290	int startOffset();
276		291
Lines 280-295 Link Here
280	* If token filtering is used then the last token may end at lower offset.	295	* If token filtering is used then the last token may end at lower offset.
281	* <br/>	296	* <br/>
282	* It's guaranteed that there will be no token ending above this offset.	297	* It's guaranteed that there will be no token ending above this offset.
		298	* <br/>
		299	* For EmbeddedTokenList a EmbeddingContainer.updateStatus() must be called
		300	* prior this method to obtain up-to-date results.
283	*/	301	*/
284	int endOffset();	302	int endOffset();
285		303
286	/**	304	/**
287	* Check if this token list is removed from token hierarchy.	305	* Check if this token list is removed from token hierarchy.
288	* <br/>	306	* <br/>
289	* Should only be called under the lock of the root token list.	307	* Should only be called under a lock of a root token list.
290	*	308	*
291	* @return true if the token list was removed or false otherwise.	309	* @return true if the token list was removed or false otherwise.
292	*/	310	*/
293	boolean isRemoved();	311	boolean isRemoved();
294		312
295	}	313	}




import java.util.List;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.ArrayUtilities;
import org.netbeans.lib.editor.util.GapList;
import org.netbeans.lib.lexer.inc.TokenHierarchyEventInfo;

/**
 * List of token lists that collects all token lists for a given language path.
 * <br/>
 * There can be both lists with/without joining of the embedded sections.
 * Non-joining TLL gets created when someone asks for TokenHierarchy.tokenSequenceList().
 * Joining TLL gets created if any of the embeddings for the particular language path
 * has LanguageEmbedding.joinSections() set to true.
 * <br/>
 * Initial implementation attempted to initialize the list of token lists lazily
 * upon asking for it by client. However there was a problem with fixing

 * <br/>
 * Non-joining embedded token lists' contents will be lexed without token list list assistance.
 * <br/>
 * JoinTokenList deals with sections joining.
 * <br/>
 * If there are mixed joining/non-joining language embedding instances for the same
 * language path then the non-joining ones can possibly become initialized (lexed)
 * without TLL if they are asked individually. Later when first joining embedding is found
 * the token list list will be created and contain both joining and non-joining
 * embeddings but the joinSections will be respected for individual lexing.
 * Non-joining sections will be lexed individually and the join sections will be lexed as joined.
 * </p>
 *
 * @author Miloslav Metelka
 */

public final class TokenListList<T extends TokenId> extends GapList<EmbeddedTokenList<T>> {

    private final TokenList<?> rootTokenList;

    private final LanguagePath languagePath;

    /**
     * Whether this token list is holding joint sections embeddings.
     * <br/>
     * If so it is mandatorily maintained.
     */
    private boolean joinSections;

    /**
     * Total count of children. It's maintained to quickly resolve
     * whether the list may be released.
     */
    private int childrenCount;
    

    public TokenListList(TokenList<?> rootTokenList, LanguagePath languagePath) {
    public TokenListList(TokenHierarchyOperation<?,?> operation, LanguagePath languagePath) {
        super(4);
        this.rootTokenList = rootTokenList;
        this.languagePath = languagePath;

        // languagePath has size >= 2
        assert (languagePath.size() >= 2);
        Language<T> language = LexerUtilsConstants.innerLanguage(languagePath);
        if (languagePath.size() > 2) {
            TokenListList<?> parentTokenList = rootTokenList.tokenHierarchyOperation().tokenListList(languagePath.parent());
            TokenListList parentTokenList = operation.tokenListList(languagePath.parent());
            for (int parentIndex = 0; parentIndex < parentTokenList.size(); parentIndex++) {
                TokenList<?> tokenList = parentTokenList.get(parentIndex);
                scanTokenList(tokenList, language);
            }
        } else { // Parent is root token list
            scanTokenList(rootTokenList, language);
        }
        
        if (joinSections) {
            JoinTokenList.init(this, 0, size());
        } else {
            // Init individual lists
            for (EmbeddedTokenList<T> etl : this) {
                etl.initAllTokens();
            }
        }
    }
    
    private void scanTokenList(TokenList<?> tokenList, Language<T> language) {
        int tokenCount = tokenList.tokenCount();
        for (int i = 0; i < tokenCount; i++) {
            // Check for embedded token list of the given language
            EmbeddedTokenList<T> etl = EmbeddingContainer.embeddedTokenList(tokenList, i, language, false);
            if (etl != null) {
                add(etl);
                if (etl.embedding().joinSections()) {
                    this.joinSections = true;
                    if (!etl.isInited()) {
                        etl.init(relexState);
                        relexState = LexerUtilsConstants.endState(etl, relexState);
                    }
                } else { // Not joining sections -> next section starts with null state
                    relexState = null;
                }
            }
        }
        return relexState;
    }
    
    public LanguagePath languagePath() {

        return (childrenCount > 0);
    }
    
    public Object relexState(int index) {
        // Find the previous non-empty section or non-joining section
        Object relexState = INVALID_STATE;
        for (int i = index - 1; i >= 0 && relexState == INVALID_STATE; i--) {
            relexState = LexerUtilsConstants.endState(get(i));
        }
        if (relexState == INVALID_STATE) // Start from real begining
            relexState = null;
        return relexState;
    }

    /**
     * Return a valid token list or null if the index is too high.
     */
    public EmbeddedTokenList<T> getOrNull(int index) {
        return (index < size()) ? get(index) : null;
    }
    
    private static final EmbeddedTokenList<?>[] EMPTY_TOKEN_LIST_ARRAY = new EmbeddedTokenList<?>[0];

    public EmbeddedTokenList<T>[] replace(int index, int removeTokenListCount, List<EmbeddedTokenList<T>> addTokenLists) {
        @SuppressWarnings("unchecked")
        EmbeddedTokenList<T>[] removed = (removeTokenListCount > 0)
                ? (EmbeddedTokenList<T>[]) new EmbeddedTokenList[removeTokenListCount]
                : (EmbeddedTokenList<T>[]) EMPTY_TOKEN_LIST_ARRAY;
        if (removeTokenListCount > 0) {
            copyElements(index, index + removeTokenListCount, removed, 0);
            remove(index, removeTokenListCount);
        }
        addAll(index, addTokenLists);
        List<EmbeddedTokenList<?>> etlLists = (List<EmbeddedTokenList<?>>)addTokenLists;
        addAll(index, etlLists);
        return removed;
    }

    public TokenList<?> rootTokenList() {
        return rootTokenList;
    }

    void childAdded() {
        throw new UnsupportedOperationException("Not yet implemented");
    }
    
    TokenHierarchyOperation operation() {
        return operation;
    }

    int modCount() {
        return operation.modCount();
    }

    public int findIndex(int offset) {
        int high = size() - 1;
        int low = 0;
        while (low <= high) {
            int mid = (low + high) >>> 1;
            EmbeddedTokenList<T> etl = get(mid);
            // Ensure that the startOffset() will be updated
            etl.embeddingContainer().updateStatus();
            int cmp = etl.startOffset() - offset;

     * were removed then these TLs then the token lists beyond the modification point
     * will be forced to update itself which may 
     */
    public int findIndexDuringUpdate(EmbeddedTokenList<T> targetEtl, TokenHierarchyEventInfo eventInfo) {
        int high = size() - 1;
        int low = 0;
        int targetStartOffset = LexerUtilsConstants.updatedStartOffset(targetEtl, eventInfo);
        if (targetStartOffset > modOffset && targetEtl.embeddingContainer().isRemoved()) {
            targetStartOffset = Math.max(targetStartOffset - removedLength, modOffset);
        }
        while (low <= high) {
            int mid = (low + high) >>> 1;
            EmbeddedTokenList<T> etl = get(mid);
            // Ensure that the startOffset() will be updated
            int startOffset = LexerUtilsConstants.updatedStartOffset(etl, eventInfo);
            int startOffset = etl.startOffset();
            if (startOffset > modOffset && etl.embeddingContainer().isRemoved()) {
                startOffset = Math.max(startOffset - removedLength, modOffset);
            }
            int cmp = startOffset - targetStartOffset;
            if (cmp < 0)
                low = mid + 1;

                // In such case these need to be searched by linear search in both directions
                // from the found one.
                if (etl != targetEtl) {
                    while (--low >= 0) {
                    while (low >= 0) {
                        etl = get(low);
                        if (etl == targetEtl) { // Quick check for match
                            return low;
                        }
                        // Check whether this was appropriate attempt for match
                        if (LexerUtilsConstants.updatedStartOffset(etl, eventInfo) != targetStartOffset)
                        startOffset = etl.startOffset();
                        if (startOffset > modOffset && etl.embeddingContainer().isRemoved()) {
                            startOffset = Math.max(startOffset - removedLength, modOffset);
                        }
                        if (startOffset != modOffset)
                            break;
                        low--;
                    }
                    
                    // Go up from mid
                    low = mid;
                    while (++low < size()) {
                        etl = get(low);
                        if (etl == targetEtl) { // Quick check for match
                            return low;
                        }
                        // Check whether this was appropriate attempt for match
                        if (LexerUtilsConstants.updatedStartOffset(etl, eventInfo) != targetStartOffset)
                        startOffset = etl.startOffset();
                        if (startOffset > modOffset && etl.embeddingContainer().isRemoved()) {
                            startOffset = Math.max(startOffset - removedLength, modOffset);
                        }
                        if (startOffset != modOffset)
                            break;
                        low++;
                    }
                }
                break;

        // Check whether the token lists are in a right order
        int lastEndOffset = 0;
        for (int i = 0; i < size(); i++) {
            EmbeddedTokenList<T> etl = get(i);
            etl.embeddingContainer().updateStatusUnsync();
            if (etl.isRemoved()) {
                return "TOKEN-LIST-LIST Removed token list at index=" + i + '\n' + this;
            }
            if (etl.startOffset() < lastEndOffset) {
                return "TOKEN-LIST-LIST Invalid start offset at index=" + i +
                        ": etl[" + i + "].startOffset()=" + etl.startOffset() +

            }
            lastEndOffset = etl.endOffset();
        }
        if (joinSections()) {
            return get(0).joinTokenList().checkConsistency();
        }
        return null;
    }


        sb.append('\n');
        int digitCount = ArrayUtilities.digitCount(size());
        for (int i = 0; i < size(); i++) {
            EmbeddedTokenList<T> etl = get(i);
            ArrayUtilities.appendBracketedIndex(sb, i, digitCount);
            etl.embeddingContainer().updateStatus();
            etl.dumpInfo(sb);
            EmbeddingContainer ec = etl.embeddingContainer();
            if (ec != null && ec.isRemoved()) {
                sb.append(", <--REMOVED-->");




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer;

import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.token.AbstractToken;

/**
 * Type for having either token or embedding.
 *
 * @author Miloslav Metelka
 */

public interface TokenOrEmbedding<T extends TokenId> {

    /**
     * Get token reference 
     * 
     * @return <code>this</code> if this is a token instance or
     *  a wrapped token if this is an embedding container.
     */
    AbstractToken<T> token();
    
    /**
     * Get non-null embedding container if this is embedding.
     * 
     * @return non-null embedding or null if this is token.
     */
    EmbeddingContainer<T> embedding();

}





import java.util.AbstractList;
import java.util.ArrayList;
import java.util.ArrayList;
import java.util.Collections;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.api.lexer.TokenSequence;

/**


public final class TokenSequenceList extends AbstractList<TokenSequence<?>> {
    
    private TokenList<?> rootTokenList;

    private final TokenListList<?> tokenListList;
    
    private final List<TokenSequence<?>> tokenSequences;


     */
    private int tokenListIndex;
    
    public TokenSequenceList(TokenList<?> rootTokenList, LanguagePath languagePath,
    int startOffset, int endOffset) {
        this.rootTokenList = rootTokenList;
        this.endOffset = endOffset;
        this.expectedModCount = rootTokenList.modCount();

        if (languagePath.size() == 1) { // Is supported too
            this.tokenListList = null;
            tokenListIndex = Integer.MAX_VALUE; // Mark no mods to tokenSequences
            TokenList<?> rootTokenList = operation.rootTokenList();
            if (rootTokenList.languagePath() == languagePath) {
                TokenSequence<?> rootTS = LexerApiPackageAccessor.get().createTokenSequence(
                            checkWrapTokenList(rootTokenList, startOffset, endOffset));

            }

        } else { // languagePath.size() >= 2
            this.tokenListList = rootTokenList.tokenHierarchyOperation().tokenListList(languagePath);
            // Possibly skip initial token lists accroding to startOffset
            int size = tokenListList.size();
            int high = size - 1;

                    int mid = (tokenListIndex + high) / 2;
                    EmbeddedTokenList<?> etl = tokenListList.get(mid);
                    // Update end offset before querying
                    etl.embeddingContainer().updateStatusUnsync();
                    int tlEndOffset = etl.endOffset(); // updateStatusImpl() just called
                    if (tlEndOffset < startOffset) {
                        tokenListIndex = mid + 1;

                firstTokenList = tokenListList.getOrNull(tokenListIndex);
                if (tokenListIndex == size) { // Right above the ones that existed at begining of bin search
                    while (firstTokenList != null) {
                        firstTokenList.embeddingContainer().updateStatusUnsync();
                        if (firstTokenList.endOffset() >= startOffset) { // updateStatusImpl() just called
                            break;
                        }

            }

            if (firstTokenList != null) {
                firstTokenList.embeddingContainer().updateStatusUnsync();
                tokenSequences = new ArrayList<TokenSequence<?>>(4);
                tokenSequences.add(LexerApiPackageAccessor.get().createTokenSequence(
                        checkWrapTokenList(firstTokenList, startOffset, endOffset)));

    }
    
    void checkForComodification() {
        if (expectedModCount != rootTokenList.modCount())
            throw new ConcurrentModificationException(
                    "Caller uses obsolete TokenSequenceList: expectedModCount=" + expectedModCount + // NOI18N
                    " != modCount=" + rootTokenList.modCount()
            );
    }





/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.batch;

import java.util.ArrayList;
import java.util.Set;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.lib.lexer.EmbeddingContainer;
import org.netbeans.lib.lexer.LAState;
import org.netbeans.lib.lexer.TokenList;
import org.netbeans.lib.lexer.LexerInputOperation;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.api.lexer.InputAttributes;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.TokenHierarchyOperation;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.token.TextToken;


/**
 * Token list used for root list for immutable inputs.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public abstract class BatchTokenList<T extends TokenId>
extends ArrayList<Object> implements TokenList<T> {
    
    /** Flag for additional correctness checks (may degrade performance). */
    private static final boolean testing = Boolean.getBoolean("netbeans.debug.lexer.test");
    
    private static boolean maintainLAState;
    
    /**
     * Check whether lookaheads and states are stored for testing purposes.
     */
    public static boolean isMaintainLAState() {
        return maintainLAState;
    }
    
    public static void setMaintainLAState(boolean maintainLAState) {
        BatchTokenList.maintainLAState = maintainLAState;
    }
    
    private final TokenHierarchyOperation<?,T> tokenHierarchyOperation;
    
    private final LanguagePath languagePath;
    
    private final Set<T> skipTokenIds;
    
    private final InputAttributes inputAttributes;
    
    /**
     * Lexer input used for lexing of the input.
     */
    private LexerInputOperation<T> lexerInputOperation;

    private LAState laState;
    
    private boolean inited;
    
    
    public BatchTokenList(TokenHierarchyOperation<?,T> tokenHierarchyOperation,
    Language<T> language, Set<T> skipTokenIds, InputAttributes inputAttributes) {
        this.tokenHierarchyOperation = tokenHierarchyOperation;
        this.languagePath = LanguagePath.get(language);
        this.skipTokenIds = skipTokenIds;
        this.inputAttributes = inputAttributes;
        if (testing) { // Maintain lookaheads and states when in test environment
            laState = LAState.empty();
        }
    }

    public abstract char childTokenCharAt(int rawOffset, int index);

    protected abstract LexerInputOperation<T> createLexerInputOperation();

    protected void init() {
        lexerInputOperation = createLexerInputOperation();
    }
    
    public TokenList<?> root() {
        return this; // this list should always be the root list of the token hierarchy
    }
    
    public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {
        return tokenHierarchyOperation;
    }
    
    public LanguagePath languagePath() {
        return languagePath;
    }
    
    public synchronized int tokenCount() {
        if (!inited) {
            init();
            inited = true;
        }
        if (lexerInputOperation != null) { // still lexing
            tokenOrEmbeddingContainerImpl(Integer.MAX_VALUE);
        }
        return size();
    }
    
    public int tokenCountCurrent() {
        return size();
    }

    public int childTokenOffset(int rawOffset) {
        // Children offsets should be absolute
        return rawOffset;
    }

    public int tokenOffset(int index) {
        Token<T> token = existingToken(index);
        int offset;
        if (token.isFlyweight()) {
            offset = 0;
            while (--index >= 0) {
                token = existingToken(index);
                offset += token.length();
                if (!token.isFlyweight()) {
                    offset += token.offset(null);
                    break;
                }
            }
        } else { // non-flyweight offset
            offset = token.offset(null);
        }
        return offset;
    }

    public synchronized Object tokenOrEmbeddingContainer(int index) {
        return tokenOrEmbeddingContainerImpl(index);
    }
    
    private Object tokenOrEmbeddingContainerImpl(int index) {
        if (!inited) {
            init();
            inited = true;
        }
        while (lexerInputOperation != null && index >= size()) {
            Token<T> token = lexerInputOperation.nextToken();
            if (token != null) { // lexer returned valid token
                add(token);
                if (laState != null) { // maintaining lookaheads and states
                    laState = laState.add(lexerInputOperation.lookahead(),
                            lexerInputOperation.lexerState());
                }
            } else { // no more tokens from lexer
                lexerInputOperation.release();
                lexerInputOperation = null;
                trimToSize();
            }
        }
        return (index < size()) ? get(index) : null;
    }
    
    private Token<T> existingToken(int index) {
        return LexerUtilsConstants.token(get(index));
    }

    public int lookahead(int index) {
        return (laState != null) ? laState.lookahead(index) : -1;
    }

    public Object state(int index) {
        return (laState != null) ? laState.state(index) : null;
    }

    public int startOffset() {
        return 0;
    }

    public int endOffset() {
        int cntM1 = tokenCount() - 1;
        if (cntM1 >= 0)
            return tokenOffset(cntM1) + LexerUtilsConstants.token(this, cntM1).length();
        return 0;
    }

    public boolean isRemoved() {
        return false;
    }

    public int modCount() {
        return -1; // immutable input
    }
    
    public synchronized AbstractToken<T> replaceFlyToken(
    int index, AbstractToken<T> flyToken, int offset) {
        TextToken<T> nonFlyToken = ((TextToken<T>)flyToken).createCopy(this, offset);
        set(index, nonFlyToken);
        return nonFlyToken;
    }

    public void wrapToken(int index, EmbeddingContainer embeddingContainer) {
        set(index, embeddingContainer);
    }

    public InputAttributes inputAttributes() {
        return inputAttributes;
    }
    
    public boolean isContinuous() {
        return (skipTokenIds == null);
    }
    
    public Set<T> skipTokenIds() {
        return skipTokenIds;
    }

}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.batch;

import java.io.Reader;
import java.util.Set;
import org.netbeans.api.lexer.Language;
import org.netbeans.lib.lexer.LexerInputOperation;
import org.netbeans.api.lexer.InputAttributes;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.TokenHierarchyOperation;


/**
 * Token list for situation when the input text must be copied.
 * It works together with SkimTokenList instances that act
 * as a filter over this token list.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class CopyTextTokenList<T extends TokenId> extends BatchTokenList<T> {
    
    /** Either reader or char sequence */
    private final Object input;
    
    public CopyTextTokenList(TokenHierarchyOperation<?,T> tokenHierarchyOperation, Reader inputReader,
    Language<T> language, Set<T> skipTokenIds, InputAttributes inputAttributes) {
        super(tokenHierarchyOperation, language, skipTokenIds, inputAttributes);
        this.input = inputReader;
    }
    
    public CopyTextTokenList(TokenHierarchyOperation<?,T> tokenHierarchyOperation, CharSequence inputText,
    Language<T> language, Set<T> skipTokenIds, InputAttributes inputAttributes) {
        super(tokenHierarchyOperation, language, skipTokenIds, inputAttributes);
        this.input = inputText;
    }
    
    public int childTokenOffset(int rawOffset) {
        // Cluster should be used so this method should never be called
        throwShouldNeverBeCalled();
        return 0; // never reached
    }

    public char childTokenCharAt(int rawOffset, int index) {
        // Cluster should be used so this method should never be called
        throwShouldNeverBeCalled();
        return ' '; // never reached
    }
    
    private void throwShouldNeverBeCalled() {
        throw new IllegalStateException("Should never be called"); // NOI18N
    }

    protected LexerInputOperation<T> createLexerInputOperation() {
        return (input instanceof Reader)
            ? new SkimLexerInputOperation<T>(this, (Reader)input)
            : new SkimLexerInputOperation<T>(this, (CharSequence)input);
    }

}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.batch;

import java.io.IOException;
import java.io.Reader;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.LexerInputOperation;
import org.netbeans.lib.lexer.TokenList;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.spi.lexer.LexerInput;

/**
 * Lexer input operation over a {@link java.io.Reader}.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class SkimLexerInputOperation<T extends TokenId> extends LexerInputOperation<T> {
    
    private static final char[] EMPTY_CHAR_ARRAY = new char[0];
    
    /**
     * Default size for reading char array.
     */
    private static final int DEFAULT_READ_CHAR_ARRAY_SIZE = 4096;
    
    /**
     * Minimum size to be read (to have space for reading).
     */
    private static final int MIN_READ_SIZE = 512;
    
    private static final int DEFAULT_CLUSTER_SIZE = 4096;
    
    /**
     * Maximum fragmentation factor for token character arrays.
     * <br>
     * If there is not enough space in the tokenCharArray
     * to copy a token's characters there then if the token's length
     * will be greater than this threshold then the token will get
     * an extra character buffer just for itself and there will
     * still be chance to use the present tokenCharArray for tokens
     * with lower length.
     */
    private static final int MAX_UNUSED_CLUSTER_SIZE_FRACTION = 50;
    

    /**
     * Reader as a primary source of characters that are further
     * copied and cached.
     */
    private Reader reader;
    
    /**
     * Array holding the read characters.
     */
    private char[] readCharArray;
    
    /**
     * Character sequence holding the characters to be read.
     */
    private CharSequence readCharSequence;
    
    /**
     * Index of a first character in the token being currently recognized.
     */
    private int readStartIndex;
    
    /**
     * End of valid chars in readCharArray (points to first invalid char).
     */
    private int readEndIndex;
    
    /**
     * Whether EOF was read from reader already or not.
     */
    private boolean eofRead;
    
    /**
     * Actual token cluster where the tokens are being placed.
     */
    private SkimTokenList<T> cluster;

    private int clusterTextEndIndex;
    
    private int defaultClusterSize = DEFAULT_CLUSTER_SIZE;
    
    /**
     * Starting offset of the cluster currently being used.
     */
    private int clusterStartOffset;
    
    /**
     * How much the offset is ahead of the token's text offset
     * in the cluster. The tokens that get skipped and flyweight tokens
     * increase this value because their text is not physically copied
     * into the clusters character data but they increase the offset.
     */
    private int offsetShift;
    
    public SkimLexerInputOperation(TokenList<T> tokenList, Reader reader) {
        super(tokenList, 0, null);
        this.reader = reader;
        this.readCharArray = new char[DEFAULT_READ_CHAR_ARRAY_SIZE];
    }
    
    public SkimLexerInputOperation(TokenList<T> tokenList, CharSequence readCharSequence) {
        super(tokenList, 0, null);
        this.readCharSequence = readCharSequence;
        this.readEndIndex = readCharSequence.length();
    }
    
    public int read(int index) { // index >= 0 is guaranteed by contract
        index += readStartIndex;
        if (index < readEndIndex) {
            return (readCharArray != null)
                ? readCharArray[index]
                : readCharSequence.charAt(index);

        } else { // must read next or return EOF
            if (!eofRead) {
                eofRead = (readCharArray != null)
                    ? readNextCharArray()
                    : true; // using readCharSequence -> no more chars

                return read(index);

            } else {
                return LexerInput.EOF;
            }
        }
    }
    
    public char readExisting(int index) {
        return (readCharArray != null)
            ? readCharArray[index]
            : readCharSequence.charAt(index);
    }
    
    public void approveToken(AbstractToken<T> token) {
        int tokenLength = token.length();
        if (isSkipToken(token)) {
            preventFlyToken();
            skipChars(tokenLength());
            
        } else if (token.isFlyweight()) {
            assert isFlyTokenAllowed();
            flyTokenAdded();
            skipChars(tokenLength);

        } else { // non-flyweight token => must be L0Token instance
            if (clusterTextEndIndex != 0) { // valid cluster exists
                // Check whether token fits into cluster's char array
                if (tokenLength + clusterTextEndIndex > cluster.getText().length) {
                    // Cannot fit the token's text into current cluster
                    finishCluster();
                }
            }

            if (clusterTextEndIndex == 0) { // allocate new cluster
                int clusterSize = defaultClusterSize;
                if (clusterSize < tokenLength) { // cluster just for one token
                    clusterSize = tokenLength;
                }
                defaultClusterSize = clusterSize;
                cluster = new SkimTokenList<T>((CopyTextTokenList<T>)tokenList(),
                        clusterStartOffset, new char[clusterSize]);
            }

            // Now it's clear that the token will fit into the cluster's text
            // TODO for DirectCharSequence use more efficient way
            char[] clusterText = cluster.getText();
            if (readCharArray != null) {
                System.arraycopy(readCharArray, readStartIndex, clusterText,
                        clusterTextEndIndex, tokenLength);
            } else { // using readCharSequence
                for (int i = 0; i < tokenLength; i++) {
                    clusterText[clusterTextEndIndex + i]
                            = readCharSequence.charAt(readStartIndex + i);
                }
            }
            
            int rawOffset = (offsetShift << 16) | clusterTextEndIndex;
            token.setTokenList(cluster);
            token.setRawOffset(rawOffset);
            clusterTextEndIndex += tokenLength;
            clearFlySequence();
        }

        readStartIndex += tokenLength;
        tokenApproved();
    }

    private void skipChars(int skipLength) {
        if (clusterTextEndIndex != 0) { // cluster already populated
            if (offsetShift + skipLength > Short.MAX_VALUE) {
                // Cannot advance offset shift without overflowing -> cluster is finished
                finishCluster();
                clusterStartOffset += skipLength;

            } else { // relOffset will fit into current cluster
                offsetShift += skipLength;
            }

        } else { // cluster is null -> can shift cluster's start offset
            clusterStartOffset += skipLength;
        }
    }        
    
    public void finish() {
        if (clusterTextEndIndex != 0) {
            finishCluster();
        }
    }

    private void finishCluster() {
        // If there would be too much unused space in the cluster's char array
        // then it will be reallocated.
        int clusterTextLength = cluster.getText().length;
        if (clusterTextLength / MAX_UNUSED_CLUSTER_SIZE_FRACTION
                > (clusterTextLength - clusterTextEndIndex)
        ) { // Fragmentation -> reallocate cluster's char array
            char[] newText = new char[clusterTextEndIndex];
            System.arraycopy(cluster.getText(), 0, newText, 0, clusterTextEndIndex);
            cluster.setText(newText);
        }
        clusterStartOffset += clusterTextEndIndex + offsetShift;
        clusterTextEndIndex = 0;
        offsetShift = 0;
        cluster = null; // cluster no longer valid
    }
    
    private boolean readNextCharArray() {
        // Copy everything from present readStartIndex till readEndIndex
        int retainLength = readEndIndex - readStartIndex;
        int minReadSize = readCharArray.length - retainLength;
        char[] newReadCharArray = readCharArray; // by default take original one
        if (minReadSize < MIN_READ_SIZE) { // allocate new
            // double the current array's size
            newReadCharArray = new char[readCharArray.length * 2];
        }
        System.arraycopy(readCharArray, readStartIndex, newReadCharArray, 0, retainLength);
        readCharArray = newReadCharArray;
        readStartIndex = 0;
        readEndIndex = retainLength;
        
        boolean eof = false;
        while (readEndIndex < readCharArray.length) {
            int readSize;
            try {
                readSize = reader.read(readCharArray, readEndIndex,
                    readCharArray.length - readEndIndex);
            } catch (IOException e) {
                // The exception is silently ignored here
                // This should generally not happen - a wrapping reader
                // should be used that will catch and process the IO exceptions.
                readSize = -1;
            }
            if (readSize == -1) {
                eof = true;
                try {
                    reader.close();
                } catch (IOException e) {
                    // The exception is silently ignored here
                    // This should generally not happen - a wrapping reader
                    // should be used that will catch and process the IO exceptions.
                }
                break;
            } else {
                readEndIndex += readSize;
            }
        }
        return eof;
    }

}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.batch;

import java.util.Set;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.lib.lexer.EmbeddingContainer;
import org.netbeans.lib.lexer.TokenList;
import org.netbeans.api.lexer.InputAttributes;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.TokenHierarchyOperation;
import org.netbeans.lib.lexer.token.AbstractToken;

/**
 * Filtering token list constructed over character array with an independent
 * start offset value.
 * <br>
 * It is constructed for batch inputs and it implements
 * a token list but it only implements translation of raw offsets
 * into real offsets and retrieving of the characters of token bodies.
 * <br>
 * Other operations are delegated to an original
 * token list that really holds the tokens.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class SkimTokenList<T extends TokenId> implements TokenList<T> {
    
    private CopyTextTokenList<T> tokenList;
    
    private int startOffset;
    
    private char[] text;
    
    
    public SkimTokenList(CopyTextTokenList<T> tokenList, int startOffset, char[] text) {
        this.tokenList = tokenList;
        this.startOffset = startOffset;
        this.text = text;
    }

    public CopyTextTokenList<T> getTokenList() {
        return tokenList;
    }
    
    public int startOffset() {
        return tokenList.startOffset();
    }
    
    public int endOffset() {
        return tokenList.endOffset();
    }
    
    public boolean isRemoved() {
        return tokenList.isRemoved();
    }

    char[] getText() {
        return text;
    }
    
    void setText(char[] text) {
        this.text = text;
    }

    public int childTokenOffset(int rawOffset) {
        int offsetShift = (rawOffset >> 16);
        return startOffset + (rawOffset & 0xFFFF) + offsetShift;
    }

    public char childTokenCharAt(int rawOffset, int index) {
        return text[((rawOffset + index) & 0xFFFF)];
    }

    public int modCount() {
        return 0;
    }

    public Object tokenOrEmbeddingContainer(int index) {
        return tokenList.tokenOrEmbeddingContainer(index);
    }
    
    public AbstractToken<T> replaceFlyToken(
    int index, AbstractToken<T> flyToken, int offset) {
        return tokenList.replaceFlyToken(index, flyToken, offset);
    }
    

    public int lookahead(int index) {
        return tokenList.lookahead(index);
    }

    public Object state(int index) {
        return tokenList.state(index);
    }

    public int tokenOffset(int index) {
        return tokenList.tokenOffset(index);
    }

    public int tokenCount() {
        return tokenList.tokenCount();
    }
    
    public int tokenCountCurrent() {
        return tokenList.tokenCountCurrent();
    }

    public TokenList<?> root() {
        return tokenList.root();
    }

    public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {
        return tokenList.tokenHierarchyOperation();
    }
    
    public LanguagePath languagePath() {
        return tokenList.languagePath();
    }

    public void wrapToken(int index, EmbeddingContainer embeddingContainer) {
        tokenList.wrapToken(index, embeddingContainer);
    }

    public InputAttributes inputAttributes() {
        return tokenList.inputAttributes();
    }
    
    public boolean isContinuous() {
        return tokenList.isContinuous();
    }

    public Set<T> skipTokenIds() {
        return tokenList.skipTokenIds();
    }

}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.batch;

import java.util.Set;
import org.netbeans.api.lexer.Language;
import org.netbeans.lib.lexer.LexerInputOperation;
import org.netbeans.lib.lexer.TextLexerInputOperation;
import org.netbeans.api.lexer.InputAttributes;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.TokenHierarchyOperation;


/**
 * Batch token list over text expressed as character sequence.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class TextTokenList<T extends TokenId> extends BatchTokenList<T> {

    private CharSequence inputText;

    public TextTokenList(TokenHierarchyOperation<?,T> tokenHierarchyOperation, CharSequence inputText,
    Language<T> language, Set<T> skipTokenIds, InputAttributes inputAttributes) {
        super(tokenHierarchyOperation, language, skipTokenIds, inputAttributes);
        this.inputText = inputText;
    }
    
    public char childTokenCharAt(int rawOffset, int index) {
        return inputText.charAt(rawOffset + index); // rawOffset is absolute
    }
    
    protected LexerInputOperation<T> createLexerInputOperation() {
        return new TextLexerInputOperation<T>(this, inputText);
    }

}

Lines 48-54 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/inc/DocumentInput.java (-1 lines)
48	import javax.swing.text.Document;	48	import javax.swing.text.Document;
49	import org.netbeans.api.lexer.InputAttributes;	49	import org.netbeans.api.lexer.InputAttributes;
50	import org.netbeans.api.lexer.Language;	50	import org.netbeans.api.lexer.Language;
51	import org.netbeans.api.lexer.TokenId;
52	import org.netbeans.lib.editor.util.swing.DocumentListenerPriority;	51	import org.netbeans.lib.editor.util.swing.DocumentListenerPriority;
53	import org.netbeans.lib.editor.util.swing.DocumentUtilities;	52	import org.netbeans.lib.editor.util.swing.DocumentUtilities;
54	import org.netbeans.lib.lexer.LanguageManager;	53	import org.netbeans.lib.lexer.LanguageManager;




import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.EmbeddingContainer;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.lib.lexer.TokenHierarchyOperation;
import org.netbeans.lib.lexer.TokenList;
import org.netbeans.lib.lexer.TokenOrEmbedding;
import org.netbeans.lib.lexer.token.AbstractToken;

/**

        return tokenOffsetDiff;
    }
    
    public TokenOrEmbedding<T> tokenOrEmbedding(int index) {
        return tokenList.tokenOrEmbedding(index);
    }

    public AbstractToken<T> replaceFlyToken(int index, AbstractToken<T> flyToken, int offset) {
        return tokenList.replaceFlyToken(index, flyToken, offset);
    }

    public int tokenOffsetByIndex(int index) {
        return tokenOffsetDiff + tokenList.tokenOffsetByIndex(index);
    }

    public int modCount() {
        return LexerUtilsConstants.MOD_COUNT_IMMUTABLE_INPUT;
    }

    public int tokenCount() {

        return tokenList.languagePath();
    }

    public int tokenOffset(AbstractToken<T> token) {
        return tokenList.tokenOffset(token);
    }

    public int[] tokenIndex(int offset) {
        return LexerUtilsConstants.tokenIndexBinSearch(this, offset, tokenCount());
    }

    public char charAt(int offset) {
        throw new IllegalStateException("Unexpected call.");
    }


        tokenList.wrapToken(index, embeddingContainer);
    }

    public TokenList<?> rootTokenList() {
        return tokenList.rootTokenList();
    }

    public CharSequence inputSourceText() {
        return rootTokenList().inputSourceText();
    }

    public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {
        return tokenList.tokenHierarchyOperation();
    }




import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.api.lexer.InputAttributes;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.TokenHierarchyOperation;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.token.TextToken;
import org.netbeans.lib.lexer.TokenOrEmbedding;
import org.netbeans.spi.lexer.MutableTextInput;



 */

public final class IncTokenList<T extends TokenId>
extends FlyOffsetGapList<TokenOrEmbedding<T>> implements MutableTokenList<T> {
    
    private final TokenHierarchyOperation<?,T> tokenHierarchyOperation;

    private LanguagePath languagePath;
    
    private CharSequence inputSourceText;
    
    /**
     * Lexer input operation used for lexing of the input.

    public void reinit() {
        if (languagePath != null) {
            MutableTextInput input = tokenHierarchyOperation.mutableTextInput();
            this.inputSourceText = LexerSpiPackageAccessor.get().text(input);
            this.lexerInputOperation = new TextLexerInputOperation<T>(this);
        } else {
            this.inputSourceText = null;
            releaseLexerInputOperation();
        }
        this.laState = LAState.empty();
    }
    
    public void releaseLexerInputOperation() {
        if (lexerInputOperation != null) {
            lexerInputOperation.release();
            lexerInputOperation = null;
        }
    }

    public void refreshLexerInputOperation() {
        releaseLexerInputOperation();
        int lastTokenIndex = tokenCountCurrent() - 1;
        lexerInputOperation = createLexerInputOperation(
                lastTokenIndex + 1,
                existingTokensEndOffset(),
                (lastTokenIndex >= 0) ? state(lastTokenIndex) : null
        );
    }
    
    public LanguagePath languagePath() {
        return languagePath;
    }

    
    public synchronized int tokenCount() {
        if (lexerInputOperation != null) { // still lexing
            tokenOrEmbeddingImpl(Integer.MAX_VALUE);
        }
        return size();
    }

    public int tokenOffset(AbstractToken<T> token) {
        int rawOffset = token.rawOffset();
        return text.charAt(index);
    }
    
    public int childTokenOffset(int rawOffset) {
        return (rawOffset < offsetGapStart()
                ? rawOffset
                : rawOffset - offsetGapLength());
    }

    public int tokenOffsetByIndex(int index) {
        return elementOffset(index);
    }
    
    public int[] tokenIndex(int offset) {
        return LexerUtilsConstants.tokenIndexLazyTokenCreation(this, offset);
    }

    /**

        rootModCount++;
    }
    
    public synchronized TokenOrEmbedding<T> tokenOrEmbedding(int index) {
        return tokenOrEmbeddingImpl(index);
    }
    
    private TokenOrEmbedding<T> tokenOrEmbeddingImpl(int index) {
        while (lexerInputOperation != null && index >= size()) {
            AbstractToken<T> token = lexerInputOperation.nextToken();
            if (token != null) { // lexer returned valid token
                updateElementOffsetAdd(token);
                add(token);
                laState = laState.add(lexerInputOperation.lookahead(),
                        lexerInputOperation.lexerState());
            } else { // no more tokens from lexer
                releaseLexerInputOperation();
                lexerInputOperation = null;
                trimToSize();
                laState.trimToSize();
            }

        return nonFlyToken;
    }

    public synchronized void wrapToken(int index, EmbeddingContainer<T> embeddingContainer) {
        set(index, embeddingContainer);
    }


        return LexerSpiPackageAccessor.get().inputAttributes(tokenHierarchyOperation.mutableTextInput());
    }
    
    protected int elementRawOffset(TokenOrEmbedding<T> elem) {
        return elem.token().rawOffset();
    }
 
    protected void setElementRawOffset(TokenOrEmbedding<T> elem, int rawOffset) {
        elem.token().setRawOffset(rawOffset);
    }
    
    protected boolean isElementFlyweight(TokenOrEmbedding<T> elem) {
        // token wrapper always contains non-flyweight token
        return (elem.embedding() == null)
            && elem.token().isFlyweight();
    }
    
    protected int elementLength(TokenOrEmbedding<T> elem) {
        return elem.token().length();
    }
    
    public TokenOrEmbedding<T> tokenOrEmbeddingUnsync(int index) {
        // Must use synced tokenOrEmbeddingContainer() because of possible change
        // of the underlying list impl when adding lazily requested tokens
        return LexerUtilsConstants.token(tokenOrEmbeddingContainer(index));
    }

    public Object tokenOrEmbeddingContainerUnsync(int index) {
        // Solely for token list updater or token hierarchy snapshots
        // having single-threaded exclusive write access
        return get(index);
    }

    public int lookahead(int index) {
        return laState.lookahead(index);
    }

        return size();
    }

    public TokenList<?> rootTokenList() {
        return this;
    }

    public CharSequence inputSourceText() {
        return inputSourceText;
    }

    public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {

    
    public LexerInputOperation<T> createLexerInputOperation(
    int tokenIndex, int relexOffset, Object relexState) {
        // Possibly release unfinished lexing - will be restarted in replaceTokens()
        // Releasing the lexer now allows to share a single backing lexer's impl instance better.
        // Do not assign null to lexerInputOperation since the replaceTokens() would not know
        // that the lexing was unfinished.
        if (lexerInputOperation != null)
            lexerInputOperation.release();

        // Used for mutable lists only so maintain LA and state
        return new TextLexerInputOperation<T>(this, tokenIndex, relexState,
                relexOffset, inputSourceText.length());
    }

    public boolean isFullyLexed() {
        return (lexerInputOperation == null);
    }

    public void replaceTokens(TokenListChange<T> change, int diffLength) {
        int index = change.index();
        // Remove obsolete tokens (original offsets are retained)
        int removeTokenCount = change.removedTokenCount();
        AbstractToken<T> firstRemovedToken = null;
        if (removeTokenCount > 0) {
            @SuppressWarnings("unchecked")
            TokenOrEmbedding<T>[] removedTokensOrEmbeddings = new TokenOrEmbedding[removeTokenCount];
            copyElements(index, index + removeTokenCount, removedTokensOrEmbeddings, 0);
            firstRemovedToken = removedTokensOrEmbeddings[0].token();
            for (int i = 0; i < removeTokenCount; i++) {
                TokenOrEmbedding<T> tokenOrEmbedding = removedTokensOrEmbeddings[i];
                // It's necessary to update-status of all removed tokens' contained embeddings
                // since otherwise (if they would not be up-to-date) they could not be updated later
                // as they lose their parent token list which the update-status relies on.
                EmbeddingContainer<T> ec = tokenOrEmbedding.embedding();
                if (ec != null) {
                    assert (ec.cachedModCount() != rootModCount) : "ModCount already updated"; // NOI18N
                    ec.updateStatusUnsyncAndMarkRemoved();
                }
                AbstractToken<T> token = tokenOrEmbedding.token();
                if (!token.isFlyweight()) {
                    updateElementOffsetRemove(token);
                    token.setTokenList(null);
                }
            }
            remove(index, removeTokenCount); // Retain original offsets
            laState.remove(index, removeTokenCount); // Remove lookaheads and states
            change.setRemovedTokens(removedTokensOrEmbeddings);
            }
            offset += token.length();
        }
        remove(index, removeTokenCount); // Retain original offsets
        laState.remove(index, removeTokenCount); // Remove lookaheads and states
        change.setRemovedTokens(removedTokensOrEmbeddingContainers);
        change.setRemovedEndOffset(offset);

        // Move and fix the gap according to the performed modification.
        // Instead of modOffset the gap is located at first relexed token's start
        // because then the already precomputed index corresponding to the given offset
        // can be reused. Otherwise there would have to be another binary search for index.
        if (offsetGapStart() != change.offset()) {
            // Minimum of the index of the first removed index and original computed index
            moveOffsetGap(change.offset(), change.index());
        }
        updateOffsetGapLength(-diffLength);

        // Add created tokens.
        List<TokenOrEmbedding<T>> addedTokensOrEmbeddings = change.addedTokenOrEmbeddings();
        if (addedTokensOrEmbeddings != null && addedTokensOrEmbeddings.size() > 0) {
            for (TokenOrEmbedding<T> tokenOrEmbedding : addedTokensOrEmbeddings) {
                updateElementOffsetAdd(tokenOrEmbedding.token());
                AbstractToken<T> token = (AbstractToken<T>)tokenOrBranch;
                updateElementOffsetAdd(token);
            }
            addAll(index, addedTokensOrEmbeddings);
            laState = laState.addAll(index, change.laState());
            change.syncAddedTokenCount();
            // Check for bounds change only
            if (removeTokenCount == 1 && addedTokensOrEmbeddings.size() == 1) {
                // Compare removed and added token ids and part types
                AbstractToken<T> removedToken = LexerUtilsConstants.token(removedTokensOrEmbeddingContainers[0]);
                AbstractToken<T> addedToken = change.addedToken(0);
                if (firstRemovedToken.id() == addedToken.id()
                    && firstRemovedToken.partType() == addedToken.partType()
                ) {
                    change.markBoundsChange();
                }
            }
        }

        // Possibly restart unfinished lexing
        if (this.lexerInputOperation != null) { // Lexing was not finished before update
            int tokenCount = tokenCountCurrent();
            lexerInputOperation = createLexerInputOperation(tokenCount, elementOrEndOffset(tokenCount),
                (tokenCount > 0) ? state(tokenCount - 1) : null);
        }
    }
    

        return null;
    }

    @Override
    public int startOffset() {
        return 0;
    }

    public int endOffset() {
        return inputSourceText.length();
    }

    public boolean isRemoved() {
        return false; // Should never become removed
    }

    public void setInputSourceText(CharSequence text) {
        this.inputSourceText = text;
    }

    @Override

        return LexerUtilsConstants.appendTokenList(null, this).toString();
    }
    
    public CharSequence text() {
        return text;
    }
    
    public void setText(CharSequence text) {
        this.text = text;
    }
    
}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.inc;

import java.util.ArrayList;
import java.util.List;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.EmbeddedJoinInfo;
import org.netbeans.lib.lexer.EmbeddedTokenList;
import org.netbeans.lib.lexer.JoinLexerInputOperation;
import org.netbeans.lib.lexer.JoinTokenListBase;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.token.JoinToken;
import org.netbeans.lib.lexer.token.PartToken;

/**
 * Token list change for join token lists.
 *
 * @author Miloslav Metelka
 */
final class JoinTokenListChange<T extends TokenId> extends TokenListChange<T> {
    
    /** ETL where character modification occurred. */
    EmbeddedTokenList<T> charModTokenList;
    
    private TokenListListUpdate<T> tokenListListUpdate;

    private int startRelexTokenListIndex;

    private List<RelexTokenListChange<T>> relexChanges;
    
    private JoinLexerInputOperation<T> joinLexerInputOperation;
    
    public JoinTokenListChange(MutableJoinTokenList<T> tokenList) {
        super(tokenList);
    }

    public List<? extends TokenListChange<T>> relexChanges() {
        return relexChanges;
    }

    public TokenListListUpdate<T> tokenListListUpdate() {
        return tokenListListUpdate;
    }
    
    public void setTokenListListUpdate(TokenListListUpdate<T> tokenListListUpdate) {
        this.tokenListListUpdate = tokenListListUpdate;
    }
    
    public void setStartInfo(JoinLexerInputOperation<T> joinLexerInputOperation, int localIndex) {
        this.joinLexerInputOperation = joinLexerInputOperation;
        this.startRelexTokenListIndex = joinLexerInputOperation.activeTokenListIndex();
        this.relexChanges = new ArrayList<RelexTokenListChange<T>>(
                tokenListListUpdate.addedTokenLists.size() + 3);
        // Add first change now to incorporate starting modified token index
        RelexTokenListChange<T> firstChange = new RelexTokenListChange<T>(
                joinLexerInputOperation.tokenList(startRelexTokenListIndex));
        // Set index in ETL to properly do replaceTokens() in ETL
        // Setting both index and offset is BTW necessary in order to properly move offset gap in ETL
        firstChange.setIndex(localIndex);
        int relexOffset = joinLexerInputOperation.lastTokenEndOffset();
        firstChange.setOffset(relexOffset);
        firstChange.setMatchOffset(relexOffset); // Due to removeLastAddedToken() and etc.
        relexChanges.add(firstChange);
    }

    @Override
    public void addToken(AbstractToken<T> token, int lookahead, Object state) {
        // Check if lexer-input-operation advanced to next list and possibly add corresponding relex change(s)
        int activeTokenListIndex = joinLexerInputOperation.activeTokenListIndex();
        while (startRelexTokenListIndex + relexChanges.size() <= activeTokenListIndex) {
            // Use JLIO.tokenList() since it already contains the removed/added ETLs.
            EmbeddedTokenList<T> etl = joinLexerInputOperation.tokenList(
                    startRelexTokenListIndex + relexChanges.size());
            RelexTokenListChange<T> relexChange = new RelexTokenListChange<T>(etl);
            int startOffset = etl.startOffset();
            relexChange.setOffset(startOffset);
            relexChanges.add(relexChange);
        }
        int relexChangeIndex = activeTokenListIndex - startRelexTokenListIndex;
        if (token.getClass() == JoinToken.class) {
            JoinToken<T> joinToken = (JoinToken<T>) token;
            List<PartToken<T>> joinedParts = joinToken.joinedParts();
            int extraTokenListSpanCount = joinToken.extraTokenListSpanCount();
            int startRelexChangeIndex = relexChangeIndex - extraTokenListSpanCount;
            int joinedPartIndex = 0;
            // Only add without the last part (will be added normally outside the loop)
            // The last ETL can not be empty (must contain the last non-empty token part)
            for (int i = 0; i < extraTokenListSpanCount; i++) {
                RelexTokenListChange<T> relexChange = relexChanges.get(startRelexChangeIndex + i);
                // Check whether token list is non-empty by checking a text length that it covers.
                // Do not use etl.tokenCount() since the tokens are just being added into ETL.
                EmbeddedTokenList<T> etl = (EmbeddedTokenList<T>) relexChange.tokenList();
                if (etl.textLength() > 0) {
                    PartToken<T> partToken = joinedParts.get(joinedPartIndex++);
                    relexChange.addToken(partToken, 0, null);
                }
                relexChange.joinTokenLastPartShift = extraTokenListSpanCount - i;
            }
            // Last part will be added normally by subsequent code
            token = joinedParts.get(joinedPartIndex); // Should be (joinedParts.size()-1)
        }
        RelexTokenListChange<T> relexChange = relexChanges.get(relexChangeIndex);
        relexChange.addToken(token, lookahead, state);
    }
    
    @Override
    public AbstractToken<T> removeLastAddedToken() {
        RelexTokenListChange<T> lastRelexChange;
        AbstractToken<T> lastAddedToken = null;
        do {
            lastRelexChange = relexChanges.get(relexChanges.size() - 1);
            int tokenCount = lastRelexChange.tokenList().tokenCountCurrent();
            if (lastRelexChange.tokenList().tokenCountCurrent() > 0) { // There might be empty ETLs
                lastAddedToken = lastRelexChange.removeLastAddedToken();
                tokenCount--;
            }
            if (tokenCount == 0) {
                relexChanges.remove(relexChanges.size() - 1);
            }
        } while (lastAddedToken.getClass() != PartToken.class ||
                ((PartToken<T>) lastAddedToken).joinToken().joinedParts().get(0) != lastAddedToken);
        matchIndex--;
        matchOffset = lastRelexChange.matchOffset;
        // If lastAddedToken is PartToken then its joinToken() should be returned
        // but the return value should currently be ignored
        return lastAddedToken;
    }
    
    public void replaceTokens(int diffLength) {
        // Determine position of matchIndex in token lists
        // if matchIndex == jtl.tokenCount() the token list index will be the last list
        //   and endLocalIndex will be its tokenCount(). Because of this
        //   there must be a check whether token list index is not among removed ETLs.
        MutableJoinTokenList<T> jtl = (MutableJoinTokenList<T>) tokenList();
        int endLocalIndex = jtl.tokenStartLocalIndex(matchIndex);
        int matchTokenListIndex = jtl.activeTokenListIndex();
        if (matchTokenListIndex >= tokenListListUpdate.modTokenListIndex + tokenListListUpdate.removedTokenListCount) {
            assert (matchIndex == jtl.tokenCountCurrent()); // Should only happen in this situation
            // Project into relexChanges
            matchTokenListIndex += tokenListListUpdate.tokenListCountDiff();
            relexChanges.get(matchTokenListIndex - startRelexTokenListIndex).setMatchIndex(endLocalIndex);
            int afterAddIndex = tokenListListUpdate.modTokenListIndex + tokenListListUpdate.addedTokenLists.size();
            while (--matchTokenListIndex >= afterAddIndex) {
                TokenListChange<T> change = relexChanges.get(matchTokenListIndex - startRelexTokenListIndex);
                change.setMatchIndex(change.tokenList().tokenCountCurrent());
            }
        }
        // Fill in the below-mod-ETLs area
        int index = tokenListListUpdate.modTokenListIndex;
        while (--index >= startRelexTokenListIndex) {
            TokenListChange<T> change = relexChanges.get(index - startRelexTokenListIndex);
            change.setMatchIndex(change.tokenList().tokenCountCurrent());
        }

        // Physically replace the token lists
        JoinTokenListBase base = jtl.base();
        if (tokenListListUpdate.isTokenListsMod()) {
            // Move gap after last ETL that was relexed (obsolete ETLs still not removed)
            int relexEndIndex = startRelexTokenListIndex + relexChanges.size();
            int relexEndOldIndex = relexEndIndex - tokenListListUpdate.tokenListCountDiff();
            base.moveIndexGap(jtl.tokenListList(), jtl.tokenListStartIndex(), relexEndOldIndex);
            // Do physical ETLs replace
            jtl.tokenListList().replace(jtl.tokenListStartIndex() + tokenListListUpdate.modTokenListIndex,
                    tokenListListUpdate.removedTokenListCount, tokenListListUpdate.addedTokenLists);
            base.tokenListModNotify(tokenListListUpdate.tokenListCountDiff());
        }

        // Remember join token count right before the first relexed ETL
        int joinTokenIndex;
        if (startRelexTokenListIndex > 0) {
            EmbeddedTokenList<T> etl = jtl.tokenList(startRelexTokenListIndex - 1);
            joinTokenIndex = etl.joinInfo.joinTokenIndex() + etl.joinTokenCount(); // Physical removal already performed
        } else {
            joinTokenIndex = 0;
        }
        // Now process each relex change and update join token count etc.
        int relexChangesSizeM1 = relexChanges.size() - 1;
        int i;
        for (i = 0; i <= relexChangesSizeM1; i++) {
            RelexTokenListChange<T> change = relexChanges.get(i);
            EmbeddedTokenList<T> etl = (EmbeddedTokenList<T>) change.tokenList();
            if (etl.joinInfo == null) {
                etl.joinInfo = new EmbeddedJoinInfo(base, joinTokenIndex, startRelexTokenListIndex + i);
            } else {
                etl.joinInfo.setRawJoinTokenIndex(joinTokenIndex);
            }
            // Set new joinTokenLastPartShift before calling etl.joinTokenCount()
            etl.joinInfo.setJoinTokenLastPartShift(change.joinTokenLastPartShift);
            // Replace tokens in the individual ETL
            int realDiffLength = (etl == charModTokenList) ? diffLength : 0;
            etl.replaceTokens(change, realDiffLength);
            // Fix join token count
            joinTokenIndex += etl.joinTokenCount();
        }
        
        // Now fix the total join token count
        int origJoinTokenIndex = (i < jtl.tokenListCount())
                ? jtl.tokenList(i).joinInfo.joinTokenIndex()
                : base.joinTokenCount();
        int joinTokenCountDiff = joinTokenIndex - origJoinTokenIndex;
        base.updateJoinTokenCount(joinTokenCountDiff);
        
        if (relexChangesSizeM1 == 0) { // Only changed single ETL
            if (relexChanges.get(0).isBoundsChange()) {
                markBoundsChange(); // Joined change treated as bounds change too
            }
        }
    }

    @Override
    public String toString() {
        return super.toString() + ", tokenListListUpdate=" + tokenListListUpdate + // NOI18N
                ", startRelexTokenListIndex=" + startRelexTokenListIndex + // NOI18N
                ", relexChanges.size()=" + relexChanges.size();
    }

    @Override
    public String toStringMods(int indent) {
        StringBuilder sb = new StringBuilder(100);
        for (RelexTokenListChange change : relexChanges) {
            sb.append(change.toStringMods(indent));
            sb.append('\n');
        }
        return sb.toString();
    }
    
    static final class RelexTokenListChange<T extends TokenId> extends TokenListChange<T> {
        
        int joinTokenLastPartShift; // New value for EmbeddedJoinInfo.joinTokenLastPartShift during relex

        RelexTokenListChange(EmbeddedTokenList<T> tokenList) {
            super(tokenList);
        }

        @Override
        public String toString() {
            return super.toString() + ", lps=" + joinTokenLastPartShift;
        }
        
    }

}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.inc;

import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.EmbeddedTokenList;
import org.netbeans.lib.lexer.JoinLexerInputOperation;
import org.netbeans.lib.lexer.JoinTokenList;

/**
 * Lexer input operation over multiple joined sections (embedded token lists).
 * <br/>
 * It produces regular tokens (to be added directly into ETL represented by
 * {@link #activeTokenList()} and also special {@link #JoinToken} instances
 * in case a token spans boundaries of multiple ETLs.
 * <br/>
 * It can either work over JoinTokenList directly or, during a modification,
 * it simulates that certain token lists are already removed/added to underlying token list.
 * <br/>
 * 
 * {@link #recognizedTokenLastInTokenList()} gives information whether the lastly
 * produced token ends right at boundary of the activeTokenList.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

class MutableJoinLexerInputOperation<T extends TokenId> extends JoinLexerInputOperation<T> {
    
    private TokenListListUpdate<T> tokenListListUpdate;

    MutableJoinLexerInputOperation(JoinTokenList<T> joinTokenList, int relexJoinIndex, Object lexerRestartState,
            int activeTokenListIndex, int relexOffset, TokenListListUpdate<T> tokenListListUpdate
    ) {
        super(joinTokenList, relexJoinIndex, lexerRestartState, activeTokenListIndex, relexOffset);
        this.tokenListListUpdate = tokenListListUpdate;
    }

    @Override
    public EmbeddedTokenList<T> tokenList(int tokenListIndex) {
        return tokenListListUpdate.afterUpdateTokenList((JoinTokenList<T>) tokenList, tokenListIndex);
    }

    @Override
    protected int tokenListCount() {
        return tokenListListUpdate.afterUpdateTokenListCount((JoinTokenList<T>) tokenList);
    }

    @Override
    public String toString() {
        return super.toString() + ", tokenListListUpdate: " + tokenListListUpdate; // NOI18N
    }

}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.inc;

import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.EmbeddedTokenList;
import org.netbeans.lib.lexer.JoinTokenList;
import org.netbeans.lib.lexer.JoinTokenListBase;
import org.netbeans.lib.lexer.LexerInputOperation;
import org.netbeans.lib.lexer.TokenListList;
import org.netbeans.lib.lexer.TokenOrEmbedding;


/**
 * Mutable join token list allows mutations by token list updater.
 * 
 * @author Miloslav Metelka
 */

class MutableJoinTokenList<T extends TokenId> extends JoinTokenList<T> implements MutableTokenList<T> {
    
    static <T extends TokenId> MutableJoinTokenList<T> create(TokenListList<T> tokenListList, int etlIndex, EmbeddedTokenList<T> etl) {
        int tokenListStartIndex = etlIndex - etl.joinInfo.tokenListIndex();
        MutableJoinTokenList<T> jtl = new MutableJoinTokenList<T>(tokenListList, etl.joinInfo.base, tokenListStartIndex);
        // Position to this etl's join index
        jtl.setActiveTokenListIndex(etlIndex - tokenListStartIndex);
        return jtl;
    }

    MutableJoinTokenList(TokenListList<T> tokenListList, JoinTokenListBase base, int tokenListStartIndex) {
        super(tokenListList, base, tokenListStartIndex);
    }

    public TokenOrEmbedding<T> tokenOrEmbeddingUnsync(int index) {
        return tokenOrEmbedding(index);
    }

    public boolean isFullyLexed() {
        return true;
    }

    public void replaceTokens(TokenListChange<T> change, int diffLength) {
        ((JoinTokenListChange<T>) change).replaceTokens(diffLength);
    }

    public LexerInputOperation<T> createLexerInputOperation(int tokenIndex, int relexOffset, Object relexState) {
        // Should never be called
        throw new IllegalStateException("Should never be called"); // NOI18N
    }

    public void resetActiveAfterUpdate() { // Update the active token list after updating
        activeTokenListIndex = 0;
        fetchActiveTokenListData();
    }

}





import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.LexerInputOperation;
import org.netbeans.lib.lexer.TokenList;
import org.netbeans.lib.lexer.TokenOrEmbedding;

/**
 * Token list that allows mutating by token list mutator.

     * Also do not perform any checks regarding index validity
     * - only items below {@link #tokenCountCurrent()} will be requested.
     */
    TokenOrEmbedding<T> tokenOrEmbeddingUnsync(int index);

    /**
     * Create lexer input operation used for relexing of the input.
     */

    /**
     * Update the token list by replacing tokens according to the given change.
     */
    void replaceTokens(TokenListChange<T> change, int diffLength);

}




import org.netbeans.lib.lexer.TokenList;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.token.TextToken;
import org.netbeans.lib.lexer.TokenOrEmbedding;

/**
 * Token list implementation holding added or removed tokens from a list.

    
    private final LanguagePath languagePath;
    
    private TokenOrEmbedding<T>[] tokenOrEmbeddings;
    
    private int removedTokensStartOffset;
    
    public RemovedTokenList(LanguagePath languagePath, TokenOrEmbedding<T>[] tokensOrBranches) {
        this.languagePath = languagePath;
        this.tokenOrEmbeddings = tokensOrBranches;
    }
    
    public LanguagePath languagePath() {
        return languagePath;
    }
    
    public TokenOrEmbedding<T> tokenOrEmbedding(int index) {
        return (index < tokenOrEmbeddings.length) ? tokenOrEmbeddings[index] : null;
    }

    public int lookahead(int index) {

        return null;
    }

    public int tokenOffsetByIndex(int index) {
        Token<?> token = existingToken(index);
        if (token.isFlyweight()) {
            int offset = 0;

        }
    }

    public int[] tokenIndex(int offset) {
        return LexerUtilsConstants.tokenIndexBinSearch(this, offset, tokenCountCurrent());
    }

    private Token<T> existingToken(int index) {
        return tokenOrEmbeddings[index].token();
    }

    public synchronized AbstractToken<T> replaceFlyToken(
    int index, AbstractToken<T> flyToken, int offset) {
        TextToken<T> nonFlyToken = ((TextToken<T>)flyToken).createCopy(this, offset);
        tokenOrEmbeddings[index] = nonFlyToken;
        return nonFlyToken;
    }


    }

    public int tokenCountCurrent() {
        return tokenOrEmbeddings.length;
    }

    public int modCount() {
        return LexerUtilsConstants.MOD_COUNT_IMMUTABLE_INPUT;
    }
    
    public int tokenOffset(AbstractToken<T> token) {
        int rawOffset = token.rawOffset();
        // Offsets of contained tokens are absolute
        return rawOffset;
    }
    
    public char charAt(int offset) {
        throw new IllegalStateException("Querying of text for removed tokens not supported"); // NOI18N
    }


        throw new IllegalStateException("Branching of removed tokens not supported"); // NOI18N
    }
    
    public TokenList<?> rootTokenList() {
        return null;
    }

    public CharSequence inputSourceText() {
        return null;
    }

    public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {
        return null;
    }


    public int startOffset() {
        if (tokenCountCurrent() > 0 || tokenCount() > 0)
            return tokenOffsetByIndex(0);
        return 0;
    }

    public int endOffset() {
        int cntM1 = tokenCount() - 1;
        if (cntM1 >= 0)
            return tokenOffsetByIndex(cntM1) + tokenOrEmbedding(cntM1).token().length();
        return 0;
    }





import java.util.Set;
import org.netbeans.api.lexer.InputAttributes;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.CompactMap;
import org.netbeans.lib.lexer.EmbeddedTokenList;

import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.lib.lexer.TokenHierarchyOperation;
import org.netbeans.lib.lexer.TokenList;
import org.netbeans.lib.lexer.TokenOrEmbedding;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.token.TextToken;


    private int liveTokenOffsetDiff;

    /** Captured original tokens or branches. */
    private TokenOrEmbedding<T>[] origTokenOrEmbeddings;

    /** Original token's offsets. The array is occupied
     * and maintained in the same way like origTokensOrBranches.

        return liveTokenList.languagePath();
    }
    
    public TokenOrEmbedding<T> tokenOrEmbedding(int index) {
        if (liveTokenGapStart == -1 || index < liveTokenGapStart) {
            return liveTokenList.tokenOrEmbedding(index);
        }
        index -= liveTokenGapStart;
        if (index < origTokenCount) {
            return origTokenOrEmbeddings[origTokenStartIndex + index];
        }
        return liveTokenList.tokenOrEmbedding(liveTokenGapEnd + index - origTokenCount);
    }

    public int lookahead(int index) {

        return null;
    }

    public int tokenOffsetByIndex(int index) {
        if (liveTokenGapStart == -1 || index < liveTokenGapStart) {
            return liveTokenList.tokenOffsetByIndex(index);
        }
        index -= liveTokenGapStart;
        if (index < origTokenCount) {

        }
        index -= origTokenCount;

        AbstractToken<T> token = liveTokenList.tokenOrEmbeddingUnsync(liveTokenGapEnd + index).token();
                tokenOrEmbeddingContainerUnsync(liveTokenGapEnd + index));
        int offset;
        if (token.isFlyweight()) {
            offset = token.length();
            while (--index >= 0) {
                token = liveTokenList.tokenOrEmbeddingUnsync(liveTokenGapEnd + index).token();
                        tokenOrEmbeddingContainerUnsync(liveTokenGapEnd + index));
                if (token.isFlyweight()) {
                    offset += token.length();
                } else { // non-flyweight element
                    offset += tokenOffset(token, liveTokenList);
                    break;
                }
            }
            if (index == -1) { // below the boundary of above-gap live tokens
                index += liveTokenGapStart + origTokenCount;
                if (index >= 0) {
                    offset += tokenOffsetByIndex(index);
                }
            }
            
        } else { // non-flyweight
            offset = tokenOffset(token, liveTokenList);
        }
        return offset;
    }

    public int[] tokenIndex(int offset) {
        return LexerUtilsConstants.tokenIndexLazyTokenCreation(this, offset);
    }

    /**

     * @return offset for the particular token.
     */
    public <TT extends TokenId> int tokenOffset(
    AbstractToken<TT> token, TokenList<TT> tokenList) {
        // The following situations can happen:
        // 1. Token instance is contained in token2offset map so the token's
        //    offset is overriden by the information in the map.

        //    needs to be corrected if necessary.
        if (tokenList.getClass() == EmbeddedTokenList.class) {
            EmbeddedTokenList<TT> etl = (EmbeddedTokenList<TT>)tokenList;
            AbstractToken<?> rootBranchToken = null; // originally etl.rootToken();
            Token2OffsetEntry<T> entry = token2offset.get(rootBranchToken);
            if (entry != null) {
                return entry.offset();// used to be: + etl.childTokenOffsetShift(rawOffset);
            } else { // no special entry => check whether the regular offset is below liveTokenGapStartOffset
                int offset = etl.tokenOffset(token);
                TokenList rootTokenList = etl.rootTokenList();
                if (rootTokenList != null && rootTokenList.getClass() == IncTokenList.class) {
                    if (offset >= liveTokenGapStartOffset) {
                        offset += liveTokenOffsetDiff;

            if (entry != null) {
                return entry.offset();
            } else {
                int offset = tokenList.tokenOffset(token);
                if (tokenList.getClass() == IncTokenList.class) {
                    if (offset >= liveTokenGapStartOffset) {
                        offset += liveTokenOffsetDiff;
                        rawOffset += liveTokenOffsetDiff;
                    }
                    return rawOffset;
                }
                return offset;
            }
        }
    }

    }

    public int modCount() {
        return LexerUtilsConstants.MOD_COUNT_IMMUTABLE_INPUT;
    }
    
    public int tokenOffset(AbstractToken<T> token) {
        int rawOffset = token.rawOffset();
        // Offset of the standalone token is absolute
        return rawOffset;
    }
    
    public char charAt(int offset) {
        // No tokens expected to be parented to this token list
        throw new IllegalStateException("Not expected to be called"); // NOI18N
    }

    public void wrapToken(int index, EmbeddingContainer<T> embeddingContainer) {
        // Allow branching
        if (liveTokenGapStart == -1 || index < liveTokenGapStart) {
            liveTokenList.wrapToken(index, embeddingContainer);
        } else {
            index -= liveTokenGapStart;
            if (index < origTokenCount) {
                origTokenOrEmbeddings[origTokenStartIndex + index] = embeddingContainer;
            } else {
                liveTokenList.wrapToken(liveTokenGapEnd + index - origTokenCount, embeddingContainer);
            }

            index -= liveTokenGapStart;
            if (index < origTokenCount) {
                nonFlyToken = ((TextToken<T>)flyToken).createCopy(this, offset);
                origTokenOrEmbeddings[origTokenStartIndex + index] = nonFlyToken;
            } else {
                nonFlyToken = liveTokenList.replaceFlyToken(
                        liveTokenGapEnd + index - origTokenCount,

        return nonFlyToken;
    }
    
    public TokenList<?> rootTokenList() {
        return this;
    }

    public CharSequence inputSourceText() {
        return rootTokenList().inputSourceText();
    }

    public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {
        return snapshot;
    }


    public int startOffset() {
        if (tokenCountCurrent() > 0 || tokenCount() > 0)
            return tokenOffsetByIndex(0);
        return 0;
    }

    public int endOffset() {
        int cntM1 = tokenCount() - 1;
        if (cntM1 >= 0)
            return tokenOffsetByIndex(cntM1) + tokenOrEmbedding(cntM1).token().length();
        return 0;
    }
    

            liveTokenGapStart = startRemovedIndex;
            liveTokenGapEnd = startRemovedIndex;
            liveTokenGapStartOffset = change.offset();
            @SuppressWarnings("unchecked")
            TokenOrEmbedding<T>[] tokenOrEmbeddings = new TokenOrEmbedding[removedTokenList.tokenCount()];
            origTokenOrEmbeddings = tokenOrEmbeddings;
            origOffsets = new int[origTokenOrEmbeddings.length];
        }

        int liveTokenIndexDiff = change.tokenChangeInfo().addedTokenCount()

            int offset = change.offset();
            liveTokenGapStartOffset = offset;
            for (index = startRemovedIndex; index < bound; index++) {
                TokenOrEmbedding<T> tokenOrEmbedding = removedTokenList.tokenOrEmbedding(index - startRemovedIndex);
                AbstractToken<T> token = tokenOrEmbedding.token();
                if (!token.isFlyweight()) {
                    TokenList<T> tokenList = token.tokenList();
                    if (tokenList == null) {
                        tokenList = null; // new StandaloneTokenList<T>(change.languagePath(),
                                // eventInfo.originalText().toCharArray(offset, offset + token.length()));
                        token.setTokenList(tokenList);
                    }
                }
                origOffsets[origTokenStartIndex] = offset;
                origTokenOrEmbeddings[origTokenStartIndex++] = tokenOrEmbedding;
                offset += token.length();
            }

            while (index < liveTokenGapStart) {
                TokenOrEmbedding<T> tokenOrEmbedding = liveTokenList.tokenOrEmbeddingUnsync(index + liveTokenIndexDiff);
                AbstractToken<T> t = tokenOrEmbedding.token();
                if (!t.isFlyweight()) {
                    token2offset.putEntry(new Token2OffsetEntry<T>(t, offset));
                }
                origOffsets[origTokenStartIndex] = offset;
                origTokenOrEmbeddings[origTokenStartIndex++] = tokenOrEmbedding;
                offset += t.length();
                index++;
            }

            int index = endRemovedIndex;
            int offset = change.removedEndOffset();
            for (index = endRemovedIndex - 1; index >= bound; index--) {
                TokenOrEmbedding<T> tokenOrEmbedding = removedTokenList.tokenOrEmbedding(index - startRemovedIndex);
                AbstractToken<T> token = tokenOrEmbedding.token();
                offset -= token.length();
                if (!token.isFlyweight()) {
                    TokenList<T> tokenList = token.tokenList();
                    if (tokenList == null) {
                        tokenList = null; // new StandaloneTokenList<T>(change.languagePath(),
                                // eventInfo.originalText().toCharArray(offset, offset + token.length()));
                        token.setTokenList(tokenList);
                    }
                }

                if (liveTokenOffsetDiff != 0) {
                    token2offset.putEntry(new Token2OffsetEntry<T>(token, origOffsets[origTokenIndex]));
                }
                origTokenOrEmbeddings[origTokenIndex--] = tokenOrEmbedding;
            }

            while (index >= liveTokenGapEnd) {
                TokenOrEmbedding<T> tokenOrEmbedding = liveTokenList.tokenOrEmbeddingUnsync(index + liveTokenIndexDiff);
                AbstractToken<T> token = tokenOrEmbedding.token();
                offset -= token.length();
                if (!token.isFlyweight()) {
                    token2offset.putEntry(new Token2OffsetEntry<T>(token, offset));
                }
                origOffsets[origTokenIndex] = offset + liveTokenOffsetDiff;
                token2offset.putEntry(new Token2OffsetEntry<T>(token, origOffsets[origTokenIndex]));
                origTokenOrEmbeddings[origTokenIndex--] = tokenOrEmbedding;
                index--;
            }
            liveTokenGapEnd = endRemovedIndex;

    }

    private void ensureOrigTokensStartCapacity(int extraOrigTokenCount) {
        if (extraOrigTokenCount > origTokenOrEmbeddings.length - origTokenCount) { // will need to reallocate
            // Could check for maximum possible token count (origTokenCount + below-and-above live token counts)
            // but would cause init of live tokens above gap which is undesirable
            @SuppressWarnings("unchecked")
            TokenOrEmbedding<T>[] newOrigTokensOrBranches = new TokenOrEmbedding[
                    (origTokenOrEmbeddings.length * 3 / 2) + extraOrigTokenCount];
            int[] newOrigOffsets = new int[newOrigTokensOrBranches.length];
            int newIndex = Math.max(extraOrigTokenCount, (newOrigTokensOrBranches.length
                    - (origTokenCount + extraOrigTokenCount)) / 2);
            System.arraycopy(origTokenOrEmbeddings, origTokenStartIndex,
                    newOrigTokensOrBranches, newIndex, origTokenCount);
            System.arraycopy(origOffsets, origTokenStartIndex,
                    newOrigOffsets, newIndex, origTokenCount);
            origTokenOrEmbeddings = newOrigTokensOrBranches;
            origOffsets = newOrigOffsets;
            origTokenStartIndex = newIndex;

        } else if (extraOrigTokenCount > origTokenStartIndex) { // only move
            // Move to the end of the array
            int newIndex = origTokenOrEmbeddings.length - origTokenCount;
            System.arraycopy(origTokenOrEmbeddings, origTokenStartIndex,
                    origTokenOrEmbeddings, newIndex, origTokenCount);
            System.arraycopy(origOffsets, origTokenStartIndex,
                    origOffsets, newIndex, origTokenCount);
            origTokenStartIndex = origTokenOrEmbeddings.length - origTokenCount;
        }
    }
    
    private void ensureOrigTokensEndCapacity(int extraOrigTokenCount) {
        if (extraOrigTokenCount > origTokenOrEmbeddings.length - origTokenCount) { // will need to reallocate
            // Could check for maximum possible token count (origTokenCount + below-and-above live token counts)
            // but would cause init of live tokens above gap which is undesirable
            @SuppressWarnings("unchecked")
            TokenOrEmbedding<T>[] newOrigTokensOrBranches = new TokenOrEmbedding[
                    (origTokenOrEmbeddings.length * 3 / 2) + extraOrigTokenCount];
            int[] newOrigOffsets = new int[newOrigTokensOrBranches.length];
            int newIndex = (newOrigTokensOrBranches.length
                    - (origTokenCount + extraOrigTokenCount)) / 2;
            System.arraycopy(origTokenOrEmbeddings, origTokenStartIndex,
                    newOrigTokensOrBranches, newIndex, origTokenCount);
            System.arraycopy(origOffsets, origTokenStartIndex,
                    newOrigOffsets, newIndex, origTokenCount);
            origTokenOrEmbeddings = newOrigTokensOrBranches;
            origOffsets = newOrigOffsets;
            origTokenStartIndex = newIndex;

        } else if (extraOrigTokenCount > origTokenOrEmbeddings.length - origTokenCount - origTokenStartIndex) { // only move
            // Move to the end of the array
            System.arraycopy(origTokenOrEmbeddings, origTokenStartIndex,
                    origTokenOrEmbeddings, 0, origTokenCount);
            System.arraycopy(origOffsets, origTokenStartIndex,
                    origOffsets, 0, origTokenCount);
            origTokenStartIndex = 0;




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.inc;

import java.util.Set;
import org.netbeans.api.lexer.InputAttributes;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.EmbeddingContainer;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.lib.lexer.TokenHierarchyOperation;
import org.netbeans.lib.lexer.TokenList;
import org.netbeans.lib.lexer.token.AbstractToken;


/**
 * Single token list maintains a text for a single token.
 * <br/>
 * It's used for token hierarchy snapshots only.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class StandaloneTokenList<T extends TokenId> implements TokenList<T> {

    private char[] tokenText;

    private LanguagePath languagePath;
    
    public StandaloneTokenList(LanguagePath languagePath, char[] tokenText) {
        this.languagePath = languagePath;
        this.tokenText = tokenText;
    }
    
    public LanguagePath languagePath() {
        return languagePath;
    }
    
    public Object tokenOrEmbeddingContainer(int index) {
        throw new IllegalStateException("Not expected to be called"); // NOI18N
    }

    public AbstractToken<T> replaceFlyToken(
    int index, AbstractToken<T> flyToken, int offset) {
        throw new IllegalStateException("Not expected to be called"); // NOI18N
    }

    public int lookahead(int index) {
        return -1;
    }

    public Object state(int index) {
        return null;
    }

    public int tokenOffset(int index) {
        throw new IllegalStateException("Not expected to be called"); // NOI18N
    }
    
    public int tokenCount() {
        return 1;
    }

    public int tokenCountCurrent() {
        return 1;
    }

    public int modCount() {
        return -1;
    }
    
    public int childTokenOffset(int rawOffset) {
        // Offset of the standalone token is absolute
        return rawOffset;
    }
    
    public char childTokenCharAt(int rawOffset, int index) {
        return tokenText[index];
    }

    public void wrapToken(int index, EmbeddingContainer embeddingContainer) {
        throw new IllegalStateException("Branching of standalone tokens not supported"); // NOI18N
    }
    
    public TokenList<?> root() {
        return this;
    }
    
    public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {
        return null;
    }
    
    public InputAttributes inputAttributes() {
        throw new IllegalStateException("Not expected to be called"); // NOI18N
    }
    
    public boolean isContinuous() {
        return true;
    }

    public Set<T> skipTokenIds() {
        return null;
    }

    public int startOffset() {
        if (tokenCountCurrent() > 0 || tokenCount() > 0)
            return tokenOffset(0);
        return 0;
    }

    public int endOffset() {
        int cntM1 = tokenCount() - 1;
        if (cntM1 >= 0)
            return tokenOffset(cntM1) + LexerUtilsConstants.token(this, cntM1).length();
        return 0;
    }
    
    public boolean isRemoved() {
        return false; // Should be used when part of a snapshot
    }

}





import org.netbeans.api.lexer.TokenChange;
import org.netbeans.api.lexer.TokenHierarchyEventType;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.CharSequenceUtilities;
import org.netbeans.lib.lexer.LexerApiPackageAccessor;
import org.netbeans.lib.lexer.LexerSpiPackageAccessor;
import org.netbeans.lib.lexer.TokenHierarchyOperation;
import org.netbeans.spi.lexer.MutableTextInput;

/**
 * Shared information for all the token list changes

    
    private TokenChange<?> tokenChange;

    private final int modOffset;

    private final int removedLength;



        this.tokenHierarchyOperation = tokenHierarchyOperation;
        this.type = type;
        this.modOffset = modificationOffset;
        this.removedLength = removedLength;
        this.removedText = removedText;
        this.insertedLength = insertedLength;

        }
    }

    public int modOffset() {
        return modOffset;
    }

    public int removedLength() {

    }
    
    public CharSequence insertedText() {
        return currentText().subSequence(modOffset(), modOffset() + insertedLength());
    }
    
    public int diffLength() {
        return insertedLength - removedLength;
    }
    
    /**

                        );
            }
            originalText = new OriginalText(currentText(),
                    modOffset, removedText, insertedLength);
        }
        return originalText;
    }

    public String modificationDescription(boolean detail) {
        StringBuilder sb = new StringBuilder(originalText().length() + 300);
        if (removedLength() > 0) {
            sb.append("TEXT REMOVED <").append(modOffset()).append(","). // NOI18N
                    append(modOffset() + removedLength()).append('>');
            sb.append(':').append(removedLength());
            if (removedText() != null) {
                sb.append(" \"");

            sb.append('\n');
        }
        if (insertedLength() > 0) {
            sb.append("TEXT INSERTED <").append(modOffset()).append(","). // NOI18N
                    append(modOffset() + insertedLength()).append(">:"). // NOI18N
                    append(insertedLength()).append(" \""); // NOI18N
            CharSequenceUtilities.debugText(sb, insertedText());
            sb.append("\"\n");

            sb.append("\n\n----------------- ORIGINAL TEXT -----------------\n" + // NOI18N
                originalText() +
                "\n----------------- BEFORE-CARET TEXT -----------------\n" + // NOI18N
                originalText().subSequence(0, modOffset()) +
                "|<--CARET\n" // NOI18N
            );
        }

    public String toString() {
        StringBuilder sb = new StringBuilder();
        sb.append("modOffset="); // NOI18N
        sb.append(modOffset());
        if (removedLength() > 0) {
            sb.append(", removedLength=");
            sb.append(removedLength());




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.inc;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.EmbeddedTokenList;
import org.netbeans.lib.lexer.EmbeddingContainer;
import org.netbeans.lib.lexer.TokenHierarchyOperation;
import org.netbeans.lib.lexer.TokenList;
import org.netbeans.lib.lexer.TokenListList;

/**
 * Request for updating of token hierarchy after text modification
 * or custom embedding creation/removal.
 * <br/>
 * This class contains all the data and methods related to updating.
 *
 * @author Miloslav Metelka
 */

public final class TokenHierarchyUpdate {
    
    // -J-Dorg.netbeans.lib.lexer.TokenHierarchyUpdate.level=FINE
    static final Logger LOG = Logger.getLogger(TokenHierarchyUpdate.class.getName());

    /**
     * Special constant value to avoid double map search for token list lists updating.
     */
    private static final UpdateItem<?> NO_ITEM = new UpdateItem<TokenId>(null);
    
    final TokenHierarchyEventInfo eventInfo;
    
    /**
     * Infos ordered from higher top levels of the hierarchy to lower levels.
     * Useful for top-down updating at the end.
     */
    private List<List<UpdateItem<?>>> itemLevels;
    
    /**
     * Mapping of LP to UpdateItem for a joined ETLs.
     */
    private Map<LanguagePath,UpdateItem<?>> path2Item;
    
    private LanguagePath lastPath2ItemPath;
    private UpdateItem<?> lastPath2ItemItem;
    
    public TokenHierarchyUpdate(TokenHierarchyEventInfo eventInfo) {
        this.eventInfo = eventInfo;
    }

    public void update() {
        TokenHierarchyOperation<?,?> operation = eventInfo.tokenHierarchyOperation();
        IncTokenList<?> incTokenList = (IncTokenList<?>) operation.rootTokenList();

        if (LOG.isLoggable(Level.FINE)) {
            if (LOG.isLoggable(Level.FINEST)) {
                // Display current state of the hierarchy by faking its text
                // through original text
                CharSequence text = incTokenList.inputSourceText();
                assert (text != null);
                incTokenList.setInputSourceText(eventInfo.originalText());
                // Dump all contents
                LOG.finest(toString());
                // Return the original text
                incTokenList.setInputSourceText(text);
            }

            StringBuilder sb = new StringBuilder(150);
            sb.append("<<<<<<<<<<<<<<<<<< LEXER CHANGE START ------------------\n"); // NOI18N
            sb.append(eventInfo.modificationDescription(false));
            TokenHierarchyUpdate.LOG.fine(sb.toString());
        }

        updateImpl(incTokenList, (operation.maxTokenListListPathSize() > 0));

        if (LOG.isLoggable(Level.FINE)) {
            LOG.fine("AFFECTED: " + eventInfo.dumpAffected() + "\n"); // NOI18N
            String extraMsg = "";
            if (LOG.isLoggable(Level.FINER)) {
                // Check consistency of the whole token hierarchy
                String error = operation.checkConsistency();
                if (error != null) {
                    String msg = "!!!CONSISTENCY-ERROR!!!: " + error + "\n";
                    if (LOG.isLoggable(Level.FINEST)) {
                        throw new IllegalStateException(msg);
                    } else {
                        LOG.finer(msg);
                    }
                } else {
                    extraMsg = "(TokenHierarchy Check OK) ";
                }
            }
            LOG.fine(">>>>>>>>>>>>>>>>>> LEXER CHANGE END " + extraMsg + "------------------\n"); // NOI18N
        }

        if (LOG.isLoggable(Level.FINEST)) {
            LOG.finest("AFTER UPDATE:\n");
            LOG.finest(toString());
        }
    }

    private <T extends TokenId> void updateImpl(IncTokenList<T> incTokenList, boolean tllChildrenMayExist) {
        incTokenList.incrementModCount();
        
        // Update starts at the top language path an goes to possibly embedded-token-lists (ETLs)
        // based on the top-level change. If there are embeddings that join sections
        // a token-list-list (TLL) exists for the given language path that maintains
        // all ETLs for the whole input source.
        // 1. The updating must always go from upper levels to more embedded levels of the token hierarchy
        //    to ensure that the tokens of the possible joined ETLs get updated properly
        //    as the tokens created/removed at upper levels may contain embeddings that will
        //    need to be added/removed from TLL of more embedded level.
        // 2. A single insert/remove may produce token updates at several
        //    places in the document due to joining of ETLs. In turn the added/removed
        //    ETLs may affect more embedded levels so the update can affect
        //    multiple places of input source.
        // 3. The algorithm must collect both removed and added ETLs
        //    and process them prior calling the TokenListUpdater to update actual tokens.
        // 4. For a removed ETL the updating must check and collect nested ETLs
        //    because some embedded tokens of the removed ETL might contain
        //    another ETL that might be maintained as TLL.
        // 5. Added ETLs must also be inspected for nested ETLs maintained in a TLL.
        //    Initialization of added ETLs is done when the particular level is processed
        //    because TLL can join sections so they must be lexed once the definite additions
        //    and removals of ETLs are known. For non-joining ETLs this could be done
        //    immediately but it is not necessary so it's done at the same time as well.
        // 6. For all TLLs their parent TLLs (for language path with last language stripped)
        //    are also maintained mandatorily.
        // 7. Algorithm maintains "item-levels" to respect top-down processing
        //    according to language-path-depth.
        
        itemLevels = new ArrayList<List<UpdateItem<?>>>(3); // Suffice for two-level embedding without realloc
        // Create root item first for root token list
        UpdateItem<T> rootItem = new UpdateItem<T>(this);
        rootItem.tokenListChange = new TokenListChange<T>(incTokenList);
        rootItem.tllChildrenMayExist = tllChildrenMayExist;
        addItem(rootItem, 0);
        processLevelInfos();
    }
    
    public <T extends TokenId> void updateCreateOrRemoveEmbedding(EmbeddedTokenList<T> addedOrRemovedTokenList, boolean add) {
        LanguagePath languagePath = addedOrRemovedTokenList.languagePath();
        int level = languagePath.size() - 1;
        itemLevels = new ArrayList<List<UpdateItem<?>>>(level + 2); // One extra level for growth
        UpdateItem<T> item = tokenListListItem(languagePath);
        if (item != null) {
            if (LOG.isLoggable(Level.FINE)) {
                LOG.fine("THU.updateCreateOrRemoveEmbedding() add=" + add + ": " + addedOrRemovedTokenList.dumpInfo(null));
            }
            if (add) {
                item.tokenListListUpdate.markAddedMember(addedOrRemovedTokenList);
            } else {
                item.tokenListListUpdate.markRemovedMember(addedOrRemovedTokenList, eventInfo);
            }
            processLevelInfos();
        }
    }
    
    private void processLevelInfos() {
        // Process item levels which can extend the list by new items at the same level
        // or in the next levels. Therefore iterate by INDEX.since size() may change.
        for (int i = 0; i < itemLevels.size(); i++) {
            List<UpdateItem<?>> items = itemLevels.get(i);
            // The "items" list should not be extended by additional items dynamically during iteration.
            for (UpdateItem<?> item : items) {
                item.update();
            }
        }
    }
    
    void addItem(UpdateItem<?> item, int level) {
        while (level >= itemLevels.size()) {
            itemLevels.add(new ArrayList<UpdateItem<?>>(3));
        }
        List<UpdateItem<?>> items = itemLevels.get(level);
        items.add(item);
    }

    void collectAddedRemovedEmbeddings(TokenListChange<?> change) {
        // Only called when tll children exist
        // First collect the removed embeddings
        TokenList<?> removedTokenList = change.tokenChangeInfo().removedTokenList();
        if (removedTokenList != null) {
            collectRemovedEmbeddings(removedTokenList);
        }
        // Now collect added embeddings
        TokenList<?> currentTokenList = change.tokenList();
        collectAddedEmbeddings(currentTokenList, change.index(), change.addedTokenOrEmbeddingsCount());
    }

    /**
     * Collect removed embeddings for the given token list recursively
     * and nest deep enough for all maintained children
     * token list lists.
     */
    void collectRemovedEmbeddings(TokenList<?> removedTokenList) {
        int tokenCount = removedTokenList.tokenCountCurrent();
        for (int i = 0; i < tokenCount; i++) { // Must go from first to last
            EmbeddingContainer<?> ec = removedTokenList.tokenOrEmbedding(i).embedding();
            if (ec != null) {
                ec.updateStatusUnsync(); // Update status since markRemoved() will need it
                EmbeddedTokenList<?> etl = ec.firstEmbeddedTokenList();
                while (etl != null && etl != EmbeddedTokenList.NO_DEFAULT_EMBEDDING) {
                    internalMarkAddedRemovedMember(etl, false);
                    etl = etl.nextEmbeddedTokenList();
                }
            }
        }
    }

    void collectAddedEmbeddings(TokenList<?> tokenList, int index, int addedCount) {
        for (int i = 0; i < addedCount; i++) {
            // Ensure that the default embedding gets possibly created
            EmbeddedTokenList<?> etl = EmbeddingContainer.embeddedTokenList(tokenList, index + i, null, false);
            while (etl != null) {
                internalMarkAddedRemovedMember(etl, true);
                etl = etl.nextEmbeddedTokenList();
            }
        }
    }

    /**
     * This code is extracted from collectAdded/RemovedEmbeddings() for convenient generification
     * over a type ET.
     */
    private <ET extends TokenId> void internalMarkAddedRemovedMember(EmbeddedTokenList<ET> etl, boolean add) {
        UpdateItem<ET> item = tokenListListItem(etl.languagePath());
        if (item != null) {
            // update-status called in caller
            if (add) {
                item.tokenListListUpdate.markAddedMember(etl);
            } else {
                item.tokenListListUpdate.markRemovedMember(etl, eventInfo);
            }
        }
    }

    /**
     * Return tll info or null if the token list list is not maintained
     * for the given language path.
     */
    private <T extends TokenId> UpdateItem<T> tokenListListItem(LanguagePath languagePath) {
        if (languagePath == lastPath2ItemPath) { // Use last queried one
            @SuppressWarnings("unchecked")
            UpdateItem<T> item = (UpdateItem<T>) lastPath2ItemItem;
            return item;

        } else { // Not last returned item
            if (path2Item == null) { // Init since it will contain either target item or noInfo()
                path2Item = new HashMap<LanguagePath,UpdateItem<?>>(4, 0.5f);
            }
            @SuppressWarnings("unchecked")
            UpdateItem<T> item = (UpdateItem<T>)path2Item.get(languagePath);
            if (item == NO_ITEM) { // Marker value for null (to query just single map - this one)
                item = null;
            } else if (item == null) {
                TokenListList<T> tokenListList = eventInfo.tokenHierarchyOperation().existingTokenListList(languagePath);
                if (tokenListList != null) {
                    item = new UpdateItem<T>(this);
                    item.setTokenListList(tokenListList);
                    item.tllChildrenMayExist = tokenListList.hasChildren();
                    int level = languagePath.size() - 1;
                    addItem(item, level); // Add item to be scheduled for processing
                    path2Item.put(languagePath, item);
                } else { // Use NO_ITEM marker value to immediately know that there's no tokenListList for the given LP
                    path2Item.put(languagePath, NO_ITEM); // NO_ITEM is of type UpdateItem<?>
                }
            } // else - regular valid item
            lastPath2ItemItem = item; // Remember unmasked value i.e. "null" directly
            return item;
        }
    }
    
    /**
     * Information about update in a particular token list or a particular token list list.
     */
    static final class UpdateItem<T extends TokenId> {

        private static final EmbeddedTokenList[] EMPTY_ETL_ARRAY = new EmbeddedTokenList[0];

        final TokenHierarchyUpdate update;

        UpdateItem<?> parentItem;

        /**
         * Token list change performed during this update.
         */
        TokenListChange<T> tokenListChange;

        TokenListListUpdate<T> tokenListListUpdate;
        
        boolean tllChildrenMayExist;

        public UpdateItem(TokenHierarchyUpdate update) {
            this.update = update;
        }

        void setParentItem(UpdateItem<?> parentItem) {
            assert (this.parentItem == null);
            this.parentItem = parentItem;
        }
        
        void setTokenListList(TokenListList<T> tokenListList) {
            this.tokenListListUpdate = new TokenListListUpdate<T>(tokenListList);
        }

        void initTokenListChange(EmbeddedTokenList<T> etl) {
            assert (tokenListChange == null);
            if (tokenListListUpdate != null) {
                // ETL managed by a TokenListList. If the TLL joins sections
                // then a JoinTokenListChange needs to be created.
                tokenListChange = tokenListListUpdate.createTokenListChange(etl);

            } else { // No child managed by TLL but want to process nested possible bounds changes as deep as possible
                // Perform change in child - it surely does not join the sections
                // since otherwise the tllItem could not be null.
                // Token list change is surely non-join since there is no TLLInfo
                tokenListChange = new TokenListChange<T>(etl);
            }
        }
        
        /**
         * Update token list(s) after added and removed embedded token lists
         * are known and in place.
         */
        void update() {
            TokenHierarchyEventInfo eventInfo = update.eventInfo;
            if (tokenListChange == null) { // Joined or unjoined ETLs
                assert (tokenListListUpdate != null);
                if (tokenListListUpdate.tokenListList.joinSections()) {
                    tokenListChange = tokenListListUpdate.createJoinTokenListChange();
                }
            } // else tokenListChange != null

            // Use always non-null List for added token lists
            if (tokenListListUpdate != null && tokenListListUpdate.addedTokenLists == null) {
                tokenListListUpdate.addedTokenLists = Collections.emptyList();
            }
            
            // Process the token list change by calling token list updater
            if (tokenListChange != null) { // Updating a concrete token list as a bounds change or joined change
                if (tokenListChange.getClass() == JoinTokenListChange.class) {
                    JoinTokenListChange<T> jChange = (JoinTokenListChange<T>) tokenListChange;
                    assert (tokenListListUpdate != null);
                    assert (tokenListListUpdate.modTokenListIndex != -1);
                    jChange.setTokenListListUpdate(tokenListListUpdate);
                    TokenListUpdater.updateJoined(jChange, eventInfo);

                } else { // non-joined update
                    TokenListUpdater.updateRegular(tokenListChange, eventInfo);
                    if (parentItem == null) {
                        eventInfo.setTokenChangeInfo(tokenListChange.tokenChangeInfo());
                    }
                }

                // Possibly process bounds change
                if (tokenListChange.isBoundsChange()) {
                    TokenListChange<T> change;
                    if (tokenListChange.getClass() == JoinTokenListChange.class) {
                        // Process the one embedded change
                        JoinTokenListChange<T> jChange = (JoinTokenListChange<T>) tokenListChange;
                        assert (jChange.relexChanges().size() == 1);
                        change = jChange.relexChanges().get(0);
                    } else {
                        change = tokenListChange;
                    }
                    processBoundsChange(change);

                } else { // Non-bounds change
                    // Mark changed area based on start of first mod.token and end of last mod.token
                    // of the root-level change
                    eventInfo.setMinAffectedStartOffset(tokenListChange.offset());
                    eventInfo.setMaxAffectedEndOffset(tokenListChange.addedEndOffset());
                    if (tllChildrenMayExist) { // If there are any possible embedded changes with TokenListList
                        if (tokenListChange.getClass() == JoinTokenListChange.class) {
                            JoinTokenListChange<T> jChange = (JoinTokenListChange<T>) tokenListChange;
                            List<? extends TokenListChange<T>> relexChanges = jChange.relexChanges();
                            for (TokenListChange<T> change : relexChanges) {
                                update.collectAddedRemovedEmbeddings(change); // Process individual changes
                            }
                        } else {
                            update.collectAddedRemovedEmbeddings(tokenListChange);
                        }
                    } // else: there is no embedding with TLL; existing ETLs will be abandoned; new one created on demand
                }

            } else if (tokenListListUpdate != null) { // Only service added/removed ETLs
                tokenListListUpdate.addRemoveTokenLists(update, tllChildrenMayExist);
            }
        }

        /**
         * Process a change where just a single token was relexed and it's the same
         * just with updated bounds.
         *
         * @param change non-null change describing the change.
         * @param parentChange parent change or null for root change.
         */
        void processBoundsChange(TokenListChange<T> change) {
            // Add an embedded change to the parent change (if exists)
            if (parentItem != null) {
                parentItem.tokenListChange.tokenChangeInfo().addEmbeddedChange(change.tokenChangeInfo());
            }
            // Go through all embedded list in a chain and check whether the embeddings are OK
            EmbeddingContainer<T> ec = change.tokenChangeInfo().removedTokenList().tokenOrEmbedding(0).embedding();
            if (ec != null) { // The only removed token had embeddings
                // Rewrap token in ec - use the added token
                ec.reinit(change.addedToken(0));
                ec.updateStatusUnsync();
                change.tokenList().wrapToken(change.index(), ec);
                // Go through all ETLs and check whether chars in start/end skip lengths weren't modified
                EmbeddedTokenList<?> etl = ec.firstEmbeddedTokenList();
                if (etl != null && etl != EmbeddedTokenList.NO_DEFAULT_EMBEDDING) {
                    // Check the text length beyond modification => end skip length must not be affected
                    TokenHierarchyEventInfo eventInfo = update.eventInfo;
                    int modRelOffset = eventInfo.modOffset() - change.offset();
                    int beyondModLength = change.addedEndOffset() - (eventInfo.modOffset() + eventInfo.diffLengthOrZero());
                    EmbeddedTokenList<?> prevEtl = null;
                    do {
                        // Check whether chars in start/end skip lengths weren't modified
                        if (processBoundsChangeEmbeddedTokenList(etl, modRelOffset, beyondModLength)) { // Embedding saved -> proceed to next ETL
                            prevEtl = etl;
                            etl = prevEtl.nextEmbeddedTokenList();
                        } else {
                            etl = ec.removeEmbeddedTokenList(prevEtl, etl);
                        }
                    } while (etl != null && etl != EmbeddedTokenList.NO_DEFAULT_EMBEDDING);
                }
            } 
        }

        /**
         * This method is extracted from processBoundsChangeEmbeddings() to allow a separate generification
         * for each ETL contained in an EC.
         * 
         * @param etl
         * @param ecTokenChange change for token wrapped by EC in which the ETL is hosted.
         * @param hasChildren
         * @param modRelOffset
         * @param beyondModLength
         * @return true if the embedding should be saved or false if it should be removed.
         */
        private <ET extends TokenId> boolean processBoundsChangeEmbeddedTokenList(
                EmbeddedTokenList<ET> etl, int modRelOffset, int beyondModLength
        ) {
            UpdateItem<ET> childItem = tllChildrenMayExist
                    ? update.<ET>tokenListListItem(etl.languagePath())
                    : null;
            // Check whether the change was not in the start or end skip lengths
            // and if so then remove the embedding
            if (modRelOffset >= etl.embedding().startSkipLength() && beyondModLength >= etl.embedding().endSkipLength()) {
                // Modification within embedding's bounds => embedding can stay
                // Embedding will be updated once the level gets processed
                if (childItem == null) {
                    childItem = new UpdateItem<ET>(update);
                    int level = etl.languagePath().size() - 1;
                    update.addItem(childItem, level);
                } else { // TokenListList exists - item already added
                    // Mark a bounds change
                    childItem.tokenListListUpdate.markChangedMember(etl);
                }
                childItem.setParentItem(this);
                childItem.initTokenListChange(etl);
                return true; // Embedding saved -> proceed with next

            } else { // Mod in start/stop skip length => Remove the etl from chain
                if (childItem != null) {
                    // update-status already done as part of rewrap-token
                    childItem.tokenListListUpdate.markRemovedMember(etl, update.eventInfo);
                }
                // Signal to remove embedding
                return false;
            }
        }

    }

}




import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.lib.lexer.TokenList;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.TokenOrEmbedding;

/**
 * Description of the change in a token list.

 * @version 1.00
 */

public class TokenListChange<T extends TokenId> {
    
    public static <T extends TokenId> TokenListChange<T> createRebuildChange(MutableTokenList<T> tokenList) {
        TokenListChange<T> change = new TokenListChange<T>(tokenList);
//        change.setIndex(0);
//        change.setOffset(0);
//        change.addedEndOffset = 0; // Tokens will be recreated lazily
        change.matchIndex = tokenList.tokenCountCurrent(); // All tokens removed
        return change;
    }
    
    private final TokenChangeInfo<T> tokenChangeInfo;
    
    /**
     * The list may store either tokens or branches as well.
     */
    private List<TokenOrEmbedding<T>> addedTokenOrEmbeddings;

    private LAState laState;

    private int offsetGapIndex;
    
    int removedEndOffset;
    
    protected int matchIndex;
    
    protected int matchOffset; // Works like addedEndOffset
    
    public TokenListChange(MutableTokenList<T> tokenList) {
        tokenChangeInfo = new TokenChangeInfo<T>(tokenList);

        return (MutableTokenList<T>)tokenChangeInfo.currentTokenList();
    }
    
    public void setMatchIndex(int matchIndex) {
        this.matchIndex = matchIndex;
    }

    public void setMatchOffset(int matchOffset) {
        this.matchOffset = matchOffset;
    }

    public int increaseMatchIndex() {
        matchOffset += tokenList().tokenOrEmbeddingUnsync(matchIndex++).token().length();
        return matchOffset;
    }

    public LanguagePath languagePath() {
        return tokenList().languagePath();
    }
    
    public int index() {
        return tokenChangeInfo.index();
    }

    public void setIndex(int index) {
        tokenChangeInfo.setIndex(index);
    }
    
    public int offset() {

    public void setOffset(int offset) {
        tokenChangeInfo.setOffset(offset);
    }

    public int removedTokenCount() {
        return matchIndex - index();
    }

    public int removedEndOffset() {
        return matchOffset; // In after-mod coordinates
    }

    public int addedEndOffset() {
        return matchOffset;
    }

    public void addToken(AbstractToken<T> token, int lookahead, Object state) {
        if (addedTokenOrEmbeddings == null) {
            addedTokenOrEmbeddings = new ArrayList<TokenOrEmbedding<T>>(2);
            laState = LAState.empty();
        }
        addedTokenOrEmbeddings.add(token);
        laState = laState.add(lookahead, state);
    }
    
    public List<TokenOrEmbedding<T>> addedTokenOrEmbeddings() {
        return addedTokenOrEmbeddings;
    }
    
    public int addedTokenOrEmbeddingsCount() {
        return (addedTokenOrEmbeddings != null) ? addedTokenOrEmbeddings.size() : 0;
    }
    
    /**
     * @return end offset of previous (retained) token.
     */
    public AbstractToken<T> removeLastAddedToken() {
        int lastIndex = addedTokenOrEmbeddings.size() - 1;
        AbstractToken<T> token = addedTokenOrEmbeddings.remove(lastIndex).token();
        laState.remove(lastIndex, 1);
        matchIndex--;
        matchOffset -= token.length();
        return token;
    }
    
    public AbstractToken<T> addedToken(int index) {
        return addedTokenOrEmbeddings.get(0).token();
    }
    
    public void syncAddedTokenCount() {
        tokenChangeInfo.setAddedTokenCount(addedTokenOrEmbeddings.size());
    }

    public void setRemovedTokens(TokenOrEmbedding<T>[] removedTokensOrBranches) {
        tokenChangeInfo.setRemovedTokenList(new RemovedTokenList<T>(
                languagePath(), removedTokensOrBranches));
    }
    
    public int removedEndOffset() {
        return removedEndOffset;
    }
    
    public void setRemovedEndOffset(int removedEndOffset) {
        this.removedEndOffset = removedEndOffset;
    }
    
    public int addedEndOffset() {
        return addedEndOffset;
    }
    
    public void setAddedEndOffset(int addedEndOffset) {
        this.addedEndOffset = addedEndOffset;
    }
    
    public boolean isBoundsChange() {


    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder();
        sb.append('"').append(languagePath().innerLanguage().mimeType());
        sb.append("\", ind=").append(index());
        sb.append(", off=").append(offset());
        sb.append(", mInd=").append(matchIndex);
        sb.append(", mOff=").append(matchOffset);
        sb.append(", Add:").append(addedTokenOrEmbeddingsCount());
        sb.append(", tCnt=").append(tokenList().tokenCountCurrent());
        if (isBoundsChange()) {
            sb.append(", BoChan");
        }
        return sb.toString();
    }
    
    public String toStringMods(int indent) {
        StringBuilder sb = new StringBuilder();
        sb.append('"');
        sb.append(languagePath().innerLanguage().mimeType());
        sb.append("\", index=");
        sb.append(index());
        sb.append(", offset=");
        sb.append(offset());
        if (isBoundsChange()) {
            sb.append(", boundsChange");
        }
        TokenList<T> removedTL = tokenChangeInfo.removedTokenList();
        if (removedTL != null && removedTL.tokenCount() > 0) {
            int digitCount = ArrayUtilities.digitCount(removedTL.tokenCount() - 1);
            for (int i = 0; i < removedTL.tokenCount(); i++) {
                sb.append('\n');
                ArrayUtilities.appendSpaces(sb, indent);
                sb.append("Rem[");
                ArrayUtilities.appendIndex(sb, i, digitCount);
                sb.append("]: ");
                LexerUtilsConstants.appendTokenInfo(sb, removedTL, i, null, false, 0, true);
            }
        }
        if (addedTokenOrEmbeddings() != null) {
            int digitCount = ArrayUtilities.digitCount(addedTokenOrEmbeddings().size() - 1);
            for (int i = 0; i < addedTokenOrEmbeddings().size(); i++) {
                sb.append('\n');
                ArrayUtilities.appendSpaces(sb, indent);
                sb.append("Add[");
                ArrayUtilities.appendIndex(sb, i, digitCount);
                sb.append("]: ");
                LexerUtilsConstants.appendTokenInfo(sb, addedTokenOrEmbeddings.get(i),
                        laState.lookahead(i), laState.state(i), null, false, 0, true);
            }
        }
        return sb.toString();




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.inc;

import java.util.ArrayList;
import java.util.List;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.EmbeddedTokenList;
import org.netbeans.lib.lexer.JoinTokenList;
import org.netbeans.lib.lexer.TokenList;
import org.netbeans.lib.lexer.TokenListList;

/**
 * Change of a particular TokenListList.
 *
 * @author Miloslav Metelka
 */

final class TokenListListUpdate<T extends TokenId> {
    
    /**
     * Token list list for the case when the particular language path
     * corresponds to joined.
     */
    final TokenListList<T> tokenListList;

    int modTokenListIndex;

    int removedTokenListCount;

    List<EmbeddedTokenList<T>> addedTokenLists;

    TokenListListUpdate(TokenListList<T> tokenListList) {
        this.tokenListList = tokenListList;
        this.modTokenListIndex = -1;
    }

    public boolean isTokenListsMod() { // If any ETL was removed/added
        return (removedTokenListCount != 0) || addedTokenLists.size() > 0;
    }

    public int tokenListCountDiff() {
        return addedTokenLists.size() - removedTokenListCount;
    }

    public EmbeddedTokenList<T> afterUpdateTokenList(JoinTokenList<T> jtl, int tokenListIndex) {
        EmbeddedTokenList<T> etl;
        if (tokenListIndex < modTokenListIndex) {
            etl = jtl.tokenList(tokenListIndex);
        } else if (tokenListIndex - modTokenListIndex < addedTokenLists.size()) {
            etl = addedTokenLists.get(tokenListIndex - modTokenListIndex);
        } else { // Last part after removed and added
            etl = jtl.tokenList(modTokenListIndex + removedTokenListCount - addedTokenLists.size());
        }
        return etl;
    }

    protected int afterUpdateTokenListCount(JoinTokenList<T> jtl) {
        return jtl.tokenListCount() - removedTokenListCount + addedTokenLists.size();
    }

    void markChangedMember(EmbeddedTokenList<T> changedTokenList) {
        assert (modTokenListIndex == -1);
        modTokenListIndex = tokenListList.findIndex(changedTokenList.startOffset());
        assert (tokenListList.get(modTokenListIndex) == changedTokenList);
    }

    void markChageBetween(int offset) { // Nothing added/removed and mod outside of bounds of an ETL
        assert (modTokenListIndex == -1);
        modTokenListIndex = tokenListList.findIndex(offset);
    }

    /**
     * Mark the given token list as removed in the token list list.
     * All removed token lists should be marked subsequently their increasing offset
     * so it should be necessary to search for the index just once.
     * <br/>
     * It's expected that updateStatusImpl() was already called
     * on the corresponding embedding container.
     */
    void markRemovedMember(EmbeddedTokenList<T> removedTokenList, TokenHierarchyEventInfo eventInfo) {
        boolean indexWasMinusOne; // Used for possible exception cause debugging
//            removedTokenList.embeddingContainer().checkStatusUpdated();
        if (modTokenListIndex == -1) {
            indexWasMinusOne = true;
            modTokenListIndex = tokenListList.findIndexDuringUpdate(removedTokenList, eventInfo);
            assert (modTokenListIndex >= 0) : "tokenListIndex=" + modTokenListIndex + " < 0"; // NOI18N
        } else { // tokenListIndex already initialized
            indexWasMinusOne = false;
        }
        TokenList<T> markedForRemoveTokenList = tokenListList.getOrNull(modTokenListIndex + removedTokenListCount);
        if (markedForRemoveTokenList != removedTokenList) {
            int realIndex = tokenListList.indexOf(removedTokenList);
            throw new IllegalStateException("Removing at tokenListIndex=" + modTokenListIndex + // NOI18N
                    " but real tokenListIndex is " + realIndex + // NOI18N
                    " (indexWasMinusOne=" + indexWasMinusOne + ").\n" + // NOI18N
                    "Wishing to remove tokenList\n" + removedTokenList + // NOI18N
                    "\nbut marked-for-remove tokenList is \n" + markedForRemoveTokenList + // NOI18N
                    "\nfrom tokenListList\n" + tokenListList + // NOI18N
                    "\n\nModification description:\n" + eventInfo.modificationDescription(true) // NOI18N
                    );
        }
        removedTokenListCount++;
    }

    /**
     * Mark the given token list to be added to this list of token lists.
     * At the end first the token lists marked for removal will be removed
     * and then the token lists marked for addition will be added.
     * <br/>
     * It's expected that updateStatusImpl() was already called
     * on the corresponding embedding container.
     */
    void markAddedMember(EmbeddedTokenList<T> addedTokenList) {
//            addedTokenList.embeddingContainer().checkStatusUpdated();
        if (addedTokenLists == null) {
            if (modTokenListIndex == -1) {
                modTokenListIndex = tokenListList.findIndex(addedTokenList.startOffset());
                assert (modTokenListIndex >= 0) : "tokenListIndex=" + modTokenListIndex + " < 0"; // NOI18N
            }
            addedTokenLists = new ArrayList<EmbeddedTokenList<T>>(4);
        }
        addedTokenLists.add(addedTokenList);
    }

    void addRemoveTokenLists(TokenHierarchyUpdate update, boolean tllChildrenMayExist) {
        assert (removedTokenListCount > 0 || addedTokenLists != null);
        EmbeddedTokenList<T>[] removedTokenLists = tokenListList.replace(
                modTokenListIndex, removedTokenListCount, addedTokenLists);
        if (tllChildrenMayExist) {
            for (int i = 0; i < removedTokenLists.length; i++) {
                update.collectRemovedEmbeddings(removedTokenLists[i]);
            }
            for (int i = 0; i < addedTokenLists.size(); i++) {
                EmbeddedTokenList<T> addedEtl = addedTokenLists.get(i);
                update.collectAddedEmbeddings(addedEtl, 0, addedEtl.tokenCountCurrent());
            }
        }
    }

    TokenListChange<T> createTokenListChange(EmbeddedTokenList<T> etl) {
        assert (etl != null);
        TokenListChange<T> etlTokenListChange;
        if (tokenListList.joinSections()) {
            MutableJoinTokenList<T> jtl = MutableJoinTokenList.create(tokenListList, modTokenListIndex, etl);
            etlTokenListChange = new JoinTokenListChange<T>(jtl);
        } else { // Non-joining
            etlTokenListChange = new TokenListChange<T>(etl);
        }
        return etlTokenListChange;
    }

    TokenListChange<T> createJoinTokenListChange() {
        assert (tokenListList.joinSections());
        // In case when adding at jtl.tokenListCount() a last ETL must be used
        int etlIndex = Math.min(modTokenListIndex, tokenListList.size() - 1);
        MutableJoinTokenList<T> jtl = MutableJoinTokenList.create(tokenListList, etlIndex, tokenListList.get(etlIndex));
        return new JoinTokenListChange<T>(jtl);
    }

    @Override
    public String toString() {
        return " modTokenListIndex=" + modTokenListIndex + // NOI18N
                "; Rem:" + removedTokenListCount + // NOI18N
                " Add:" + addedTokenLists.size() + // NOI18N
                " Size:" + tokenListList.size(); // NOI18N
    }

}




import java.util.logging.Level;
import java.util.logging.Logger;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.CharSequenceUtilities;
import org.netbeans.lib.lexer.EmbeddedJoinInfo;
import org.netbeans.lib.lexer.EmbeddedTokenList;
import org.netbeans.lib.lexer.JoinLexerInputOperation;
import org.netbeans.lib.lexer.JoinTokenList;
import org.netbeans.lib.lexer.LexerInputOperation;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.token.PartToken;


/**
 * Token list updater fixes a list of tokens constructed for a document
 * after text of the document gets modified.
 * <br>
 * Subclasses need to define all the abstract methods

 * Updater looks similar to list iterator
 * but there are differences in the semantics
 * of iterator's modification operations.
 * <br/>
 * The algorithm used in the {@link #update(int, int)}
 * is based on "General Incremental Lexical Analysis" written
 * by Tim A. Wagner and Susan L. Graham, University
 * of California, Berkeley. It's available online
 * at <a href="http://www.cs.berkeley.edu/Research/Projects/harmonia/papers/twagner-lexing.pdf">
 * twagner-lexing.pdf</a>.
 * <br/>
 * Ending <code>EOF</code> token is not used but the lookahead
 * of the ending token(s) is increased by one (past the end of the input)
 * if they have reached the EOF.
 * <br/>
 * Non-startable tokens are not supported.
 * <br/>
 * When updating a token with lookback one as a result
 * of modification the lookahead of the preceding token is inspected
 * to find out whether the modification has really affected it.
 * This can often save the previous token from being relexed.
 * <br/>
 * Currently the algorithm computes the lookback values on the fly
 * and it does not store the lookback in the tokens. For typical languages
 * the lookback is reasonably small (0, 1 or 2) so it's usually not worth

 * There would also be an additional overhead of updating the lookback
 * values in the tokens after the modification and the algorithm code would
 * be somewhat less readable.
 * </p>
 *
 * <p>
 * The algorithm removes the affected tokens in the natural order as they
 * follow in the token stream. That can be used when the removed tokens
 * need to be collected (e.g. in an array).
 * <br/>
 * If the offset and state after token recognition matches
 * the end offset and state after recognition of the originally present
 * token then the relexing is stopped because a match was found and the newly
 * produced tokens would match the present ones.
 * <br/>
 * Otherwise the token(s) in the list are removed and replaced
 * by the relexed token and the relexing continues until a match is reached.
 * </p>
 * 
 * <p>
 * When using token list updater with JoinTokenList.Mutable there is a special treatment
 * of offsets independent of the underlying JoinTokenListChange and LexerInputOperation.
 * The updater treats the modOffset to be relative (in the number of characters)
 * to the relexOffset point (which is a real first relexed token's offset; it's necessary
 * for restarting of the lexer input operation) so when going over a JoinToken
 * the modOffset must be recomputed to not contain the gaps between individual join token parts.
 * </p>
 *
 * @author Miloslav Metelka
 * @version 1.00

    private static final Logger LOG = Logger.getLogger(TokenListUpdater.class.getName());

    /**
     * Use incremental algorithm to update a regular list of tokens (IncTokenList or EmbeddedTokenList)
     * after a modification done in the underlying storage.
     * 
     * @param change non-null change that will incorporate the performed chagnes.
     *  or embedded token list.
     * @param modOffset offset where the modification occurred.
     * @param insertedLength number of characters inserted at modOffset.
     * @param removedLength number of characters removed at modOffset.
     * @param change non-null change that will incorporate the performed chagnes.
     * @param zeroIndexRelexState state used for relexing at index 0.
     */
    public static <T extends TokenId> void updateRegular(TokenListChange<T> change, TokenHierarchyEventInfo eventInfo) {
        MutableTokenList<T> tokenList = change.tokenList();
        int tokenCount = tokenList.tokenCountCurrent();
        // Fetch offset where the modification occurred
        LanguageOperation<T> languageOperation = LexerUtilsConstants.innerLanguageOperation(
                tokenList.languagePath());

        int tokenCount = tokenList.tokenCountCurrent(); // presently created token count
        // Now determine which token is the first to be relexed.
        // If it would be either modified token or previous-of-modified token
        // (for modification right at the begining of modified token)
        // then the token will be attempted to be validated (without running
        // a lexer).
        AbstractToken<T> modToken;
        // modTokenOffset holds begining of the token in which the modification occurred.
        int modTokenOffset;
        // index points to the modified token
        int index;

        boolean loggable = LOG.isLoggable(Level.FINE);
        if (loggable) {
            logModification(tokenList.inputSourceText(), eventInfo, tokenCount, false);
        }
        
        // Find modified token by binary search in existing tokens
        // Use LexerUtilsConstants.tokenIndexBinSearch() to NOT lazily create new tokens here
        int[] indexAndTokenOffset = LexerUtilsConstants.tokenIndexBinSearch(tokenList, eventInfo.modOffset(), tokenCount);
        // Index and offset from which the relexing will start
        int relexIndex = indexAndTokenOffset[0];
        // relexOffset points to begining of a token in which the modification occurred
        // or which is affected by a modification (its lookahead points beyond modification point).
        int relexOffset = indexAndTokenOffset[1];
        if (relexIndex == -1) { // No tokens at all
            relexIndex = 0;
            relexOffset = tokenList.startOffset();
        }

        // Index of token before which the relexing will end (or == tokenCount)
        int matchIndex = relexIndex;
        // Offset of token at matchIndex
        int matchOffset = relexOffset;

        if (relexIndex == tokenCount) { // Change right at end of last token or beyond it (if not fully lexed)
            // relexOffset set to end offset of the last token
            if (!tokenList.isFullyLexed() && eventInfo.modOffset() >= relexOffset +
                    ((relexIndex > 0) ? tokenList.lookahead(relexIndex - 1) : 0)
            ) { // Do nothing if beyond last token's lookahed
                // Check whether the last token could be affected at all
                // by checking whether the modification was performed
                // in the last token's lookahead.
                // For fully lexed inputs the characters added to the end
                // must be properly lexed and notified (even if the last present
                // token has zero lookahead).
                if (loggable) {
                    LOG.log(Level.FINE, "TLU.updateRegular() FINISHED: Not fully lexed yet. rOff=" +
                            relexOffset + ", mOff=" + eventInfo.modOffset() + "\n");
                }
                change.setIndex(relexIndex);
                change.setOffset(relexOffset);
                change.setMatchIndex(matchIndex);
                change.setMatchOffset(matchOffset);
                tokenList.replaceTokens(change, eventInfo.diffLength());
                return; // not affected at all
            } // change.setIndex() will be performed later in relex()

            // Leave matchOffset as is (will possibly end relexing at tokenCount and unfinished relexing
            // will be continued by replaceTokens()).
            // For fully lexed lists it is necessary to lex till the end of input.
            if (tokenList.isFullyLexed())
                matchOffset = Integer.MAX_VALUE;
            modTokenOffset = tokenList.tokenOffset(index);
            if (modOffset >= modTokenOffset) { // inside or above the last token?
                modToken = token(tokenList, index);
                int modTokenEndOffset = modTokenOffset + modToken.length();
                if (modOffset >= modTokenEndOffset) { // above last token
                    // Modification was right at the end boundary of the last token
                    // or above it (token list can be created lazily so that is valid case).
                    // Check whether the last token could be affected at all
                    // by checking the last token's lookahead.
                    // For fully lexed inputs the characters added to the end
                    // must be properly lexed and notified (even if the last present
                    // token has zero lookahead).
                    if (!tokenList.isFullyLexed()
                        && modOffset >= modTokenEndOffset + tokenList.lookahead(index)
                    ) {
                        if (loggable) {
                            LOG.log(Level.FINE, "TokenListUpdater.update() FINISHED: Not fully lexed yet. modTokenOffset="
                                    + modTokenOffset + ", modToken.length()=" + modToken.length() + "\n");
                        }
                        return; // not affected at all
                    }

        } else { // relexIndex < tokenCount
            // Possibly increase matchIndex and matchOffset by skipping the tokens in the removed area
            if (eventInfo.removedLength() > 0) { // At least remove token at relexOffset
                matchOffset += tokenList.tokenOrEmbeddingUnsync(matchIndex++).token().length();
                int removedEndOffset = eventInfo.modOffset() + eventInfo.removedLength();
                while (matchOffset < removedEndOffset && matchIndex < tokenCount) {
                    matchOffset += tokenList.tokenOrEmbeddingUnsync(matchIndex++).token().length();
                int low = 0; // use index as 'high'
                while (low <= index) {
                    int mid = (low + index) / 2;
                    int midStartOffset = tokenList.tokenOffset(mid);

                    if (midStartOffset < modOffset) {
                        low = mid + 1;
                    } else if (midStartOffset > modOffset) {
                        index = mid - 1;
                    } else {
                        // Token starting exactly at modOffset found
                        index = mid;
                        modTokenOffset = midStartOffset;
                        break;
                    }
                }
            } else { // For inside-token inserts match on the next token
                if (matchOffset < eventInfo.modOffset()) {
                    matchOffset += tokenList.tokenOrEmbeddingUnsync(matchIndex++).token().length();
                modToken = token(tokenList, index);
                if (loggable) {
                    LOG.log(Level.FINE, "BIN-SEARCH: index=" + index
                            + ", modTokenOffset=" + modTokenOffset
                            + ", modToken.id()=" + modToken.id() + "\n");
                }
            }
            // Update the matchOffset so that it corresponds to the state
            // after the modification
            matchOffset += eventInfo.diffLength();
        }

        // Check whether modification affected previous token
        while (relexIndex > 0 && relexOffset + tokenList.lookahead(relexIndex - 1) > eventInfo.modOffset()) {
            relexIndex--;
            if (loggable) {
                LOG.log(Level.FINE, "    Token at rInd=" + relexIndex + " affected (la=" + // NOI18N
                        tokenList.lookahead(relexIndex) + ") => relex it\n"); // NOI18N
        // Index and offset from which the relexing will start.
        int relexIndex;
        int relexOffset;
        // Whether the token validation should be attempted or not.
        boolean attemptValidation = false;

        if (index == 0) { // modToken is first in the list
            relexIndex = index;
            relexOffset = modTokenOffset;
            // Can validate modToken if removal does not span whole token
            if (modToken != null && removedLength < modToken.length()) {
                attemptValidation = true;
            }
            AbstractToken<T> token = tokenList.tokenOrEmbeddingUnsync(relexIndex).token();
            relexOffset -= token.length();
            // Check for insert-only right at the end of the previous token
            if (modOffset == modTokenOffset && removedLength == 0) {
                index--; // move to previous token
                modToken = token(tokenList, index);
                modTokenOffset -= modToken.length();
            }

            // Check whether modification affected previous token
            if (index == 0 || modTokenOffset + tokenList.lookahead(index - 1) <= modOffset) {
                // Modification did not affect previous token
                relexIndex = index;
                relexOffset = modTokenOffset;
                // Check whether modification was localized to modToken only
                if (modOffset + removedLength < modTokenOffset + modToken.length()) {
                    attemptValidation = true;
                }

            } else { // at least previous token affected
                relexOffset = modTokenOffset - token(tokenList, index - 1).length();
                relexIndex = index - 2; // Start with token below previous token
                
                // Go back and mark all affected tokens for removals
                while (relexIndex >= 0) {
                    AbstractToken<T> token = token(tokenList, relexIndex);
                    // Check if token was not affected by modification
                    if (relexOffset + tokenList.lookahead(relexIndex) <= modOffset) {
                        break;
                    }
                    relexIndex--;
                    relexOffset -= token.length();
                }
                relexIndex++; // Next token will be relexed
            }
        }
        
        // Check whether actual relexing is necessary
        // (the relexing may end at higher offset if the relexed
        // tokens will end at different boundaries than the original
        // tokens or if the states after the tokens' recognition
        // will differ from the original states in the original tokens.
        int matchOffset;

        // Perform token validation of modToken if possible.
        // The index variable will hold the token index right before the matching point.
        if (attemptValidation) {
            matchOffset = modTokenOffset + modToken.length();
            TokenValidator tokenValidator = languageOperation.tokenValidator(modToken.id());
            if (tokenValidator != null && (tokenList.getClass() != IncTokenList.class)) {
                    
//                if (tokenValidator.validateToken(modToken, modOffset - modTokenOffset, modRelOffset,
//                        removedLength, insertedLength)
//                ) {
//                    // Update positions
//                                change.initRemovedAddedOffsets()

//                    return; // validated successfully
//                }
            }

        } else { // Validation cannot be attempted
            // Need to compute matchOffset and matchIndex
            // by iterating forward
            if (index < tokenCount) {
                matchOffset = modTokenOffset + modToken.length();
                int removeEndOffset = modOffset + removedLength;
                while (matchOffset < removeEndOffset && index + 1 < tokenCount) {
                    index++;
                    matchOffset += token(tokenList, index).length();
                }

            } else // After last token
                matchOffset = modTokenOffset;
        }

        // State from which the lexer can be started
        Object relexState = (relexIndex > 0) ? tokenList.state(relexIndex - 1) : null;
        change.setIndex(relexIndex);
        // after the modification
        matchOffset += insertedLength - removedLength;
        change.setOffset(relexOffset);
        change.setMatchIndex(matchIndex);
        change.setMatchOffset(matchOffset);
        
        // Variables' values:
        // 'index' - points to modified token. Or index == tokenCount for modification
        //     past the last token.
        // 'tokenCount' - token count in the original token list.
        // 'relexIndex' - points to the token that will be relexed as first.
        // 'relexOffset' - points to begining of the token that will be relexed as first.
        // 'matchOffset' - points to end of token after which the fixed token list could
        //     possibly match the original token list. Points to end of token at 'index'
        //     variable if 'index < tokenCount' and to the end of the last token
        //     if 'index == tokenCount'.

        // Check whether relexing is necessary.
        // Necessary condition for no-relexing is a removal at token's boundary
        // and the token right before modOffset must have zero lookahead (if lookahead would be >0 
        // then the token would be affected) and the states before relexIndex must equal
        // to the state before matchIndex.
        // In case of removal starting and ending at token boundaries
        // the relexing might not be necessary.
        boolean relex = (relexOffset != matchOffset)
                || (eventInfo.insertedLength() > 0)
                || (matchIndex == 0) // ensure the tokenList.state(matchIndex - 1) will not fail with IOOBE
                || !LexerUtilsConstants.statesEqual(relexState, tokenList.state(matchIndex - 1));

        // There is an extra condition that the lookahead of the matchToken
        // must not span the next (retained) token. This condition helps to ensure

        // As the empty tokens are not allowed the situation may only occur
        // for lookahead > 1.
        int lookahead;
        if (!relex && (lookahead = tokenList.lookahead(matchIndex - 1)) > 1 && matchIndex < tokenCount) {
            // Check whether lookahead of the token before match point exceeds the whole token right after match point
            relex = (lookahead > tokenList.tokenOrEmbeddingUnsync(matchIndex).token().length()); // check next token
        }

        if (loggable) {
            StringBuilder sb = new StringBuilder(200);
            sb.append("BEFORE-RELEX: relex=").append(relex);
            sb.append(", rInd=").append(relexIndex).append(", rOff=").append(relexOffset);
            sb.append(", mInd=").append(matchIndex).append(", mOff=").append(matchOffset).append('\n');
            sb.append(", rSta=").append(relexState).append(", tokenList-part:\n");
            LexerUtilsConstants.appendTokenList(sb, tokenList, matchIndex, matchIndex - 3, matchIndex + 3, false, 4, false);
            sb.append('\n');
            LOG.log(Level.FINE, sb.toString());
        }

        assert (relexIndex >= 0);
        if (relex) {
            // Create lexer input operation for the given token list
            LexerInputOperation<T> lexerInputOperation
                    = tokenList.createLexerInputOperation(relexIndex, relexOffset, relexState);
            relex(change, lexerInputOperation, tokenCount);
        }

        tokenList.replaceTokens(change, eventInfo.diffLength());
        if (loggable) {
            LOG.log(Level.FINE, "TLU.updateRegular() FINISHED: change:" + change + "\nMods:" + change.toStringMods(4));
        }
    }


    /**
     * Use incremental algorithm to update a JoinTokenList after a modification done in the underlying storage.
     * <br>
     * The assumption is that there may only be two states:
     * <ul>
     *   <li> There is a local input source modification bounded to a particular ETL.
     *        In such case there should be NO token lists removed/added.
     *   </li>
     *   <li> The modification spans multiple ETLs and all the affected ETLs will be removed.
     *        The modification is "bounded" by the removed ETLs i.e.
     *            modOffset &gt;= first-removed-ETL.startOffset()
     *        and modOffset + removedLength &lt;= last-removed-ETL.endOffset()
     *   </li>
     * </ul>
     * 
     * @param change non-null change that will incorporate the performed chagnes.
     * @param modOffset offset where the modification occurred.
     *  For join token lists if modification is done inside a JoinToken
     *  the modOffset must be a logical distance from token's begining
     *  that corresponds to the modificaion's point (i.e. like if the token
     *  would be continuous).
     * @param insertedLength number of characters inserted at modOffset.
     * @param removedLength number of characters removed at modOffset.
     */
    public static <T extends TokenId> void updateJoined(JoinTokenListChange<T> change, TokenHierarchyEventInfo eventInfo) {
        MutableJoinTokenList<T> jtl = (MutableJoinTokenList<T>) change.tokenList();
        TokenListListUpdate<T> tokenListListUpdate = change.tokenListListUpdate();
        int tokenCount = jtl.tokenCount();
        boolean loggable = LOG.isLoggable(Level.FINE);
        if (loggable) {
            logModification(jtl.inputSourceText(), eventInfo, tokenCount, true);
        }
        
        // First determine what area is affected by removed/added ETLs
        int relexJoinIndex;
        int modOffset = eventInfo.modOffset();
        int relexTokenListIndex = tokenListListUpdate.modTokenListIndex; // Index of ETL where a change occurred.
        // Relative distance of mod against relex point (or point of ETLs added/removed)
        int relModOffset;
        if (tokenListListUpdate.isTokenListsMod()) {
            // Find relexJoinIndex by examining ETL at relexTokenListIndex-1.
            // This way the code is more uniform than examining ETL at relexTokenListIndex.
            if (relexTokenListIndex > 0) { // non-first ETL
                relexTokenListIndex--;
                jtl.setActiveTokenListIndex(relexTokenListIndex);
                EmbeddedTokenList<T> relexEtl = jtl.activeTokenList();
                EmbeddedJoinInfo joinInfo = relexEtl.joinInfo;
                relexJoinIndex = jtl.activeEndJoinIndex();
                if (joinInfo.joinTokenLastPartShift() > 0) { // Mod points inside join token
                    // Find first non-empty ETL below to determine partTextOffset()
                    while (relexEtl.tokenCountCurrent() == 0) { // No tokens in ETL
                        jtl.setActiveTokenListIndex(--relexTokenListIndex);
                        relexEtl = jtl.activeTokenList();
                    }
                    // relexEtl is non-empty - last token is PartToken
                    PartToken<T> partToken = (PartToken<T>) relexEtl.tokenOrEmbeddingUnsync(
                            relexEtl.tokenCountCurrent() - 1).token();
                    relModOffset = partToken.partTextOffset();
                } else { // Not a join token => use first token at relexTokenListIndex
                    relexTokenListIndex++;
                    relModOffset = 0;
                }
            } else { // (relexTokenListIndex == 0)
                relexJoinIndex = 0;
                jtl.setActiveTokenListIndex(0);
                relModOffset = 0;
            }

        } else { // No token list mod
            assert ((eventInfo.insertedLength() > 0) || (eventInfo.removedLength() > 0)) : "No modification";
            jtl.setActiveTokenListIndex(relexTokenListIndex);
            EmbeddedTokenList<T> relexEtl = jtl.activeTokenList();
            change.charModTokenList = relexEtl;
            // Search within releEtl only - can use binary search safely (unlike on JTL with removed ETLs)
            int[] indexAndTokenOffset = relexEtl.tokenIndex(modOffset); // Index could be -1 TBD
            relexJoinIndex = relexEtl.joinInfo.joinTokenIndex() + indexAndTokenOffset[0];
            relModOffset = modOffset - indexAndTokenOffset[1];
        }

        // Matching point index and offset. Matching point vars are assigned early
        // and relex-vars are possibly shifted down first and then the match-vars are updated.
        // That's because otherwise the "working area" of JTL (above/below token list mod)
        // would have to be switched below and above.
        int matchJoinIndex = relexJoinIndex;
        int matchOffset = modOffset - relModOffset; // Suitable for single-ETL update (will be corrected later)
        
        // Update relex-vars according to lookahead of tokens before relexJoinIndex
        while (relexJoinIndex > 0 && jtl.lookahead(relexJoinIndex - 1) > relModOffset) {
            AbstractToken<T> relexToken = jtl.tokenOrEmbeddingUnsync(--relexJoinIndex).token();
            relModOffset += relexToken.length(); // User regular token.length() here
            if (loggable) {
                LOG.log(Level.FINE, "    Token at rInd=" + relexJoinIndex + " affected (la=" + // NOI18N
                        jtl.lookahead(relexJoinIndex) + ") => relex it\n"); // NOI18N
            }
        }

        // Create lexer input operation now since JTL should be positioned before removed ETLs
        // and JLIO needs to scan tokens backwards for fly sequence length.
        Object relexState = (relexJoinIndex > 0) ? jtl.state(relexJoinIndex - 1) : null;
        int relexLocalIndex = jtl.tokenStartLocalIndex(relexJoinIndex);
        relexTokenListIndex = jtl.activeTokenListIndex();
        int relexOffset = jtl.activeTokenList().tokenOffsetByIndex(relexLocalIndex);
        JoinLexerInputOperation<T> lexerInputOperation = new MutableJoinLexerInputOperation<T>(
                jtl, relexJoinIndex, relexState, relexTokenListIndex, relexOffset, tokenListListUpdate);
        lexerInputOperation.init();
        change.setIndex(relexJoinIndex);
        change.setOffset(relexOffset);
        change.setStartInfo(lexerInputOperation, relexLocalIndex);
        // setMatchIndex() and setMatchOffset() called later below

        // Index of token before which the relexing will end (or == tokenCount)
        if (tokenListListUpdate.isTokenListsMod()) { // Assign first token after last removed ETL
            int afterModTokenListIndex = tokenListListUpdate.modTokenListIndex + tokenListListUpdate.removedTokenListCount;
            if (afterModTokenListIndex == jtl.tokenListCount()) { // Removed till end
                matchJoinIndex = tokenCount;
                matchOffset = Integer.MAX_VALUE;
            } else { // Removed inside 
                EmbeddedTokenList<T> afterModEtl = jtl.tokenList(afterModTokenListIndex);
                matchJoinIndex = afterModEtl.joinInfo.joinTokenIndex();
                // Check if the first token of afterModEtl is not an end of join token
                // or that the afterModEtl does not participate in a join token (may be empty)
                if (afterModEtl.tokenCountCurrent() > 0) {
                    AbstractToken<T> token = afterModEtl.tokenOrEmbeddingUnsync(0).token();
                    if (token.getClass() == PartToken.class) {
                        matchJoinIndex++;
                        matchOffset = afterModEtl.startOffset() + token.length();
                    } else {
                        matchOffset = afterModEtl.startOffset();
                    }
                } else { // No tokens in this ETL
                    int joinTokenLastPartShift = afterModEtl.joinInfo.joinTokenLastPartShift();
                    if (joinTokenLastPartShift > 0) { // Part of join token
                        afterModTokenListIndex += joinTokenLastPartShift;
                        matchJoinIndex++;
                        matchOffset = afterModEtl.startOffset();
                    } else { // Empty ETL but not a part of JoinToken - ending empty ETL(s)
                        matchOffset = afterModEtl.startOffset();
                    }
                }
                // Move jtl past removed/added token lists
                jtl.setActiveTokenListIndex(afterModTokenListIndex);

            }
        } else { // No token ETLs removed/added
            // matchOffset already initialized to (modOffset - orig-relModOffset)
            if (eventInfo.removedLength() > 0) { // At least remove token at relexOffset
                matchOffset += jtl.tokenOrEmbeddingUnsync(matchJoinIndex++).token().length();
                int removedEndOffset = eventInfo.modOffset() + eventInfo.removedLength();
                while (matchOffset < removedEndOffset) {
                    matchOffset += jtl.tokenOrEmbeddingUnsync(matchJoinIndex++).token().length();
                }
            } else { // For inside-token inserts match on the next token
                if (matchOffset < eventInfo.modOffset()) {
                    matchOffset += jtl.tokenOrEmbeddingUnsync(matchJoinIndex++).token().length();
                }
            }
            // Update the matchOffset so that it corresponds to the state
            // after the modification
            matchOffset += eventInfo.diffLength();
        }

        // TBD relexing necessity optimizations like in updateRegular()
        change.setMatchIndex(matchJoinIndex);
        change.setMatchOffset(matchOffset);
        relex(change, lexerInputOperation, tokenCount);
        jtl.replaceTokens(change, eventInfo.diffLength());
        if (loggable) {
            LOG.log(Level.FINE, "TLU.updateRegular() FINISHED: change:" + change + // NOI18N
                    "\nMods:" + change.toStringMods(4)); // NOI18N
        }
    }


    /**
     * Relex part of input to create new tokens. This method may sometimes be skipped e.g. for removal of chars
     * corresponding to a single token preceded by a token with zero lookahead.
     * <br/>
     * This code is common for both updateRegular() and updateJoined().
     * 
     * @param tokenList non-null token list that is being updated. It may be top-level list
     *  or embedded token list.
     * @param change token list change into which the created tokens are being added.
     * @param tokenCount current token count in tokenList.
     */
    private static <T extends TokenId> void relex(TokenListChange<T> change,
            LexerInputOperation<T> lexerInputOperation, int tokenCount
    ) {
        boolean loggable = LOG.isLoggable(Level.FINE);
        MutableTokenList<T> tokenList = change.tokenList();
        // Remember the match index below which the comparison of extra relexed tokens
        // (matching the original ones) cannot go.
        int lowestMatchIndex = change.matchIndex;

        AbstractToken<T> token;
        int relexOffset = lexerInputOperation.lastTokenEndOffset();
        while ((token = lexerInputOperation.nextToken()) != null) {
            // Get lookahead and state; Will certainly use them both since updater runs for inc token lists only
            int lookahead = lexerInputOperation.lookahead();
            Object state = lexerInputOperation.lexerState();
            if (loggable) {
                StringBuilder sb = new StringBuilder(100);
                sb.append("LEXED-TOKEN: ");
                int tokenEndOffset = lexerInputOperation.lastTokenEndOffset();
                CharSequence inputSourceText = tokenList.inputSourceText();
                if (tokenEndOffset > inputSourceText.length()) {
                    sb.append(tokenEndOffset).append("!! => ");
                    tokenEndOffset = inputSourceText.length();
                    sb.append(tokenEndOffset);
                }
                sb.append('"');
                CharSequenceUtilities.debugText(sb, inputSourceText.subSequence(relexOffset, tokenEndOffset));
                sb.append('"');
                sb.append(" ").append(token.id());
                sb.append(", <").append(relexOffset);
                sb.append(", ").append(relexOffset + token.length());
                sb.append("> LA=").append(lookahead);
                sb.append(", state=").append(state);
                sb.append(", IHC=").append(System.identityHashCode(token));
                sb.append("\n");
                LOG.log(Level.FINE, sb.toString());
            }

            change.addToken(token, lookahead, state);
            // Here add regular token length even for JoinToken instances
            // since this is used solely for comparing with matchOffset which
            // also uses the per-input-chars coordinates. Real token's offset is independent value
            // assigned by the underlying TokenListChange and LexerInputOperation.
            relexOffset = lexerInputOperation.lastTokenEndOffset();
            // Marks all original tokens that would cover the area of just lexed token as removed.
            // 'matchIndex' will point right above the last token that was removed
            // 'matchOffset' will point to the end of the last removed token
            if (relexOffset > change.matchOffset) {
                do { // Mark all tokens below
                    if (change.matchIndex == tokenCount) { // index == tokenCount
                        if (tokenList.isFullyLexed()) {
                            change.matchOffset = Integer.MAX_VALUE; // Force lexing till end of input
                        } else { // Not fully lexed -> stop now
                            // Fake the conditions to break the relexing loop
                            change.matchOffset = relexOffset;
                            state = tokenList.state(change.matchIndex - 1);
                        }
                        break;
                    }
                    // Skip the token at matchIndex and also increase matchOffset
                    // The default (increasing matchOffset by token.length()) is overriden for join token list.
                    change.increaseMatchIndex();
                } while (relexOffset > change.matchOffset);
            }

            // Check whether the new token ends at matchOffset with the same state
            // like the original which typically means end of relexing
            if (relexOffset == change.matchOffset
                && LexerUtilsConstants.statesEqual(state, 
                    (change.matchIndex > 0) ? tokenList.state(change.matchIndex - 1) : null)
            ) {
                // Here it's a potential match and the relexing could end.
                // However there are additional SAME-LOOKAHEAD requirements
                // that are checked here and if not satisfied the relexing will continue.
                // SimpleLexerRandomTest.test() contains detailed description.
                
                // If there are no more original tokens to be removed then stop since
                // there are no tokens ahead that would possibly have to be relexed because of LA differences.
                if (change.matchIndex == tokenCount)
                    break;

                int matchPointOrigLookahead = (change.matchIndex > 0)
                        ? tokenList.lookahead(change.matchIndex - 1)
                        : 0;
                // If old and new LAs are the same it should be safe to stop relexing.
                // Also since all tokens are non-empty it's enough to just check
                // LA > 1 (because LA <= 1 cannot span more than one token).
                // The same applies for current LA.
                if (lookahead == matchPointOrigLookahead ||
                    matchPointOrigLookahead <= 1 && lookahead <= 1
                ) {
                    break;
                }
                
                int afterMatchPointTokenLength = tokenList.tokenOrEmbeddingUnsync(change.matchIndex).token().length();
                if (matchPointOrigLookahead <= afterMatchPointTokenLength &&
                    lookahead <= afterMatchPointTokenLength
                ) {
                    // Here both the original and relexed before-match-point token
                    // have their LAs ending within bounds of the after-match-point token so it's OK
                    break;
                }

                // It's true that nothing can be generally predicted about LA if the token after match point
                // would be relexed (compared to the original's token LA). However the following criteria
                // should possibly suffice.
                int afterMatchPointOrigTokenLookahead = tokenList.lookahead(change.matchIndex);
                if (lookahead - afterMatchPointTokenLength <= afterMatchPointOrigTokenLookahead &&
                    (matchPointOrigLookahead <= afterMatchPointTokenLength ||
                        lookahead >= matchPointOrigLookahead)
                ) {
                    // The orig LA of after-match-point token cannot be lower than the currently lexed  LA's projection into it.
                    // Also check that the orig lookahead ended in the after-match-point token
                    // or otherwise require the relexed before-match-point token to have >= lookahead of the original
                    // before-match-point token).
                    break;
                // 'index' will point to the last token that was removed
                // 'matchOffset' will point to the end of the last removed token
                if (relexOffset > matchOffset && index < tokenCount) {
                    attemptValidation = false;
                    do {
                        index++;
                        if (index == tokenCount) {
                            // Make sure the number of removed tokens will be computed properly later
                            modToken = null;
                            // Check whether it should lex till the end
                            // or whether 'Match at anything' should be done
                            if (tokenList.isFullyLexed()) {
                                // Will lex till the end of input
                                matchOffset = Integer.MAX_VALUE;
                            } else {
                                // Force stop lexing
                                relex = false;
                            }
                            break;
                        }
                        matchOffset += token(tokenList, index).length();
                    } while (relexOffset > matchOffset);
                }

                // The token at matchIndex must be relexed
                if (loggable) {
                    LOG.log(Level.FINE, "    EXTRA-RELEX: mInd=" + change.matchIndex + ", LA=" + lookahead + "\n");
                }
                // Skip the token at matchIndex
                change.increaseMatchIndex();
                // Continue by fetching next token
            }
        }
        lexerInputOperation.release();
                    //  See initial part of SimpleRandomTest.test() verifies this.
                    // 2. Algorithm attempts to have the same lookaheads in tokens
                    //  like the regular batch scanning would produce.
                    //  Although not strictly necessary requirement
                    //  it helps to simplify the debugging in case the lexer does not work
                    //  well in the incremental setup.
                    //  The following code checks that the lookahead of the original match token
                    //  (i.e. the token right before matchOffset) does "end" inside
                    //  the next token - if not then relexing the next token is done.
                    //  The second part of SimpleRandomTest.test() verifies this.

        // If at least two tokens were lexed it's possible that e.g. the last added token
        // will be the same like the last removed token and in such case
        // the addition of the last token should be 'undone'.
        // This all may happen due to the fact that for larger lookaheads
        // the algorithm must relex the token(s) within lookahead (see the code above).
        int lastAddedTokenIndex = change.addedTokenOrEmbeddingsCount() - 1;
        // There should remain at least one added token since that one
        // may not be the same like the original removed one because
        // token lengths would differ because of the input source modification.
        
        if (change.matchOffset != Integer.MAX_VALUE) { // would not make sense when lexing past end of existing tokens
            while (lastAddedTokenIndex >= 1 && // At least one token added
                    change.matchIndex > lowestMatchIndex // At least one token removed
            ) {
                AbstractToken<T> lastAddedToken = change.addedTokenOrEmbeddings().get(lastAddedTokenIndex).token();
                AbstractToken<T> lastRemovedToken = tokenList.tokenOrEmbeddingUnsync(change.matchIndex - 1).token();
                if (lastAddedToken.id() != lastRemovedToken.id()
                    || lastAddedToken.length() != lastRemovedToken.length()
                    || change.laState().lookahead(lastAddedTokenIndex) != tokenList.lookahead(change.matchIndex - 1)
                            if (lookahead != tokenList.lookahead(i)
                                    || matchTokenLookahead > 0
                            ) {
                                // This token must be relexed
                                if (loggable) {
                                    LOG.log(Level.FINE, "EXTRA-RELEX: index=" + index + ", lookahead=" + lookahead
                                            + ", tokenLength=" + tokenLength + "\n");
                                }
                                index = i;
                                matchOffset += tokenLength;
                                relex = true;
                                // Continue - further tokens may be affected
                            }
                            i++;
                        }
                    }

                    if (!relex) {
                        if (attemptValidation) {
//                            if (modToken.id() == token.id()
//                                    && tokenList.lookahead(index) == lookahead
//                                    && !modToken.isFlyweight()
//                                    && !token.isFlyweight()
//                                    && (tokenList.getClass() != IncTokenList.class
//                                        || change.tokenHierarchyOperation().canModifyToken(index, modToken))
//                                    && LexerSpiTokenPackageAccessor.get().restoreToken(
//                                            languageOperation.tokenHandler(),
//                                            modToken, token)
//                            ) {
//                                // Restored successfully
//                                // TODO implement - fix token's length and return
//                                // now default in fact to failed validation
//                            }
                            attemptValidation = false;
                        }
                    }
                }
            } while (relex); // End of the relexing loop
            lexerInputOperation.release();

            // If at least two tokens were lexed it's possible that e.g. the last added token
            // will be the same like the last removed token and in such case
            // the addition of the last token should be 'undone'.
            // This all may happen due to the fact that for larger lookaheads
            // the algorithm must relex the token(s) within lookahead (see the code above).
            int lastAddedTokenIndex = change.addedTokensOrBranchesCount() - 1;
            // There should remain at least one added token since that one
            // may not be the same like the original removed one because
            // token lengths would differ because of the input source modification.
            while (lastAddedTokenIndex >= 1 && index > relexIndex && index < tokenCount) {
                AbstractToken<T> addedToken = LexerUtilsConstants.token(
                        change.addedTokensOrBranches().get(lastAddedTokenIndex));
                AbstractToken<T> removedToken = token(tokenList, index);
                if (addedToken.id() != removedToken.id()
                    || addedToken.length() != removedToken.length()
                    || change.laState().lookahead(lastAddedTokenIndex) != tokenList.lookahead(index)
                    || !LexerUtilsConstants.statesEqual(change.laState().state(lastAddedTokenIndex),
                        tokenList.state(change.matchIndex - 1))
                ) {
                    break;
                }
                // Last removed and added tokens are the same so undo the addition
                if (loggable) {
                    LOG.log(Level.FINE, "    RETAIN-ORIGINAL at (mInd-1)=" + (change.matchIndex-1) +
                            ", id=" + lastRemovedToken.id() + "\n");
                }
                lastAddedTokenIndex--;
                change.removeLastAddedToken(); // Includes decreasing of matchIndex and matchOffset
                relexOffset -= addedToken.length();
                change.removeLastAddedToken();
            }
        } else { // matchOffset == Integer.MAX_VALUE
            // Fix matchOffset to point to end of last token since it's used
            //   as last-added-token-end-offset in event notifications
            change.setMatchOffset(relexOffset);
        }
    }

    private static void logModification(CharSequence inputSourceText, TokenHierarchyEventInfo eventInfo,
            int tokenCount, boolean updateJoined
    ) {
        int modOffset = eventInfo.modOffset();
        int removedLength = eventInfo.removedLength();
        int insertedLength = eventInfo.insertedLength();
        String insertedText = "";
        if (insertedLength > 0) {
            insertedText = ", insTxt:\"" + CharSequenceUtilities.debugText(
                    inputSourceText.subSequence(modOffset, modOffset + insertedLength)) + '"';
        }
        // Debug 10 chars around modOffset
        int afterInsertOffset = modOffset + insertedLength;
        CharSequence beforeText = inputSourceText.subSequence(Math.max(afterInsertOffset - 5, 0), afterInsertOffset);
        CharSequence afterText = inputSourceText.subSequence(afterInsertOffset,
                Math.min(afterInsertOffset + 5, inputSourceText.length()));
        StringBuilder sb = new StringBuilder(200);
        sb.append("TLU.update");
        sb.append(updateJoined ? "Joined" : "Regular");
        sb.append("() modOff=").append(modOffset);
        sb.append(", text-around:\"").append(beforeText).append('|');
        sb.append(afterText).append("\", insLen=");
        sb.append(insertedLength).append(insertedText);
        sb.append(", remLen=").append(removedLength);
        sb.append(", tCnt=").append(tokenCount).append('\n');
        LOG.log(Level.FINE, sb.toString());
    }

}





package org.netbeans.lib.lexer.token;

import org.netbeans.lib.lexer.TokenOrEmbedding;
import java.util.List;
import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.CharSequenceUtilities;
import org.netbeans.lib.lexer.EmbeddedTokenList;
import org.netbeans.lib.lexer.EmbeddingContainer;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.lib.lexer.TokenList;

/**
 * Abstract token is base class of all token implementations used in the lexer module.
 * <br/>
 * Two descendants of AbstractToken:
 * <ul>
 *   <li>{@link DefaultToken} - by default does not contain a text but points
 *       into a text storage of its token list instead. It may however cache
 *       its text as string in itself.
 *       <ul>
 *           <li></li>
 *       </ul>
 *   </li>
 *   <li>{@link TextToken} - contains text that it represents; may act as flyweight token.
 *       {@link CustomTextToken} allows a token to have a custom text independent
 *       of text of an actual storage.
 *   </li>
 * 
 * 
 *
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public abstract class AbstractToken<T extends TokenId> extends Token<T>
implements TokenOrEmbedding<T> {
    
    private final T id; // 12 bytes (8-super + 4)

    protected TokenList<T> tokenList; // 16 bytes
    
    protected int rawOffset; // 20 bytes

    /**
     * @id non-null token id.

        this.rawOffset = rawOffset;
    }
    
    public abstract int length();
    
    /**
     * Get identification of this token.
     *

    @Override
    public final T id() {
        return id;
    }

    /**
     * Get text represented by this token.
     */
    @Override
    public CharSequence text() {
        if (tokenList != null) {
            if (tokenList.getClass() == EmbeddedTokenList.class) {
                EmbeddedTokenList<?> etl = (EmbeddedTokenList<?>)tokenList;
                return etl.embeddingContainer().updateStatus() ? this : null;
            }
            return this;
        } else {
            return null;
        }
    }

    /**

    }

    @Override
    public int offset(TokenHierarchy<?> tokenHierarchy) {
        if (tokenList != null) {
            if (tokenList.getClass() == EmbeddedTokenList.class) // Sync status first
                ((EmbeddedTokenList)tokenList).embeddingContainer().updateStatus();
            return tokenList.tokenOffset(this);
        }
        return rawOffset; // Covers the case of flyweight token that will return -1
        return (tokenList != null)
                ? tokenList.childTokenOffset(rawOffset)
                : rawOffset;
//        if (tokenHierarchy != null) {
//            return LexerApiPackageAccessor.get().tokenHierarchyOperation(
//                    tokenHierarchy).tokenOffset(this, tokenList, rawOffset);

//                : rawOffset;
//        }
    }

    @Override
    public boolean hasProperties() {
        return false;

        return null;
    }

    @Override
    public Token<T> joinToken() {
        return null;
     */
    public final char charAt(int index) {
        if (index < 0 || index >= length()) {
            throw new IndexOutOfBoundsException(
                "index=" + index + ", length=" + length() // NOI18N
            );
        }
        if (tokenList == null) { // Should normally not happen
            // A bit strange to throw IOOBE but it's more practical since
            // TokenHierarchy's dump can overcome IOOBE and deliver a useful debug but not NPEs etc.
            throw new IndexOutOfBoundsException("index=" + index + ", length=" + length() +
                    " but tokenList==null for token " + dumpInfo(null));
        }
        return tokenList.childTokenCharAt(rawOffset, index);
    }

    @Override
    public List<? extends Token<T>> joinedParts() {
        return null;
    }

    // Implements TokenOrEmbedding
    public final AbstractToken<T> token() {
        return this;
    }
    
    // Implements TokenOrEmbedding
    public final EmbeddingContainer<T> embedding() {
        return null;
    }

    @Override
    public boolean isRemoved() {
        if (tokenList != null) {
            if (tokenList.getClass() == EmbeddedTokenList.class)
                ((EmbeddedTokenList)tokenList).embeddingContainer().updateStatus();
            return tokenList.isRemoved();
        }
        return !isFlyweight();
    }

    public String dumpInfo() {
        return dumpInfo(null, null, true, 0).toString();
    }

    /**

     *
     * @param tokenHierarchy <code>null</code> should be passed
     *  (the parameter is reserved for future use when token hierarchy snapshots will be implemented).
     * @param dumpText whether text should be dumped (not for TokenListUpdater
     *  when text is already shifted).
     * @return dump of the thorough token information. If token's text is longer
     *  than 400 characters it will be shortened.
     */
    public StringBuilder dumpInfo(StringBuilder sb, TokenHierarchy<?> tokenHierarchy, boolean dumpTokenText, int indent) {
        if (sb == null) {
            sb = new StringBuilder(50);
        }
        if (dumpTokenText) {
            CharSequence text = text();
            if (text != null) {
                sb.append('"');
                int textLength = text.length();
                for (int i = 0; i < textLength; i++) {
                    if (textLength > 400 && i >= 200 && i < textLength - 200) {
                        i = textLength - 200;
                        sb.append(" ...<TEXT-SHORTENED>... "); // NOI18N
                        continue;
                    }
                    try {
                        CharSequenceUtilities.debugChar(sb, text.charAt(i));
                    } catch (IndexOutOfBoundsException e) {
                        // For debugging purposes it's better than to completely fail
                        sb.append("IOOBE at index=").append(i).append("!!!"); // NOI18N
                        break;
                    }
                }
                sb.append('"');
            } else {
                sb.append("<null-text>"); // NOI18N
                    // For debugging purposes it's better than to completely fail
                    sb.append("IOOBE at index=").append(i).append("!!!"); // NOI18N
                    break;
                }
            }
            sb.append(' ');
        } else {
            sb.append("<null-text>"); // NOI18N
        }
        sb.append(' ');
        if (isFlyweight()) {
            sb.append("F(").append(length()).append(')');
        } else {

        }
        sb.append(' ').append(id != null ? id.name() + '[' + id.ordinal() + ']' : "<null-id>"); // NOI18N
        sb.append(" ").append(dumpInfoTokenType());
        return sb;
    }
    
    protected String dumpInfoTokenType() {




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.token;

import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.spi.lexer.TokenPropertyProvider;

/**
 * Token that may hold custom text and also additional properties.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class ComplexToken<T extends TokenId> extends CustomTextToken<T> {

    private final TokenPropertyProvider propertyProvider; // 36 bytes
    
    public ComplexToken(T id, int length, CharSequence customText, PartType partType,
    TokenPropertyProvider propertyProvider) {
        super(id, length, customText, partType);
        this.propertyProvider = propertyProvider;
    }

    @Override
    public boolean hasProperties() {
        return (propertyProvider != null);
    }

    @Override
    public Object getProperty(Object key) {
        return (propertyProvider != null) ? propertyProvider.getValue(this, key) : null;
    }
    
    @Override
    protected String dumpInfoTokenType() {
        return "ComT"; // NOI18N "ComplexToken"
    }
    
}





package org.netbeans.lib.lexer.token;

import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.TokenId;

/**
 * Token with a custom text and the token length likely different
 * from text's length. It can be used to shrink size of the input chars
 * being referenced from skim token list by referencing some fixed characters.
 * <br/>
 * Token with the custom text cannot be branched by a language embedding.
 *

 * @version 1.00
 */

public class CustomTextToken<T extends TokenId> extends TextToken<T> {
    
    private final int length; // 28 bytes (24-super + 4)
    
    private final PartType partType; // 32 bytes
    
    /**
     * @param id non-null identification of the token.
     * @param text non-null text of the token.
     * @param length length of the token.
     * @param text non-null text of the token.
     */
    public CustomTextToken(T id, CharSequence text, int length) {
        super(id, text);
        this.length = length;
        this.text = text;
        this.partType = partType;
    }
    
    @Override
    public boolean isCustomText() {
        return true;
    }

    @Override
    public final int length() {
        return length;
    }
    
    @Override

Lines 42-47 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/token/DefaultToken.java (-5 / +206 lines)
42	package org.netbeans.lib.lexer.token;	42	package org.netbeans.lib.lexer.token;
43		43
44	import org.netbeans.api.lexer.TokenId;	44	import org.netbeans.api.lexer.TokenId;
		45	import org.netbeans.lib.editor.util.CharSequenceUtilities;
		46	import org.netbeans.lib.lexer.LexerUtilsConstants;
45		47
46	/**	48	/**
47	* Default token which by default obtains text from its background storage.	49	* Default token which by default obtains text from its background storage.
Lines 61-67 Link Here
61		63
62	public class DefaultToken<T extends TokenId> extends AbstractToken<T> implements CharSequence {	64	public class DefaultToken<T extends TokenId> extends AbstractToken<T> implements CharSequence {
63		65
64	private final int length; // 24 bytes (20-super + 4)	66	/**
		67	* Used in Token.text() to decide whether "this" should be returned and
		68	* a text.charAt() will be slower or, for larger tokens,
		69	* a subsequence of input source text should be created and returned instead..
		70	*/
		71	private static final int INPUT_SOURCE_SUBSEQUENCE_THRESHOLD = 30;
		72
		73	/**
		74	* Field that is of type CharSequence and is either length of the token
		75	* or cached text of the token as String.
		76	*/
		77	private CharSequence tokenLengthOrCachedText; // 24 bytes (20-super + 4)
65		78
66	/**	79	/**
67	* Construct new default token.	80	* Construct new default token.
Lines 69-75 Link Here
69	public DefaultToken(T id, int length) {	82	public DefaultToken(T id, int length) {
70	super(id);	83	super(id);
71	assert (length > 0) : "Token length=" + length + " <= 0"; // NOI18N	84	assert (length > 0) : "Token length=" + length + " <= 0"; // NOI18N
72	this.length = length;	85	this.tokenLengthOrCachedText = TokenLength.get(length);
73	}	86	}
74		87
75	/**	88	/**
Lines 77-88 Link Here
77	*/	90	*/
78	public DefaultToken(T id) {	91	public DefaultToken(T id) {
79	super(id);	92	super(id);
80	this.length = 0;	93	this.tokenLengthOrCachedText = TokenLength.get(0);
81	}	94	}
82		95
83	@Override	96	@Override
84	public final int length() {	97	public int length() {
85	return length;	98	return tokenLengthOrCachedText.length();
86	}	99	}
87		100
88	@Override	101	@Override
Lines 90-93 Link Here
90	return "DefT"; // NOI18N "TextToken" or "FlyToken"	103	return "DefT"; // NOI18N "TextToken" or "FlyToken"
91	}	104	}
92		105
		106	/**
		107	* Get text represented by this token.
		108	*/
		109	@Override
		110	public CharSequence text() {
		111	CharSequence text;
		112	if (tokenLengthOrCachedText.getClass() == TokenLength.class) {
		113	if (!isRemoved()) { // Updates status for EmbeddedTokenList; tokenList != null
		114	int len = tokenLengthOrCachedText.length();
		115	if (len >= INPUT_SOURCE_SUBSEQUENCE_THRESHOLD) {
		116	// Create subsequence of input source text
		117	CharSequence inputSourceText = tokenList.inputSourceText();
		118	int tokenOffset = tokenList.tokenOffset(this);
		119	text = new InputSourceSubsequence(this, inputSourceText,
		120	tokenOffset, tokenOffset + len, tokenOffset, tokenOffset + len);
		121	} else { // Small token
		122	text = this;
		123	}
		124	} else { // Token is removed
		125	text = null;
		126	}
		127	} else { // tokenLength contains cached text
		128	text = tokenLengthOrCachedText;
		129	}
		130	return text;
		131	}
		132
		133	/**
		134	* Implementation of <code>CharSequence.charAt()</code>
		135	* for case when this token is used as token's text char sequence.
		136	*/
		137	public final char charAt(int index) {
		138	if (index < 0 \|\| index >= length()) {
		139	throw new IndexOutOfBoundsException(
		140	"index=" + index + ", length=" + length() // NOI18N
		141	);
		142	}
		143	if (tokenList == null) { // Should normally not happen
		144	// A bit strange to throw IOOBE but it's more practical since
		145	// TokenHierarchy's dump can overcome IOOBE and deliver a useful debug but not NPEs etc.
		146	throw new IndexOutOfBoundsException("index=" + index + ", length=" + length() +
		147	" but tokenList==null for token " + dumpInfo(null, null, false, 0));
		148	}
		149	int tokenOffset = tokenList.tokenOffset(this);
		150	return tokenList.inputSourceText().charAt(tokenOffset + index);
		151	}
		152
		153	/**
		154	* Implementation of <code>CharSequence.subSequence()</code>
		155	* for case when this token is used as token's text char sequence.
		156	*/
		157	public final CharSequence subSequence(int start, int end) {
		158	// Create subsequence of token's text
		159	CharSequence text;
		160	int textLength = tokenLengthOrCachedText.length();
		161	CharSequenceUtilities.checkIndexesValid(start, end, textLength);
		162
		163	if (tokenLengthOrCachedText.getClass() == TokenLength.class) {
		164	// If calling this.subSequence() then this.text() was already called
		165	// so the status should be updated already and also the token is not removed.
		166	// For simplicity always make a subsequence of the input source text.
		167	CharSequence inputSourceText = tokenList.inputSourceText();
		168	int tokenOffset = tokenList.tokenOffset(this);
		169	text = new InputSourceSubsequence(this, inputSourceText,
170	tokenOffset + start, tokenOffset + end, tokenOffset, tokenOffset + textLength);
171
172	} else { // tokenLength contains cached text
173	text = tokenLengthOrCachedText.subSequence(start, end);
174	}
175	return text;
176	}
177
178	/**
179	* Implementation of <code>CharSequence.toString()</code>
180	* for case when this token is used as token's text char sequence.
181	*/
182	@Override
183	public String toString() {
184	// In reality this method can either be called as result of calling Token.text().toString()
185	// or just calling Token.toString() for debugging purposes
186	String textStr;
187	if (tokenLengthOrCachedText.getClass() == TokenLength.class) {
188	if (!isRemoved()) { // Updates status for EmbeddedTokenList; tokenList != null
189	TokenLength tokenLength = (TokenLength) tokenLengthOrCachedText;
190	CharSequence inputSourceText = tokenList.inputSourceText();
191	int nextCacheFactor = tokenLength.nextCacheFactor();
192	int threshold = (inputSourceText.getClass() == String.class)
193	? LexerUtilsConstants.INPUT_TEXT_STRING_THRESHOLD
194	: LexerUtilsConstants.CACHE_TOKEN_TO_STRING_THRESHOLD;
195	int tokenOffset = tokenList.tokenOffset(this);
196	textStr = inputSourceText.subSequence(tokenOffset,
197	tokenOffset + tokenLength.length()).toString();
198	if (nextCacheFactor < threshold) {
199	tokenLengthOrCachedText = tokenLength.next(nextCacheFactor);
200	} else { // Should become cached
201	tokenLengthOrCachedText = textStr;
202	}
203	setTokenLengthOrCachedText(tokenLengthOrCachedText);
204	} else { // Token already removed
205	textStr = "<null>";
206	}
207
208	} else { // tokenLength contains cached text
209	textStr = tokenLengthOrCachedText.toString();
210	}
211	return textStr;
212	}
213
214	synchronized CharSequence tokenLengthOrCachedText() {
215	return tokenLengthOrCachedText;
216	}
217
218	synchronized void setTokenLengthOrCachedText(CharSequence tokenLengthOrCachedText) {
219	this.tokenLengthOrCachedText = tokenLengthOrCachedText;
220	}
221
222
223	private static final class InputSourceSubsequence implements CharSequence {
224
225	private final DefaultToken token; // (8-super + 4) = 12 bytes
226
227	private final CharSequence inputSourceText; // 16 bytes
228
229	private final int start; // 20 bytes
230
231	private final int end; // 24 bytes
232
233	private final int tokenStart; // 28 bytes
234
235	private final int tokenEnd; // 32 bytes
236
237	public InputSourceSubsequence(DefaultToken token, CharSequence text,
238	int start, int end, int tokenStart, int tokenEnd
239	) {
240	this.token = token;
241	this.inputSourceText = text;
242	this.start = start;
243	this.end = end;
244	this.tokenStart = tokenStart;
245	this.tokenEnd = tokenEnd;
246	}
247
248	public int length() {
249	return end - start;
250	}
251
252	public char charAt(int index) {
253	CharSequenceUtilities.checkIndexValid(index, length());
254	return inputSourceText.charAt(start + index);
255	}
256
257	public CharSequence subSequence(int start, int end) {
258	CharSequenceUtilities.checkIndexesValid(this, start, end);
259	return new InputSourceSubsequence(token, inputSourceText,
260	this.start + start, this.start + end, tokenStart, tokenEnd);
261	}
262
263	@Override
264	public String toString() {
265	String textStr;
266	// Increase usage
267	CharSequence tokenLengthOrCachedText = token.tokenLengthOrCachedText();
268	if (tokenLengthOrCachedText.getClass() == TokenLength.class) {
269	TokenLength tokenLength = (TokenLength) tokenLengthOrCachedText;
270	int nextCacheFactor = tokenLength.nextCacheFactor();
271	int threshold = (inputSourceText.getClass() == String.class)
272	? LexerUtilsConstants.INPUT_TEXT_STRING_THRESHOLD
273	: LexerUtilsConstants.CACHE_TOKEN_TO_STRING_THRESHOLD;
274	if (nextCacheFactor < threshold) {
275	textStr = inputSourceText.subSequence(start, end).toString();
276	tokenLengthOrCachedText = tokenLength.next(nextCacheFactor);
277	} else { // Should become cached
278	// Create cached text
279	String tokenTextString = inputSourceText.subSequence(tokenStart, tokenEnd).toString();
280	tokenLengthOrCachedText = tokenTextString;
281	// Substring returns this for start == 0 && end == length()
282	textStr = tokenTextString.substring(start - tokenStart, end - tokenStart);
283	}
284	token.setTokenLengthOrCachedText(tokenLengthOrCachedText);
285
286	} else { // Already cached text
287	textStr = tokenLengthOrCachedText.subSequence(start - tokenStart, end - tokenStart).toString();
288	}
289	return textStr;
290	}
291
292	}
293
93	}	294	}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.token;

import java.util.List;
import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.ArrayUtilities;
import org.netbeans.spi.lexer.TokenPropertyProvider;

/**
 * Token consisting of multiple parts.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class JoinToken<T extends TokenId> extends PropertyToken<T> {

    private List<PartToken<T>> joinedParts; // 32 bytes (28-super + 4)
    
    private int completeLength; // 36 bytes
    
    /**
     * Number of ETLs spanned including empty ETLs except a first part.
     */
    private int extraTokenListSpanCount; // 40 bytes

    public JoinToken(T id, int length, TokenPropertyProvider<T> propertyProvider, PartType partType) {
        super(id, length, propertyProvider, partType);
    }

    @Override
    public List<PartToken<T>> joinedParts() {
        return joinedParts;
    }
    
    public void setJoinedParts(List<PartToken<T>> joinedParts, int extraTokenListSpanCount) {
        assert (joinedParts != null) : "joinedParts expected to be non-null";
        this.joinedParts = joinedParts;
        for (PartToken partToken : joinedParts) {
            completeLength += partToken.length();
        }
        this.extraTokenListSpanCount = extraTokenListSpanCount;
    }

    public PartToken<T> lastPart() {
        return joinedParts.get(joinedParts.size() - 1);
    }

    public int extraTokenListSpanCount() {
        return extraTokenListSpanCount;
    }

    @Override
    public int offset(TokenHierarchy<?> tokenHierarchy) {
        return joinedParts.get(0).offset(tokenHierarchy);
    }
    
    @Override
    public int length() {
        return completeLength;
    }

    @Override
    public CharSequence text() {
        return new JoinTokenText<T>(joinedParts, completeLength);
    }

    @Override
    public boolean isRemoved() {
        // Check whether last part of token is removed - this needs to be improved
        // for the case when token is just partially recreated.
        return lastPart().isRemoved();
    }

    @Override
    public StringBuilder dumpInfo(StringBuilder sb, TokenHierarchy<?> tokenHierarchy, boolean dumpTokenText, int indent) {
        super.dumpInfo(sb, tokenHierarchy, dumpTokenText, indent);
        sb.append(", ").append(joinedParts.size()).append(" parts");
        int digitCount = String.valueOf(joinedParts.size() - 1).length();
        for (int i = 0; i < joinedParts.size(); i++) {
            sb.append('\n');
            ArrayUtilities.appendSpaces(sb, indent + 2);
            ArrayUtilities.appendBracketedIndex(sb, i, digitCount);
            joinedParts.get(i).dumpInfo(sb, tokenHierarchy, dumpTokenText, indent + 4);
        }
        return sb;
    }

    @Override
    protected String dumpInfoTokenType() {
        return "JoiT"; // NOI18N
    }

}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.token;

import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.CharSequenceUtilities;

/**
 * Char sequence over join token parts.
 * 
 * @author Miloslav Metelka
 */

public final class JoinTokenText<T extends TokenId> implements CharSequence {
    
    private static final Logger LOG = Logger.getLogger(JoinTokenText.class.getName());

    private List<PartToken<T>> joinedParts;

    private int activePartIndex;
    
    private CharSequence activeInputText;

    private int activeStartCharIndex;
    
    private int activeEndCharIndex;
    
    private int length;
    
    public JoinTokenText(List<PartToken<T>> joinedParts, int length) {
        this.joinedParts = joinedParts;
        this.activeInputText = joinedParts.get(0).text();
        // Implicit: this.activeStartCharIndex = 0;
        this.activeEndCharIndex = activeInputText.length();
        this.length = length;
    }

    public synchronized char charAt(int index) {
        if (index < activeStartCharIndex) { // Find non-empty previous
            if (index < 0)
                throw new IndexOutOfBoundsException("index=" + index + " < 0");
            do {
                activePartIndex--;
                if (activePartIndex < 0) { // Should never happen
                    LOG.log(Level.WARNING, "Internal error: index=" + index + ", " + dumpState());
                }
                activeInputText = joinedParts.get(activePartIndex).text();
                int len = activeInputText.length();
                activeEndCharIndex = activeStartCharIndex;
                activeStartCharIndex -= len;
            } while (index < activeStartCharIndex);
        } else if (index >= activeEndCharIndex) { // Find non-empty next
            if (index >= length)
                throw new IndexOutOfBoundsException("index=" + index + " >= length()=" + length);
            do {
                activePartIndex++;
                activeInputText = joinedParts.get(activePartIndex).text();
                int len = activeInputText.length();
                activeStartCharIndex = activeEndCharIndex;
                activeEndCharIndex += len;
            } while (index >= activeEndCharIndex);
        }

        // Valid char within current segment
        return activeInputText.charAt(index - activeStartCharIndex);
    }

    public int length() {
        return length;
    }
    
    public CharSequence subSequence(int start, int end) {
        return CharSequenceUtilities.toString(this, start, end);
    }
    
    @Override
    public synchronized String toString() {
        return CharSequenceUtilities.toString(this);
    }
    
    private String dumpState() {
        return "activeTokenListIndex=" + activePartIndex +
                ", activeStartCharIndex=" + activeStartCharIndex +
                ", activeEndCharIndex=" + activeEndCharIndex +
                ", length=" + length;
    }

}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.token;

import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.TokenOrEmbedding;
import org.netbeans.spi.lexer.TokenPropertyProvider;

/**
 * Part of a {@link JoinToken}.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class PartToken<T extends TokenId> extends PropertyToken<T> {

    private TokenOrEmbedding<T> joinTokenOrEmbedding; // 32 bytes (28-super + 4)
    
    private int partTokenIndex; // Index of this part inside 
    
    private int partTextOffset; // Offset of this part's text among all parts that comprise the complete token

    public PartToken(T id, int length, TokenPropertyProvider<T> propertyProvider, PartType partType,
            TokenOrEmbedding<T> joinToken, int partTokenIndex, int partTextOffset
    ) {
        super(id, length, propertyProvider, partType);
        setJoinTokenOrEmbedding(joinToken);
        this.partTokenIndex = partTokenIndex;
        this.partTextOffset = partTextOffset;
    }

    @Override
    public JoinToken<T> joinToken() {
        return (JoinToken<T>)joinTokenOrEmbedding.token();
    }
    
    public boolean isLastPart() {
        return (joinToken().lastPart() == this);
    }
    
    public TokenOrEmbedding<T> joinTokenOrEmbedding() {
        return joinTokenOrEmbedding;
    }
    
    public void setJoinTokenOrEmbedding(TokenOrEmbedding<T> joinTokenOrEmbedding) {
        this.joinTokenOrEmbedding = joinTokenOrEmbedding;
    }

    public int partTokenIndex() {
        return partTokenIndex;
    }

    public int partTextOffset() {
        return partTextOffset;
    }

    @Override
    protected String dumpInfoTokenType() {
        return "ParT[" + partTokenIndex + "]"; // NOI18N
    }

}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.token;

import java.util.EnumSet;
import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.spi.lexer.TokenPropertyProvider;

/**
 * Property provider that stores {@link org.netbeans.api.lexer.PartType} information.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class PartTypePropertyProvider implements TokenPropertyProvider<TokenId> {
    
    private static final PartTypePropertyProvider[] partTypeOrdinal2Provider
            = new PartTypePropertyProvider[PartType.class.getEnumConstants().length];

    static {
        for (PartType partType : EnumSet.allOf(PartType.class)) {
            partTypeOrdinal2Provider[partType.ordinal()] = new PartTypePropertyProvider(partType);
        }
    }
    
    public static <T extends TokenId> TokenPropertyProvider<T> get(PartType partType) {
        return (TokenPropertyProvider<T>)partTypeOrdinal2Provider[partType.ordinal()];
    }
    
    public static <T extends TokenId> TokenPropertyProvider<T> createDelegating(
            PartType partType, TokenPropertyProvider<T> delegate
    ) {
        return new Delegating<T>(partType, delegate);
    }
    
    private PartType partType;

    public PartTypePropertyProvider(PartType partType) {
        this.partType = partType;
    }

    public Object getValue(Token<TokenId> token, Object key) {
        if (key == PartType.class) {
            return partType;
        }
        return null;
    }
    
    private static final class Delegating<T extends TokenId> implements TokenPropertyProvider<T> {
        
        private final PartType partType;
        
        private final TokenPropertyProvider<T> delegate;
        
        Delegating(PartType partType, TokenPropertyProvider<T> delegate) {
            assert (delegate != null) : "delegate expected to be non-null. Use PartTypePropertyProvider.get() instead."; // NOTICES
            this.partType = partType;
            this.delegate = delegate;
        }

        public Object getValue(Token<T> token, Object key) {
            if (key == PartType.class) {
                return partType;
            }
            return delegate.getValue(token, key);
        }

    }
    
}




package org.netbeans.lib.lexer.token;

import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.spi.lexer.TokenPropertyProvider;

/**
 * Token with associated properties. It may also act as a token part but without
 * a reference to a complete token e.g. suitable for java's incomplete block comment.
 * <p>
 * Instances of this token are more costly than other token types
 * because in addition to regular information they store preprocessed
 * text of the token.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public class PropertyToken<T extends TokenId> extends DefaultToken<T> {

    private final TokenPropertyProvider<T> propertyProvider; // 28 bytes (24-super + 4)

    public PropertyToken(T id, int length, TokenPropertyProvider<T> propertyProvider, PartType partType) {
    
    public PropertyToken(T id, int length,
    TokenPropertyProvider propertyProvider, PartType partType) {
        super(id, length);
        assert (partType != null);
        this.propertyProvider = (propertyProvider != null)
                ? PartTypePropertyProvider.createDelegating(partType, propertyProvider)
                : PartTypePropertyProvider.<T>get(partType);
    }

    @Override
    public boolean hasProperties() {
        return (propertyProvider != null);

    public Object getProperty(Object key) {
        return (propertyProvider != null) ? propertyProvider.getValue(this, key) : null;
    }

    @Override
    public PartType partType() {
        return (PartType) getProperty(PartType.class);
    }

    @Override
    protected String dumpInfoTokenType() {
        return "ProT"; // NOI18N
    }

}




import org.netbeans.lib.lexer.TokenList;

/**
 * Token with an explicit text - either serving a flyweight token
 * or a non-flyweight replacement for a flyweight token.
 * <br/>
 * The represented text should be the same like the original content
 * of the recognized text input portion.
 * <br/>
 * Token with the custom text cannot be branched by a language embedding.
 *
 * <p>
 * The text token can act as a flyweight token by calling

     * is expected to correspond to the recognized input portion
     * (i.e. the text is not custom).
     * <br/>
     * The token can be made flyweight by using <code>makeFlyweight()</code>.
     *
     * @param id non-null identification of the token.
     * @param text non-null text of the token.

    }

    @Override
    public int length() {
        return text.length();
    }


    }
    
    public final TextToken<T> createCopy(TokenList<T> tokenList, int rawOffset) {
        return new TextToken<T>(id(), tokenList, rawOffset, text);
    }
    
    @Override

        return isFlyweight() ? "FlyT" : "TexT"; // NOI18N "TextToken" or "FlyToken"
    }

    @Override
    public String toString() {
        return text.toString();
    }




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.token;

import org.netbeans.lib.lexer.LexerUtilsConstants;

/**
 * Improves performance of doing Token.text().toString().
 * by using a cache factor which gets increased by every access to that method.
 * <br/>
 * Once a cache factor exceeds a threshold the result of Token.text().toString()
 * will be cached.
 * <br/>
 * TBD values of constants used by this class should be reviewed to match 
 * a real complexity of the particular operations.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class TokenLength implements CharSequence {
    
    private static final TokenLength[][] CACHE = new TokenLength[
            LexerUtilsConstants.MAX_CACHED_TOKEN_LENGTH + 1][];
    
    
    public static TokenLength get(int length) {
        TokenLength tokenLength;
        if (length <= LexerUtilsConstants.MAX_CACHED_TOKEN_LENGTH) {
            synchronized (CACHE) {
                TokenLength[] tokenLengths = CACHE[length];
                if (tokenLengths == null) {
                    tokenLengths = new TokenLength[1];
                    CACHE[length] = tokenLengths;
                }
                tokenLength = tokenLengths[0];
                if (tokenLength == null) {
                    tokenLength = new TokenLength(length, 
                            LexerUtilsConstants.CACHE_TOKEN_TO_STRING_THRESHOLD, (short)1);
                    tokenLengths[0] = tokenLength;
                }
            }
        } else { // length too high - not cached
            tokenLength = new TokenLength(length,
                    LexerUtilsConstants.CACHE_TOKEN_TO_STRING_THRESHOLD, (short)1);
        }
        return tokenLength;
    }
    
    /**
     * Length of a token.
     */
    private final int length; // 12 bytes (8-super + 4)
    
    /**
     * Cache factor of this item.
     */
    private final short cacheFactor; // 14 bytes
    
    /**
     * Index of a next item in array of token lengths with the same length in CACHE.
     */
    private final short nextArrayIndex; // 16 bytes
    
    TokenLength(int length, short cacheFactor, short nextArrayIndex) {
        this.length = length;
        this.cacheFactor = cacheFactor;
        this.nextArrayIndex = nextArrayIndex;
    }
    
    public int length() {
        return length;
    }
    
    public short cacheFactor() {
        return cacheFactor;
    }
    
    public int nextCacheFactor() {
        return cacheFactor + length + LexerUtilsConstants.TOKEN_LENGTH_STRING_CREATION_FACTOR;
    }
    
    public TokenLength next(int nextCacheFactor) {
        TokenLength tokenLength;
        if (length <= LexerUtilsConstants.MAX_CACHED_TOKEN_LENGTH) {
            synchronized (CACHE) {
                TokenLength[] tokenLengths = CACHE[length];
                if (tokenLengths == null || tokenLengths.length <= nextArrayIndex) {
                    TokenLength[] tmp = new TokenLength[nextArrayIndex + 1];
                    if (tokenLengths != null) {
                        System.arraycopy(tokenLengths, 0, tmp, 0, tokenLengths.length);
                    }
                    tokenLengths = tmp;
                    CACHE[length] = tokenLengths;
                }
                tokenLength = tokenLengths[nextArrayIndex];
                if (tokenLength == null) {
                    tokenLength = new TokenLength(length, (short)nextCacheFactor, (short)(nextArrayIndex + 1));
                    tokenLengths[nextArrayIndex] = tokenLength;
                }
            }
        } else { // length too high - not cached
            tokenLength = new TokenLength(length, (short)nextCacheFactor, (short)(nextArrayIndex + 1));
        }
        return tokenLength;
    }

    public char charAt(int index) {
        throw new IllegalStateException("Should never be called.");
    }

    public CharSequence subSequence(int start, int end) {
        throw new IllegalStateException("Should never be called.");
    }

}

Lines 205-213 Link Here

(-)a/lexer/src/org/netbeans/spi/lexer/LanguageEmbedding.java (-1 / +2 lines)
205	return joinSections;	205	return joinSections;
206	}	206	}
207		207
		208	@Override
208	public String toString() {	209	public String toString() {
209	return "language: " + language() + ", skip[" + startSkipLength() // NOI18N	210	return "language: " + language() + ", skip[" + startSkipLength() // NOI18N
210	+ ", " + endSkipLength + "]"; // NOI18N	211	+ ", " + endSkipLength + "];" + (joinSections ? "join" : "no-join"); // NOI18N
211	}	212	}
212		213
213	}	214	}




import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.CharProvider;
import org.netbeans.lib.lexer.LexerApiPackageAccessor;
import org.netbeans.lib.lexer.LexerInputOperation;
import org.netbeans.lib.lexer.LexerSpiPackageAccessor;

    }

    /**
     * This feature is currently not supported - Token.text()
     * will return null for non-flyweight tokens.
     * <br/>
     * Determine whether the text of the token with the particular id should
     * be retained after the token has been removed from the token list
     * because of the underlying mutable input source modification.

            return languageHierarchy.isRetainTokenText(id);
        }

        public LexerInput createLexerInput(LexerInputOperation<?> operation) {
            return new LexerInput(operation);
        }

        public Language<?> language(MutableTextInput<?> mti) {




package org.netbeans.spi.lexer;

import org.netbeans.lib.editor.util.AbstractCharSequence;
import org.netbeans.lib.lexer.LexerInputOperation;
import org.netbeans.lib.lexer.LexerUtilsConstants;

/**

    public static final int EOF = -1;
    
    /**
     * LexerInputOperation on which this lexer input delegates.
     * its operation.
     */
    private LexerInputOperation<?> operation;
    
    /**
     * Character sequence that corresponds

    /**
     * Construct instance of the lexer input.
     *
     * @param operation non-null character provider for this lexer input.
     */
    LexerInput(LexerInputOperation operation) {
        this.operation = operation;
    }
    
    /**

     *   - all of them will return EOF.
     */
    public int read() {
        int c = operation.read();
        if (c == EOF) {
            eof = 1;
        }

            eof = 0; // backup EOF
            count--;
        }
        operation.backup(count);
    }
    
    /**

     *   If {@link LexerInput#EOF} was read then it is not counted into read length.
     */
    public int readLength() {
        return operation.readLength();
    }
    
    /**

            if (index < 0 || index >= length) {
                throw new IndexOutOfBoundsException("index=" + index + ", length=" + length); // NOI18N
            }
            return operation.readExistingAtIndex(index);
        }
        
    }





package org.netbeans.spi.lexer;

import java.util.Set;
import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.CharSequenceUtilities;
import org.netbeans.lib.lexer.LanguageOperation;
import org.netbeans.lib.lexer.LexerInputOperation;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.lib.lexer.token.CustomTextToken;
import org.netbeans.lib.lexer.token.DefaultToken;
import org.netbeans.lib.lexer.token.ComplexToken;
import org.netbeans.lib.lexer.token.ComplexToken;
import org.netbeans.lib.lexer.token.PropertyToken;
import org.netbeans.lib.lexer.token.TextToken;

/**
 * Lexer should delegate all the token instances creation to this class.


public final class TokenFactory<T extends TokenId> {

    /** Flag for additional correctness checks (may degrade performance). */
    private static final boolean testing = Boolean.getBoolean("netbeans.debug.lexer.test");

    /**
     * Token instance that the token creation methods in this class produce
     * if there is an active filtering of certain token ids
     * and the just recognized token-id should be skipped.
     * Normally lexers do not need to check for this except some specific cases
     * in which the {@link #isSkipToken(Token)} is a better typed alternative
     * to this field.
     *
     * @deprecated Use {@link #isSkipToken(Token)} instead.
     */
    public static final Token SKIP_TOKEN = LexerUtilsConstants.SKIP_TOKEN;
        = new TextToken<TokenId>(
            new TokenIdImpl("skip-token-id; special id of TokenFactory.SKIP_TOKEN; " + // NOI18N
                    " It should never be part of token sequence", 0, null), // NOI18N
            "" // empty skip token text NOI18N
        );
    
    private final LexerInputOperation<T> operation;
    

     * @see #createToken(TokenId, int)
     */
    public Token<T> createToken(T id) {
        return createToken(id, operation.readLength());
    }

    /**

     *  because of token id filter.
     */
    public Token<T> createToken(T id, int length) {
        return operation.createToken(id, length);
            operation.tokenRecognized(length, true);
            return skipToken();
        } else { // Do not skip the token
            if (operation.tokenRecognized(length, false)) { // Create preprocessed token
//                return new PreprocessedTextToken<T>(id, operation.tokenLength());
                return new DefaultToken<T>(id, operation.tokenLength());
            } else {
                return new DefaultToken<T>(id, operation.tokenLength());
            }
        }
    }

    /**
     * Create regular token instance with an explicit length and part type.
     * <br/>
     * This is suitable e.g. for unfinished block comment when a COMMENT token
     * and PartType.START arguments would be used.
     *
     * @param id non-null token id recognized by the lexer.
     * @param length >=0 length of the token to be created. The length must not

     *  because of token id filter.
     */
    public Token<T> createToken(T id, int length, PartType partType) {
        return operation.createToken(id, length, partType);
        if (partType == PartType.COMPLETE)
            return createToken(id, length);

        if (isSkipToken(id)) {
            operation.tokenRecognized(length, true);
            return skipToken();
        } else { // Do not skip the token
            if (operation.tokenRecognized(length, false)) { // Create preprocessed token
//                return new ComplexToken<T>(id, operation.tokenLength(), null, partType, null);
                return new PropertyToken<T>(id, operation.tokenLength(), null, partType);
            } else {
                return new PropertyToken<T>(id, operation.tokenLength(), null, partType);
            }
        }
    }

    /**

     *  because of token id filter.
     */
    public Token<T> getFlyweightToken(T id, String text) {
        return operation.getFlyweightToken(id, text);
        // Compare each recognized char with the corresponding char in text
        if (testing) {
            for (int i = 0; i < text.length(); i++) {
                if (text.charAt(i) != operation.readExisting(i)) {
                    throw new IllegalArgumentException("Flyweight text in " + // NOI18N
                            "TokenFactory.getFlyweightToken(" + id + ", \"" + // NOI18N
                            CharSequenceUtilities.debugText(text) + "\") " + // NOI18N
                            "differs from recognized text: '" + // NOI18N
                            CharSequenceUtilities.debugChar(operation.readExisting(i)) +
                            "' != '" + CharSequenceUtilities.debugChar(text.charAt(i)) + // NOI18N
                            "' at index=" + i // NOI18N
                    );
                }
            }
        }

        // Check whether token with given id should be created
        if (isSkipToken(id)) {
            operation.tokenRecognized(text.length(), true);
            return skipToken();
        } else { // Do not skip the token
            if (operation.tokenRecognized(text.length(), false)) { // Create preprocessed token
//                return new PreprocessedTextToken<T>(id, operation.tokenLength());
                return new DefaultToken<T>(id, operation.tokenLength());
            } else if (operation.isFlyTokenAllowed()) {
                LanguageOperation<T> langOp = operation.languageOperation();
                return langOp.getFlyweightToken(id, text);
            } else { // return non-flyweight token
                return new DefaultToken<T>(id, operation.tokenLength());
            }
        }
    }
    
    /**
     * Create complete token with properties.
     *
     * @param id non-null token id.
     * @param length >=0 length of the token to be created. The length must not
     *  exceed the number of characters read from the lexer input.
     * @param propertyProvider token property provider or null if there are no extra properties.
     * @param partType whether this token is complete or just a part of complete token.
     *  See {@link TokenPropertyProvider} for examples how this parameter may be used.
     * @return non-null property token instance.
     *  <br/>

     *  if tokens for the given token id should be skipped
     *  because of token id filter.
     */
    public Token<T> createPropertyToken(T id, int length, TokenPropertyProvider<T> propertyProvider) {
        return operation.createPropertyToken(id, length, propertyProvider, PartType.COMPLETE);
        checkPartTypeNonNull(partType);
        if (isSkipToken(id)) {
            operation.tokenRecognized(length, true);
            return skipToken();
        } else { // Do not skip the token
            if (operation.tokenRecognized(length, false)) { // Create preprocessed token
//                return new ComplexToken<T>(id, operation.tokenLength(),
//                    propertyProvider, null, partType);
                return new PropertyToken<T>(id, operation.tokenLength(),
                    propertyProvider, partType);
            } else {
                return new PropertyToken<T>(id, operation.tokenLength(),
                    propertyProvider, partType);
            }
        }
    }

    /**
     * Create token with properties.
     *
     * @param id non-null token id.
     * @param length >=0 length of the token to be created. The length must not
     *  exceed the number of characters read from the lexer input.
     * @param propertyProvider token property provider or null if there are no extra properties.
     *  See {@link TokenPropertyProvider} for examples how this parameter may be used.
     * @param partType whether this token is complete or just a part of complete token.
     *  Null may be passed which implies {@link PartType#COMPLETE}.
     * @return non-null property token instance.
     *  <br/>
     *  {@link #SKIP_TOKEN} will be returned
     *  if tokens for the given token id should be skipped
     *  because of token id filter.
     */
    public Token<T> createPropertyToken(T id, int length,
    TokenPropertyProvider<T> propertyProvider, PartType partType) {
        return operation.createPropertyToken(id, length, propertyProvider, partType);
    }

    /**
     * Create token with a custom text that possibly differs in length and content
     * from the text represented by the token in the input text.
     * <br/>
     * <b>Note: This method should not be used. It is planned to be removed completely.</b>
     * The custom text tokens no longer
     * save space by not refrencing the original characters (when read e.g. from a Reader).
     * <br/>
     * Having token's text to always match the input's text is more systematic
     * and simplifies the lexer module's design.
     * <br/>
     * Therefore the only benefit of custom text tokens would be if certain tools
     * e.g. parsers would require a different text than the one present naturally
     * in the token. In such case the token should have a property
     * (the key can be e.g. a CharSequence.class) that will return a char sequence
     * with the desired text. If the text is a sub sequence of original token's text
     * the token property provider can even be made flyweight:
     * <pre>
     * StripFirstAndLastCharTokenPropertyProvider implements TokenPropertyProvider {
     *     public TokenPropertyProvider INSTANCE = new StripFirstAndLastCharTokenPropertyProvider();
     *     public Object getValue(Token token, Object key) {
     *         if (key == CharSequence.class) {
     *             return token.text().subSequence(1, token.length() - 1);
     *         }
     *         return null;
     *     }
     * }
     * </pre>
     * 
     * <p>
     * </p>
     * 
     * @param id non-null token id of the token being created.
     * @param text non-null custom text assigned to the token.
     * @param length recognized characters corresponding to the token being created.
     * @param partType should always be null otherwise this method would throw
     *  an exception.
     * @deprecated This method is deprecated without replacement - see description
     *  how a similar effect can be obtained.
     */
    public Token<T> createCustomTextToken(T id, CharSequence text, int length, PartType partType) {
        if (partType != null) {
            throw new IllegalArgumentException("This method is deprecated and it should" + 
                    " only be used with partType==null (see its javadoc).");
            return skipToken();
        } else { // Do not skip the token
            if (operation.tokenRecognized(length, false)) { // Create preprocessed token
                return new CustomTextToken<T>(id, operation.tokenLength(), text, partType);
//                return new ComplexToken<T>(id, operation.tokenLength(), null, text, partType);
            } else {
                return new CustomTextToken<T>(id, operation.tokenLength(), text, partType);
            }
        }
        return operation.createCustomTextToken(id, length, text);
    
    private boolean isSkipToken(T id) {
        Set<? extends TokenId> skipTokenIds = operation.skipTokenIds();
        return (skipTokenIds != null) && skipTokenIds.contains(id);
    }

    /**
     * Check whether a token (produced by one of the token creation methods)
     * is a special flyweight token used in cases
     * when there is an active filtering of certain token ids (e.g. comments and whitespace)
     * and the just recognized token-id should be skipped.
     *
     * @param token non-null token.
     * @return true if the token is a skip-token.
     */
    public boolean isSkipToken(Token<T> token) {
        return token == SKIP_TOKEN;
    }

    private void checkPartTypeNonNull(PartType partType) {
        if (partType == null)
            throw new IllegalArgumentException("partType must be non-null");
    }
    
}

Lines 83-89 Link Here

(-)a/lexer/src/org/netbeans/spi/lexer/TokenPropertyProvider.java (-2 / +2 lines)
83	* this.value = value;	83	* this.value = value;
84	* }	84	* }
85	*	85	*
86	* public Object getValue(Token token, Object key) {	86	* public Object getValue(Token<T> token, Object key) {
87	* if ("key".equals(key)) {	87	* if ("key".equals(key)) {
88	* return value;	88	* return value;
89	* }	89	* }
Lines 109-114 Link Here
109	* @param key non-null key for which the value should be retrieved.	109	* @param key non-null key for which the value should be retrieved.
110	* @return value of the property or null if there is no value for the given key.	110	* @return value of the property or null if there is no value for the given key.
111	*/	111	*/
112	Object getValue(Token token, Object key);	112	Object getValue(Token<T> token, Object key);
113		113
114	}	114	}





package org.netbeans.api.lexer;

import java.util.List;
import org.netbeans.junit.NbTestCase;

/**

        public PartType partType() {
            return null;
        }

        public boolean isRemoved() {
            return false;
        }

        public Token<T> joinToken() {
            return null;
        }

        public List<? extends Token<T>> joinedParts() {
            return null;
        }
        
    }


Lines 53-61 Link Here

(-)a/lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java (-6 / +27 lines)
53	import org.netbeans.lib.lexer.TokenList;	53	import org.netbeans.lib.lexer.TokenList;
54	import org.netbeans.lib.lexer.test.LexerTestUtilities;	54	import org.netbeans.lib.lexer.test.LexerTestUtilities;
55	import org.netbeans.lib.lexer.test.ModificationTextDocument;	55	import org.netbeans.lib.lexer.test.ModificationTextDocument;
56	import org.netbeans.lib.lexer.test.simple.*;
57	import org.netbeans.lib.lexer.token.DefaultToken;	56	import org.netbeans.lib.lexer.token.DefaultToken;
58	import org.netbeans.lib.lexer.token.TextToken;	57	import org.netbeans.lib.lexer.token.TextToken;
		58	import org.netbeans.lib.lexer.token.TokenLength;
59		59
60	/**	60	/**
61	* Test methods of token sequence.	61	* Test methods of token sequence.
Lines 403-409 Link Here
403	}	403	}
404		404
405	public void testTokenSize() {	405	public void testTokenSize() {
406	String text = "abc+";	406	String text = "abc+def";
407	TokenHierarchy<?> hi = TokenHierarchy.create(text,TestTokenId.language());	407	TokenHierarchy<?> hi = TokenHierarchy.create(text,TestTokenId.language());
408	TokenSequence<?> ts = hi.tokenSequence();	408	TokenSequence<?> ts = hi.tokenSequence();
409		409
Lines 411-428 Link Here
411	LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "abc", 0);	411	LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "abc", 0);
412	assertTrue(ts.moveNext());	412	assertTrue(ts.moveNext());
413	LexerTestUtilities.assertTokenEquals(ts,TestTokenId.PLUS, "+", 3);	413	LexerTestUtilities.assertTokenEquals(ts,TestTokenId.PLUS, "+", 3);
		414	assertTrue(ts.moveNext());
		415	LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "def", 4);
414	assertFalse(ts.moveNext());	416	assertFalse(ts.moveNext());
415		417
416	TokenList tokenList = LexerTestUtilities.tokenList(ts);	418	TokenList tokenList = LexerTestUtilities.tokenList(ts);
417	ts.moveIndex(0); // move before "abc"	419	ts.moveIndex(0); // move before "abc"
418	assertTrue(ts.moveNext());	420	assertTrue(ts.moveNext());
419	// Test DefaultToken size	421	// Test DefaultToken size
420	assertSame(DefaultToken.class, ts.token().getClass());	422	Token<?> token = ts.token();
421	assertSize("Token instance too big", Collections.singletonList(ts.token()), 24,new Object[] { tokenList,TestTokenId.IDENTIFIER});	423	// Exclude TokenLength since it should be cached - verify later
		424	TokenLength cachedTokenLength = TokenLength.get(token.length());
		425	assertSame(DefaultToken.class, token.getClass());
		426	assertSize("Token instance too big", Collections.singletonList(token), 24,
		427	new Object[] { tokenList, TestTokenId.IDENTIFIER, cachedTokenLength });
		428
		429	// Check that TokenLength is cached for small tokens
		430	assertSame("TokenLength instances not cached for small tokens",
		431	cachedTokenLength, TokenLength.get(token.length()));
		432
422	// Test TextToken size	433	// Test TextToken size
423	assertTrue(ts.moveNext());	434	assertTrue(ts.moveNext());
424	assertSame(TextToken.class, ts.token().getClass());	435	token = ts.token();
425	assertSize("Token instance too big", Collections.singletonList(ts.token()), 24,new Object[] { tokenList,TestTokenId.PLUS, "+"});	436	assertSame(TextToken.class, token.getClass());
		437	assertSize("Token instance too big", Collections.singletonList(token), 24,
		438	new Object[] { tokenList, TestTokenId.PLUS, "+" });
		439
		440	// Test DefaultToken size
		441	assertTrue(ts.moveNext());
		442	token = ts.token();
		443	assertSame(DefaultToken.class, token.getClass());
		444	// Verify that the TokenLength is cached for small tokens - use tokenLength3 directly
		445	assertSize("Token instance too big", Collections.singletonList(token), 24,
		446	new Object[] { tokenList, TestTokenId.IDENTIFIER, cachedTokenLength });
426	}	447	}
427		448
428	public void testSubSequenceInUnfinishedTH() throws Exception {	449	public void testSubSequenceInUnfinishedTH() throws Exception {




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */
package org.netbeans.lib.lexer;

import java.util.List;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenSequence;
import org.netbeans.junit.NbTestCase;
import org.netbeans.lib.lexer.lang.TestJoinSectionsTextTokenId;
import org.netbeans.lib.lexer.test.LexerTestUtilities;
import org.netbeans.lib.lexer.lang.TestJoinSectionsTopTokenId;
import org.netbeans.lib.lexer.test.ModificationTextDocument;

/**
 * Test embedded sections that should be lexed together.
 *
 * @author Miloslav Metelka
 */
public class JoinSectionsTest extends NbTestCase {
    
    public JoinSectionsTest(String testName) {
        super(testName);
    }

    protected void setUp() throws Exception {
    }

    public void testJoinSections() throws Exception {
        // Turn on detailed checking
//        Logger.getLogger(TokenHierarchyOperation.class.getName()).setLevel(Level.FINEST);

        //             000000000011111111112222222222
        //             012345678901234567890123456789
        String text = "a{b<cd>e}f<gh>i{j<kl>m}n";
        ModificationTextDocument doc = new ModificationTextDocument();
        doc.insertString(0, text, null);
        doc.putProperty(Language.class,TestJoinSectionsTopTokenId.language());
        
        TokenHierarchy<?> hi = TokenHierarchy.get(doc);
        TokenSequence<?> ts = hi.tokenSequence();
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "a{b", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<cd>", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "e}f", -1);
        
        // Get embedded tokens within TEXT tokens. There should be "a" then BRACES start "{b" then BRACES end "e}|" then "f"
        LanguagePath innerLP = LanguagePath.get(TestJoinSectionsTopTokenId.language()).
                embedded(TestJoinSectionsTextTokenId.language());
        List<TokenSequence<?>> tsList = hi.tokenSequenceList(innerLP, 0, Integer.MAX_VALUE);
        checkInitialTokens(tsList);
        
        
        // Use iterator for fetching token sequences
        int i = 0;
        for (TokenSequence<?> ts2 : tsList) {
            assertSame(ts2, tsList.get(i++));
        }

        LexerTestUtilities.assertConsistency(hi);
        
        // Check tokenSequenceList() with explicit offsets
        // Check correct TSs bounds
        tsList = hi.tokenSequenceList(innerLP, 0, 7);
        assertEquals(1, tsList.size());
        tsList = hi.tokenSequenceList(innerLP, 0, 8);
        assertEquals(2, tsList.size());
        
        
        // Do modifications
        // Remove second closing brace '}'
        doc.remove(8, 1);
        LexerTestUtilities.assertConsistency(hi);
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        // before:    "a{b<cd>e}f<gh>i{j<kl>m}n";
        // after:     "a{b<cd>ef<gh>i{j<kl>m}n";
        tsList = hi.tokenSequenceList(innerLP, 0, Integer.MAX_VALUE);
        assertEquals(4, tsList.size()); // 2 sections

        // 1.section "a{b"
        ts = tsList.get(0);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "a", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "{b", -1);
        Token<?> token = ts.token();
        assertEquals(PartType.START, token.partType());
        assertFalse(ts.moveNext());
        
        // 2.section "ef"
        ts = tsList.get(1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "ef", -1);
        token = ts.token();
        assertEquals(PartType.MIDDLE, token.partType());
        assertFalse(ts.moveNext());
        
        // 3.section "i{j"
        ts = tsList.get(2);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "i{j", -1);
        token = ts.token();
        assertEquals(PartType.MIDDLE, token.partType());
        assertFalse(ts.moveNext());
        
        // 4.section "m}n"
        ts = tsList.get(3);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "m}", -1);
        token = ts.token();
        assertEquals(PartType.END, token.partType());
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "n", -1);
        assertFalse(ts.moveNext());
        

        // Re-add second closing brace '}'
        doc.insertString(8, "}", null);
        LexerTestUtilities.assertConsistency(hi);
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        // before:    "a{b<cd>ef<gh>i{j<kl>m}n";
        // after:     "a{b<cd>e}f<gh>i{j<kl>m}n";
        tsList = hi.tokenSequenceList(innerLP, 0, Integer.MAX_VALUE);
        checkInitialTokens(tsList);
        
        doc.remove(0, doc.getLength());
        LexerTestUtilities.assertConsistency(hi);
        ts = hi.tokenSequence();
        assertFalse(ts.moveNext());
        doc.insertString(0, text, null);

    }

    private void checkInitialTokens(List<TokenSequence<?>> tsList) {
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        // text:      "a{b<cd>e}f<gh>i{j<kl>m}n";
        assertEquals(4, tsList.size()); // 4 sections

        // 1.section
        TokenSequence<?> ts = tsList.get(0);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "a", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "{b", -1);
        Token<?> token = ts.token();
        assertEquals(PartType.START, token.partType());
        assertFalse(ts.moveNext());
        
        // 2.section
        ts = tsList.get(1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "e}", -1);
        token = ts.token();
        assertEquals(PartType.END, token.partType());
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "f", -1);
        assertFalse(ts.moveNext());
        
        // 3.section
        ts = tsList.get(2);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "i", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "{j", -1);
        token = ts.token();
        assertEquals(PartType.START, token.partType());
        assertFalse(ts.moveNext());
        
        // 4.section
        ts = tsList.get(3);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "m}", -1);
        token = ts.token();
        assertEquals(PartType.END, token.partType());
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "n", -1);
        assertFalse(ts.moveNext());
    }

}

Lines 41-47 Link Here

(-)a/lexer/test/unit/src/org/netbeans/lib/lexer/lang/TestJavadocLexer.java (-1 lines)
41		41
42	package org.netbeans.lib.lexer.lang;	42	package org.netbeans.lib.lexer.lang;
43		43
44	import org.netbeans.lib.lexer.lang.TestJavadocTokenId;
45	import org.netbeans.api.lexer.Token;	44	import org.netbeans.api.lexer.Token;
46	import org.netbeans.spi.lexer.Lexer;	45	import org.netbeans.spi.lexer.Lexer;
47	import org.netbeans.spi.lexer.LexerInput;	46	import org.netbeans.spi.lexer.LexerInput;

Lines 42-48 Link Here

(-)a/lexer/test/unit/src/org/netbeans/lib/lexer/lang/TestJoinSectionsTextLexer.java (-1 lines)
42	package org.netbeans.lib.lexer.lang;	42	package org.netbeans.lib.lexer.lang;
43		43
44	import org.netbeans.api.lexer.PartType;	44	import org.netbeans.api.lexer.PartType;
45	import org.netbeans.lib.lexer.lang.TestJoinSectionsTextTokenId;
46	import org.netbeans.api.lexer.Token;	45	import org.netbeans.api.lexer.Token;
47	import org.netbeans.spi.lexer.Lexer;	46	import org.netbeans.spi.lexer.Lexer;
48	import org.netbeans.spi.lexer.LexerInput;	47	import org.netbeans.spi.lexer.LexerInput;

Lines 60-67 Link Here

(-)a/lexer/test/unit/src/org/netbeans/lib/lexer/lang/TestJoinSectionsTextTokenId.java (-2 / +2 lines)
60	*/	60	*/
61	public enum TestJoinSectionsTextTokenId implements TokenId {	61	public enum TestJoinSectionsTextTokenId implements TokenId {
62		62
63	TEXT(), // Text except of text within braces	63	BRACES(), // "{...}" i.e. text within braces
64	BRACES(); // "{ ... }" i.e. text within braces	64	TEXT(); // Text except of text within braces
65		65
66	private TestJoinSectionsTextTokenId() {	66	private TestJoinSectionsTextTokenId() {
67	}	67	}

Lines 60-67 Link Here

(-)a/lexer/test/unit/src/org/netbeans/lib/lexer/lang/TestJoinSectionsTopTokenId.java (-2 / +2 lines)
60	*/	60	*/
61	public enum TestJoinSectionsTopTokenId implements TokenId {	61	public enum TestJoinSectionsTopTokenId implements TokenId {
62		62
63	TEXT(),	63	TAG(), // Text enclosed in <..> including '<' and '>'
64	TAG();	64	TEXT(); // Any text not enclosed in <...>
65		65
66	private TestJoinSectionsTopTokenId() {	66	private TestJoinSectionsTopTokenId() {
67	}	67	}




    }
    
    /**
     * @see #assertTokenSequencesEqual(String,TokenSequence,TokenHierarchy,TokenSequence,TokenHierarchy,boolean)
     */
    public static void assertTokenSequencesEqual(
    TokenSequence<?> expected, TokenHierarchy<?> expectedHi,
    TokenSequence<?> actual, TokenHierarchy<?> actualHi,
    boolean testLookaheadAndState) {
        assertTokenSequencesEqual(null, expected, expectedHi, actual, actualHi, testLookaheadAndState);
    }

    /**
     * Compare contents of the given token sequences by moving through all their
     * tokens.
     * <br/>

    public static void assertTokenSequencesEqual(String message,
    TokenSequence<?> expected, TokenHierarchy<?> expectedHi,
    TokenSequence<?> actual, TokenHierarchy<?> actualHi,
    boolean testLookaheadAndState, boolean dumpWholeHi) {
        String prefix = messagePrefix(message);
        TestCase.assertEquals(prefix + "Move previous: ", expected.movePrevious(), actual.movePrevious());
        int i = 0;
        while (expected.moveNext()) {
            String prefixI = prefix + "->[" + i + "]";
            TestCase.assertTrue(prefixI + ": Cannot moveNext() in test token sequence", actual.moveNext());
            assertTokensEqual(prefixI, expected, expectedHi, actual, actualHi, testLookaheadAndState);
            i++;
            TestCase.assertFalse(prefix + "Move next not disabled", actual.moveNext());
            success = true;
        } finally {
            if (!success) {
                System.err.println("Expected token sequence dump:\n" + expected);
                System.err.println("Test token sequence dump:\n" + actual);
            }
        }
        TestCase.assertFalse(prefix + "moveNext() possible at end of test token sequence", actual.moveNext());
    }

    private static void assertTokensEqual(String message,

    }
    
    public static void incCheck(Document doc, boolean nested) {
        TokenHierarchy<?> incHi = TokenHierarchy.get(doc);
        assertConsistency(incHi);

        Language<?> language = (Language<?>)
                doc.getProperty(Language.class);
        String docText = null;

            e.printStackTrace();
            TestCase.fail("BadLocationException occurred");
        }
        TokenHierarchy<?> batchHi = TokenHierarchy.create(docText, language);
        TokenSequence<?> batchTS = batchHi.tokenSequence();
        TokenSequence<?> incTS = incHi.tokenSequence();
        try {
            // Compare lookaheads and states as well
            assertTokenSequencesEqual("TOP", batchTS, batchHi, incTS, incHi, true, false);
        } catch (Throwable t) {
            // Go forward two tokens to have an extra tokens context
            batchTS.moveNext();
            batchTS.moveNext();
            StringBuilder sb = new StringBuilder(512);
            sb.append("BATCH token sequence dump:\n").append(batchTS);
            sb.append("\n\nTEST token sequence dump:\n").append(incTS);
            TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);
            if (lastHi != null) {
//                    System.err.println("PREVIOUS batch token sequence dump:\n" + lastHi.tokenSequence());
            }
            throw new IllegalStateException(sb.toString(), t);
        }
        
        if (nested) {
            batchTS.moveStart();
            incTS.moveStart();
            try {
                incCheckNested("TOP", doc, batchTS, batchHi, incTS, incHi);
            } catch (Throwable t) { // Re-throw with hierarchy info
                StringBuilder sb = new StringBuilder(512);
                sb.append("BATCH token hierarchy:\n").append(batchHi);
                sb.append("\n\n\nTEST token hierarchy:\n").append(incHi);
                throw new IllegalStateException(sb.toString(), t);
            }
        }

        // Check the change since last modification
        TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);
        if (lastHi != null) {
            // TODO comparison
        }
        doc.putProperty(LAST_TOKEN_HIERARCHY, batchHi); // new last batch token hierarchy
    }

    public static void incCheckNested(String message, Document doc,
            TokenSequence<?> batch, TokenHierarchy<?> batchTH,
            TokenSequence<?> inc, TokenHierarchy<?> incTH
    ) {
        int i = 0;
        while (inc.moveNext()) {
            TestCase.assertTrue("No more tokens in batch token sequence", batch.moveNext());
            TokenSequence<?> batchE = batch.embedded();
            TokenSequence<?> incE = inc.embedded();
            String messageE = message + "->[" + i + "]";
            if (incE != null) {
                TestCase.assertNotNull("Inc embedded sequence is null", batchE);
                assertTokenSequencesEqual(messageE, batchE, batchTH, incE, incTH, true, true);

                incCheckNested(messageE, doc, batchE, batchTH, incE, incTH);
            } else { // Inc embedded is null
                TestCase.assertNull("Batch embedded sequence non-null", batchE);
            }
            i++;
        }
    }

    /**
     * Get lookahead for the token to which the token sequence is positioned.
     * <br/>

        return tokenList(ts).lookahead(ts.index());
    }

        /**
     * Get state for the token to which the token sequence is positioned.
     * <br/>
     * The method uses reflection to get reference to tokenList field in token sequence.




import javax.swing.text.Document;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.CharSequenceUtilities;



 * @author mmetelka
 */
public class TestRandomModify {
    
    private final Class hostClass;

    private boolean debugOperation;


    
    private List<SnapshotDescription> snapshots = new ArrayList<SnapshotDescription>();

    public TestRandomModify(Class hostClass) {
        this(0, hostClass);
    }
    
    public TestRandomModify(long seed, Class hostClass) {
        this.hostClass = hostClass;
        this.doc = new javax.swing.text.PlainDocument();

        this.random = new Random();
        if (seed == 0) { // Use currentTimeMillis() (btw nanoTime() in 1.5 instead)
            seed = System.currentTimeMillis();
        }
        System.err.println(hostClass.getName() + " with SEED=" + seed + "L");
        random.setSeed(seed);
    }
    

    public void insertText(int offset, String text) throws Exception {
        if (text.length() > 0) {
            if (isDebugOperation()) {
                int beforeTextStartOffset = Math.max(offset - 5, 0);
                String beforeText = document().getText(beforeTextStartOffset, offset - beforeTextStartOffset);
                int afterTextEndOffset = Math.min(offset + 5, document().getLength());
                String afterText = doc.getText(offset, afterTextEndOffset - offset);
                System.err.println(opIdString() + " INSERT(" + offset +
                        ", " + text.length() +"): \"" +
                        CharSequenceUtilities.debugText(text) +"\" text-around: \"" +
                        CharSequenceUtilities.debugText(beforeText) + '|' + 
                        CharSequenceUtilities.debugText(afterText) + "\""
                );
                if (isDebugDocumentText()) {
                    StringBuilder sb = new StringBuilder();

    
    public void clearDocument() throws Exception {
        doc.remove(0, doc.getLength());
        // Verify that there are no tokens
        LexerTestUtilities.incCheck(doc, false);
    }
    
    public final Language<?> language() {

                    System.err.println("Comparing snapshot " + i + " of " + snapshots.size());
                }
                // Check snapshot without comparing lookaheads and states
                LexerTestUtilities.assertTokenSequencesEqual(null, bm.tokenSequence(), bm,
                        s.tokenSequence(), s, false, false);
            }
        }
    }

Lines 58-64 Link Here

(-)a/lexer/test/unit/src/org/netbeans/lib/lexer/test/dump/TokenDumpCheck.java (-1 / +1 lines)
58	import org.netbeans.api.lexer.TokenUtilities;	58	import org.netbeans.api.lexer.TokenUtilities;
59	import org.netbeans.junit.NbTestCase;	59	import org.netbeans.junit.NbTestCase;
60	import org.netbeans.lib.editor.util.CharSequenceUtilities;	60	import org.netbeans.lib.editor.util.CharSequenceUtilities;
61	import org.netbeans.lib.lexer.batch.BatchTokenList;	61	import org.netbeans.lib.lexer.BatchTokenList;
62	import org.netbeans.lib.lexer.test.LexerTestUtilities;	62	import org.netbeans.lib.lexer.test.LexerTestUtilities;
63		63
64	/**	64	/**





import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.spi.lexer.Lexer;
import org.netbeans.spi.lexer.LexerInput;
import org.netbeans.spi.lexer.LexerRestartInfo;

                case EOF:
                    input.backup(1);
                    return tokenFactory.createPropertyToken(id, input.readLength(),
                        new UnicodeCharValueProvider<TokenDumpTokenId>(new Character(ch)), PartType.COMPLETE);
            }
        }
        input.backup(1);

    public void release() {
    }

    private static final class UnicodeCharValueProvider<T extends TokenId> implements TokenPropertyProvider<T> {

        private Character ch;

        UnicodeCharValueProvider(Character ch) {
            this.ch = ch;
        }

        public Object getValue(Token<T> token, Object key) {
            if (TokenDumpTokenId.UNICODE_CHAR_TOKEN_PROPERTY.equals(key))
                return ch;
            return null; // no non-tokenStore value
        }

    }

}

Lines 41-53 Link Here

(-)a/lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenListUpdaterTest.java (-2 / +20 lines)
41		41
42	package org.netbeans.lib.lexer.test.inc;	42	package org.netbeans.lib.lexer.test.inc;
43		43
		44	import java.io.PrintStream;
44	import java.util.ConcurrentModificationException;	45	import java.util.ConcurrentModificationException;
		46	import java.util.logging.Level;
		47	import java.util.logging.Logger;
45	import javax.swing.text.Document;	48	import javax.swing.text.Document;
46	import junit.framework.TestCase;	49	import junit.framework.TestCase;
47	import org.netbeans.api.lexer.Language;	50	import org.netbeans.api.lexer.Language;
48	import org.netbeans.api.lexer.TokenHierarchy;	51	import org.netbeans.api.lexer.TokenHierarchy;
49	import org.netbeans.api.lexer.TokenId;
50	import org.netbeans.api.lexer.TokenSequence;	52	import org.netbeans.api.lexer.TokenSequence;
		53	import org.netbeans.junit.NbTestCase;
51	import org.netbeans.lib.lexer.test.LexerTestUtilities;	54	import org.netbeans.lib.lexer.test.LexerTestUtilities;
52	import org.netbeans.lib.lexer.test.ModificationTextDocument;	55	import org.netbeans.lib.lexer.test.ModificationTextDocument;
53	import org.netbeans.lib.lexer.lang.TestTokenId;	56	import org.netbeans.lib.lexer.lang.TestTokenId;
Lines 57-63 Link Here
57	*	60	*
58	* @author mmetelka	61	* @author mmetelka
59	*/	62	*/
60	public class TokenListUpdaterTest extends TestCase {	63	public class TokenListUpdaterTest extends NbTestCase {
61		64
62	public TokenListUpdaterTest(String testName) {	65	public TokenListUpdaterTest(String testName) {
63	super(testName);	66	super(testName);
Lines 67-72 Link Here
67	}	70	}
68		71
69	protected void tearDown() throws java.lang.Exception {	72	protected void tearDown() throws java.lang.Exception {
		73	}
		74
		75	@Override
		76	public PrintStream getLog() {
		77	return System.out;
		78	// return super.getLog();
		79	}
		80
		81	@Override
		82	protected Level logLevel() {
		83	return Level.INFO;
		84	// return super.logLevel();;
70	}	85	}
71		86
72	public void testInsertUnfinishedLexing() throws Exception {	87	public void testInsertUnfinishedLexing() throws Exception {
Lines 176-182 Link Here
176	LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "a", 0);	191	LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "a", 0);
177		192
178	// Remove "b"	193	// Remove "b"
		194	// Logger.getLogger(org.netbeans.lib.lexer.inc.TokenListUpdater.class.getName()).setLevel(Level.FINE); // Extra logging
179	doc.remove(2, 1);	195	doc.remove(2, 1);
		196	// Logger.getLogger(org.netbeans.lib.lexer.inc.TokenListUpdater.class.getName()).setLevel(Level.WARNING); // End of extra logging
180	try {	197	try {
181	ts.moveNext();	198	ts.moveNext();
182	fail("Should not get there");	199	fail("Should not get there");
Lines 186-191 Link Here
186		203
187	ts = hi.tokenSequence();	204	ts = hi.tokenSequence();
188	assertTrue(ts.moveNext());	205	assertTrue(ts.moveNext());
		206	CharSequence tokenText = ts.token().text();
189	LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "a", 0);	207	LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "a", 0);
190	assertTrue(ts.moveNext());	208	assertTrue(ts.moveNext());
191	LexerTestUtilities.assertTokenEquals(ts,TestTokenId.PLUS, "+", 1);	209	LexerTestUtilities.assertTokenEquals(ts,TestTokenId.PLUS, "+", 1);




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */
package org.netbeans.lib.lexer.test.join;

import java.io.PrintStream;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenSequence;
import org.netbeans.junit.NbTestCase;
import org.netbeans.lib.lexer.lang.TestJoinSectionsTextTokenId;
import org.netbeans.lib.lexer.test.LexerTestUtilities;
import org.netbeans.lib.lexer.lang.TestJoinSectionsTopTokenId;
import org.netbeans.lib.lexer.test.ModificationTextDocument;

/**
 * Test embedded sections that should be lexed together.
 * 
 * <p>
 * Top lexer recognizes TestJoinSectionsTopTokenId.TAG  (text v zobacich)
 *                  and TestJoinSectionsTopTokenId.TEXT (everything else).
 *
 * TestJoinSectionsTopTokenId.TAG is branched into
 *                      TestJoinSectionsTextTokenId.BRACES "{...}"
 *                  and TestJoinSectionsTextTokenId.TEXT (everything else)
 * 
 * </p>
 * 
 *
 * @author Miloslav Metelka
 */
public class JoinSectionsMod1Test extends NbTestCase {
    
    public JoinSectionsMod1Test(String testName) {
        super(testName);
    }

    protected void setUp() throws Exception {
    }

    @Override
    public PrintStream getLog() {
        return System.out;
//        return super.getLog();
    }

    @Override
    protected Level logLevel() {
        return Level.INFO;
//        return super.logLevel();;
    }

    public void testShortDocMod() throws Exception {
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        String text = "xay<b>zc";
        ModificationTextDocument doc = new ModificationTextDocument();
        doc.insertString(0, text, null);
        doc.putProperty(Language.class, TestJoinSectionsTopTokenId.language());
        LexerTestUtilities.incCheck(doc, true); // Ensure the whole embedded hierarchy gets created

        Logger.getLogger(org.netbeans.lib.lexer.inc.TokenListUpdater.class.getName()).setLevel(Level.FINE); // Extra logging
      doc.remove(6, 1);
        LexerTestUtilities.incCheck(doc, true);
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        //     text = "xay<b>c";
        //                   \yz<uv>hk
      doc.insertString(6, "yz<uv>hk", null);
        LexerTestUtilities.incCheck(doc, true);
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        //     text = "xay<b>yz<uv>hkc";
      doc.remove(12, 3);
        LexerTestUtilities.incCheck(doc, true);
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        //     text = "xay<b>yz<uv>";
      doc.insertString(12, "hkc", null);
        LexerTestUtilities.incCheck(doc, true);
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        //     text = "xay<b>yz<uv>hkc";

    }

    public void testJoinSections() throws Exception {
        if (true)
            return;
        // Turn on detailed checking
//        Logger.getLogger(TokenHierarchyOperation.class.getName()).setLevel(Level.FINEST);

        //             000000000011111111112222222222
        //             012345678901234567890123456789
        String text = "a{b<cd>e}f<gh>i{j<kl>m}n";
        ModificationTextDocument doc = new ModificationTextDocument();
        doc.insertString(0, text, null);
        doc.putProperty(Language.class,TestJoinSectionsTopTokenId.language());
        
        TokenHierarchy<?> hi = TokenHierarchy.get(doc);
        TokenSequence<?> ts = hi.tokenSequence();
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "a{b", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<cd>", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "e}f", -1);
        
        // Get embedded tokens within TEXT tokens. There should be "a" then BRACES start "{b" then BRACES end "e}|" then "f"
        LanguagePath innerLP = LanguagePath.get(TestJoinSectionsTopTokenId.language()).
                embedded(TestJoinSectionsTextTokenId.language());
        List<TokenSequence<?>> tsList = hi.tokenSequenceList(innerLP, 0, Integer.MAX_VALUE);
        checkInitialTokens(tsList);
        
        
        // Use iterator for fetching token sequences
        int i = 0;
        for (TokenSequence<?> ts2 : tsList) {
            assertSame(ts2, tsList.get(i++));
        }

        LexerTestUtilities.assertConsistency(hi);
        
        // Check tokenSequenceList() with explicit offsets
        // Check correct TSs bounds
        tsList = hi.tokenSequenceList(innerLP, 0, 7);
        assertEquals(1, tsList.size());
        tsList = hi.tokenSequenceList(innerLP, 0, 8);
        assertEquals(2, tsList.size());
        
        
        // Do modifications
        // Remove second closing brace '}'

//        Logger.getLogger(org.netbeans.lib.lexer.inc.TokenListUpdater.class.getName()).setLevel(Level.FINE); // Extra logging
        doc.remove(8, 1);
//        Logger.getLogger(org.netbeans.lib.lexer.inc.TokenListUpdater.class.getName()).setLevel(Level.WARNING); // End of extra logging
        LexerTestUtilities.assertConsistency(hi);
        LexerTestUtilities.incCheck(doc, true);
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        // before:    "a{b<cd>e}f<gh>i{j<kl>m}n";
        // after:     "a{b<cd>ef<gh>i{j<kl>m}n";
        //             i0     i1    i2     i3
        tsList = hi.tokenSequenceList(innerLP, 0, Integer.MAX_VALUE);
        assertEquals(4, tsList.size()); // 2 sections

        // 1.section "a{b"
        ts = tsList.get(0);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "a", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "{b", -1);
        Token<?> token = ts.token();
        assertEquals(PartType.START, token.partType());
        assertFalse(ts.moveNext());
        
        // 2.section "ef"
        ts = tsList.get(1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "ef", -1);
        token = ts.token();
        assertEquals(PartType.MIDDLE, token.partType());
        assertFalse(ts.moveNext());
        
        // 3.section "i{j"
        ts = tsList.get(2);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "i{j", -1);
        token = ts.token();
        assertEquals(PartType.MIDDLE, token.partType());
        assertFalse(ts.moveNext());
        
        // 4.section "m}n"
        ts = tsList.get(3);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "m}", -1);
        token = ts.token();
        assertEquals(PartType.END, token.partType());
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "n", -1);
        assertFalse(ts.moveNext());
        

        // Re-add second closing brace '}'
        doc.insertString(8, "}", null);
        LexerTestUtilities.assertConsistency(hi);
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        // before:    "a{b<cd>ef<gh>i{j<kl>m}n";
        // after:     "a{b<cd>e}f<gh>i{j<kl>m}n";
        tsList = hi.tokenSequenceList(innerLP, 0, Integer.MAX_VALUE);
        checkInitialTokens(tsList);
        
        doc.remove(0, doc.getLength());
        LexerTestUtilities.assertConsistency(hi);
        ts = hi.tokenSequence();
        assertFalse(ts.moveNext());
        doc.insertString(0, text, null);

    }

    private void checkInitialTokens(List<TokenSequence<?>> tsList) {
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        // text:      "a{b<cd>e}f<gh>i{j<kl>m}n";
        assertEquals(4, tsList.size()); // 4 sections

        // 1.section
        TokenSequence<?> ts = tsList.get(0);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "a", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "{b", -1);
        Token<?> token = ts.token();
        assertEquals(PartType.START, token.partType());
        assertFalse(ts.moveNext());
        
        // 2.section
        ts = tsList.get(1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "e}", -1);
        token = ts.token();
        assertEquals(PartType.END, token.partType());
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "f", -1);
        assertFalse(ts.moveNext());
        
        // 3.section
        ts = tsList.get(2);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "i", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "{j", -1);
        token = ts.token();
        assertEquals(PartType.START, token.partType());
        assertFalse(ts.moveNext());
        
        // 4.section
        ts = tsList.get(3);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "m}", -1);
        token = ts.token();
        assertEquals(PartType.END, token.partType());
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "n", -1);
        assertFalse(ts.moveNext());
    }

}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.test.join;

import java.util.List;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.PartType;
import org.netbeans.lib.lexer.lang.TestJoinSectionsTopTokenId;
import org.netbeans.lib.lexer.lang.TestTokenId;
import org.netbeans.api.lexer.TokenChange;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenHierarchyEvent;
import org.netbeans.api.lexer.TokenHierarchyListener;
import org.netbeans.api.lexer.TokenSequence;
import org.netbeans.junit.NbTestCase;
import org.netbeans.lib.lexer.lang.TestJoinSectionsTextTokenId;
import org.netbeans.lib.lexer.test.LexerTestUtilities;
import org.netbeans.lib.lexer.lang.TestPlainTokenId;
import org.netbeans.lib.lexer.test.ModificationTextDocument;
import org.netbeans.spi.lexer.LanguageEmbedding;

/**
 * Test several simple lexer impls.
 *
 * <p>
 * Top lexer recognizes TestJoinSectionsTopTokenId.TAG  (text v zobacich)
 *                  and TestJoinSectionsTopTokenId.TEXT (everything else).
 *
 * TestJoinSectionsTopTokenId.TAG is branched into
 *                      TestJoinSectionsTextTokenId.BRACES "{...}"
 *                  and TestJoinSectionsTextTokenId.TEXT (everything else)
 * 
 * </p>
 *
 * @author mmetelka
 */
public class JoinSectionsMod2Test extends NbTestCase {
    
    public JoinSectionsMod2Test(String testName) {
        super(testName);
    }
    
    protected void setUp() throws java.lang.Exception {
    }

    protected void tearDown() throws java.lang.Exception {
    }

    public void testMove() throws Exception { // TokenSequence.move() and moveIndex()
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        String text = "a{b<cd>e}f<gh>i{}{j<kl>m}n<o>p<q>r";
        ModificationTextDocument doc = new ModificationTextDocument();
        doc.insertString(0, text, null);
        doc.putProperty(Language.class,TestJoinSectionsTopTokenId.language());
        
        TokenHierarchy<?> hi = TokenHierarchy.get(doc);
        TokenSequence<?> ts = hi.tokenSequence();
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "a{b", 0);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<cd>", 3);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "e}f", 7);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<gh>", 10);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "i{}{j", 14);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<kl>", 19);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "m}n", 23);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<o>", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "p", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<q>", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "r", -1);
        assertFalse(ts.moveNext());
        
        // Check regular TS.embedded()
        ts.moveStart();
        assertTrue(ts.moveNext()); // Over "a{b"
        TokenSequence<?> tse = ts.embedded();
        assertTrue(tse.moveNext());
        LexerTestUtilities.assertTokenEquals(tse,TestJoinSectionsTextTokenId.TEXT, "a", 0);
        assertTrue(tse.moveNext());
        LexerTestUtilities.assertTokenEquals(tse,TestJoinSectionsTextTokenId.BRACES, "{b", 1);
        assertEquals(tse.token().partType(), PartType.START);
        assertFalse(tse.moveNext());
        
        assertTrue(ts.moveNext()); // Over "<cd>"
        assertTrue(ts.moveNext()); // Over "e}f"
        TokenSequence<?> tse2 = ts.embedded();
        assertTrue(tse2.moveNext());
        LexerTestUtilities.assertTokenEquals(tse2,TestJoinSectionsTextTokenId.BRACES, "e}", 7);
        assertEquals(tse2.token().partType(), PartType.END);
        assertTrue(tse2.moveNext());
        LexerTestUtilities.assertTokenEquals(tse2,TestJoinSectionsTextTokenId.TEXT, "f", 9);
        assertEquals(tse2.token().partType(), PartType.START);
        assertFalse(tse2.moveNext());
        
        assertTrue(ts.moveNext()); // Over "<gh>"
        assertTrue(ts.moveNext()); // Over "i{}{j"
        TokenSequence<?> tse3 = ts.embedded();
        assertTrue(tse3.moveNext());
        LexerTestUtilities.assertTokenEquals(tse3,TestJoinSectionsTextTokenId.TEXT, "i", 14);
        assertEquals(tse3.token().partType(), PartType.END);
        assertTrue(tse3.moveNext());
        LexerTestUtilities.assertTokenEquals(tse3,TestJoinSectionsTextTokenId.BRACES, "{}", 15);
        assertEquals(tse3.token().partType(), PartType.COMPLETE);
        assertTrue(tse3.moveNext());
        LexerTestUtilities.assertTokenEquals(tse3,TestJoinSectionsTextTokenId.BRACES, "{j", 17);
        assertEquals(tse3.token().partType(), PartType.START);
        assertFalse(tse3.moveNext());


        // Check TS.embeddedJoin()
        TokenSequence<?> tsej = ts.embeddedJoined();
        assertEquals(2, tsej.index());
        assertTrue(tsej.moveNext());
        int o = tsej.offset();
        LexerTestUtilities.assertTokenEquals(tsej,TestJoinSectionsTextTokenId.TEXT, "fi", 9);
        assertEquals(9, tsej.token().offset(null)); // Assert also token.offset() besides TS.offset()
        assertEquals(tsej.token().partType(), PartType.COMPLETE);
        assertTrue(tsej.moveNext());
        LexerTestUtilities.assertTokenEquals(tsej,TestJoinSectionsTextTokenId.BRACES, "{}", 15);
        assertEquals(tsej.token().partType(), PartType.COMPLETE);
        assertTrue(tsej.moveNext());
        LexerTestUtilities.assertTokenEquals(tsej,TestJoinSectionsTextTokenId.BRACES, "{jm}", 17);
        assertEquals(tsej.token().partType(), PartType.COMPLETE);
//        assertFalse(tsej.moveNext());
        
    }
        
}




/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer.test.join;

import java.util.List;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.PartType;
import org.netbeans.lib.lexer.lang.TestJoinSectionsTopTokenId;
import org.netbeans.lib.lexer.lang.TestTokenId;
import org.netbeans.api.lexer.TokenChange;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenHierarchyEvent;
import org.netbeans.api.lexer.TokenHierarchyListener;
import org.netbeans.api.lexer.TokenSequence;
import org.netbeans.junit.NbTestCase;
import org.netbeans.lib.lexer.lang.TestJoinSectionsTextTokenId;
import org.netbeans.lib.lexer.test.LexerTestUtilities;
import org.netbeans.lib.lexer.lang.TestPlainTokenId;
import org.netbeans.lib.lexer.test.ModificationTextDocument;
import org.netbeans.spi.lexer.LanguageEmbedding;

/**
 * Test several simple lexer impls.
 *
 * <p>
 * Top lexer recognizes TestJoinSectionsTopTokenId.TAG  (text v zobacich)
 *                  and TestJoinSectionsTopTokenId.TEXT (everything else).
 *
 * TestJoinSectionsTopTokenId.TAG is branched into
 *                      TestJoinSectionsTextTokenId.BRACES "{...}"
 *                  and TestJoinSectionsTextTokenId.TEXT (everything else)
 * 
 * </p>
 *
 * @author mmetelka
 */
public class JoinSectionsPositioningTest extends NbTestCase {
    
    public JoinSectionsPositioningTest(String testName) {
        super(testName);
    }
    
    protected void setUp() throws java.lang.Exception {
    }

    protected void tearDown() throws java.lang.Exception {
    }

    private ModificationTextDocument initDocument() throws Exception {
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        String text = "a{b<cd>e}f<gh>i{}{j<kl>m}n<o>p<q>r";
        ModificationTextDocument doc = new ModificationTextDocument();
        doc.insertString(0, text, null);
        doc.putProperty(Language.class,TestJoinSectionsTopTokenId.language());
        return doc;
    }
    
    public void testTokensAndEmbeddings() throws Exception { // TokenSequence.move() and moveIndex()
        ModificationTextDocument doc = initDocument();
        TokenHierarchy<?> hi = TokenHierarchy.get(doc);
        TokenSequence<?> ts = hi.tokenSequence();
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "a{b", 0);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<cd>", 3);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "e}f", 7);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<gh>", 10);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "i{}{j", 14);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<kl>", 19);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "m}n", 23);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<o>", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "p", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<q>", -1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "r", -1);
        assertFalse(ts.moveNext());
        assertEquals(11, ts.tokenCount());
        
        // Check regular TS.embedded()
        ts.moveStart();
        assertTrue(ts.moveNext()); // Over "a{b"
        TokenSequence<?> tse = ts.embedded();
        assertTrue(tse.moveNext());
        LexerTestUtilities.assertTokenEquals(tse,TestJoinSectionsTextTokenId.TEXT, "a", 0);
        assertTrue(tse.moveNext());
        LexerTestUtilities.assertTokenEquals(tse,TestJoinSectionsTextTokenId.BRACES, "{b", 1);
        assertEquals(tse.token().partType(), PartType.START);
        assertFalse(tse.moveNext());
        
        assertTrue(ts.moveNext()); // Over "<cd>"
        assertTrue(ts.moveNext()); // Over "e}f"
        TokenSequence<?> tse2 = ts.embedded();
        assertTrue(tse2.moveNext());
        LexerTestUtilities.assertTokenEquals(tse2,TestJoinSectionsTextTokenId.BRACES, "e}", 7);
        assertEquals(tse2.token().partType(), PartType.END);
        assertTrue(tse2.moveNext());
        LexerTestUtilities.assertTokenEquals(tse2,TestJoinSectionsTextTokenId.TEXT, "f", 9);
        assertEquals(tse2.token().partType(), PartType.START);
        assertFalse(tse2.moveNext());
        
        assertTrue(ts.moveNext()); // Over "<gh>"
        assertTrue(ts.moveNext()); // Over "i{}{j"
        TokenSequence<?> tse3 = ts.embedded();
        assertTrue(tse3.moveNext());
        LexerTestUtilities.assertTokenEquals(tse3,TestJoinSectionsTextTokenId.TEXT, "i", 14);
        assertEquals(tse3.token().partType(), PartType.END);
        assertTrue(tse3.moveNext());
        LexerTestUtilities.assertTokenEquals(tse3,TestJoinSectionsTextTokenId.BRACES, "{}", 15);
        assertEquals(tse3.token().partType(), PartType.COMPLETE);
        assertTrue(tse3.moveNext());
        LexerTestUtilities.assertTokenEquals(tse3,TestJoinSectionsTextTokenId.BRACES, "{j", 17);
        assertEquals(tse3.token().partType(), PartType.START);
        assertFalse(tse3.moveNext());


        // Check TS.embeddedJoin()
        TokenSequence<?> tsej = ts.embeddedJoined();
        assertEquals(2, tsej.index());
        assertTrue(tsej.moveNext());
        LexerTestUtilities.assertTokenEquals(tsej,TestJoinSectionsTextTokenId.TEXT, "fi", 9);
        assertEquals(9, tsej.token().offset(null)); // Assert also token.offset() besides TS.offset()
        assertEquals(tsej.token().partType(), PartType.COMPLETE);
        assertTrue(tsej.moveNext());
        LexerTestUtilities.assertTokenEquals(tsej,TestJoinSectionsTextTokenId.BRACES, "{}", 15);
        assertEquals(tsej.token().partType(), PartType.COMPLETE);
        assertTrue(tsej.moveNext());
        LexerTestUtilities.assertTokenEquals(tsej,TestJoinSectionsTextTokenId.BRACES, "{jm}", 17);
        assertEquals(tsej.token().partType(), PartType.COMPLETE);

        tsej.moveStart();
        assertTrue(tsej.moveNext());
        LexerTestUtilities.assertTokenEquals(tsej,TestJoinSectionsTextTokenId.TEXT, "a", 0);
        assertEquals(0, tsej.token().offset(null)); // Assert also token.offset() besides TS.offset()
        assertEquals(tsej.token().partType(), PartType.COMPLETE);
        assertTrue(tsej.moveNext());
        LexerTestUtilities.assertTokenEquals(tsej,TestJoinSectionsTextTokenId.BRACES, "{be}", 1);
        assertEquals(1, tsej.token().offset(null)); // Assert also token.offset() besides TS.offset()
        assertEquals(tsej.token().partType(), PartType.COMPLETE);
//        assertFalse(tsej.moveNext());
        
    }

    public void testTSMove() throws Exception {
        ModificationTextDocument doc = initDocument();
        TokenHierarchy<?> hi = TokenHierarchy.get(doc);
        TokenSequence<?> ts = hi.tokenSequence();
        assertEquals(1, ts.move(8));
        ts.moveStart();
        assertTrue(ts.moveNext());
        
        // Test TS.move() on embeddedJoin()
        TokenSequence<?> tsej = ts.embeddedJoined();
        assertEquals(-3, tsej.move(-3)); // Token starts at offset == 1
        assertEquals(0, tsej.index());

        assertEquals(7, tsej.move(8)); // Token starts at offset == 1
        assertEquals(1, tsej.index());

        assertEquals(6, tsej.move(7)); // Token starts at offset == 1
        assertEquals(1, tsej.index());
        
        assertEquals(5, tsej.move(6)); // Token starts at offset == 1
        assertEquals(1, tsej.index());
        
        assertEquals(2, tsej.move(3)); // Token starts at offset == 1
        assertEquals(1, tsej.index());
        
        assertEquals(1, tsej.move(2)); // Token starts at offset == 1
        assertEquals(1, tsej.index());
        
        assertEquals(1, tsej.move(2)); // Token starts at offset == 1
        assertEquals(1, tsej.index());

        assertEquals(1, tsej.move(2)); // Token starts at offset == 1
        assertEquals(1, tsej.index());

        assertEquals(1, tsej.move(16)); // Token starts at offset == 15
        assertEquals(3, tsej.index());
        
        assertEquals(0, tsej.move(15)); // Token starts at offset == 15
        assertEquals(3, tsej.index());
        
        assertEquals(0, tsej.move(17)); // Token starts at offset == 15
        assertEquals(4, tsej.index());
        
    }

    public void testShortDoc() throws Exception {
        //             000000000011111111112222222222
        //             012345678901234567890123456789
        String text = "a<b>c";
        ModificationTextDocument doc = new ModificationTextDocument();
        doc.insertString(0, text, null);
        doc.putProperty(Language.class, TestJoinSectionsTopTokenId.language());
        TokenHierarchy<?> hi = TokenHierarchy.get(doc);
        TokenSequence<?> ts = hi.tokenSequence();
        assertTrue(ts.moveNext());
        ts.embedded(); // Creates JTL
    }

}

Lines 106-112 Link Here

(-)a/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/CustomEmbeddingTest.java (-2 / +2 lines)
106	assertEquals(0, etc.embeddedChangeCount());	106	assertEquals(0, etc.embeddedChangeCount());
107		107
108	// Test the contents of the embedded sequence	108	// Test the contents of the embedded sequence
109	TokenSequence<?> ets = ts.embedded();	109	TokenSequence<?> ets = ts.embedded(); // Over "// line comment"
110	assertTrue(ets.moveNext());	110	assertTrue(ets.moveNext());
111	LexerTestUtilities.assertTokenEquals(ets,TestTokenId.IDENTIFIER, "line", 18);	111	LexerTestUtilities.assertTokenEquals(ets,TestTokenId.IDENTIFIER, "line", 18);
112	assertTrue(ets.moveNext());	112	assertTrue(ets.moveNext());
Lines 158-164 Link Here
158		158
159		159
160	// Check token sequence list	160	// Check token sequence list
161	// Create custm embedding again	161	// Create custom embedding again
162	assertTrue(ts.createEmbedding(TestTokenId.language(), 2, 2));	162	assertTrue(ts.createEmbedding(TestTokenId.language(), 2, 2));
163	LanguagePath lpe = LanguagePath.get(TestTokenId.language()).embedded(TestTokenId.language());	163	LanguagePath lpe = LanguagePath.get(TestTokenId.language()).embedded(TestTokenId.language());
164	List<TokenSequence<?>> tsl = hi.tokenSequenceList(lpe, 0, Integer.MAX_VALUE);	164	List<TokenSequence<?>> tsl = hi.tokenSequenceList(lpe, 0, Integer.MAX_VALUE);

Lines 174-180 Link Here

(-)a/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java (-1 / +1 lines)
174	}	174	}
175	tm = System.currentTimeMillis() - tm;	175	tm = System.currentTimeMillis() - tm;
176	assertTrue("Timeout tm = " + tm + "msec", tm < 1000); // Should be fast	176	assertTrue("Timeout tm = " + tm + "msec", tm < 1000); // Should be fast
177	System.out.println("Lexed input " + text.length()	177	System.out.println("SimpleLexerBatchTest.testPerf(): Lexed input " + text.length()
178	+ " chars long and created " + cntr + " tokens in " + tm + " ms.");	178	+ " chars long and created " + cntr + " tokens in " + tm + " ms.");
179	}	179	}
180		180





package org.netbeans.lib.lexer.test.simple;

import java.io.PrintStream;
import java.util.logging.Level;
import org.netbeans.lib.lexer.lang.TestTokenId;
import java.util.ConcurrentModificationException;
import java.util.logging.Logger;
import javax.swing.text.Document;
import junit.framework.TestCase;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.api.lexer.TokenSequence;
import org.netbeans.junit.NbTestCase;
import org.netbeans.lib.lexer.test.LexerTestUtilities;
import org.netbeans.lib.lexer.test.ModificationTextDocument;


 *
 * @author mmetelka
 */
public class SimpleLexerIncTest extends NbTestCase {
    
    public SimpleLexerIncTest(String testName) {
        super(testName);
    }
    
    @Override
    protected void setUp() throws java.lang.Exception {
    }

    @Override
    protected void tearDown() throws java.lang.Exception {
    }

    @Override
    public PrintStream getLog() {
        return System.out;
//        return super.getLog();
    }

    @Override
    protected Level logLevel() {
        return Level.INFO;
//        return super.logLevel();;
    }

    public void test() throws Exception {

        doc.putProperty(Language.class,TestTokenId.language());
        TokenHierarchy<?> hi = TokenHierarchy.get(doc);
        assertNotNull("Null token hierarchy for document", hi);

        // Check insertion of text that produces token with LA=0
        doc.insertString(0, "+", null);
        LexerTestUtilities.incCheck(doc, false);
        doc.remove(0, doc.getLength());
        LexerTestUtilities.incCheck(doc, false);
        
        TokenSequence<?> ts = hi.tokenSequence();
        assertFalse(ts.moveNext());
        
        // Insert text into document
        String commentText = "/* test comment  */";
        //             0123456789
        String text = "abc+uv-xy +-+" + commentText + "def";
        int commentTextStartOffset = 13;
        doc.insertString(0, text, null);

        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "def", offset);
        assertFalse(ts.moveNext());

        LexerTestUtilities.incCheck(doc, false);
        
        // Check TokenSequence.move()

        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "abc", 0);

        relOffset = ts.move(5); // to "uv"
        assertEquals(relOffset, 1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "uv", 4);

        } catch (ConcurrentModificationException e) {
            // Expected exception
        }
        LexerTestUtilities.incCheck(doc, false);
        
        
        ts = hi.tokenSequence();
        assertTrue(ts.moveNext());

        
        // Remove added 'd' to become "abc" again
        doc.remove(2, 1); // should be "abc" again
        LexerTestUtilities.incCheck(doc, false);
        

        ts = hi.tokenSequence();
        assertTrue(ts.moveNext());

        
        // Now insert right at the end of first token - identifier with lookahead 1
        doc.insertString(3, "x", null); // should become "abcx"
        LexerTestUtilities.incCheck(doc, false);
        
        ts = hi.tokenSequence();
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "abcx", 0);
        LexerTestUtilities.incCheck(doc, false);

        doc.remove(3, 1); // return back to "abc"
        LexerTestUtilities.incCheck(doc, false);

        ts = hi.tokenSequence();
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "abc", 0);
        LexerTestUtilities.incCheck(doc, false);

        
        // Now insert right at the end of "+" token - operator with lookahead 1 (because of "+-+" operator)
        doc.insertString(4, "z", null); // should become "abc" "+" "zuv"
        LexerTestUtilities.incCheck(doc, false);
        
        ts = hi.tokenSequence();
        assertTrue(ts.moveNext());

        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "zuv", 4);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.MINUS, "-", 7);
        

        doc.remove(4, 1); // return back to "abc" "+" "uv"

        LexerTestUtilities.incCheck(doc, false);

        // Now insert right after "-" - operator with lookahead 0
        doc.insertString(7, "z", null);
        LexerTestUtilities.incCheck(doc, false);
        
        ts = hi.tokenSequence();
        assertTrue(ts.moveNext());

        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.MINUS, "-", 6);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "zxy", 7);
        LexerTestUtilities.incCheck(doc, false);

        doc.remove(7, 1); // return back to "abc" "+" "uv"

        LexerTestUtilities.incCheck(doc, false);

        // Now insert between "+-" and "+" in "+-+" - operator with lookahead 0

        doc.insertString(doc.getLength(), "-", null);
        LexerTestUtilities.incCheck(doc, false);
        // Insert again "-" at the end of the document (now lookahead of preceding is zero)
//        Logger.getLogger(org.netbeans.lib.lexer.inc.TokenListUpdater.class.getName()).setLevel(Level.FINE); // Extra logging
        doc.insertString(doc.getLength(), "-", null);
//        Logger.getLogger(org.netbeans.lib.lexer.inc.TokenListUpdater.class.getName()).setLevel(Level.WARNING); // End of extra logging
        LexerTestUtilities.incCheck(doc, false);
        // Insert again "+-+" at the end of the document (now lookahead of preceding is zero)
        doc.insertString(doc.getLength(), "+-+", null);

Lines 41-48 Link Here

(-)a/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerRandomTest.java (-19 / +66 lines)
41		41
42	package org.netbeans.lib.lexer.test.simple;	42	package org.netbeans.lib.lexer.test.simple;
43		43
		44	import java.io.PrintStream;
		45	import java.util.logging.Level;
		46	import java.util.logging.Logger;
		47	import org.netbeans.junit.NbTestCase;
44	import org.netbeans.lib.lexer.lang.TestTokenId;	48	import org.netbeans.lib.lexer.lang.TestTokenId;
45	import junit.framework.TestCase;
46	import org.netbeans.lib.lexer.test.FixedTextDescriptor;	49	import org.netbeans.lib.lexer.test.FixedTextDescriptor;
47	import org.netbeans.lib.lexer.test.LexerTestUtilities;	50	import org.netbeans.lib.lexer.test.LexerTestUtilities;
48	import org.netbeans.lib.lexer.test.RandomCharDescriptor;	51	import org.netbeans.lib.lexer.test.RandomCharDescriptor;
Lines 55-61 Link Here
55	*	58	*
56	* @author mmetelka	59	* @author mmetelka
57	*/	60	*/
58	public class SimpleLexerRandomTest extends TestCase {	61	public class SimpleLexerRandomTest extends NbTestCase {
59		62
60	public SimpleLexerRandomTest(String testName) {	63	public SimpleLexerRandomTest(String testName) {
61	super(testName);	64	super(testName);
Lines 69-74 Link Here
69	protected void tearDown() throws java.lang.Exception {	72	protected void tearDown() throws java.lang.Exception {
70	}	73	}
71		74
		75	@Override
		76	public PrintStream getLog() {
		77	return System.out;
		78	// return super.getLog();
		79	}
		80
		81	@Override
		82	protected Level logLevel() {
		83	return Level.INFO;
		84	// return super.logLevel();;
		85	}
		86
72	public void testRandom() throws Exception {	87	public void testRandom() throws Exception {
73	test(0);	88	test(0);
74	}	89	}
Lines 78-84 Link Here
78	}	93	}
79		94
80	private void test(long seed) throws Exception {	95	private void test(long seed) throws Exception {
81	TestRandomModify randomModify = new TestRandomModify(seed);	96	TestRandomModify randomModify = new TestRandomModify(seed, this.getClass());
82	randomModify.setLanguage(TestTokenId.language());	97	randomModify.setLanguage(TestTokenId.language());
83		98
84	//randomModify.setDebugOperation(true);	99	//randomModify.setDebugOperation(true);
Lines 87-114 Link Here
87		102
88	// Check for incorrect lookahead counting problem	103	// Check for incorrect lookahead counting problem
89	// after one of the larger updates of the LexerInputOperation's code	104	// after one of the larger updates of the LexerInputOperation's code
90	randomModify.insertText(0, "+--+"); // "+"[2]; "-"[0]; "-"[0]; "+"[1];	105	randomModify.insertText(0, "+--+"); // "+"[2]; "-"[1]; "-"[0]; "+"[1];
91	randomModify.removeText(2, 1); // "+-+": "+-+"[0];	106	randomModify.removeText(2, 1); // "+-+": "+-+"[0];
92
93	randomModify.clearDocument();	107	randomModify.clearDocument();
94		108
95	// Check that token list updater respects that the lookaheads	109	// Check for error with querying laState.lookahead(-1) after implementing lexer input
		110	// Logger.getLogger(org.netbeans.lib.lexer.inc.TokenListUpdater.class.getName()).setLevel(Level.FINE); // Extra logging
		111	randomModify.insertText(0, "--");
		112	randomModify.insertText(0, "-");
		113	// Logger.getLogger(org.netbeans.lib.lexer.inc.TokenListUpdater.class.getName()).setLevel(Level.WARNING); // End of extra logging
		114	randomModify.clearDocument();
		115
		116	// Check for incorrect backward elimination of extra relexed tokens.
		117	// This required to establish lowestMatchIndex in TokenListUpdater.relex().
		118	randomModify.insertText(0, "+ +");
		119	randomModify.insertText(2, "\n-\n");
		120	randomModify.clearDocument();
		121
		122
		123	// -------------------- SAME-LOOKAHEAD REQUIREMENTS -------------------------
		124	// Check that the token list updater respects the rule
		125	// that the lookahead of the incrementally created tokens
		126	// is the same like in a regular batch lexing.
		127	// This may not be strictly necessary for correctness but very beneficial because
		128	// then the incrementally patched tokens can be compared after each modification
		129	// with a token list obtained by batch lexing and everything including
		130	// lookaheads and states can be required to be the same.
		131	//
		132	// -------------------- SAME-LOOKAHEAD REQUIREMENT #1 -------------------------
		133	// Check that when token list updater finds a match (token boundary and states match)
		134	// that also the lookahead of
96	// of subsequent tokens (already present in the token list)	135	// of subsequent tokens (already present in the token list)
97	// correspond to the lookahead of the relexed token.	136	// correspond to the lookahead of the relexed token.
98	// In the following example a "+-+" token must be created.	137	// In the following example a "+-+" token must be created.
99	randomModify.insertText(0, "---+"); // "-"[0]; "-"[0]; "-"[0]; "+"[1];	138	randomModify.insertText(0, "---+"); // "-"[0]; "-"[0]; "-"[0]; "+"[1]; <- see the second token LA=0
100	randomModify.insertText(1, "+");	139	randomModify.insertText(1, "+"); // "-+--+": "-"[0]; "+"[2]; "-"[1]; "-"[0]; "+"[1]; <- seems only "+"[2] was added
101	randomModify.removeText(3, 1);	140	// BUT note the next token "-"[1] has to be relexed too since the original had LA=0
102		141
		142	// Now in addition check that "+-+" will be created.
		143	randomModify.removeText(3, 1); // "-+-+": "-"[0]; "+-+"[0]
103	randomModify.clearDocument();	144	randomModify.clearDocument();
104		145
105	// Check that the token list updater checks respects the rule
106	// that the lookahead of the incrementally created tokens
107	// is the same like in a regular batch lexing.
108	randomModify.insertText(0, "+--+--");
109	randomModify.removeText(2, 1);
110		146
		147	// -------------------- SAME-LOOKAHEAD REQUIREMENT #2 -------------------------
		148	// Here check that an original token after the match point would not have unnecesarily high LA.
		149	// This could happen if the original token before the match point would have LA longer than
		150	// the length of the token that follows it (the one right after the match point) and so it would affect
		151	// LA of that next token too. Now if the newly relexed token (before match point) would have small LA
		152	// the retained token after match point would still hold the extra LA unnecesarilly.
		153	randomModify.insertText(0, "+--+--"); // "+"[2]; "-"[1]; "-"[0]; "+"[2]; "-"[1]; "-"[0]
		154	randomModify.removeText(2, 1); // "+-+--": Without extra check: "+-+"[0]; "-"[1]; "-"[0]
111	randomModify.clearDocument();	155	randomModify.clearDocument();
		156	// BUT in batch lexing the second token would have LA=0.
		157	// A potential fix is to check that when lexing stops that the
		158
		159
112		160
113	// Check for the previous case but this time the relexing would normally	161	// Check for the previous case but this time the relexing would normally
114	// be skipped but this would lead to lookahead 1 for the "-" token	162	// be skipped but this would lead to lookahead 1 for the "-" token
Lines 116-136 Link Here
116	randomModify.insertText(0, "-+--"); // "-"[0]; "+"[2]; "-"[1]; "-"[0];	164	randomModify.insertText(0, "-+--"); // "-"[0]; "+"[2]; "-"[1]; "-"[0];
117	// Without extra care it would become "-"[0]; "-"[1]; "-"[0];	165	// Without extra care it would become "-"[0]; "-"[1]; "-"[0];
118	randomModify.removeText(1, 1); // "---": "-"[0]; "-"[0]; "-"[0];	166	randomModify.removeText(1, 1); // "---": "-"[0]; "-"[0]; "-"[0];
119
120	randomModify.clearDocument();	167	randomModify.clearDocument();
121		168
122	// Similar case to the previous one but with more tokens	169	// Similar case to the previous one but with more tokens
123	randomModify.insertText(0, "-+-++--");	170	randomModify.insertText(0, "-+-++--");
124	randomModify.removeText(1, 4);	171	randomModify.removeText(1, 4);
125
126	randomModify.clearDocument();	172	randomModify.clearDocument();
		173
		174
127		175
128	// Check for the case when token validation cannot be performed	176	// Check for the case when token validation cannot be performed
129	// because although the lenghth of the removal is less than	177	// because although the lenghth of the removal is less than
130	// the "+-+" token's length the removal spans token boundaries	178	// the "+-+" token's length the removal spans token boundaries
131	randomModify.insertText(0, "-+-+ --");	179	randomModify.insertText(0, "-+-+ --");
132	randomModify.removeText(3, 2);	180	randomModify.removeText(3, 2);
133
134	randomModify.clearDocument();	181	randomModify.clearDocument();
135		182
136		183
Lines 161-167 Link Here
161		208
162	randomModify.test(	209	randomModify.test(
163	new RandomModifyDescriptor[] {	210	new RandomModifyDescriptor[] {
164	new RandomModifyDescriptor(200, plusMinusTextProvider,	211	new RandomModifyDescriptor(200, textProvider,
165	0.2, 0.2, 0.1,	212	0.2, 0.2, 0.1,
166	0.2, 0.2,	213	0.2, 0.2,
167	0.0, 0.0), // snapshots create/destroy	214	0.0, 0.0), // snapshots create/destroy

Return to bug 117450

Lines 516-536 Link Here

(-)a/editor.util/src/org/netbeans/lib/editor/util/CharSequenceUtilities.java (-7 / +20 lines)
516	/**	516	/**
517	* Ensure that the given start and end parameters are valid indices	517	* Ensure that the given start and end parameters are valid indices
518	* of the given text.	518	* of the given text.
519	* @throws IndexOutOfBoundsException if the start or end are not within bounds	519	* @param start must be >=0 and <=end.
520	* of the given text.	520	* @param end must be >=start and <=textLength.
		521	* @param length total length of a charsequence.
		522	* @throws IndexOutOfBoundsException if the start or end are not within bounds.
521	*/	523	*/
522	public static void checkIndexesValid(CharSequence text, int start, int end) {	524	public static void checkIndexesValid(int start, int end, int length) {
523	if (start < 0) {	525	if (start < 0) {
524	throw new IndexOutOfBoundsException("start=" + start + " < 0"); // NOI18N	526	throw new IndexOutOfBoundsException("start=" + start + " < 0"); // NOI18N
525	}	527	}
526	if (end < start) {	528	if (end < start) {
527	throw new IndexOutOfBoundsException("end=" + end + " < start=" + start); // NOI18N	529	throw new IndexOutOfBoundsException("end=" + end + " < start=" + start); // NOI18N
528	}	530	}
529	if (end > text.length()) {	531	if (end > length) {
530	throw new IndexOutOfBoundsException("end=" + end // NOI18N	532	throw new IndexOutOfBoundsException("end=" + end + " > length()=" + length); // NOI18N
531	+ " > text.length()=" + text.length()); // NOI18N
532	}	533	}
533	}	534	}
534		535
		536	/**
		537	* Ensure that the given start and end parameters are valid indices
		538	* of the given text.
		539	* @param text non-null char sequence.
		540	* @param start must be >=0 and <=end.
		541	* @param end must be >=start and <=<code>text.length()</code>.
		542	* @throws IndexOutOfBoundsException if the start or end are not within bounds
		543	* of the given text.
		544	*/
		545	public static void checkIndexesValid(CharSequence text, int start, int end) {
		546	checkIndexesValid(start, end, text.length());
		547	}
535		548
536	}	549	}

Lines 113-118 Link Here

(-)a/lexer/apichanges.xml (+19 lines)
113	<!-- ACTUAL CHANGES BEGIN HERE: -->	113	<!-- ACTUAL CHANGES BEGIN HERE: -->
114		114
115	<changes>	115	<changes>
		116	<change id="Joined-sections-Lexing">
		117	<api name="api"/>
		118	<summary>Joined Sections Lexing</summary>
		119	<version major="1" minor="28"/>
		120	<date day="28" month="5" year="2008"/>
		121	<author login="mmetelka"/>
		122	<compatibility source="compatible" binary="compatible" semantic="compatible" deletion="no" addition="yes" modification="no"/>
		123	<description>
		124	<p>
		125	Embeddings that request input sections to be joined before lexing
		126	are now lexed as a single section.
		127	<br/>
		128	<code>Token.isRemoved()</code> was added to check whether a particular token
		129	is still present in token hierarchy or whether it was removed as part of a modification.
		130	</p>
		131	</description>
		132	<issue number="117450"/>
		133	</change>
		134
116	<change id="Lexer-API-Cleanup">	135	<change id="Lexer-API-Cleanup">
117	<api name="api"/>	136	<api name="api"/>
118	<summary>Lexer API Cleanup</summary>	137	<summary>Lexer API Cleanup</summary>

Added Link Here

(-)06a7890f802e (+247 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer;
43
44	import java.util.ArrayList;
45	import java.util.Set;
46	import org.netbeans.api.lexer.Language;
47	import org.netbeans.api.lexer.LanguagePath;
48	import org.netbeans.api.lexer.InputAttributes;
49	import org.netbeans.api.lexer.TokenId;
50	import org.netbeans.lib.lexer.token.AbstractToken;
51	import org.netbeans.lib.lexer.token.TextToken;
52
53
54	/**
55	* Token list used for immutable inputs.
56	*
57	* @author Miloslav Metelka
58	* @version 1.00
59	*/
60
61	public final class BatchTokenList<T extends TokenId>
62	extends ArrayList<TokenOrEmbedding<T>> implements TokenList<T> {
63
64	/** Flag for additional correctness checks (may degrade performance). */
65	private static final boolean testing = Boolean.getBoolean("netbeans.debug.lexer.test");
66
67	private static boolean maintainLAState;
68
69	/**
70	* Check whether lookaheads and states are stored for testing purposes.
71	*/
72	public static boolean isMaintainLAState() {
73	return maintainLAState;
74	}
75
76	public static void setMaintainLAState(boolean maintainLAState) {
77	BatchTokenList.maintainLAState = maintainLAState;
78	}
79
80	private final TokenHierarchyOperation<?,T> tokenHierarchyOperation;
81
82	private final CharSequence inputSourceText;
83
84	private final LanguagePath languagePath;
85
86	private final Set<T> skipTokenIds;
87
88	private final InputAttributes inputAttributes;
89
90	/**
91	* Lexer input used for lexing of the input.
92	*/
93	private LexerInputOperation<T> lexerInputOperation;
94
95	private LAState laState;
96
97
98	public BatchTokenList(TokenHierarchyOperation<?,T> tokenHierarchyOperation, CharSequence inputText,
99	Language<T> language, Set<T> skipTokenIds, InputAttributes inputAttributes) {
100	this.tokenHierarchyOperation = tokenHierarchyOperation;
101	this.inputSourceText = inputText;
102	this.languagePath = LanguagePath.get(language);
103	this.skipTokenIds = skipTokenIds;
104	this.inputAttributes = inputAttributes;
105	if (testing) { // Maintain lookaheads and states when in test environment
106	laState = LAState.empty();
107	}
108	this.lexerInputOperation = createLexerInputOperation();
109	}
110
111	protected LexerInputOperation<T> createLexerInputOperation() {
112	return new TextLexerInputOperation<T>(this);
113	}
114
115	public TokenList<?> rootTokenList() {
116	return this; // this list should always be the root list of the token hierarchy
117	}
118
119	public CharSequence inputSourceText() {
120	return inputSourceText;
121	}
122
123	public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {
124	return tokenHierarchyOperation;
125	}
126
127	public LanguagePath languagePath() {
128	return languagePath;
129	}
130
131	public synchronized int tokenCount() {
132	if (lexerInputOperation != null) { // still lexing
133	tokenOrEmbeddingImpl(Integer.MAX_VALUE);
134	}
135	return size();
136	}
137
138	public int tokenCountCurrent() {
139	return size();
140	}
141
142	public int tokenOffset(AbstractToken<T> token) {
143	int rawOffset = token.rawOffset();
144	// Children offsets should be absolute
145	return rawOffset;
146	}
147
148	public int tokenOffsetByIndex(int index) {
149	AbstractToken<T> token = existingToken(index);
150	int offset;
151	if (token.isFlyweight()) {
152	offset = 0;
153	while (--index >= 0) {
154	token = existingToken(index);
155	offset += token.length();
156	if (!token.isFlyweight()) {
157	offset += token.offset(null);
158	break;
159	}
160	}
161	} else { // non-flyweight offset
162	offset = token.offset(null);
163	}
164	return offset;
165	}
166
167	public int[] tokenIndex(int offset) {
168	return LexerUtilsConstants.tokenIndexLazyTokenCreation(this, offset);
169	}
170
171	public synchronized TokenOrEmbedding<T> tokenOrEmbedding(int index) {
172	return tokenOrEmbeddingImpl(index);
173	}
174
175	private TokenOrEmbedding<T> tokenOrEmbeddingImpl(int index) {
176	while (lexerInputOperation != null && index >= size()) {
177	AbstractToken<T> token = lexerInputOperation.nextToken();
178	if (token != null) { // lexer returned valid token
179	add(token);
180	if (laState != null) { // maintaining lookaheads and states
181	laState = laState.add(lexerInputOperation.lookahead(),
182	lexerInputOperation.lexerState());
183	}
184	} else { // no more tokens from lexer
185	lexerInputOperation.release();
186	lexerInputOperation = null;
187	trimToSize();
188	}
189	}
190	return (index < size()) ? get(index) : null;
191	}
192
193	private AbstractToken<T> existingToken(int index) {
194	return get(index).token();
195	}
196
197	public int lookahead(int index) {
198	return (laState != null) ? laState.lookahead(index) : -1;
199	}
200
201	public Object state(int index) {
202	return (laState != null) ? laState.state(index) : null;
203	}
204
205	public int startOffset() {
206	return 0;
207	}
208
209	public int endOffset() {
210	int cntM1 = tokenCount() - 1;
211	if (cntM1 >= 0)
212	return tokenOffsetByIndex(cntM1) + tokenOrEmbeddingImpl(cntM1).token().length();
213	return 0;
214	}
215
216	public boolean isRemoved() {
217	return false;
218	}
219
220	public int modCount() {
221	return LexerUtilsConstants.MOD_COUNT_IMMUTABLE_INPUT; // immutable input
222	}
223
224	public synchronized AbstractToken<T> replaceFlyToken(
225	int index, AbstractToken<T> flyToken, int offset) {
226	TextToken<T> nonFlyToken = ((TextToken<T>)flyToken).createCopy(this, offset);
227	set(index, nonFlyToken);
228	return nonFlyToken;
229	}
230
231	public void wrapToken(int index, EmbeddingContainer<T> embeddingContainer) {
232	set(index, embeddingContainer);
233	}
234
235	public InputAttributes inputAttributes() {
236	return inputAttributes;
237	}
238
239	public boolean isContinuous() {
240	return (skipTokenIds == null);
241	}
242
243	public Set<T> skipTokenIds() {
244	return skipTokenIds;
245	}
246
247	}

Lines 241-248 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/CharPreprocessorOperation.java (-6 / +6 lines)
241	public void notifyError(String errorMessage) {	241	public void notifyError(String errorMessage) {
242	if (lexerInputOperation != null) {	242	if (lexerInputOperation != null) {
243	int parentIndex = parent.readIndex(); // Get the	243	int parentIndex = parent.readIndex(); // Get the
244	lexerInputOperation.notifyPreprocessorError(	244	// lexerInputOperation.notifyPreprocessorError(
245	new CharPreprocessorError(errorMessage, parent.deepRawLength(parentIndex)));	245	// new CharPreprocessorError(errorMessage, parent.deepRawLength(parentIndex)));
246	}	246	}
247	}	247	}
248		248
Lines 288-297 Link Here
288	return tokenLength;	288	return tokenLength;
289	}	289	}
290		290
291	public void tokenRecognized(int tokenLength) {	291	public void assignTokenLength(int tokenLength, boolean skipToken) {
292	this.tokenLength = tokenLength;	292	this.tokenLength = tokenLength;
293	// Modify tokenLength for preprocessed characters	293	// Modify tokenLength for preprocessed characters
294	parent.tokenRecognized(parentLength(tokenLength));	294	parent.assignTokenLength(parentLength(tokenLength), skipToken);
295	}	295	}
296		296
297	public PreprocessedTextStorage createPreprocessedTextStorage(CharSequence rawText,	297	public PreprocessedTextStorage createPreprocessedTextStorage(CharSequence rawText,
Lines 390-396 Link Here
390	* This method is called after the token has been recognized	390	* This method is called after the token has been recognized
391	* to clear internal data related to processing of token's characters.	391	* to clear internal data related to processing of token's characters.
392	*/	392	*/
393	public void tokenApproved() {	393	public void consumeTokenLength() {
394	if (prepStartIndex != lookaheadIndex) { // some prep chars (may be after token length)	394	if (prepStartIndex != lookaheadIndex) { // some prep chars (may be after token length)
395	if (prepStartIndex < tokenLength) { // prep chars before token end	395	if (prepStartIndex < tokenLength) { // prep chars before token end
396	if (prepEndIndex <= tokenLength) { // no preprocessed chars past token end	396	if (prepEndIndex <= tokenLength) { // no preprocessed chars past token end
Lines 417-423 Link Here
417		417
418	readIndex -= tokenLength;	418	readIndex -= tokenLength;
419	lookaheadIndex -= tokenLength;	419	lookaheadIndex -= tokenLength;
420	parent.tokenApproved();	420	parent.consumeTokenLength();
421		421
422	if (testing)	422	if (testing)
423	consistencyCheck();	423	consistencyCheck();

Lines 101-118 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/CharProvider.java (-7 / +8 lines)
101	* token length in the root lexer input operation due to character	101	* token length in the root lexer input operation due to character
102	* preprocessing.	102	* preprocessing.
103	* <br/>	103	* <br/>
104	* The tokenLength should be cached by this provider.	104	* The tokenLength at a particular level should be cached by the corresponding provider.
105	* @param skip whether the token will be skipped due to filtering of its id.	105	*
		106	* @param skipToken whether the token will be skipped due to filtering of its id.
106	* @return true if the token is preprocessed or false otherwise.	107	* @return true if the token is preprocessed or false otherwise.
107	*/	108	*/
108	void tokenRecognized(int tokenLength);	109	void assignTokenLength(int tokenLength, boolean skipToken);
109		110
110	/**	111	/**
111	* Notify this provider that the token was approved and	112	* Notify this provider that the token was created and
112	* that the tokenLength number of characters should be skipped	113	* that the tokenLength number of characters should be consumed
113	* (tokenLength should be cached by the provider).	114	* (tokenLength should continue to be held by the provider).
114	*/	115	*/
115	void tokenApproved();	116	void consumeTokenLength();
116		117
117	/**	118	/**
118	* Collect extra preprocessed characters from the parent providers.	119	* Collect extra preprocessed characters from the parent providers.

Added Link Here

(-)06a7890f802e (+122 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer;
43
44	/**
45	* Class that wraps a each embedded token list contained in join token list.
46	*
47	* @author Miloslav Metelka
48	*/
49
50	public final class EmbeddedJoinInfo {
51
52	public EmbeddedJoinInfo(JoinTokenListBase base, int rawJoinTokenIndex, int rawTokenListIndex) {
53	assert (base != null);
54	this.base = base;
55	this.rawJoinTokenIndex = rawJoinTokenIndex;
56	this.rawTokenListIndex = rawTokenListIndex;
57	}
58
59	/**
60	* Reference to join token list base as a join-related extension
61	* of this ETL.
62	* In fact this is the only field through which the join token list base instance
63	* is referenced.
64	*/
65	public final JoinTokenListBase base; // 12 bytes (8-super + 4)
66
67	/**
68	* Index in terms of join token list
69	* that corresponds to first token of wrapped ETL.
70	* <br/>
71	* The index must be gap-preprocessed.
72	*/
73	int rawJoinTokenIndex; // 16 bytes
74
75	/**
76	* Index of related ETL in a join token list (base).
77	* <br/>
78	* The index must be gap-preprocessed.
79	*/
80	int rawTokenListIndex; // 20 bytes
81
82	/**
83	* Number of items to go forward to reach last part of a join token.
84	* Zero otherwise.
85	*/
86	private int joinTokenLastPartShift; // 24 bytes
87
88	public int joinTokenIndex() {
89	return base.joinTokenIndex(rawJoinTokenIndex);
90	}
91
92	public void setRawJoinTokenIndex(int rawJoinTokenIndex) {
93	this.rawJoinTokenIndex = rawJoinTokenIndex;
94	}
95
96	public int tokenListIndex() {
97	return base.tokenListIndex(rawTokenListIndex);
98	}
99
100	public int joinTokenLastPartShift() {
101	return joinTokenLastPartShift;
102	}
103
104	public void setJoinTokenLastPartShift(int joinTokenLastPartShift) {
105	this.joinTokenLastPartShift = joinTokenLastPartShift;
106	}
107
108	public StringBuilder dumpInfo(StringBuilder sb) {
109	if (sb == null)
110	sb = new StringBuilder(70);
111	sb.append("jti=").append(joinTokenIndex());
112	sb.append(", tli=").append(tokenListIndex());
113	sb.append(", lps=").append(joinTokenLastPartShift());
114	return sb;
115	}
116
117	@Override
118	public String toString() {
119	return dumpInfo(null).toString();
120	}
121
122	}

Added Link Here

(-)06a7890f802e (+415 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer;
43
44	import java.util.List;
45	import org.netbeans.api.lexer.PartType;
46	import org.netbeans.api.lexer.TokenId;
47	import org.netbeans.lib.editor.util.ArrayUtilities;
48	import org.netbeans.spi.lexer.LexerInput;
49	import org.netbeans.lib.lexer.token.AbstractToken;
50	import org.netbeans.lib.lexer.token.JoinToken;
51	import org.netbeans.lib.lexer.token.PartToken;
52	import org.netbeans.spi.lexer.TokenPropertyProvider;
53
54	/**
55	* Lexer input operation over multiple joined sections (embedded token lists).
56	* <br/>
57	* It produces regular tokens (to be added directly into ETL represented by
58	* {@link #activeTokenList()} and also special {@link #JoinToken} instances
59	* in case a token spans boundaries of multiple ETLs.
60	* <br/>
61	* It can either work over JoinTokenList directly or, during a modification,
62	* it simulates that certain token lists are already removed/added to underlying token list.
63	* <br/>
64	*
65	* {@link #recognizedTokenLastInTokenList()} gives information whether the lastly
66	* produced token ends right at boundary of the activeTokenList.
67	*
68	* @author Miloslav Metelka
69	* @version 1.00
70	*/
71
72	public class JoinLexerInputOperation<T extends TokenId> extends LexerInputOperation<T> {
73
74	CharSequence inputSourceText;
75
76	private TokenListText readText; // For servicing read()
77
78	private TokenListText readExistingText;
79
80	/**
81	* Token list in which the last recognized token started.
82	*/
83	private EmbeddedTokenList<T> activeTokenList;
84
85	/**
86	* Index of activeTokenList in JTL.
87	*/
88	private int activeTokenListIndex;
89
90	/**
91	* End offset of the active token list.
92	*/
93	private int activeTokenListEndOffset;
94
95	/**
96	* Real token's start offset used to derive the token's offset in ETL.
97	* Since tokenStartOffset is affected by TokenListList.readOffsetShift
98	* it cannot be used for this purpose.
99	*/
100	private int realTokenStartOffset;
101
102	private boolean recognizedTokenJoined; // Whether recognized token will consist of parts
103
104
105	public JoinLexerInputOperation(JoinTokenList<T> joinTokenList, int relexJoinIndex, Object lexerRestartState,
106	int activeTokenListIndex, int relexOffset
107	) {
108	super(joinTokenList, relexJoinIndex, lexerRestartState);
109	this.inputSourceText = joinTokenList.inputSourceText();
110	this.activeTokenListIndex = activeTokenListIndex;
111	tokenStartOffset = relexOffset;
112	readOffset = relexOffset;
113	// Assign realTokenStartOffset after fetchActiveTokenList() since it would overwrite it
114	realTokenStartOffset = relexOffset;
115	}
116
117	public final void init() {
118	// Following code uses tokenList() method overriden in MutableJoinLexerInputOperation
119	// so the following code would fail when placed in constructor since the constructor of MJLIO would not yet run.
120	fetchActiveTokenList();
121	readText = new TokenListText();
122	readText.init();
123	}
124
125	/**
126	* Get active ETL into which the last produced token should be added.
127	* For join tokens there is an ETL into which a last part of JT should be added.
128	*/
129	public EmbeddedTokenList<T> activeTokenList() {
130	return activeTokenList;
131	}
132
133	/**
134	* Get index of active ETL into which the last produced token should be added.
135	* For join tokens there is an index of the last ETL into which a last part of JT should be added.
136	*/
137	public int activeTokenListIndex() {
138	return activeTokenListIndex;
139	}
140
141	/**
142	* True if the last returned token is last in {@link #activeTokenList()}.
143	* For join tokens this applies to the last part of join token.
144	*/
145	public boolean recognizedTokenLastInTokenList() {
146	// realTokenStartOffset is set to the end of last recognized token
147	return (realTokenStartOffset == activeTokenListEndOffset);
148	}
149
150	@Override
151	public int lastTokenEndOffset() {
152	return realTokenStartOffset;
153	}
154
155	public int read(int offset) { // index >= 0 is guaranteed by contract
156	return readText.read(offset);
157	}
158
159	public char readExisting(int offset) {
160	if (readText.isInBounds(offset)) {
161	return readText.inBoundsChar(offset);
162	}
163	if (readExistingText == null) {
164	readExistingText = new TokenListText();
165	readExistingText.initFrom(readText);
166	}
167	return readExistingText.existingChar(offset);
168	}
169
170	@Override
171	public void assignTokenLength(int tokenLength) {
172	super.assignTokenLength(tokenLength);
173	// Check whether activeTokenList needs to be changed due to various flags
174	if (recognizedTokenLastInTokenList()) { // Advance to next token list
175	// Since this is done when recognizing a next token it should be ok when recognizing
176	// last token in the last ETL (it should not go beyond last ETL).
177	do {
178	activeTokenListIndex++;
179	fetchActiveTokenList();
180	} while (realTokenStartOffset == activeTokenListEndOffset); // Skip empty ETLs
181	}
182	// Advance to end of currently recognized token
183	realTokenStartOffset += tokenLength;
184	// Joined token past ETL's boundary
185	recognizedTokenJoined = (realTokenStartOffset > activeTokenListEndOffset);
186	}
187
188	private void fetchActiveTokenList() {
189	activeTokenList = tokenList(activeTokenListIndex);
190	realTokenStartOffset = activeTokenList.startOffset();
191	activeTokenListEndOffset = activeTokenList.endOffset();
192	}
193
194	public EmbeddedTokenList<T> tokenList(int tokenListIndex) { // Also used by JoinTokenListChange
195	return ((JoinTokenList<T>) tokenList).tokenList(tokenListIndex);
196	}
197
198	protected int tokenListCount() {
199	return ((JoinTokenList<T>) tokenList).tokenListCount();
200	}
201
202	protected void fillTokenData(AbstractToken<T> token) {
203	if (!recognizedTokenJoined) {
204	token.setTokenList(activeTokenList);
205	// Subtract tokenLength since this is already advanced to end of token
206	token.setRawOffset(realTokenStartOffset - tokenLength);
207	}
208	}
209
210	@Override
211	protected boolean isFlyTokenAllowed() {
212	return super.isFlyTokenAllowed() && !recognizedTokenJoined;
213	}
214
215	@Override
216	protected AbstractToken<T> createDefaultTokenInstance(T id) {
217	if (recognizedTokenJoined) {
218	return createJoinToken(id, null, PartType.COMPLETE);
219	} else { // Regular case
220	return super.createDefaultTokenInstance(id);
221	}
222	}
223
224	@Override
225	protected AbstractToken<T> createPropertyTokenInstance(T id,
226	TokenPropertyProvider<T> propertyProvider, PartType partType) {
227	if (recognizedTokenJoined) {
228	return createJoinToken(id, null, partType);
229	} else { // Regular case
230	return super.createPropertyTokenInstance(id, propertyProvider, partType);
231	}
232	}
233
234	private AbstractToken<T> createJoinToken(T id,
235	TokenPropertyProvider<T> propertyProvider, PartType partType) {
236	// Create join token
237	// realTokenStartOffset is already advanced by tokenLength so first decrease it
238	realTokenStartOffset -= tokenLength;
239	JoinToken<T> joinToken = new JoinToken<T>(id, tokenLength, propertyProvider, partType);
240	int joinPartCountEstimate = readText.tokenListIndex - activeTokenListIndex + 1;
241	@SuppressWarnings("unchecked")
242	PartToken<T>[] parts = new PartToken[joinPartCountEstimate];
243	int partLength = activeTokenListEndOffset - realTokenStartOffset;
244	PartToken<T> partToken = new PartToken<T>(id, partLength, propertyProvider, PartType.START, joinToken, 0, 0);
245	partToken.setTokenList(activeTokenList);
246	partToken.setRawOffset(realTokenStartOffset); // realTokenStartOffset already decreased by tokenLength
247	parts[0] = partToken;
248	int partIndex = 1;
249	int partTextOffset = partLength; // Length of created parts so far
250	int firstPartTokenListIndex = activeTokenListIndex;
251	do {
252	activeTokenListIndex++;
253	fetchActiveTokenList();
254	// realTokenStartOffset set to start activeTokenList
255	PartType partPartType;
256	// Attempt total ETL's length as partLength
257	partLength = activeTokenListEndOffset - realTokenStartOffset;
258	if (partLength == 0) {
259	continue;
260	}
261	if (partTextOffset + partLength >= tokenLength) { // Last part
262	partLength = tokenLength - partTextOffset;
263	// If the partType of the join token is not complete then this will be PartType.MIDDLE
264	partPartType = (partType == PartType.START) ? PartType.MIDDLE : PartType.END;
265	} else { // Non-last part
266	partPartType = PartType.MIDDLE;
267	}
268
269	partToken = new PartToken<T>(id, partLength, propertyProvider, partPartType, joinToken, partIndex, partTextOffset);
270	// realTokenStartOffset still points to start of activeTokenList
271	partToken.setRawOffset(realTokenStartOffset); // ETL.startOffset() will be subtracted upon addition to ETL
272	partToken.setTokenList(activeTokenList);
273	partTextOffset += partLength;
274	parts[partIndex++] = partToken;
275	} while (partTextOffset < tokenLength);
276	// Update realTokenStartOffset which pointed to start of activeTokenList
277	realTokenStartOffset += partLength;
278	// Check that the array does not have any extra items
279	if (partIndex < parts.length) {
280	@SuppressWarnings("unchecked")
281	PartToken<T>[] tmp = new PartToken[partIndex];
282	System.arraycopy(parts, 0, tmp, 0, partIndex);
283	parts = tmp;
284	}
285	List<PartToken<T>> partList = ArrayUtilities.unmodifiableList(parts);
286	joinToken.setJoinedParts(partList, activeTokenListIndex - firstPartTokenListIndex);
287	// joinToken.setTokenList() makes no sense - JoinTokenList instances are temporary
288	// joinToken.setRawOffset() makes no sense - offset taken from initial part
289	return joinToken;
290	}
291
292	/**
293	* Class for reading of text of subsequent ETLs - it allows to see their text
294	* as a consecutive character sequence (inputSourceText is used as a backing char sequence)
295	* with an increasing readIndex (it's not decremented after token's recognition).
296	*/
297	final class TokenListText {
298
299	int tokenListIndex;
300
301	int tokenListStartOffset;
302
303	int tokenListEndOffset;
304
305	/**
306	* A constant added to readOffset to allow a smoothly increasing reading offset
307	* when reading through multiple ETLs with gaps among them.
308	*/
309	int readOffsetShift;
310
311	void init() {
312	EmbeddedTokenList<T> etl = tokenList(activeTokenListIndex);
313	tokenListStartOffset = etl.startOffset();
314	tokenListEndOffset = etl.endOffset();
315	// No extra shift for first token
316	}
317
318	void initFrom(TokenListText text) {
319	this.tokenListIndex = text.tokenListIndex;
320	this.tokenListStartOffset = text.tokenListStartOffset;
321	this.tokenListEndOffset = text.tokenListEndOffset;
322	this.readOffsetShift = text.readOffsetShift;
323	}
324
325	/**
326	* Read next char or return EOF.
327	*/
328	int read(int offset) {
329	offset += readOffsetShift;
330	if (offset < tokenListEndOffset) {
331	return inputSourceText.charAt(offset);
332	} else {
333	while (++tokenListIndex < tokenListCount()) {
334	EmbeddedTokenList etl = tokenList(tokenListIndex);
335	tokenListStartOffset = etl.startOffset();
336	// Increase offset shift by the size of gap between ETLs
337	readOffsetShift += tokenListStartOffset - tokenListEndOffset;
338	// Also shift given offset value
339	offset += tokenListStartOffset - tokenListEndOffset;
340	tokenListEndOffset = etl.endOffset();
341	if (readOffset < tokenListEndOffset) { // ETL might be empty
342	return inputSourceText.charAt(offset);
343	}
344	}
345	tokenListIndex--; // Return to (tokenListCount() - 1)
346	return LexerInput.EOF;
347	}
348	}
349
350	/**
351	* Check whether currently set text covers the given relative index.
352	*
353	* @param index index in the same metrics as readIndex.
354	* @return whether the given index is within current bounds.
355	*/
356	boolean isInBounds(int offset) {
357	offset += readOffsetShift;
358	return offset >= tokenListStartOffset && offset < tokenListEndOffset;
359	}
360
361	/**
362	* Get char that was previously verified to be within bounds.
363	*/
364	char inBoundsChar(int offset) {
365	offset += readOffsetShift;
366	return inputSourceText.charAt(offset);
367	}
368
369	char existingChar(int offset) {
370	offset += readOffsetShift;
371	if (offset < tokenListStartOffset) {
372	while (true) { // Char should exist
373	tokenListIndex--;
374	EmbeddedTokenList etl = tokenList(tokenListIndex);
375	tokenListEndOffset = etl.endOffset();
376	// Decrease offset shift by the size of gap between ETLs
377	readOffsetShift -= tokenListStartOffset - tokenListEndOffset;
378	// Also shift given offset value
379	offset -= tokenListStartOffset - tokenListEndOffset;
380	tokenListStartOffset = etl.startOffset();
381	if (readOffset >= tokenListStartOffset) { // ETL might be empty
382	return inputSourceText.charAt(offset);
383	}
384	}
385
386	} else if (offset >= tokenListEndOffset) {
387	while (true) { // Char should exist
388	tokenListIndex++;
389	EmbeddedTokenList etl = tokenList(tokenListIndex);
390	tokenListStartOffset = etl.startOffset();
391	// Increase offset shift by the size of gap between ETLs
392	readOffsetShift += tokenListStartOffset - tokenListEndOffset;
393	// Also shift given offset value
394	offset += tokenListStartOffset - tokenListEndOffset;
395	tokenListEndOffset = etl.endOffset();
396	if (readOffset < tokenListEndOffset) { // ETL might be empty
397	return inputSourceText.charAt(offset);
398	}
399	}
400
401	}
402	// Index within current bounds
403	return inputSourceText.charAt(offset);
404	}
405
406	}
407
408	@Override
409	public String toString() {
410	return super.toString() + ", realTokenStartOffset=" + realTokenStartOffset + // NOI18N
411	", activeTokenListIndex=" + activeTokenListIndex + // NOI18N
412	", activeTokenListEndOffset=" + activeTokenListEndOffset; // NOI18N
413	}
414
415	}

Added Link Here

(-)06a7890f802e (+174 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer;
43
44	/**
45	* Structure holding a token list count that bases a JoinTokenList.
46	* <br/>
47	* There is also an information regarding a join token index gap inside the series
48	* of contained embedded token lists.
49	* <br/>
50	* Join token list can be created as a wrapper around this class with a knowledge
51	* of token list list and an token list index at which the join token list would start.
52	*
53	* @author Miloslav Metelka
54	*/
55
56	public final class JoinTokenListBase {
57
58	private static final int INDEX_GAP_LENGTH_INITIAL_SIZE = (Integer.MAX_VALUE >> 1);
59
60	/**
61	* Number of embedded token lists contained in join token list.
62	*/
63	int tokenListCount; // 12 bytes (8-super + 4)
64
65	/**
66	* Total count of tokens contained in JoinTokenList.
67	*/
68	int joinTokenCount; // 16 bytes
69
70	/**
71	* Last modCount for which the JoinTokenList was updated.
72	* <br/>
73	* JoinTokenList.updateStatus() may be used to update.
74	*/
75	int lastModCount; // 20 bytes
76
77	/**
78	* Index among contained embedded token lists where both index-related gaps are located.
79	*/
80	int indexGapsIndex; // 24 bytes
81
82	/**
83	* Length of an index gap for computation of indexes in a JoinTokenList
84	* based on ETLs.
85	* <br/>
86	* The above gap checking is done by checking whether the index is above gap length
87	* since the initial gap length is so high that the indexes should never reach
88	* its size (even decreased by added items).
89	*/
90	int joinTokenIndexGapLength = INDEX_GAP_LENGTH_INITIAL_SIZE; // 28 bytes
91
92	/**
93	* Length of an index gap for computation of index of ETL in a JoinTokenList
94	* which is useful for finding of a start-token-list-index of the join token list.
95	* <br/>
96	* The above gap checking is done by checking whether the index is above gap length
97	* since the initial gap length is so high that the indexes should never reach
98	* its size (even decreased by added items).
99	*/
100	int tokenListIndexGapLength = INDEX_GAP_LENGTH_INITIAL_SIZE; // 32 bytes
101
102	JoinTokenListBase(int tokenListCount, int lastModCount) {
103	this.tokenListCount = tokenListCount;
104	this.lastModCount = lastModCount;
105	// Move index gap to be at the end of all contained token lists
106	this.indexGapsIndex = tokenListCount;
107	}
108
109	int joinTokenIndex(int rawJoinTokenIndex) {
110	return (rawJoinTokenIndex < joinTokenIndexGapLength)
111	? rawJoinTokenIndex
112	: rawJoinTokenIndex - joinTokenIndexGapLength;
113	}
114
115	int tokenListIndex(int rawTokenListIndex) {
116	return (rawTokenListIndex < tokenListIndexGapLength)
117	? rawTokenListIndex
118	: rawTokenListIndex - tokenListIndexGapLength;
119	}
120
121	/**
122	* Move both gaps in sync so that ETL.JoinInfo in an ETL at "index" is above both gaps.
123	*
124	* @param tokenListList non-null TLL.
125	* @param tokenListStartIndex points to first list belonging to a JTL.
126	* @param index index to which the gaps should be moved.
127	*/
128
129	public void moveIndexGap(TokenListList<?> tokenListList, int tokenListStartIndex, int index) {
130	if (index < indexGapsIndex) {
131	// Items above index should be moved to be above gap
132	do {
133	EmbeddedJoinInfo joinInfo = tokenListList.get(tokenListStartIndex + index).joinInfo;
134	joinInfo.rawTokenListIndex += tokenListIndexGapLength;
135	joinInfo.rawJoinTokenIndex += joinTokenIndexGapLength;
136	} while (++index < indexGapsIndex);
137	} else if (index > indexGapsIndex) {
138	// Items above index should be moved to be above gap
139	do {
140	index--;
141	EmbeddedJoinInfo joinInfo = tokenListList.get(tokenListStartIndex + index).joinInfo;
142	joinInfo.rawTokenListIndex += tokenListIndexGapLength;
143	joinInfo.rawJoinTokenIndex += joinTokenIndexGapLength;
144	} while (index > indexGapsIndex);
145	}
146	}
147
148	public void tokenListModNotify(int tokenListCountDiff) {
149	// Gap assumed to be above last added token list or above
150	indexGapsIndex += tokenListCountDiff; // Move gap above added
151	tokenListCount += tokenListCountDiff;
152	tokenListIndexGapLength -= tokenListCountDiff;
153	}
154
155	public int joinTokenCount() {
156	return joinTokenCount;
157	}
158
159	public void updateJoinTokenCount(int joinTokenCountDiff) {
160	joinTokenCount += joinTokenCountDiff;
161	joinTokenIndexGapLength -= joinTokenCountDiff;
162	}
163
164	@Override
165	public String toString() {
166	StringBuilder sb = new StringBuilder(70);
167	sb.append("tokenListCount=").append(tokenListCount);
168	sb.append(", tokenCount=").append(joinTokenCount);
169	sb.append(", lastModCount=").append(lastModCount);
170	sb.append(", indexGapsIndex=").append(indexGapsIndex);
171	return sb.toString();
172	}
173
174	}

Lines 52-70 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/LAState.java (-9 / +13 lines)
52		52
53	private static final LAState EMPTY = new NoState(0);	53	private static final LAState EMPTY = new NoState(0);
54		54
55	private static final LAState INIT_STATE = new NoState(0);
56
57	public static LAState empty() {	55	public static LAState empty() {
58	return EMPTY;	56	return EMPTY;
59	}	57	}
60		58
61	/**
62	* Special state for marking that an embedded token list was not inited yet.
63	*/
64	public static LAState initState() {
65	return INIT_STATE;
66	}
67
68	static int withExtraCapacity(int capacity) {	59	static int withExtraCapacity(int capacity) {
69	return capacity * 3 / 2 + 4;	60	return capacity * 3 / 2 + 4;
70	}	61	}
Lines 105-110 Link Here
105	return capacity() - gapLength;	96	return capacity() - gapLength;
106	}	97	}
107		98
		99	/**
		100	* Add a particular lookahead and state.
		101	*
		102	* @param lookahead
		103	* @param state
		104	* @return either same or a new LAState containing the given lookahead and state.
		105	*/
108	public final LAState add(int lookahead, Object state) {	106	public final LAState add(int lookahead, Object state) {
109	LAState ret;	107	LAState ret;
110	if (gapLength > 0) { // enough space	108	if (gapLength > 0) { // enough space
Lines 154-159 Link Here
154		152
155	protected abstract LAState upgrade(int capacity, Class laStateClass);	153	protected abstract LAState upgrade(int capacity, Class laStateClass);
156		154
		155	/**
		156	* Whether an upgrade is necessary when the given laStateClass needs to be used.
		157	*
		158	* @param laStateClass non-null requested laStateClass
		159	* @return true if upgrade is necessary.
		160	*/
157	protected abstract boolean isUpgrade(Class laStateClass);	161	protected abstract boolean isUpgrade(Class laStateClass);
158		162
159	protected abstract Class addToGapStart(int lookahead, Object state);	163	protected abstract Class addToGapStart(int lookahead, Object state);

Lines 109-116 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/LanguageOperation.java (-37 / +41 lines)
109	if (!existingLanguagePaths.contains(lp)) {	109	if (!existingLanguagePaths.contains(lp)) {
110	newLanguagePaths.add(lp);	110	newLanguagePaths.add(lp);
111	}	111	}
112	@SuppressWarnings("unchecked")	112	Language<T> language = (Language<T>)LexerUtilsConstants.<T>innerLanguage(lp);
113	Language<T> language = (Language<T>)lp.innerLanguage();
114	if (!exploredLanguages.contains(language)) {	113	if (!exploredLanguages.contains(language)) {
115	exploredLanguages.add(language);	114	exploredLanguages.add(language);
116	Set<T> ids = language.tokenIds();	115	Set<T> ids = language.tokenIds();
Lines 162-167 Link Here
162	WeakListeners.create(PropertyChangeListener.class, this, LanguageManager.getInstance()));	161	WeakListeners.create(PropertyChangeListener.class, this, LanguageManager.getInstance()));
163	}	162	}
164		163
		164	public Language<T> language() {
		165	return language;
		166	}
		167
165	public synchronized TokenValidator<T> tokenValidator(T id) {	168	public synchronized TokenValidator<T> tokenValidator(T id) {
166	if (tokenValidators == null) {	169	if (tokenValidators == null) {
167	tokenValidators = allocateTokenValidatorArray(language.maxOrdinal() + 1);	170	tokenValidators = allocateTokenValidatorArray(language.maxOrdinal() + 1);
Lines 188-218 Link Here
188	}	191	}
189	FlyItem<T> item = flyItems[id.ordinal()];	192	FlyItem<T> item = flyItems[id.ordinal()];
190	if (item == null) {	193	if (item == null) {
191	token = new TextToken<T>(id, text); // create flyweight token	194	item = new FlyItem<T>(id, text);
192	token.makeFlyweight();	195	flyItems[id.ordinal()] = item;
193	flyItems[id.ordinal()] = new FlyItem<T>(token);
194	} else { // already a valid item
195	token = item.token();
196	if (token.text() != text) {
197	token = item.token2();
198	if (token == null \|\| token.text() != text) {
199	token = item.token();
200	if (!CharSequenceUtilities.textEquals(token.text(), text)) {
201	token = item.token2();
202	if (token == null \|\| !CharSequenceUtilities.textEquals(token.text(), text)) {
203	// Create new token
204	token = new TextToken<T>(id, text);
205	token.makeFlyweight();
206	}
207	item.pushToken(token);
208	}
209	} else { // found token2
210	item.pushToken(token);
211	}
212	}
213	}	196	}
214	assert (token != null); // Should return non-null token	197	return item.flyToken(id, text);
215	return token;
216	}	198	}
217		199
218	public synchronized EmbeddingPresence embeddingPresence(T id) {	200	public synchronized EmbeddingPresence embeddingPresence(T id) {
Lines 322-348 Link Here
322	return (TokenValidator<T>[]) new TokenValidator[length];	304	return (TokenValidator<T>[]) new TokenValidator[length];
323	}	305	}
324		306
		307
325	private static final class FlyItem<T extends TokenId> {	308	private static final class FlyItem<T extends TokenId> {
326		309
327	private TextToken<T> token;	310	private TextToken<T> token; // Most used (first candidate)
328		311
329	private TextToken<T> token2;	312	private TextToken<T> token2; // Second most used
330		313
331	public FlyItem(TextToken<T> token) {	314	FlyItem(T id, String text) {
332	this.token = token;	315	newToken(id, text);
		316	token2 = token; // Make both item non-null
333	}	317	}
334		318
335	public TextToken<T> token() {	319	TextToken<T> flyToken(T id, String text) {
		320	// First do a quick check for equality only in both items
		321	if (token.text() != text) {
		322	if (token2.text() == text) {
		323	// Swap token and token2 => token will contain the right value
		324	swap();
		325	} else { // token.text() != text && token2.text() != text
		326	// Now deep-compare of text of both tokens
		327	if (!CharSequenceUtilities.textEquals(token.text(), text)) {
		328	if (!CharSequenceUtilities.textEquals(token2.text(), text)) {
		329	token2 = token;
		330	newToken(id, text);
		331	} else { // swap
		332	swap();
		333	}
		334	}
		335	}
		336	}
336	return token;	337	return token;
337	}	338	}
338		339
339	public TextToken<T> token2() {	340	void newToken(T id, String text) {
340	return token2;	341	// Create new token
		342	token = new TextToken<T>(id, text);
		343	token.makeFlyweight();
341	}	344	}
342		345
343	public void pushToken(TextToken<T> token) {	346	private void swap() {
344	this.token2 = this.token;	347	TextToken<T> tmp = token;
345	this.token = token;	348	token = token2;
		349	token2 = tmp;
346	}	350	}
347		351
348	}	352	}

Lines 64-100 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/PreprocessedTextLexerInputOperation.java (-26 / +10 lines)
64	private int lastRawLengthShift;	64	private int lastRawLengthShift;
65		65
66	private int tokenEndRawLengthShift;	66	private int tokenEndRawLengthShift;
67		67
68	public PreprocessedTextLexerInputOperation(TokenList<T> tokenList, PreprocessedTextStorage prepText) {	68	private int tokenStartIndex; // Extra added to compile
69	this(tokenList, 0, null, prepText, 0, 0, prepText.length());
70	}
71		69
72	public PreprocessedTextLexerInputOperation(TokenList<T> tokenList, int tokenIndex,	70	public PreprocessedTextLexerInputOperation(TokenList<T> tokenList, int tokenIndex,
73	Object lexerRestartState, PreprocessedTextStorage prepText, int prepTextStartOffset,	71	Object lexerRestartState, PreprocessedTextStorage prepText, int prepTextStartOffset,
74	int startOffset, int endOffset) {	72	int startOffset, int endOffset) {
75	super(tokenList, tokenIndex, lexerRestartState, prepText,	73	super(tokenList, tokenIndex, lexerRestartState, startOffset, endOffset);
76	prepTextStartOffset, startOffset, endOffset);
77	this.preprocessedText = prepText;	74	this.preprocessedText = prepText;
78	int index = startOffset - prepTextStartOffset;	75	int index = startOffset - prepTextStartOffset;
79	if (index > 0) {	76	if (index > 0) {
80	tokenStartRawLengthShift = preprocessedText.rawLengthShift(index);	77	tokenStartRawLengthShift = preprocessedText.rawLengthShift(index);
81	lastRawLengthShift = tokenStartRawLengthShift;	78	lastRawLengthShift = tokenStartRawLengthShift;
82	}	79	}
83	preprocessingLevelCount++; // extra level of preprocessing
84	}
85
86	public int deepRawLength(int length) {
87	return length + preprocessedText.rawLengthShift(tokenStartIndex() + length - 1)
88	- tokenStartRawLengthShift;
89	}
90
91	public int deepRawLengthShift(int index) {
92	return preprocessedText.rawLengthShift(tokenStartIndex() + index)
93	- tokenStartRawLengthShift;
94	}	80	}
95		81
96	public int read(int index) { // index >= 0 is guaranteed by contract	82	public int read(int index) { // index >= 0 is guaranteed by contract
97	index += tokenStartIndex();	83	index += tokenStartIndex;
98	if (index < readEndIndex()) {	84	if (index < readEndIndex()) {
99	// Check whether the char is preprocessed	85	// Check whether the char is preprocessed
100	int rls = preprocessedText.rawLengthShift(index);	86	int rls = preprocessedText.rawLengthShift(index);
Lines 111-123 Link Here
111	}	97	}
112	}	98	}
113		99
114	public void tokenRecognized(int tokenLength) {	100	public void assignTokenLength(int tokenLength) {
115	super.tokenRecognized(tokenLength);
116	tokenEndRawLengthShift = preprocessedText.rawLengthShift(	101	tokenEndRawLengthShift = preprocessedText.rawLengthShift(
117	tokenStartIndex() + tokenLength() - 1);	102	tokenStartIndex + tokenLength - 1);
118	}	103	}
119		104
120	public void tokenApproved() {	105	protected void tokenApproved() {
121	// Increase base raw length shift by the token's last-char shift	106	// Increase base raw length shift by the token's last-char shift
122	tokenStartRawLengthShift += tokenEndRawLengthShift;	107	tokenStartRawLengthShift += tokenEndRawLengthShift;
123		108
Lines 135-141 Link Here
135	prepEndIndex -= tokenLength();	120	prepEndIndex -= tokenLength();
136	}	121	}
137	}	122	}
138	super.tokenApproved();
139	}	123	}
140		124
141	public void collectExtraPreprocessedChars(CharProvider.ExtraPreprocessedChars epc,	125	public void collectExtraPreprocessedChars(CharProvider.ExtraPreprocessedChars epc,
Lines 153-159 Link Here
153	// for the present token and the ending chars could possibly	137	// for the present token and the ending chars could possibly
154	// be non-preprocessed (prepEndIndex > tokenLength)	138	// be non-preprocessed (prepEndIndex > tokenLength)
155	while (--i >= prepStartIndex && postCount > 0	139	while (--i >= prepStartIndex && postCount > 0
156	&& preprocessedText.rawLengthShift(i + tokenStartIndex()) == tokenEndRawLengthShift	140	&& preprocessedText.rawLengthShift(i + tokenStartIndex) == tokenEndRawLengthShift
157	) { // not preprocessed	141	) { // not preprocessed
158	postCount--;	142	postCount--;
159	}	143	}
Lines 167-177 Link Here
167	assert (preCount >= 0 && postCount >= 0);	151	assert (preCount >= 0 && postCount >= 0);
168	epc.ensureExtraLength(preCount + postCount);	152	epc.ensureExtraLength(preCount + postCount);
169	while (--preCount >= 0) {	153	while (--preCount >= 0) {
170	epc.insert(readExisting(prepStartIndex - 1), deepRawLength(prepStartIndex) - prepStartIndex);	154	// epc.insert(readExisting(prepStartIndex - 1), deepRawLength(prepStartIndex) - prepStartIndex);
171	prepStartIndex--;	155	prepStartIndex--;
172	}	156	}
173	while (--postCount >= 0) {	157	while (--postCount >= 0) {
174	epc.append(readExisting(prepEndIndex), deepRawLength(prepEndIndex) - topPrepEndIndex);	158	// epc.append(readExisting(prepEndIndex), deepRawLength(prepEndIndex) - topPrepEndIndex);
175	prepEndIndex++;	159	prepEndIndex++;
176	topPrepEndIndex++;	160	topPrepEndIndex++;
177	}	161	}

Lines 110-193 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java (-90 / +36 lines)
110	// No upper bound for end index so use tokenCount() (can be improved if desired)	110	// No upper bound for end index so use tokenCount() (can be improved if desired)
111	limitEndIndex = tokenList.tokenCount();	111	limitEndIndex = tokenList.tokenCount();
112	} else { // Valid limit end offset	112	} else { // Valid limit end offset
113	limitEndIndex = tokenList.tokenCountCurrent(); // presently created token count	113	int[] indexAndTokenOffset = tokenList.tokenIndex(limitEndOffset);
114	if (limitEndIndex == 0) { // no tokens yet -> attempt to create at least one	114	limitEndIndex = indexAndTokenOffset[0];
115	if (tokenList.tokenOrEmbeddingContainer(0) != null) { // some tokens exist	115	if (limitEndIndex != -1) {
116	// Re-get the present token count (could be created a chunk of tokens at once)	116	// If the limitStartOffset is "inside" a token and it's not at or beyond end of TL
117	limitEndIndex = tokenList.tokenCountCurrent();	117	if (limitEndOffset != indexAndTokenOffset[1] && limitEndIndex < tokenList.tokenCountCurrent()) {
		118	limitEndIndex++; // Include the token that contains the offset
118	}	119	}
119	}	120	} else { // No tokens at all
120		121	limitEndIndex = 0;
121	if (limitEndIndex > 0) {
122	// tokenCount surely >0
123	int tokenOffset = tokenList.tokenOffset(limitEndIndex - 1);
124	if (limitEndOffset > tokenOffset) { // may need to create further tokens if they do not exist
125	// Force token list to create subsequent tokens
126	// Cannot subtract offset by each token's length because
127	// there may be gaps between tokens due to token id filter use.
128	AbstractToken<?> token = token(limitEndIndex - 1);
129	int tokenLength = token.length();
130	while (limitEndOffset > tokenOffset + tokenLength) { // above present token
131	Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(limitEndIndex);
132	if (tokenOrEmbeddingContainer != null) {
133	token = LexerUtilsConstants.token(tokenOrEmbeddingContainer);
134	if (tokenList.isContinuous() \|\| token.isFlyweight()) {
135	tokenOffset += tokenLength;
136	} else { // retrieve offset
137	tokenOffset = tokenList.tokenOffset(limitEndIndex);
138	}
139	tokenLength = token.length();
140	limitEndIndex++;
141	} else { // no more tokens => break
142	break;
143	}
144	}
145
146	} else { // end index within existing tokens
147	// The offset is within the currently recognized tokens
148	// Use binary search
149	int low = 0;
150	limitEndIndex--;
151
152	while (low <= limitEndIndex) {
153	int mid = (low + limitEndIndex) / 2;
154	int midStartOffset = tokenList.tokenOffset(mid);
155
156	if (midStartOffset < limitEndOffset) {
157	low = mid + 1;
158	} else if (midStartOffset > limitEndOffset) {
159	limitEndIndex = mid - 1;
160	} else { // Token starting exactly at offset found
161	limitEndIndex = mid - 1;
162	break;
163	}
164	}
165	limitEndIndex++; // Increase from 'high' to end index
166	}
167	}	122	}
168	}	123	}
169		124
170	// Compute limitStartIndex (currently == 0)	125	// Compute limitStartIndex (currently == 0)
171	if (limitEndIndex > 0 && limitStartOffset > 0) {	126	if (limitEndIndex > 0 && limitStartOffset > 0) {
172	int high = limitEndIndex - 1;	127	// Although the binary search could only be in <0,limitEndIndex> bounds
173	while (limitStartIndex <= high) {	128	// use regular TL.tokenIndex() because it has substantially better performance
174	int mid = (limitStartIndex + high) / 2;	129	// e.g. in JoinTokenList.
175	int midStartOffset = tokenList.tokenOffset(mid);	130	int[] indexAndTokenOffset = tokenList.tokenIndex(limitStartOffset);
176		131	limitStartIndex = indexAndTokenOffset[0];
177	if (midStartOffset < limitStartOffset) {	132	// Check if the limitStartOffset is not in gap after end of token at limitStartIndex
178	limitStartIndex = mid + 1;	133	if (limitStartIndex < tokenList.tokenCountCurrent() &&
179	} else if (midStartOffset > limitStartOffset) {	134	indexAndTokenOffset[1] + tokenList.tokenOrEmbedding(limitStartIndex).token().length() <= limitStartOffset
180	high = mid - 1;
181	} else { // Token starting exactly at offset found
182	limitStartIndex = mid + 1;
183	break;
184	}
185	}
186	// Include previous token if it "includes" limitStartOffset (also handles gaps between tokens properly)
187	if (limitStartIndex > 0 &&
188	tokenList.tokenOffset(limitStartIndex - 1) + token(limitStartIndex - 1).length() > limitStartOffset
189	) {	135	) {
190	limitStartIndex--;	136	limitStartIndex++;
191	}	137	}
192	}	138	}
193	}	139	}
Lines 204-225 Link Here
204	return limitEndOffset;	150	return limitEndOffset;
205	}	151	}
206		152
207	public Object tokenOrEmbeddingContainer(int index) {	153	public TokenOrEmbedding<T> tokenOrEmbedding(int index) {
208	index += limitStartIndex;	154	index += limitStartIndex;
209	return (index < limitEndIndex)	155	return (index < limitEndIndex)
210	? tokenList.tokenOrEmbeddingContainer(index)	156	? tokenList.tokenOrEmbedding(index)
211	: null;	157	: null;
212	}	158	}
213		159
214	public int tokenOffset(int index) {	160	public int tokenOffsetByIndex(int index) {
215	index += limitStartIndex;	161	index += limitStartIndex;
216	if (index >= limitEndIndex)	162	if (index >= limitEndIndex)
217	throw new IndexOutOfBoundsException("index=" + index + " >= limitEndIndex=" + limitEndIndex);	163	throw new IndexOutOfBoundsException("index=" + index + " >= limitEndIndex=" + limitEndIndex);
218	return tokenList.tokenOffset(index);	164	return tokenList.tokenOffsetByIndex(index);
		165	}
		166
		167	public int[] tokenIndex(int offset) {
		168	return LexerUtilsConstants.tokenIndexBinSearch(this, offset, tokenCountCurrent());
219	}	169	}
220		170
221	public int tokenCount() {	171	public int tokenCount() {
222	return limitEndIndex - limitStartIndex;	172	return tokenCountCurrent();
223	}	173	}
224		174
225	public int tokenCountCurrent() {	175	public int tokenCountCurrent() {
Lines 238-257 Link Here
238	return tokenList.languagePath();	188	return tokenList.languagePath();
239	}	189	}
240		190
241	public int childTokenOffset(int rawOffset) {	191	public int tokenOffset(AbstractToken<T> token) {
242	throw new IllegalStateException("Unexpected call.");	192	return tokenList.tokenOffset(token);
243	}
244
245	public char childTokenCharAt(int rawOffset, int index) {
246	throw new IllegalStateException("Unexpected call.");
247	}	193	}
248		194
249	public void wrapToken(int index, EmbeddingContainer<T> embeddingContainer) {	195	public void wrapToken(int index, EmbeddingContainer<T> embeddingContainer) {
250	tokenList.wrapToken(limitStartIndex + index, embeddingContainer);	196	tokenList.wrapToken(limitStartIndex + index, embeddingContainer);
251	}	197	}
252		198
253	public TokenList<?> root() {	199	public TokenList<?> rootTokenList() {
254	return tokenList.root();	200	return tokenList.rootTokenList();
		201	}
		202
		203	public CharSequence inputSourceText() {
		204	return rootTokenList().inputSourceText();
255	}	205	}
256		206
257	public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {	207	public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {
Lines 281-294 Link Here
281		231
282	public int startOffset() {	232	public int startOffset() {
283	if (tokenCountCurrent() > 0 \|\| tokenCount() > 0)	233	if (tokenCountCurrent() > 0 \|\| tokenCount() > 0)
284	return tokenOffset(0);	234	return tokenOffsetByIndex(0);
285	return limitStartOffset;	235	return limitStartOffset;
286	}	236	}
287		237
288	public int endOffset() {	238	public int endOffset() {
289	int cntM1 = tokenCount() - 1;	239	int cntM1 = tokenCount() - 1;
290	if (cntM1 >= 0)	240	if (cntM1 >= 0)
291	return tokenOffset(cntM1) + token(cntM1).length();	241	return tokenOffsetByIndex(cntM1) + tokenList.tokenOrEmbedding(cntM1).token().length();
292	return limitStartOffset;	242	return limitStartOffset;
293	}	243	}
294		244
Lines 296-303 Link Here
296	return tokenList.isRemoved();	246	return tokenList.isRemoved();
297	}	247	}
298		248
299	private AbstractToken<T> token(int index) {
300	return LexerUtilsConstants.token(tokenList, index);
301	}
302
303	}	249	}

Lines 51-126 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/TextLexerInputOperation.java (-45 / +27 lines)
51	* @author Miloslav Metelka	51	* @author Miloslav Metelka
52	* @version 1.00	52	* @version 1.00
53	*/	53	*/
54
55	public class TextLexerInputOperation<T extends TokenId> extends LexerInputOperation<T> {	54	public class TextLexerInputOperation<T extends TokenId> extends LexerInputOperation<T> {
56		55
57	/**	56	/**
58	* Input text from which the reading of characters is done.	57	* Input text from which the reading of characters is done.
59	*/	58	*/
60	private final CharSequence inputText;	59	private final CharSequence inputSourceText;
61		60
62	private final int inputTextStartOffset;
63
64	/**	61	/**
65	* End of valid chars in readCharArray (points to first invalid char).	62	* Point beyond which the reading cannot go.
66	*/	63	*/
67	private int readEndIndex;	64	private int readEndOffset;
68
69		65
70	public TextLexerInputOperation(TokenList<T> tokenList, CharSequence inputText) {	66
71	this(tokenList, 0, null, inputText, 0, 0, inputText.length());	67	public TextLexerInputOperation(TokenList<T> tokenList) {
		68	this(tokenList, 0, null, 0, -1);
72	}	69	}
73		70
74	public TextLexerInputOperation(TokenList<T> tokenList, int tokenIndex,	71	public TextLexerInputOperation(TokenList<T> tokenList, int tokenIndex,
75	Object lexerRestartState, CharSequence inputText, int inputTextStartOffset,	72	Object lexerRestartState, int startOffset, int endOffset) {
76	int startOffset, int endOffset) {
77	super(tokenList, tokenIndex, lexerRestartState);	73	super(tokenList, tokenIndex, lexerRestartState);
78	this.inputText = inputText;	74	this.inputSourceText = tokenList.inputSourceText();
79	this.inputTextStartOffset = inputTextStartOffset;	75	if (endOffset == -1) {
80		76	endOffset = inputSourceText.length();
81	// Make the offsets relative to the input start offset	77	}
82	startOffset -= inputTextStartOffset;
83	endOffset -= inputTextStartOffset;
84	assert (0 <= startOffset) && (startOffset <= endOffset)	78	assert (0 <= startOffset) && (startOffset <= endOffset)
85	&& (endOffset <= inputText.length())	79	&& (endOffset <= inputSourceText.length())
86	: "startOffset=" + startOffset + ", endOffset=" + endOffset	80	: "startOffset=" + startOffset + ", endOffset=" + endOffset
87	+ ", inputText.length()=" + inputText.length();	81	+ ", inputSourceText.length()=" + inputSourceText.length();
88	setTokenStartIndex(startOffset);	82	tokenStartOffset = startOffset;
89	readEndIndex = endOffset;	83	readOffset = tokenStartOffset;
		84	readEndOffset = endOffset;
90	}	85	}
91		86
92	public int read(int index) { // index >= 0 is guaranteed by contract	87	public int read(int offset) {
93	index += tokenStartIndex();	88	if (offset < readEndOffset) {
94	if (index < readEndIndex) {	89	return inputSourceText.charAt(offset);
95	return inputText.charAt(index);
96	} else { // must read next or return EOF	90	} else { // must read next or return EOF
97	return LexerInput.EOF;	91	return LexerInput.EOF;
98	}	92	}
99	}	93	}
100		94
101	public char readExisting(int index) {	95	public char readExisting(int offset) {
102	return inputText.charAt(tokenStartIndex() + index);	96	return inputSourceText.charAt(offset);
103	}	97	}
104		98
105	public void approveToken(AbstractToken<T> token) {	99	protected void fillTokenData(AbstractToken<T> token) {
106	if (isSkipToken(token)) {	100	token.setTokenList(tokenList);
107	preventFlyToken();	101	token.setRawOffset(tokenStartOffset);
108
109	} else if (token.isFlyweight()) {
110	assert isFlyTokenAllowed();
111	flyTokenAdded();
112
113	} else { // non-flyweight token
114	token.setTokenList(tokenList());
115	token.setRawOffset(inputTextStartOffset + tokenStartIndex());
116	clearFlySequence();
117	}
118
119	tokenApproved();
120	}	102	}
121		103
122	protected final int readEndIndex() {	104	protected final int readEndIndex() {
123	return readEndIndex;	105	return readEndOffset;
124	}	106	}
125		107
126	}	108	}

Added Link Here

(-)06a7890f802e (+70 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer;
43
44	import org.netbeans.api.lexer.TokenId;
45	import org.netbeans.lib.lexer.token.AbstractToken;
46
47	/**
48	* Type for having either token or embedding.
49	*
50	* @author Miloslav Metelka
51	*/
52
53	public interface TokenOrEmbedding<T extends TokenId> {
54
55	/**
56	* Get token reference
57	*
58	* @return <code>this</code> if this is a token instance or
59	* a wrapped token if this is an embedding container.
60	*/
61	AbstractToken<T> token();
62
63	/**
64	* Get non-null embedding container if this is embedding.
65	*
66	* @return non-null embedding or null if this is token.
67	*/
68	EmbeddingContainer<T> embedding();
69
70	}

Lines 43-56 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/TokenSequenceList.java (-14 / +11 lines)
43		43
44	import java.util.AbstractList;	44	import java.util.AbstractList;
45	import java.util.ArrayList;	45	import java.util.ArrayList;
46	import java.util.ArrayList;
47	import java.util.Collections;	46	import java.util.Collections;
48	import java.util.ConcurrentModificationException;	47	import java.util.ConcurrentModificationException;
49	import java.util.Iterator;	48	import java.util.Iterator;
50	import java.util.List;	49	import java.util.List;
51	import java.util.NoSuchElementException;	50	import java.util.NoSuchElementException;
52	import org.netbeans.api.lexer.LanguagePath;	51	import org.netbeans.api.lexer.LanguagePath;
53	import org.netbeans.api.lexer.TokenId;
54	import org.netbeans.api.lexer.TokenSequence;	52	import org.netbeans.api.lexer.TokenSequence;
55		53
56	/**	54	/**
Lines 61-69 Link Here
61		59
62	public final class TokenSequenceList extends AbstractList<TokenSequence<?>> {	60	public final class TokenSequenceList extends AbstractList<TokenSequence<?>> {
63		61
64	private TokenHierarchyOperation<?,?> operation;	62	private TokenList<?> rootTokenList;
65		63
66	private final TokenListList tokenListList;	64	private final TokenListList<?> tokenListList;
67		65
68	private final List<TokenSequence<?>> tokenSequences;	66	private final List<TokenSequence<?>> tokenSequences;
69		67
Lines 78-93 Link Here
78	*/	76	*/
79	private int tokenListIndex;	77	private int tokenListIndex;
80		78
81	public TokenSequenceList(TokenHierarchyOperation<?,?> operation, LanguagePath languagePath,	79	public TokenSequenceList(TokenList<?> rootTokenList, LanguagePath languagePath,
82	int startOffset, int endOffset) {	80	int startOffset, int endOffset) {
83	this.operation = operation;	81	this.rootTokenList = rootTokenList;
84	this.endOffset = endOffset;	82	this.endOffset = endOffset;
85	this.expectedModCount = operation.modCount();	83	this.expectedModCount = rootTokenList.modCount();
86		84
87	if (languagePath.size() == 1) { // Is supported too	85	if (languagePath.size() == 1) { // Is supported too
88	this.tokenListList = null;	86	this.tokenListList = null;
89	tokenListIndex = Integer.MAX_VALUE; // Mark no mods to tokenSequences	87	tokenListIndex = Integer.MAX_VALUE; // Mark no mods to tokenSequences
90	TokenList<?> rootTokenList = operation.rootTokenList();
91	if (rootTokenList.languagePath() == languagePath) {	88	if (rootTokenList.languagePath() == languagePath) {
92	TokenSequence<?> rootTS = LexerApiPackageAccessor.get().createTokenSequence(	89	TokenSequence<?> rootTS = LexerApiPackageAccessor.get().createTokenSequence(
93	checkWrapTokenList(rootTokenList, startOffset, endOffset));	90	checkWrapTokenList(rootTokenList, startOffset, endOffset));
Lines 97-103 Link Here
97	}	94	}
98		95
99	} else { // languagePath.size() >= 2	96	} else { // languagePath.size() >= 2
100	this.tokenListList = operation.tokenListList(languagePath);	97	this.tokenListList = rootTokenList.tokenHierarchyOperation().tokenListList(languagePath);
101	// Possibly skip initial token lists accroding to startOffset	98	// Possibly skip initial token lists accroding to startOffset
102	int size = tokenListList.size();	99	int size = tokenListList.size();
103	int high = size - 1;	100	int high = size - 1;
Lines 108-114 Link Here
108	int mid = (tokenListIndex + high) / 2;	105	int mid = (tokenListIndex + high) / 2;
109	EmbeddedTokenList<?> etl = tokenListList.get(mid);	106	EmbeddedTokenList<?> etl = tokenListList.get(mid);
110	// Update end offset before querying	107	// Update end offset before querying
111	etl.embeddingContainer().updateStatusImpl();	108	etl.embeddingContainer().updateStatusUnsync();
112	int tlEndOffset = etl.endOffset(); // updateStatusImpl() just called	109	int tlEndOffset = etl.endOffset(); // updateStatusImpl() just called
113	if (tlEndOffset < startOffset) {	110	if (tlEndOffset < startOffset) {
114	tokenListIndex = mid + 1;	111	tokenListIndex = mid + 1;
Lines 123-129 Link Here
123	firstTokenList = tokenListList.getOrNull(tokenListIndex);	120	firstTokenList = tokenListList.getOrNull(tokenListIndex);
124	if (tokenListIndex == size) { // Right above the ones that existed at begining of bin search	121	if (tokenListIndex == size) { // Right above the ones that existed at begining of bin search
125	while (firstTokenList != null) {	122	while (firstTokenList != null) {
126	firstTokenList.embeddingContainer().updateStatusImpl();	123	firstTokenList.embeddingContainer().updateStatusUnsync();
127	if (firstTokenList.endOffset() >= startOffset) { // updateStatusImpl() just called	124	if (firstTokenList.endOffset() >= startOffset) { // updateStatusImpl() just called
128	break;	125	break;
129	}	126	}
Lines 136-142 Link Here
136	}	133	}
137		134
138	if (firstTokenList != null) {	135	if (firstTokenList != null) {
139	firstTokenList.embeddingContainer().updateStatusImpl();	136	firstTokenList.embeddingContainer().updateStatusUnsync();
140	tokenSequences = new ArrayList<TokenSequence<?>>(4);	137	tokenSequences = new ArrayList<TokenSequence<?>>(4);
141	tokenSequences.add(LexerApiPackageAccessor.get().createTokenSequence(	138	tokenSequences.add(LexerApiPackageAccessor.get().createTokenSequence(
142	checkWrapTokenList(firstTokenList, startOffset, endOffset)));	139	checkWrapTokenList(firstTokenList, startOffset, endOffset)));
Lines 207-216 Link Here
207	}	204	}
208		205
209	void checkForComodification() {	206	void checkForComodification() {
210	if (expectedModCount != operation.modCount())	207	if (expectedModCount != rootTokenList.modCount())
211	throw new ConcurrentModificationException(	208	throw new ConcurrentModificationException(
212	"Caller uses obsolete TokenSequenceList: expectedModCount=" + expectedModCount + // NOI18N	209	"Caller uses obsolete TokenSequenceList: expectedModCount=" + expectedModCount + // NOI18N
213	" != modCount=" + operation.modCount()	210	" != modCount=" + rootTokenList.modCount()
214	);	211	);
215	}	212	}
216		213

Removed Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/batch/BatchTokenList.java (-255 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer.batch;
43
44	import java.util.ArrayList;
45	import java.util.Set;
46	import org.netbeans.api.lexer.Language;
47	import org.netbeans.api.lexer.LanguagePath;
48	import org.netbeans.lib.lexer.EmbeddingContainer;
49	import org.netbeans.lib.lexer.LAState;
50	import org.netbeans.lib.lexer.TokenList;
51	import org.netbeans.lib.lexer.LexerInputOperation;
52	import org.netbeans.lib.lexer.LexerUtilsConstants;
53	import org.netbeans.api.lexer.InputAttributes;
54	import org.netbeans.api.lexer.Token;
55	import org.netbeans.api.lexer.TokenId;
56	import org.netbeans.lib.lexer.TokenHierarchyOperation;
57	import org.netbeans.lib.lexer.token.AbstractToken;
58	import org.netbeans.lib.lexer.token.TextToken;
59
60
61	/**
62	* Token list used for root list for immutable inputs.
63	*
64	* @author Miloslav Metelka
65	* @version 1.00
66	*/
67
68	public abstract class BatchTokenList<T extends TokenId>
69	extends ArrayList<Object> implements TokenList<T> {
70
71	/** Flag for additional correctness checks (may degrade performance). */
72	private static final boolean testing = Boolean.getBoolean("netbeans.debug.lexer.test");
73
74	private static boolean maintainLAState;
75
76	/**
77	* Check whether lookaheads and states are stored for testing purposes.
78	*/
79	public static boolean isMaintainLAState() {
80	return maintainLAState;
81	}
82
83	public static void setMaintainLAState(boolean maintainLAState) {
84	BatchTokenList.maintainLAState = maintainLAState;
85	}
86
87	private final TokenHierarchyOperation<?,T> tokenHierarchyOperation;
88
89	private final LanguagePath languagePath;
90
91	private final Set<T> skipTokenIds;
92
93	private final InputAttributes inputAttributes;
94
95	/**
96	* Lexer input used for lexing of the input.
97	*/
98	private LexerInputOperation<T> lexerInputOperation;
99
100	private LAState laState;
101
102	private boolean inited;
103
104
105	public BatchTokenList(TokenHierarchyOperation<?,T> tokenHierarchyOperation,
106	Language<T> language, Set<T> skipTokenIds, InputAttributes inputAttributes) {
107	this.tokenHierarchyOperation = tokenHierarchyOperation;
108	this.languagePath = LanguagePath.get(language);
109	this.skipTokenIds = skipTokenIds;
110	this.inputAttributes = inputAttributes;
111	if (testing) { // Maintain lookaheads and states when in test environment
112	laState = LAState.empty();
113	}
114	}
115
116	public abstract char childTokenCharAt(int rawOffset, int index);
117
118	protected abstract LexerInputOperation<T> createLexerInputOperation();
119
120	protected void init() {
121	lexerInputOperation = createLexerInputOperation();
122	}
123
124	public TokenList<?> root() {
125	return this; // this list should always be the root list of the token hierarchy
126	}
127
128	public TokenHierarchyOperation<?,?> tokenHierarchyOperation() {
129	return tokenHierarchyOperation;
130	}
131
132	public LanguagePath languagePath() {
133	return languagePath;
134	}
135
136	public synchronized int tokenCount() {
137	if (!inited) {
138	init();
139	inited = true;
140	}
141	if (lexerInputOperation != null) { // still lexing
142	tokenOrEmbeddingContainerImpl(Integer.MAX_VALUE);
143	}
144	return size();
145	}
146
147	public int tokenCountCurrent() {
148	return size();
149	}
150
151	public int childTokenOffset(int rawOffset) {
152	// Children offsets should be absolute
153	return rawOffset;
154	}
155
156	public int tokenOffset(int index) {
157	Token<T> token = existingToken(index);
158	int offset;
159	if (token.isFlyweight()) {
160	offset = 0;
161	while (--index >= 0) {
162	token = existingToken(index);
163	offset += token.length();
164	if (!token.isFlyweight()) {
165	offset += token.offset(null);
166	break;
167	}
168	}
169	} else { // non-flyweight offset
170	offset = token.offset(null);
171	}
172	return offset;
173	}
174
175	public synchronized Object tokenOrEmbeddingContainer(int index) {
176	return tokenOrEmbeddingContainerImpl(index);
177	}
178
179	private Object tokenOrEmbeddingContainerImpl(int index) {
180	if (!inited) {
181	init();
182	inited = true;
183	}
184	while (lexerInputOperation != null && index >= size()) {
185	Token<T> token = lexerInputOperation.nextToken();
186	if (token != null) { // lexer returned valid token
187	add(token);
188	if (laState != null) { // maintaining lookaheads and states
189	laState = laState.add(lexerInputOperation.lookahead(),
190	lexerInputOperation.lexerState());
191	}
192	} else { // no more tokens from lexer
193	lexerInputOperation.release();
194	lexerInputOperation = null;
195	trimToSize();
196	}
197	}
198	return (index < size()) ? get(index) : null;
199	}
200
201	private Token<T> existingToken(int index) {
202	return LexerUtilsConstants.token(get(index));
203	}
204
205	public int lookahead(int index) {
206	return (laState != null) ? laState.lookahead(index) : -1;
207	}
208
209	public Object state(int index) {
210	return (laState != null) ? laState.state(index) : null;
211	}
212
213	public int startOffset() {
214	return 0;
215	}
216
217	public int endOffset() {
218	int cntM1 = tokenCount() - 1;
219	if (cntM1 >= 0)
220	return tokenOffset(cntM1) + LexerUtilsConstants.token(this, cntM1).length();
221	return 0;
222	}
223
224	public boolean isRemoved() {
225	return false;
226	}
227
228	public int modCount() {
229	return -1; // immutable input
230	}
231
232	public synchronized AbstractToken<T> replaceFlyToken(
233	int index, AbstractToken<T> flyToken, int offset) {
234	TextToken<T> nonFlyToken = ((TextToken<T>)flyToken).createCopy(this, offset);
235	set(index, nonFlyToken);
236	return nonFlyToken;
237	}
238
239	public void wrapToken(int index, EmbeddingContainer embeddingContainer) {
240	set(index, embeddingContainer);
241	}
242
243	public InputAttributes inputAttributes() {
244	return inputAttributes;
245	}
246
247	public boolean isContinuous() {
248	return (skipTokenIds == null);
249	}
250
251	public Set<T> skipTokenIds() {
252	return skipTokenIds;
253	}
254
255	}

Added Link Here

(-)06a7890f802e (+282 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer.inc;
43
44	import java.util.ArrayList;
45	import java.util.List;
46	import org.netbeans.api.lexer.TokenId;
47	import org.netbeans.lib.lexer.EmbeddedJoinInfo;
48	import org.netbeans.lib.lexer.EmbeddedTokenList;
49	import org.netbeans.lib.lexer.JoinLexerInputOperation;
50	import org.netbeans.lib.lexer.JoinTokenListBase;
51	import org.netbeans.lib.lexer.token.AbstractToken;
52	import org.netbeans.lib.lexer.token.JoinToken;
53	import org.netbeans.lib.lexer.token.PartToken;
54
55	/**
56	* Token list change for join token lists.
57	*
58	* @author Miloslav Metelka
59	*/
60	final class JoinTokenListChange<T extends TokenId> extends TokenListChange<T> {
61
62	/** ETL where character modification occurred. */
63	EmbeddedTokenList<T> charModTokenList;
64
65	private TokenListListUpdate<T> tokenListListUpdate;
66
67	private int startRelexTokenListIndex;
68
69	private List<RelexTokenListChange<T>> relexChanges;
70
71	private JoinLexerInputOperation<T> joinLexerInputOperation;
72
73	public JoinTokenListChange(MutableJoinTokenList<T> tokenList) {
74	super(tokenList);
75	}
76
77	public List<? extends TokenListChange<T>> relexChanges() {
78	return relexChanges;
79	}
80
81	public TokenListListUpdate<T> tokenListListUpdate() {
82	return tokenListListUpdate;
83	}
84
85	public void setTokenListListUpdate(TokenListListUpdate<T> tokenListListUpdate) {
86	this.tokenListListUpdate = tokenListListUpdate;
87	}
88
89	public void setStartInfo(JoinLexerInputOperation<T> joinLexerInputOperation, int localIndex) {
90	this.joinLexerInputOperation = joinLexerInputOperation;
91	this.startRelexTokenListIndex = joinLexerInputOperation.activeTokenListIndex();
92	this.relexChanges = new ArrayList<RelexTokenListChange<T>>(
93	tokenListListUpdate.addedTokenLists.size() + 3);
94	// Add first change now to incorporate starting modified token index
95	RelexTokenListChange<T> firstChange = new RelexTokenListChange<T>(
96	joinLexerInputOperation.tokenList(startRelexTokenListIndex));
97	// Set index in ETL to properly do replaceTokens() in ETL
98	// Setting both index and offset is BTW necessary in order to properly move offset gap in ETL
99	firstChange.setIndex(localIndex);
100	int relexOffset = joinLexerInputOperation.lastTokenEndOffset();
101	firstChange.setOffset(relexOffset);
102	firstChange.setMatchOffset(relexOffset); // Due to removeLastAddedToken() and etc.
103	relexChanges.add(firstChange);
104	}
105
106	@Override
107	public void addToken(AbstractToken<T> token, int lookahead, Object state) {
108	// Check if lexer-input-operation advanced to next list and possibly add corresponding relex change(s)
109	int activeTokenListIndex = joinLexerInputOperation.activeTokenListIndex();
110	while (startRelexTokenListIndex + relexChanges.size() <= activeTokenListIndex) {
111	// Use JLIO.tokenList() since it already contains the removed/added ETLs.
112	EmbeddedTokenList<T> etl = joinLexerInputOperation.tokenList(
113	startRelexTokenListIndex + relexChanges.size());
114	RelexTokenListChange<T> relexChange = new RelexTokenListChange<T>(etl);
115	int startOffset = etl.startOffset();
116	relexChange.setOffset(startOffset);
117	relexChanges.add(relexChange);
118	}
119	int relexChangeIndex = activeTokenListIndex - startRelexTokenListIndex;
120	if (token.getClass() == JoinToken.class) {
121	JoinToken<T> joinToken = (JoinToken<T>) token;
122	List<PartToken<T>> joinedParts = joinToken.joinedParts();
123	int extraTokenListSpanCount = joinToken.extraTokenListSpanCount();
124	int startRelexChangeIndex = relexChangeIndex - extraTokenListSpanCount;
125	int joinedPartIndex = 0;
126	// Only add without the last part (will be added normally outside the loop)
127	// The last ETL can not be empty (must contain the last non-empty token part)
128	for (int i = 0; i < extraTokenListSpanCount; i++) {
129	RelexTokenListChange<T> relexChange = relexChanges.get(startRelexChangeIndex + i);
130	// Check whether token list is non-empty by checking a text length that it covers.
131	// Do not use etl.tokenCount() since the tokens are just being added into ETL.
132	EmbeddedTokenList<T> etl = (EmbeddedTokenList<T>) relexChange.tokenList();
133	if (etl.textLength() > 0) {
134	PartToken<T> partToken = joinedParts.get(joinedPartIndex++);
135	relexChange.addToken(partToken, 0, null);
136	}
137	relexChange.joinTokenLastPartShift = extraTokenListSpanCount - i;
138	}
139	// Last part will be added normally by subsequent code
140	token = joinedParts.get(joinedPartIndex); // Should be (joinedParts.size()-1)
141	}
142	RelexTokenListChange<T> relexChange = relexChanges.get(relexChangeIndex);
143	relexChange.addToken(token, lookahead, state);
144	}
145
146	@Override
147	public AbstractToken<T> removeLastAddedToken() {
148	RelexTokenListChange<T> lastRelexChange;
149	AbstractToken<T> lastAddedToken = null;
150	do {
151	lastRelexChange = relexChanges.get(relexChanges.size() - 1);
152	int tokenCount = lastRelexChange.tokenList().tokenCountCurrent();
153	if (lastRelexChange.tokenList().tokenCountCurrent() > 0) { // There might be empty ETLs
154	lastAddedToken = lastRelexChange.removeLastAddedToken();
155	tokenCount--;
156	}
157	if (tokenCount == 0) {
158	relexChanges.remove(relexChanges.size() - 1);
159	}
160	} while (lastAddedToken.getClass() != PartToken.class \|\|
161	((PartToken<T>) lastAddedToken).joinToken().joinedParts().get(0) != lastAddedToken);
162	matchIndex--;
163	matchOffset = lastRelexChange.matchOffset;
164	// If lastAddedToken is PartToken then its joinToken() should be returned
165	// but the return value should currently be ignored
166	return lastAddedToken;
167	}
168
169	public void replaceTokens(int diffLength) {
170	// Determine position of matchIndex in token lists
171	// if matchIndex == jtl.tokenCount() the token list index will be the last list
172	// and endLocalIndex will be its tokenCount(). Because of this
173	// there must be a check whether token list index is not among removed ETLs.
174	MutableJoinTokenList<T> jtl = (MutableJoinTokenList<T>) tokenList();
175	int endLocalIndex = jtl.tokenStartLocalIndex(matchIndex);
176	int matchTokenListIndex = jtl.activeTokenListIndex();
177	if (matchTokenListIndex >= tokenListListUpdate.modTokenListIndex + tokenListListUpdate.removedTokenListCount) {
178	assert (matchIndex == jtl.tokenCountCurrent()); // Should only happen in this situation
179	// Project into relexChanges
180	matchTokenListIndex += tokenListListUpdate.tokenListCountDiff();
181	relexChanges.get(matchTokenListIndex - startRelexTokenListIndex).setMatchIndex(endLocalIndex);
182	int afterAddIndex = tokenListListUpdate.modTokenListIndex + tokenListListUpdate.addedTokenLists.size();
183	while (--matchTokenListIndex >= afterAddIndex) {
184	TokenListChange<T> change = relexChanges.get(matchTokenListIndex - startRelexTokenListIndex);
185	change.setMatchIndex(change.tokenList().tokenCountCurrent());
186	}
187	}
188	// Fill in the below-mod-ETLs area
189	int index = tokenListListUpdate.modTokenListIndex;
190	while (--index >= startRelexTokenListIndex) {
191	TokenListChange<T> change = relexChanges.get(index - startRelexTokenListIndex);
192	change.setMatchIndex(change.tokenList().tokenCountCurrent());
193	}
194
195	// Physically replace the token lists
196	JoinTokenListBase base = jtl.base();
197	if (tokenListListUpdate.isTokenListsMod()) {
198	// Move gap after last ETL that was relexed (obsolete ETLs still not removed)
199	int relexEndIndex = startRelexTokenListIndex + relexChanges.size();
200	int relexEndOldIndex = relexEndIndex - tokenListListUpdate.tokenListCountDiff();
201	base.moveIndexGap(jtl.tokenListList(), jtl.tokenListStartIndex(), relexEndOldIndex);
202	// Do physical ETLs replace
203	jtl.tokenListList().replace(jtl.tokenListStartIndex() + tokenListListUpdate.modTokenListIndex,
204	tokenListListUpdate.removedTokenListCount, tokenListListUpdate.addedTokenLists);
205	base.tokenListModNotify(tokenListListUpdate.tokenListCountDiff());
206	}
207
208	// Remember join token count right before the first relexed ETL
209	int joinTokenIndex;
210	if (startRelexTokenListIndex > 0) {
211	EmbeddedTokenList<T> etl = jtl.tokenList(startRelexTokenListIndex - 1);
212	joinTokenIndex = etl.joinInfo.joinTokenIndex() + etl.joinTokenCount(); // Physical removal already performed
213	} else {
214	joinTokenIndex = 0;
215	}
216	// Now process each relex change and update join token count etc.
217	int relexChangesSizeM1 = relexChanges.size() - 1;
218	int i;
219	for (i = 0; i <= relexChangesSizeM1; i++) {
220	RelexTokenListChange<T> change = relexChanges.get(i);
221	EmbeddedTokenList<T> etl = (EmbeddedTokenList<T>) change.tokenList();
222	if (etl.joinInfo == null) {
223	etl.joinInfo = new EmbeddedJoinInfo(base, joinTokenIndex, startRelexTokenListIndex + i);
224	} else {
225	etl.joinInfo.setRawJoinTokenIndex(joinTokenIndex);
226	}
227	// Set new joinTokenLastPartShift before calling etl.joinTokenCount()
228	etl.joinInfo.setJoinTokenLastPartShift(change.joinTokenLastPartShift);
229	// Replace tokens in the individual ETL
230	int realDiffLength = (etl == charModTokenList) ? diffLength : 0;
231	etl.replaceTokens(change, realDiffLength);
232	// Fix join token count
233	joinTokenIndex += etl.joinTokenCount();
234	}
235
236	// Now fix the total join token count
237	int origJoinTokenIndex = (i < jtl.tokenListCount())
238	? jtl.tokenList(i).joinInfo.joinTokenIndex()
239	: base.joinTokenCount();
240	int joinTokenCountDiff = joinTokenIndex - origJoinTokenIndex;
241	base.updateJoinTokenCount(joinTokenCountDiff);
242
243	if (relexChangesSizeM1 == 0) { // Only changed single ETL
244	if (relexChanges.get(0).isBoundsChange()) {
245	markBoundsChange(); // Joined change treated as bounds change too
246	}
247	}
248	}
249
250	@Override
251	public String toString() {
252	return super.toString() + ", tokenListListUpdate=" + tokenListListUpdate + // NOI18N
253	", startRelexTokenListIndex=" + startRelexTokenListIndex + // NOI18N
254	", relexChanges.size()=" + relexChanges.size();
255	}
256
257	@Override
258	public String toStringMods(int indent) {
259	StringBuilder sb = new StringBuilder(100);
260	for (RelexTokenListChange change : relexChanges) {
261	sb.append(change.toStringMods(indent));
262	sb.append('\n');
263	}
264	return sb.toString();
265	}
266
267	static final class RelexTokenListChange<T extends TokenId> extends TokenListChange<T> {
268
269	int joinTokenLastPartShift; // New value for EmbeddedJoinInfo.joinTokenLastPartShift during relex
270
271	RelexTokenListChange(EmbeddedTokenList<T> tokenList) {
272	super(tokenList);
273	}
274
275	@Override
276	public String toString() {
277	return super.toString() + ", lps=" + joinTokenLastPartShift;
278	}
279
280	}
281
282	}

Added Link Here

(-)06a7890f802e (+93 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer.inc;
43
44	import org.netbeans.api.lexer.TokenId;
45	import org.netbeans.lib.lexer.EmbeddedTokenList;
46	import org.netbeans.lib.lexer.JoinLexerInputOperation;
47	import org.netbeans.lib.lexer.JoinTokenList;
48
49	/**
50	* Lexer input operation over multiple joined sections (embedded token lists).
51	* <br/>
52	* It produces regular tokens (to be added directly into ETL represented by
53	* {@link #activeTokenList()} and also special {@link #JoinToken} instances
54	* in case a token spans boundaries of multiple ETLs.
55	* <br/>
56	* It can either work over JoinTokenList directly or, during a modification,
57	* it simulates that certain token lists are already removed/added to underlying token list.
58	* <br/>
59	*
60	* {@link #recognizedTokenLastInTokenList()} gives information whether the lastly
61	* produced token ends right at boundary of the activeTokenList.
62	*
63	* @author Miloslav Metelka
64	* @version 1.00
65	*/
66
67	class MutableJoinLexerInputOperation<T extends TokenId> extends JoinLexerInputOperation<T> {
68
69	private TokenListListUpdate<T> tokenListListUpdate;
70
71	MutableJoinLexerInputOperation(JoinTokenList<T> joinTokenList, int relexJoinIndex, Object lexerRestartState,
72	int activeTokenListIndex, int relexOffset, TokenListListUpdate<T> tokenListListUpdate
73	) {
74	super(joinTokenList, relexJoinIndex, lexerRestartState, activeTokenListIndex, relexOffset);
75	this.tokenListListUpdate = tokenListListUpdate;
76	}
77
78	@Override
79	public EmbeddedTokenList<T> tokenList(int tokenListIndex) {
80	return tokenListListUpdate.afterUpdateTokenList((JoinTokenList<T>) tokenList, tokenListIndex);
81	}
82
83	@Override
84	protected int tokenListCount() {
85	return tokenListListUpdate.afterUpdateTokenListCount((JoinTokenList<T>) tokenList);
86	}
87
88	@Override
89	public String toString() {
90	return super.toString() + ", tokenListListUpdate: " + tokenListListUpdate; // NOI18N
91	}
92
93	}

Added Link Here

(-)06a7890f802e (+96 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer.inc;
43
44	import org.netbeans.api.lexer.TokenId;
45	import org.netbeans.lib.lexer.EmbeddedTokenList;
46	import org.netbeans.lib.lexer.JoinTokenList;
47	import org.netbeans.lib.lexer.JoinTokenListBase;
48	import org.netbeans.lib.lexer.LexerInputOperation;
49	import org.netbeans.lib.lexer.TokenListList;
50	import org.netbeans.lib.lexer.TokenOrEmbedding;
51
52
53	/**
54	* Mutable join token list allows mutations by token list updater.
55	*
56	* @author Miloslav Metelka
57	*/
58
59	class MutableJoinTokenList<T extends TokenId> extends JoinTokenList<T> implements MutableTokenList<T> {
60
61	static <T extends TokenId> MutableJoinTokenList<T> create(TokenListList<T> tokenListList, int etlIndex, EmbeddedTokenList<T> etl) {
62	int tokenListStartIndex = etlIndex - etl.joinInfo.tokenListIndex();
63	MutableJoinTokenList<T> jtl = new MutableJoinTokenList<T>(tokenListList, etl.joinInfo.base, tokenListStartIndex);
64	// Position to this etl's join index
65	jtl.setActiveTokenListIndex(etlIndex - tokenListStartIndex);
66	return jtl;
67	}
68
69	MutableJoinTokenList(TokenListList<T> tokenListList, JoinTokenListBase base, int tokenListStartIndex) {
70	super(tokenListList, base, tokenListStartIndex);
71	}
72
73	public TokenOrEmbedding<T> tokenOrEmbeddingUnsync(int index) {
74	return tokenOrEmbedding(index);
75	}
76
77	public boolean isFullyLexed() {
78	return true;
79	}
80
81	public void replaceTokens(TokenListChange<T> change, int diffLength) {
82	((JoinTokenListChange<T>) change).replaceTokens(diffLength);
83	}
84
85	public LexerInputOperation<T> createLexerInputOperation(int tokenIndex, int relexOffset, Object relexState) {
86	// Should never be called
87	throw new IllegalStateException("Should never be called"); // NOI18N
88	}
89
90	public void resetActiveAfterUpdate() { // Update the active token list after updating
91	activeTokenListIndex = 0;
92	fetchActiveTokenListData();
93	}
94
95	}
96

Lines 43-54 Link Here

(-)a/lexer/src/org/netbeans/lib/lexer/inc/TokenHierarchyEventInfo.java (-15 / +16 lines)
43		43
44	import org.netbeans.api.lexer.TokenChange;	44	import org.netbeans.api.lexer.TokenChange;
45	import org.netbeans.api.lexer.TokenHierarchyEventType;	45	import org.netbeans.api.lexer.TokenHierarchyEventType;
46	import org.netbeans.api.lexer.TokenId;
47	import org.netbeans.lib.editor.util.CharSequenceUtilities;	46	import org.netbeans.lib.editor.util.CharSequenceUtilities;
48	import org.netbeans.lib.lexer.LexerApiPackageAccessor;	47	import org.netbeans.lib.lexer.LexerApiPackageAccessor;
49	import org.netbeans.lib.lexer.LexerSpiPackageAccessor;
50	import org.netbeans.lib.lexer.TokenHierarchyOperation;	48	import org.netbeans.lib.lexer.TokenHierarchyOperation;
51	import org.netbeans.spi.lexer.MutableTextInput;
52		49
53	/**	50	/**
54	* Shared information for all the token list changes	51	* Shared information for all the token list changes
Lines 66-72 Link Here
66		63
67	private TokenChange<?> tokenChange;	64	private TokenChange<?> tokenChange;
68		65
69	private final int modificationOffset;	66	private final int modOffset;
70		67
71	private final int removedLength;	68	private final int removedLength;
72		69
Lines 97-103 Link Here
97		94
98	this.tokenHierarchyOperation = tokenHierarchyOperation;	95	this.tokenHierarchyOperation = tokenHierarchyOperation;
99	this.type = type;	96	this.type = type;
100	this.modificationOffset = modificationOffset;	97	this.modOffset = modificationOffset;
101	this.removedLength = removedLength;	98	this.removedLength = removedLength;
102	this.removedText = removedText;	99	this.removedText = removedText;
103	this.insertedLength = insertedLength;	100	this.insertedLength = insertedLength;
Lines 142-149 Link Here
142	}	139	}
143	}	140	}
144		141
145	public int modificationOffset() {	142	public int modOffset() {
146	return modificationOffset;	143	return modOffset;
147	}	144	}
148		145
149	public int removedLength() {	146	public int removedLength() {
Lines 159-165 Link Here
159	}	156	}
160		157
161	public CharSequence insertedText() {	158	public CharSequence insertedText() {
162	return currentText().subSequence(modificationOffset(), modificationOffset() + insertedLength());	159	return currentText().subSequence(modOffset(), modOffset() + insertedLength());
		160	}
		161
		162	public int diffLength() {
		163	return insertedLength - removedLength;
163	}	164	}
164		165
165	/**	166	/**
Lines 181-187 Link Here
181	);	182	);
182	}	183	}
183	originalText = new OriginalText(currentText(),	184	originalText = new OriginalText(currentText(),
184	modificationOffset, removedText, insertedLength);	185	modOffset, removedText, insertedLength);
185	}	186	}
186	return originalText;	187	return originalText;
187	}	188	}
Lines 193-200 Link Here
193	public String modificationDescription(boolean detail) {	194	public String modificationDescription(boolean detail) {
194	StringBuilder sb = new StringBuilder(originalText().length() + 300);	195	StringBuilder sb = new StringBuilder(originalText().length() + 300);
195	if (removedLength() > 0) {	196	if (removedLength() > 0) {
196	sb.append("TEXT REMOVED <").append(modificationOffset()).append(","). // NOI18N	197	sb.append("TEXT REMOVED <").append(modOffset()).append(","). // NOI18N
197	append(modificationOffset() + removedLength()).append('>');	198	append(modOffset() + removedLength()).append('>');
198	sb.append(':').append(removedLength());	199	sb.append(':').append(removedLength());
199	if (removedText() != null) {	200	if (removedText() != null) {
200	sb.append(" \"");	201	sb.append(" \"");
Lines 204-211 Link Here
204	sb.append('\n');	205	sb.append('\n');
205	}	206	}
206	if (insertedLength() > 0) {	207	if (insertedLength() > 0) {
207	sb.append("TEXT INSERTED <").append(modificationOffset()).append(","). // NOI18N	208	sb.append("TEXT INSERTED <").append(modOffset()).append(","). // NOI18N
208	append(modificationOffset() + insertedLength()).append(">:"). // NOI18N	209	append(modOffset() + insertedLength()).append(">:"). // NOI18N
209	append(insertedLength()).append(" \""); // NOI18N	210	append(insertedLength()).append(" \""); // NOI18N
210	CharSequenceUtilities.debugText(sb, insertedText());	211	CharSequenceUtilities.debugText(sb, insertedText());
211	sb.append("\"\n");	212	sb.append("\"\n");
Lines 214-220 Link Here
214	sb.append("\n\n----------------- ORIGINAL TEXT -----------------\n" + // NOI18N	215	sb.append("\n\n----------------- ORIGINAL TEXT -----------------\n" + // NOI18N
215	originalText() +	216	originalText() +
216	"\n----------------- BEFORE-CARET TEXT -----------------\n" + // NOI18N	217	"\n----------------- BEFORE-CARET TEXT -----------------\n" + // NOI18N
217	originalText().subSequence(0, modificationOffset()) +	218	originalText().subSequence(0, modOffset()) +
218	"\|<--CARET\n" // NOI18N	219	"\|<--CARET\n" // NOI18N
219	);	220	);
220	}	221	}
Lines 234-240 Link Here
234	public String toString() {	235	public String toString() {
235	StringBuilder sb = new StringBuilder();	236	StringBuilder sb = new StringBuilder();
236	sb.append("modOffset="); // NOI18N	237	sb.append("modOffset="); // NOI18N
237	sb.append(modificationOffset());	238	sb.append(modOffset());
238	if (removedLength() > 0) {	239	if (removedLength() > 0) {
239	sb.append(", removedLength=");	240	sb.append(", removedLength=");
240	sb.append(removedLength());	241	sb.append(removedLength());

Added Link Here

(-)06a7890f802e (+208 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer.inc;
43
44	import java.util.ArrayList;
45	import java.util.List;
46	import org.netbeans.api.lexer.TokenId;
47	import org.netbeans.lib.lexer.EmbeddedTokenList;
48	import org.netbeans.lib.lexer.JoinTokenList;
49	import org.netbeans.lib.lexer.TokenList;
50	import org.netbeans.lib.lexer.TokenListList;
51
52	/**
53	* Change of a particular TokenListList.
54	*
55	* @author Miloslav Metelka
56	*/
57
58	final class TokenListListUpdate<T extends TokenId> {
59
60	/**
61	* Token list list for the case when the particular language path
62	* corresponds to joined.
63	*/
64	final TokenListList<T> tokenListList;
65
66	int modTokenListIndex;
67
68	int removedTokenListCount;
69
70	List<EmbeddedTokenList<T>> addedTokenLists;
71
72	TokenListListUpdate(TokenListList<T> tokenListList) {
73	this.tokenListList = tokenListList;
74	this.modTokenListIndex = -1;
75	}
76
77	public boolean isTokenListsMod() { // If any ETL was removed/added
78	return (removedTokenListCount != 0) \|\| addedTokenLists.size() > 0;
79	}
80
81	public int tokenListCountDiff() {
82	return addedTokenLists.size() - removedTokenListCount;
83	}
84
85	public EmbeddedTokenList<T> afterUpdateTokenList(JoinTokenList<T> jtl, int tokenListIndex) {
86	EmbeddedTokenList<T> etl;
87	if (tokenListIndex < modTokenListIndex) {
88	etl = jtl.tokenList(tokenListIndex);
89	} else if (tokenListIndex - modTokenListIndex < addedTokenLists.size()) {
90	etl = addedTokenLists.get(tokenListIndex - modTokenListIndex);
91	} else { // Last part after removed and added
92	etl = jtl.tokenList(modTokenListIndex + removedTokenListCount - addedTokenLists.size());
93	}
94	return etl;
95	}
96
97	protected int afterUpdateTokenListCount(JoinTokenList<T> jtl) {
98	return jtl.tokenListCount() - removedTokenListCount + addedTokenLists.size();
99	}
100
101	void markChangedMember(EmbeddedTokenList<T> changedTokenList) {
102	assert (modTokenListIndex == -1);
103	modTokenListIndex = tokenListList.findIndex(changedTokenList.startOffset());
104	assert (tokenListList.get(modTokenListIndex) == changedTokenList);
105	}
106
107	void markChageBetween(int offset) { // Nothing added/removed and mod outside of bounds of an ETL
108	assert (modTokenListIndex == -1);
109	modTokenListIndex = tokenListList.findIndex(offset);
110	}
111
112	/**
113	* Mark the given token list as removed in the token list list.
114	* All removed token lists should be marked subsequently their increasing offset
115	* so it should be necessary to search for the index just once.
116	* <br/>
117	* It's expected that updateStatusImpl() was already called
118	* on the corresponding embedding container.
119	*/
120	void markRemovedMember(EmbeddedTokenList<T> removedTokenList, TokenHierarchyEventInfo eventInfo) {
121	boolean indexWasMinusOne; // Used for possible exception cause debugging
122	// removedTokenList.embeddingContainer().checkStatusUpdated();
123	if (modTokenListIndex == -1) {
124	indexWasMinusOne = true;
125	modTokenListIndex = tokenListList.findIndexDuringUpdate(removedTokenList, eventInfo);
126	assert (modTokenListIndex >= 0) : "tokenListIndex=" + modTokenListIndex + " < 0"; // NOI18N
127	} else { // tokenListIndex already initialized
128	indexWasMinusOne = false;
129	}
130	TokenList<T> markedForRemoveTokenList = tokenListList.getOrNull(modTokenListIndex + removedTokenListCount);
131	if (markedForRemoveTokenList != removedTokenList) {
132	int realIndex = tokenListList.indexOf(removedTokenList);
133	throw new IllegalStateException("Removing at tokenListIndex=" + modTokenListIndex + // NOI18N
134	" but real tokenListIndex is " + realIndex + // NOI18N
135	" (indexWasMinusOne=" + indexWasMinusOne + ").\n" + // NOI18N
136	"Wishing to remove tokenList\n" + removedTokenList + // NOI18N
137	"\nbut marked-for-remove tokenList is \n" + markedForRemoveTokenList + // NOI18N
138	"\nfrom tokenListList\n" + tokenListList + // NOI18N
139	"\n\nModification description:\n" + eventInfo.modificationDescription(true) // NOI18N
140	);
141	}
142	removedTokenListCount++;
143	}
144
145	/**
146	* Mark the given token list to be added to this list of token lists.
147	* At the end first the token lists marked for removal will be removed
148	* and then the token lists marked for addition will be added.
149	* <br/>
150	* It's expected that updateStatusImpl() was already called
151	* on the corresponding embedding container.
152	*/
153	void markAddedMember(EmbeddedTokenList<T> addedTokenList) {
154	// addedTokenList.embeddingContainer().checkStatusUpdated();
155	if (addedTokenLists == null) {
156	if (modTokenListIndex == -1) {
157	modTokenListIndex = tokenListList.findIndex(addedTokenList.startOffset());
158	assert (modTokenListIndex >= 0) : "tokenListIndex=" + modTokenListIndex + " < 0"; // NOI18N
159	}
160	addedTokenLists = new ArrayList<EmbeddedTokenList<T>>(4);
161	}
162	addedTokenLists.add(addedTokenList);
163	}
164
165	void addRemoveTokenLists(TokenHierarchyUpdate update, boolean tllChildrenMayExist) {
166	assert (removedTokenListCount > 0 \|\| addedTokenLists != null);
167	EmbeddedTokenList<T>[] removedTokenLists = tokenListList.replace(
168	modTokenListIndex, removedTokenListCount, addedTokenLists);
169	if (tllChildrenMayExist) {
170	for (int i = 0; i < removedTokenLists.length; i++) {
171	update.collectRemovedEmbeddings(removedTokenLists[i]);
172	}
173	for (int i = 0; i < addedTokenLists.size(); i++) {
174	EmbeddedTokenList<T> addedEtl = addedTokenLists.get(i);
175	update.collectAddedEmbeddings(addedEtl, 0, addedEtl.tokenCountCurrent());
176	}
177	}
178	}
179
180	TokenListChange<T> createTokenListChange(EmbeddedTokenList<T> etl) {
181	assert (etl != null);
182	TokenListChange<T> etlTokenListChange;
183	if (tokenListList.joinSections()) {
184	MutableJoinTokenList<T> jtl = MutableJoinTokenList.create(tokenListList, modTokenListIndex, etl);
185	etlTokenListChange = new JoinTokenListChange<T>(jtl);
186	} else { // Non-joining
187	etlTokenListChange = new TokenListChange<T>(etl);
188	}
189	return etlTokenListChange;
190	}
191
192	TokenListChange<T> createJoinTokenListChange() {
193	assert (tokenListList.joinSections());
194	// In case when adding at jtl.tokenListCount() a last ETL must be used
195	int etlIndex = Math.min(modTokenListIndex, tokenListList.size() - 1);
196	MutableJoinTokenList<T> jtl = MutableJoinTokenList.create(tokenListList, etlIndex, tokenListList.get(etlIndex));
197	return new JoinTokenListChange<T>(jtl);
198	}
199
200	@Override
201	public String toString() {
202	return " modTokenListIndex=" + modTokenListIndex + // NOI18N
203	"; Rem:" + removedTokenListCount + // NOI18N
204	" Add:" + addedTokenLists.size() + // NOI18N
205	" Size:" + tokenListList.size(); // NOI18N
206	}
207
208	}

Added Link Here

(-)06a7890f802e (+136 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer.token;
43
44	import java.util.List;
45	import org.netbeans.api.lexer.PartType;
46	import org.netbeans.api.lexer.TokenHierarchy;
47	import org.netbeans.api.lexer.TokenId;
48	import org.netbeans.lib.editor.util.ArrayUtilities;
49	import org.netbeans.spi.lexer.TokenPropertyProvider;
50
51	/**
52	* Token consisting of multiple parts.
53	*
54	* @author Miloslav Metelka
55	* @version 1.00
56	*/
57
58	public final class JoinToken<T extends TokenId> extends PropertyToken<T> {
59
60	private List<PartToken<T>> joinedParts; // 32 bytes (28-super + 4)
61
62	private int completeLength; // 36 bytes
63
64	/**
65	* Number of ETLs spanned including empty ETLs except a first part.
66	*/
67	private int extraTokenListSpanCount; // 40 bytes
68
69	public JoinToken(T id, int length, TokenPropertyProvider<T> propertyProvider, PartType partType) {
70	super(id, length, propertyProvider, partType);
71	}
72
73	@Override
74	public List<PartToken<T>> joinedParts() {
75	return joinedParts;
76	}
77
78	public void setJoinedParts(List<PartToken<T>> joinedParts, int extraTokenListSpanCount) {
79	assert (joinedParts != null) : "joinedParts expected to be non-null";
80	this.joinedParts = joinedParts;
81	for (PartToken partToken : joinedParts) {
82	completeLength += partToken.length();
83	}
84	this.extraTokenListSpanCount = extraTokenListSpanCount;
85	}
86
87	public PartToken<T> lastPart() {
88	return joinedParts.get(joinedParts.size() - 1);
89	}
90
91	public int extraTokenListSpanCount() {
92	return extraTokenListSpanCount;
93	}
94
95	@Override
96	public int offset(TokenHierarchy<?> tokenHierarchy) {
97	return joinedParts.get(0).offset(tokenHierarchy);
98	}
99
100	@Override
101	public int length() {
102	return completeLength;
103	}
104
105	@Override
106	public CharSequence text() {
107	return new JoinTokenText<T>(joinedParts, completeLength);
108	}
109
110	@Override
111	public boolean isRemoved() {
112	// Check whether last part of token is removed - this needs to be improved
113	// for the case when token is just partially recreated.
114	return lastPart().isRemoved();
115	}
116
117	@Override
118	public StringBuilder dumpInfo(StringBuilder sb, TokenHierarchy<?> tokenHierarchy, boolean dumpTokenText, int indent) {
119	super.dumpInfo(sb, tokenHierarchy, dumpTokenText, indent);
120	sb.append(", ").append(joinedParts.size()).append(" parts");
121	int digitCount = String.valueOf(joinedParts.size() - 1).length();
122	for (int i = 0; i < joinedParts.size(); i++) {
123	sb.append('\n');
124	ArrayUtilities.appendSpaces(sb, indent + 2);
125	ArrayUtilities.appendBracketedIndex(sb, i, digitCount);
126	joinedParts.get(i).dumpInfo(sb, tokenHierarchy, dumpTokenText, indent + 4);
127	}
128	return sb;
129	}
130
131	@Override
132	protected String dumpInfoTokenType() {
133	return "JoiT"; // NOI18N
134	}
135
136	}

Added Link Here

(-)06a7890f802e (+130 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer.token;
43
44	import java.util.List;
45	import java.util.logging.Level;
46	import java.util.logging.Logger;
47	import org.netbeans.api.lexer.TokenId;
48	import org.netbeans.lib.editor.util.CharSequenceUtilities;
49
50	/**
51	* Char sequence over join token parts.
52	*
53	* @author Miloslav Metelka
54	*/
55
56	public final class JoinTokenText<T extends TokenId> implements CharSequence {
57
58	private static final Logger LOG = Logger.getLogger(JoinTokenText.class.getName());
59
60	private List<PartToken<T>> joinedParts;
61
62	private int activePartIndex;
63
64	private CharSequence activeInputText;
65
66	private int activeStartCharIndex;
67
68	private int activeEndCharIndex;
69
70	private int length;
71
72	public JoinTokenText(List<PartToken<T>> joinedParts, int length) {
73	this.joinedParts = joinedParts;
74	this.activeInputText = joinedParts.get(0).text();
75	// Implicit: this.activeStartCharIndex = 0;
76	this.activeEndCharIndex = activeInputText.length();
77	this.length = length;
78	}
79
80	public synchronized char charAt(int index) {
81	if (index < activeStartCharIndex) { // Find non-empty previous
82	if (index < 0)
83	throw new IndexOutOfBoundsException("index=" + index + " < 0");
84	do {
85	activePartIndex--;
86	if (activePartIndex < 0) { // Should never happen
87	LOG.log(Level.WARNING, "Internal error: index=" + index + ", " + dumpState());
88	}
89	activeInputText = joinedParts.get(activePartIndex).text();
90	int len = activeInputText.length();
91	activeEndCharIndex = activeStartCharIndex;
92	activeStartCharIndex -= len;
93	} while (index < activeStartCharIndex);
94	} else if (index >= activeEndCharIndex) { // Find non-empty next
95	if (index >= length)
96	throw new IndexOutOfBoundsException("index=" + index + " >= length()=" + length);
97	do {
98	activePartIndex++;
99	activeInputText = joinedParts.get(activePartIndex).text();
100	int len = activeInputText.length();
101	activeStartCharIndex = activeEndCharIndex;
102	activeEndCharIndex += len;
103	} while (index >= activeEndCharIndex);
104	}
105
106	// Valid char within current segment
107	return activeInputText.charAt(index - activeStartCharIndex);
108	}
109
110	public int length() {
111	return length;
112	}
113
114	public CharSequence subSequence(int start, int end) {
115	return CharSequenceUtilities.toString(this, start, end);
116	}
117
118	@Override
119	public synchronized String toString() {
120	return CharSequenceUtilities.toString(this);
121	}
122
123	private String dumpState() {
124	return "activeTokenListIndex=" + activePartIndex +
125	", activeStartCharIndex=" + activeStartCharIndex +
126	", activeEndCharIndex=" + activeEndCharIndex +
127	", length=" + length;
128	}
129
130	}

Added Link Here

(-)06a7890f802e (+103 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer.token;
43
44	import org.netbeans.api.lexer.PartType;
45	import org.netbeans.api.lexer.TokenId;
46	import org.netbeans.lib.lexer.TokenOrEmbedding;
47	import org.netbeans.spi.lexer.TokenPropertyProvider;
48
49	/**
50	* Part of a {@link JoinToken}.
51	*
52	* @author Miloslav Metelka
53	* @version 1.00
54	*/
55
56	public final class PartToken<T extends TokenId> extends PropertyToken<T> {
57
58	private TokenOrEmbedding<T> joinTokenOrEmbedding; // 32 bytes (28-super + 4)
59
60	private int partTokenIndex; // Index of this part inside
61
62	private int partTextOffset; // Offset of this part's text among all parts that comprise the complete token
63
64	public PartToken(T id, int length, TokenPropertyProvider<T> propertyProvider, PartType partType,
65	TokenOrEmbedding<T> joinToken, int partTokenIndex, int partTextOffset
66	) {
67	super(id, length, propertyProvider, partType);
68	setJoinTokenOrEmbedding(joinToken);
69	this.partTokenIndex = partTokenIndex;
70	this.partTextOffset = partTextOffset;
71	}
72
73	@Override
74	public JoinToken<T> joinToken() {
75	return (JoinToken<T>)joinTokenOrEmbedding.token();
76	}
77
78	public boolean isLastPart() {
79	return (joinToken().lastPart() == this);
80	}
81
82	public TokenOrEmbedding<T> joinTokenOrEmbedding() {
83	return joinTokenOrEmbedding;
84	}
85
86	public void setJoinTokenOrEmbedding(TokenOrEmbedding<T> joinTokenOrEmbedding) {
87	this.joinTokenOrEmbedding = joinTokenOrEmbedding;
88	}
89
90	public int partTokenIndex() {
91	return partTokenIndex;
92	}
93
94	public int partTextOffset() {
95	return partTextOffset;
96	}
97
98	@Override
99	protected String dumpInfoTokenType() {
100	return "ParT[" + partTokenIndex + "]"; // NOI18N
101	}
102
103	}

Added Link Here

(-)06a7890f802e (+112 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer.token;
43
44	import java.util.EnumSet;
45	import org.netbeans.api.lexer.PartType;
46	import org.netbeans.api.lexer.Token;
47	import org.netbeans.api.lexer.TokenId;
48	import org.netbeans.spi.lexer.TokenPropertyProvider;
49
50	/**
51	* Property provider that stores {@link org.netbeans.api.lexer.PartType} information.
52	*
53	* @author Miloslav Metelka
54	* @version 1.00
55	*/
56
57	public final class PartTypePropertyProvider implements TokenPropertyProvider<TokenId> {
58
59	private static final PartTypePropertyProvider[] partTypeOrdinal2Provider
60	= new PartTypePropertyProvider[PartType.class.getEnumConstants().length];
61
62	static {
63	for (PartType partType : EnumSet.allOf(PartType.class)) {
64	partTypeOrdinal2Provider[partType.ordinal()] = new PartTypePropertyProvider(partType);
65	}
66	}
67
68	public static <T extends TokenId> TokenPropertyProvider<T> get(PartType partType) {
69	return (TokenPropertyProvider<T>)partTypeOrdinal2Provider[partType.ordinal()];
70	}
71
72	public static <T extends TokenId> TokenPropertyProvider<T> createDelegating(
73	PartType partType, TokenPropertyProvider<T> delegate
74	) {
75	return new Delegating<T>(partType, delegate);
76	}
77
78	private PartType partType;
79
80	public PartTypePropertyProvider(PartType partType) {
81	this.partType = partType;
82	}
83
84	public Object getValue(Token<TokenId> token, Object key) {
85	if (key == PartType.class) {
86	return partType;
87	}
88	return null;
89	}
90
91	private static final class Delegating<T extends TokenId> implements TokenPropertyProvider<T> {
92
93	private final PartType partType;
94
95	private final TokenPropertyProvider<T> delegate;
96
97	Delegating(PartType partType, TokenPropertyProvider<T> delegate) {
98	assert (delegate != null) : "delegate expected to be non-null. Use PartTypePropertyProvider.get() instead."; // NOTICES
99	this.partType = partType;
100	this.delegate = delegate;
101	}
102
103	public Object getValue(Token<T> token, Object key) {
104	if (key == PartType.class) {
105	return partType;
106	}
107	return delegate.getValue(token, key);
108	}
109
110	}
111
112	}

Added Link Here

(-)06a7890f802e (+155 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer.token;
43
44	import org.netbeans.lib.lexer.LexerUtilsConstants;
45
46	/**
47	* Improves performance of doing Token.text().toString().
48	* by using a cache factor which gets increased by every access to that method.
49	* <br/>
50	* Once a cache factor exceeds a threshold the result of Token.text().toString()
51	* will be cached.
52	* <br/>
53	* TBD values of constants used by this class should be reviewed to match
54	* a real complexity of the particular operations.
55	*
56	* @author Miloslav Metelka
57	* @version 1.00
58	*/
59
60	public final class TokenLength implements CharSequence {
61
62	private static final TokenLength[][] CACHE = new TokenLength[
63	LexerUtilsConstants.MAX_CACHED_TOKEN_LENGTH + 1][];
64
65
66	public static TokenLength get(int length) {
67	TokenLength tokenLength;
68	if (length <= LexerUtilsConstants.MAX_CACHED_TOKEN_LENGTH) {
69	synchronized (CACHE) {
70	TokenLength[] tokenLengths = CACHE[length];
71	if (tokenLengths == null) {
72	tokenLengths = new TokenLength[1];
73	CACHE[length] = tokenLengths;
74	}
75	tokenLength = tokenLengths[0];
76	if (tokenLength == null) {
77	tokenLength = new TokenLength(length,
78	LexerUtilsConstants.CACHE_TOKEN_TO_STRING_THRESHOLD, (short)1);
79	tokenLengths[0] = tokenLength;
80	}
81	}
82	} else { // length too high - not cached
83	tokenLength = new TokenLength(length,
84	LexerUtilsConstants.CACHE_TOKEN_TO_STRING_THRESHOLD, (short)1);
85	}
86	return tokenLength;
87	}
88
89	/**
90	* Length of a token.
91	*/
92	private final int length; // 12 bytes (8-super + 4)
93
94	/**
95	* Cache factor of this item.
96	*/
97	private final short cacheFactor; // 14 bytes
98
99	/**
100	* Index of a next item in array of token lengths with the same length in CACHE.
101	*/
102	private final short nextArrayIndex; // 16 bytes
103
104	TokenLength(int length, short cacheFactor, short nextArrayIndex) {
105	this.length = length;
106	this.cacheFactor = cacheFactor;
107	this.nextArrayIndex = nextArrayIndex;
108	}
109
110	public int length() {
111	return length;
112	}
113
114	public short cacheFactor() {
115	return cacheFactor;
116	}
117
118	public int nextCacheFactor() {
119	return cacheFactor + length + LexerUtilsConstants.TOKEN_LENGTH_STRING_CREATION_FACTOR;
120	}
121
122	public TokenLength next(int nextCacheFactor) {
123	TokenLength tokenLength;
124	if (length <= LexerUtilsConstants.MAX_CACHED_TOKEN_LENGTH) {
125	synchronized (CACHE) {
126	TokenLength[] tokenLengths = CACHE[length];
127	if (tokenLengths == null \|\| tokenLengths.length <= nextArrayIndex) {
128	TokenLength[] tmp = new TokenLength[nextArrayIndex + 1];
129	if (tokenLengths != null) {
130	System.arraycopy(tokenLengths, 0, tmp, 0, tokenLengths.length);
131	}
132	tokenLengths = tmp;
133	CACHE[length] = tokenLengths;
134	}
135	tokenLength = tokenLengths[nextArrayIndex];
136	if (tokenLength == null) {
137	tokenLength = new TokenLength(length, (short)nextCacheFactor, (short)(nextArrayIndex + 1));
138	tokenLengths[nextArrayIndex] = tokenLength;
139	}
140	}
141	} else { // length too high - not cached
142	tokenLength = new TokenLength(length, (short)nextCacheFactor, (short)(nextArrayIndex + 1));
143	}
144	return tokenLength;
145	}
146
147	public char charAt(int index) {
148	throw new IllegalStateException("Should never be called.");
149	}
150
151	public CharSequence subSequence(int start, int end) {
152	throw new IllegalStateException("Should never be called.");
153	}
154
155	}

Lines 42-48 Link Here

(-)a/lexer/src/org/netbeans/spi/lexer/LexerInput.java (-11 / +10 lines)
42	package org.netbeans.spi.lexer;	42	package org.netbeans.spi.lexer;
43		43
44	import org.netbeans.lib.editor.util.AbstractCharSequence;	44	import org.netbeans.lib.editor.util.AbstractCharSequence;
45	import org.netbeans.lib.lexer.CharProvider;	45	import org.netbeans.lib.lexer.LexerInputOperation;
46	import org.netbeans.lib.lexer.LexerUtilsConstants;	46	import org.netbeans.lib.lexer.LexerUtilsConstants;
47		47
48	/**	48	/**
Lines 81-90 Link Here
81	public static final int EOF = -1;	81	public static final int EOF = -1;
82		82
83	/**	83	/**
84	* Character provider to which this lexer input delegates	84	* LexerInputOperation on which this lexer input delegates.
85	* its operation.
86	*/	85	*/
87	private CharProvider charProvider;	86	private LexerInputOperation<?> operation;
88		87
89	/**	88	/**
90	* Character sequence that corresponds	89	* Character sequence that corresponds
Lines 101-110 Link Here
101	/**	100	/**
102	* Construct instance of the lexer input.	101	* Construct instance of the lexer input.
103	*	102	*
104	* @param charProvider non-null character provider for this lexer input.	103	* @param operation non-null character provider for this lexer input.
105	*/	104	*/
106	LexerInput(CharProvider charProvider) {	105	LexerInput(LexerInputOperation operation) {
107	this.charProvider = charProvider;	106	this.operation = operation;
108	}	107	}
109		108
110	/**	109	/**
Lines 116-122 Link Here
116	* - all of them will return EOF.	115	* - all of them will return EOF.
117	*/	116	*/
118	public int read() {	117	public int read() {
119	int c = charProvider.read();	118	int c = operation.read();
120	if (c == EOF) {	119	if (c == EOF) {
121	eof = 1;	120	eof = 1;
122	}	121	}
Lines 158-164 Link Here
158	eof = 0; // backup EOF	157	eof = 0; // backup EOF
159	count--;	158	count--;
160	}	159	}
161	charProvider.backup(count);	160	operation.backup(count);
162	}	161	}
163		162
164	/**	163	/**
Lines 178-184 Link Here
178	* If {@link LexerInput#EOF} was read then it is not counted into read length.	177	* If {@link LexerInput#EOF} was read then it is not counted into read length.
179	*/	178	*/
180	public int readLength() {	179	public int readLength() {
181	return charProvider.readIndex();	180	return operation.readLength();
182	}	181	}
183		182
184	/**	183	/**
Lines 332-338 Link Here
332	if (index < 0 \|\| index >= length) {	331	if (index < 0 \|\| index >= length) {
333	throw new IndexOutOfBoundsException("index=" + index + ", length=" + length); // NOI18N	332	throw new IndexOutOfBoundsException("index=" + index + ", length=" + length); // NOI18N
334	}	333	}
335	return charProvider.readExisting(index);	334	return operation.readExistingAtIndex(index);
336	}	335	}
337		336
338	}	337	}

Lines 182-197 Link Here

(-)a/lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java (-47 / +67 lines)
182	}	182	}
183		183
184	/**	184	/**
185	* @see #assertTokenSequencesEqual(String,TokenSequence,TokenHierarchy,TokenSequence,TokenHierarchy,boolean)
186	*/
187	public static void assertTokenSequencesEqual(
188	TokenSequence<?> expected, TokenHierarchy<?> expectedHi,
189	TokenSequence<?> actual, TokenHierarchy<?> actualHi,
190	boolean testLookaheadAndState) {
191	assertTokenSequencesEqual(null, expected, expectedHi, actual, actualHi, testLookaheadAndState);
192	}
193
194	/**
195	* Compare contents of the given token sequences by moving through all their	185	* Compare contents of the given token sequences by moving through all their
196	* tokens.	186	* tokens.
197	* <br/>	187	* <br/>
Lines 210-232 Link Here
210	public static void assertTokenSequencesEqual(String message,	200	public static void assertTokenSequencesEqual(String message,
211	TokenSequence<?> expected, TokenHierarchy<?> expectedHi,	201	TokenSequence<?> expected, TokenHierarchy<?> expectedHi,
212	TokenSequence<?> actual, TokenHierarchy<?> actualHi,	202	TokenSequence<?> actual, TokenHierarchy<?> actualHi,
213	boolean testLookaheadAndState) {	203	boolean testLookaheadAndState, boolean dumpWholeHi) {
214	boolean success = false;	204	String prefix = messagePrefix(message);
215	try {	205	TestCase.assertEquals(prefix + "Move previous: ", expected.movePrevious(), actual.movePrevious());
216	String prefix = messagePrefix(message);	206	int i = 0;
217	TestCase.assertEquals(prefix + "Move previous: ", expected.movePrevious(), actual.movePrevious());	207	while (expected.moveNext()) {
218	while (expected.moveNext()) {	208	String prefixI = prefix + "->[" + i + "]";
219	TestCase.assertTrue(prefix + "Move next: ", actual.moveNext());	209	TestCase.assertTrue(prefixI + ": Cannot moveNext() in test token sequence", actual.moveNext());
220	assertTokensEqual(message, expected, expectedHi, actual, actualHi, testLookaheadAndState);	210	assertTokensEqual(prefixI, expected, expectedHi, actual, actualHi, testLookaheadAndState);
221	}	211	i++;
222	TestCase.assertFalse(prefix + "Move next not disabled", actual.moveNext());
223	success = true;
224	} finally {
225	if (!success) {
226	System.err.println("Expected token sequence dump:\n" + expected);
227	System.err.println("Test token sequence dump:\n" + actual);
228	}
229	}	212	}
		213	TestCase.assertFalse(prefix + "moveNext() possible at end of test token sequence", actual.moveNext());
230	}	214	}
231		215
232	private static void assertTokensEqual(String message,	216	private static void assertTokensEqual(String message,
Lines 339-345 Link Here
339	}	323	}
340		324
341	public static void incCheck(Document doc, boolean nested) {	325	public static void incCheck(Document doc, boolean nested) {
342	TokenHierarchy<?> thInc = TokenHierarchy.get(doc);	326	TokenHierarchy<?> incHi = TokenHierarchy.get(doc);
		327	assertConsistency(incHi);
		328
343	Language<?> language = (Language<?>)	329	Language<?> language = (Language<?>)
344	doc.getProperty(Language.class);	330	doc.getProperty(Language.class);
345	String docText = null;	331	String docText = null;
Lines 349-383 Link Here
349	e.printStackTrace();	335	e.printStackTrace();
350	TestCase.fail("BadLocationException occurred");	336	TestCase.fail("BadLocationException occurred");
351	}	337	}
352	TokenHierarchy<?> thBatch = TokenHierarchy.create(docText, language);	338	TokenHierarchy<?> batchHi = TokenHierarchy.create(docText, language);
353	boolean success = false;	339	TokenSequence<?> batchTS = batchHi.tokenSequence();
354	TokenSequence<?> batchTS = thBatch.tokenSequence();	340	TokenSequence<?> incTS = incHi.tokenSequence();
355	try {	341	try {
356	// Compare lookaheads and states as well	342	// Compare lookaheads and states as well
357	assertTokenSequencesEqual(batchTS, thBatch,	343	assertTokenSequencesEqual("TOP", batchTS, batchHi, incTS, incHi, true, false);
358	thInc.tokenSequence(), thInc, true);	344	} catch (Throwable t) {
359	success = true;	345	// Go forward two tokens to have an extra tokens context
360	} finally {	346	batchTS.moveNext();
361	if (!success) {	347	batchTS.moveNext();
362	// Go forward two tokens to have an extra tokens context	348	StringBuilder sb = new StringBuilder(512);
363	batchTS.moveNext();	349	sb.append("BATCH token sequence dump:\n").append(batchTS);
364	batchTS.moveNext();	350	sb.append("\n\nTEST token sequence dump:\n").append(incTS);
365	System.err.println("BATCH token sequence dump:\n" + thBatch.tokenSequence());	351	TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);
366	TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);	352	if (lastHi != null) {
367	if (lastHi != null) {	353	// System.err.println("PREVIOUS batch token sequence dump:\n" + lastHi.tokenSequence());
368	System.err.println("PREVIOUS batch token sequence dump:\n" + lastHi.tokenSequence());	354	}
369	}	355	throw new IllegalStateException(sb.toString(), t);
		356	}
		357
		358	if (nested) {
		359	batchTS.moveStart();
		360	incTS.moveStart();
		361	try {
		362	incCheckNested("TOP", doc, batchTS, batchHi, incTS, incHi);
		363	} catch (Throwable t) { // Re-throw with hierarchy info
		364	StringBuilder sb = new StringBuilder(512);
		365	sb.append("BATCH token hierarchy:\n").append(batchHi);
		366	sb.append("\n\n\nTEST token hierarchy:\n").append(incHi);
		367	throw new IllegalStateException(sb.toString(), t);
370	}	368	}
371	}	369	}
372		370
373	// Check the change since last modification	371	// Check the change since last modification
374	TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);	372	TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);
375	if (lastHi != null) {	373	if (lastHi != null) {
376	// TODO comparison	374	// TODO comparison
377	}	375	}
378	doc.putProperty(LAST_TOKEN_HIERARCHY, thBatch); // new last batch token hierarchy	376	doc.putProperty(LAST_TOKEN_HIERARCHY, batchHi); // new last batch token hierarchy
379	}	377	}
380		378
		379	public static void incCheckNested(String message, Document doc,
		380	TokenSequence<?> batch, TokenHierarchy<?> batchTH,
		381	TokenSequence<?> inc, TokenHierarchy<?> incTH
		382	) {
		383	int i = 0;
		384	while (inc.moveNext()) {
		385	TestCase.assertTrue("No more tokens in batch token sequence", batch.moveNext());
		386	TokenSequence<?> batchE = batch.embedded();
		387	TokenSequence<?> incE = inc.embedded();
		388	String messageE = message + "->[" + i + "]";
		389	if (incE != null) {
		390	TestCase.assertNotNull("Inc embedded sequence is null", batchE);
		391	assertTokenSequencesEqual(messageE, batchE, batchTH, incE, incTH, true, true);
		392
		393	incCheckNested(messageE, doc, batchE, batchTH, incE, incTH);
		394	} else { // Inc embedded is null
		395	TestCase.assertNull("Batch embedded sequence non-null", batchE);
		396	}
		397	i++;
		398	}
		399	}
		400
381	/**	401	/**
382	* Get lookahead for the token to which the token sequence is positioned.	402	* Get lookahead for the token to which the token sequence is positioned.
383	* <br/>	403	* <br/>
Lines 387-393 Link Here
387	return tokenList(ts).lookahead(ts.index());	407	return tokenList(ts).lookahead(ts.index());
388	}	408	}
389		409
390	/**	410	/**
391	* Get state for the token to which the token sequence is positioned.	411	* Get state for the token to which the token sequence is positioned.
392	* <br/>	412	* <br/>
393	* The method uses reflection to get reference to tokenList field in token sequence.	413	* The method uses reflection to get reference to tokenList field in token sequence.

Added Link Here

(-)06a7890f802e (+293 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41	package org.netbeans.lib.lexer.test.join;
42
43	import java.io.PrintStream;
44	import java.util.List;
45	import java.util.logging.Level;
46	import java.util.logging.Logger;
47	import org.netbeans.api.lexer.Language;
48	import org.netbeans.api.lexer.LanguagePath;
49	import org.netbeans.api.lexer.PartType;
50	import org.netbeans.api.lexer.Token;
51	import org.netbeans.api.lexer.TokenHierarchy;
52	import org.netbeans.api.lexer.TokenSequence;
53	import org.netbeans.junit.NbTestCase;
54	import org.netbeans.lib.lexer.lang.TestJoinSectionsTextTokenId;
55	import org.netbeans.lib.lexer.test.LexerTestUtilities;
56	import org.netbeans.lib.lexer.lang.TestJoinSectionsTopTokenId;
57	import org.netbeans.lib.lexer.test.ModificationTextDocument;
58
59	/**
60	* Test embedded sections that should be lexed together.
61	*
62	* <p>
63	* Top lexer recognizes TestJoinSectionsTopTokenId.TAG (text v zobacich)
64	* and TestJoinSectionsTopTokenId.TEXT (everything else).
65	*
66	* TestJoinSectionsTopTokenId.TAG is branched into
67	* TestJoinSectionsTextTokenId.BRACES "{...}"
68	* and TestJoinSectionsTextTokenId.TEXT (everything else)
69	*
70	* </p>
71	*
72	*
73	* @author Miloslav Metelka
74	*/
75	public class JoinSectionsMod1Test extends NbTestCase {
76
77	public JoinSectionsMod1Test(String testName) {
78	super(testName);
79	}
80
81	protected void setUp() throws Exception {
82	}
83
84	@Override
85	public PrintStream getLog() {
86	return System.out;
87	// return super.getLog();
88	}
89
90	@Override
91	protected Level logLevel() {
92	return Level.INFO;
93	// return super.logLevel();;
94	}
95
96	public void testShortDocMod() throws Exception {
97	// 000000000011111111112222222222
98	// 012345678901234567890123456789
99	String text = "xay<b>zc";
100	ModificationTextDocument doc = new ModificationTextDocument();
101	doc.insertString(0, text, null);
102	doc.putProperty(Language.class, TestJoinSectionsTopTokenId.language());
103	LexerTestUtilities.incCheck(doc, true); // Ensure the whole embedded hierarchy gets created
104
105	Logger.getLogger(org.netbeans.lib.lexer.inc.TokenListUpdater.class.getName()).setLevel(Level.FINE); // Extra logging
106	doc.remove(6, 1);
107	LexerTestUtilities.incCheck(doc, true);
108	// 000000000011111111112222222222
109	// 012345678901234567890123456789
110	// text = "xay<b>c";
111	// \yz<uv>hk
112	doc.insertString(6, "yz<uv>hk", null);
113	LexerTestUtilities.incCheck(doc, true);
114	// 000000000011111111112222222222
115	// 012345678901234567890123456789
116	// text = "xay<b>yz<uv>hkc";
117	doc.remove(12, 3);
118	LexerTestUtilities.incCheck(doc, true);
119	// 000000000011111111112222222222
120	// 012345678901234567890123456789
121	// text = "xay<b>yz<uv>";
122	doc.insertString(12, "hkc", null);
123	LexerTestUtilities.incCheck(doc, true);
124	// 000000000011111111112222222222
125	// 012345678901234567890123456789
126	// text = "xay<b>yz<uv>hkc";
127
128	}
129
130	public void testJoinSections() throws Exception {
131	if (true)
132	return;
133	// Turn on detailed checking
134	// Logger.getLogger(TokenHierarchyOperation.class.getName()).setLevel(Level.FINEST);
135
136	// 000000000011111111112222222222
137	// 012345678901234567890123456789
138	String text = "a{b<cd>e}f<gh>i{j<kl>m}n";
139	ModificationTextDocument doc = new ModificationTextDocument();
140	doc.insertString(0, text, null);
141	doc.putProperty(Language.class,TestJoinSectionsTopTokenId.language());
142
143	TokenHierarchy<?> hi = TokenHierarchy.get(doc);
144	TokenSequence<?> ts = hi.tokenSequence();
145	assertTrue(ts.moveNext());
146	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "a{b", -1);
147	assertTrue(ts.moveNext());
148	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<cd>", -1);
149	assertTrue(ts.moveNext());
150	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "e}f", -1);
151
152	// Get embedded tokens within TEXT tokens. There should be "a" then BRACES start "{b" then BRACES end "e}\|" then "f"
153	LanguagePath innerLP = LanguagePath.get(TestJoinSectionsTopTokenId.language()).
154	embedded(TestJoinSectionsTextTokenId.language());
155	List<TokenSequence<?>> tsList = hi.tokenSequenceList(innerLP, 0, Integer.MAX_VALUE);
156	checkInitialTokens(tsList);
157
158
159	// Use iterator for fetching token sequences
160	int i = 0;
161	for (TokenSequence<?> ts2 : tsList) {
162	assertSame(ts2, tsList.get(i++));
163	}
164
165	LexerTestUtilities.assertConsistency(hi);
166
167	// Check tokenSequenceList() with explicit offsets
168	// Check correct TSs bounds
169	tsList = hi.tokenSequenceList(innerLP, 0, 7);
170	assertEquals(1, tsList.size());
171	tsList = hi.tokenSequenceList(innerLP, 0, 8);
172	assertEquals(2, tsList.size());
173
174
175	// Do modifications
176	// Remove second closing brace '}'
177
178	// Logger.getLogger(org.netbeans.lib.lexer.inc.TokenListUpdater.class.getName()).setLevel(Level.FINE); // Extra logging
179	doc.remove(8, 1);
180	// Logger.getLogger(org.netbeans.lib.lexer.inc.TokenListUpdater.class.getName()).setLevel(Level.WARNING); // End of extra logging
181	LexerTestUtilities.assertConsistency(hi);
182	LexerTestUtilities.incCheck(doc, true);
183	// 000000000011111111112222222222
184	// 012345678901234567890123456789
185	// before: "a{b<cd>e}f<gh>i{j<kl>m}n";
186	// after: "a{b<cd>ef<gh>i{j<kl>m}n";
187	// i0 i1 i2 i3
188	tsList = hi.tokenSequenceList(innerLP, 0, Integer.MAX_VALUE);
189	assertEquals(4, tsList.size()); // 2 sections
190
191	// 1.section "a{b"
192	ts = tsList.get(0);
193	assertTrue(ts.moveNext());
194	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "a", -1);
195	assertTrue(ts.moveNext());
196	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "{b", -1);
197	Token<?> token = ts.token();
198	assertEquals(PartType.START, token.partType());
199	assertFalse(ts.moveNext());
200
201	// 2.section "ef"
202	ts = tsList.get(1);
203	assertTrue(ts.moveNext());
204	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "ef", -1);
205	token = ts.token();
206	assertEquals(PartType.MIDDLE, token.partType());
207	assertFalse(ts.moveNext());
208
209	// 3.section "i{j"
210	ts = tsList.get(2);
211	assertTrue(ts.moveNext());
212	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "i{j", -1);
213	token = ts.token();
214	assertEquals(PartType.MIDDLE, token.partType());
215	assertFalse(ts.moveNext());
216
217	// 4.section "m}n"
218	ts = tsList.get(3);
219	assertTrue(ts.moveNext());
220	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "m}", -1);
221	token = ts.token();
222	assertEquals(PartType.END, token.partType());
223	assertTrue(ts.moveNext());
224	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "n", -1);
225	assertFalse(ts.moveNext());
226
227
228	// Re-add second closing brace '}'
229	doc.insertString(8, "}", null);
230	LexerTestUtilities.assertConsistency(hi);
231	// 000000000011111111112222222222
232	// 012345678901234567890123456789
233	// before: "a{b<cd>ef<gh>i{j<kl>m}n";
234	// after: "a{b<cd>e}f<gh>i{j<kl>m}n";
235	tsList = hi.tokenSequenceList(innerLP, 0, Integer.MAX_VALUE);
236	checkInitialTokens(tsList);
237
238	doc.remove(0, doc.getLength());
239	LexerTestUtilities.assertConsistency(hi);
240	ts = hi.tokenSequence();
241	assertFalse(ts.moveNext());
242	doc.insertString(0, text, null);
243
244	}
245
246	private void checkInitialTokens(List<TokenSequence<?>> tsList) {
247	// 000000000011111111112222222222
248	// 012345678901234567890123456789
249	// text: "a{b<cd>e}f<gh>i{j<kl>m}n";
250	assertEquals(4, tsList.size()); // 4 sections
251
252	// 1.section
253	TokenSequence<?> ts = tsList.get(0);
254	assertTrue(ts.moveNext());
255	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "a", -1);
256	assertTrue(ts.moveNext());
257	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "{b", -1);
258	Token<?> token = ts.token();
259	assertEquals(PartType.START, token.partType());
260	assertFalse(ts.moveNext());
261
262	// 2.section
263	ts = tsList.get(1);
264	assertTrue(ts.moveNext());
265	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "e}", -1);
266	token = ts.token();
267	assertEquals(PartType.END, token.partType());
268	assertTrue(ts.moveNext());
269	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "f", -1);
270	assertFalse(ts.moveNext());
271
272	// 3.section
273	ts = tsList.get(2);
274	assertTrue(ts.moveNext());
275	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "i", -1);
276	assertTrue(ts.moveNext());
277	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "{j", -1);
278	token = ts.token();
279	assertEquals(PartType.START, token.partType());
280	assertFalse(ts.moveNext());
281
282	// 4.section
283	ts = tsList.get(3);
284	assertTrue(ts.moveNext());
285	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.BRACES, "m}", -1);
286	token = ts.token();
287	assertEquals(PartType.END, token.partType());
288	assertTrue(ts.moveNext());
289	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTextTokenId.TEXT, "n", -1);
290	assertFalse(ts.moveNext());
291	}
292
293	}

Added Link Here

(-)06a7890f802e (+178 lines)
1	/*
2	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3	*
4	* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5	*
6	* The contents of this file are subject to the terms of either the GNU
7	* General Public License Version 2 only ("GPL") or the Common
8	* Development and Distribution License("CDDL") (collectively, the
9	* "License"). You may not use this file except in compliance with the
10	* License. You can obtain a copy of the License at
11	* http://www.netbeans.org/cddl-gplv2.html
12	* or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13	* specific language governing permissions and limitations under the
14	* License. When distributing the software, include this License Header
15	* Notice in each file and include the License file at
16	* nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
17	* particular file as subject to the "Classpath" exception as provided
18	* by Sun in the GPL Version 2 section of the License file that
19	* accompanied this code. If applicable, add the following below the
20	* License Header, with the fields enclosed by brackets [] replaced by
21	* your own identifying information:
22	* "Portions Copyrighted [year] [name of copyright owner]"
23	*
24	* Contributor(s):
25	*
26	* The Original Software is NetBeans. The Initial Developer of the Original
27	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
28	* Microsystems, Inc. All Rights Reserved.
29	*
30	* If you wish your version of this file to be governed by only the CDDL
31	* or only the GPL Version 2, indicate your decision by adding
32	* "[Contributor] elects to include this software in this distribution
33	* under the [CDDL or GPL Version 2] license." If you do not indicate a
34	* single choice of license, a recipient has the option to distribute
35	* your version of this file under either the CDDL, the GPL Version 2 or
36	* to extend the choice of license to its licensees as provided above.
37	* However, if you add GPL Version 2 code and therefore, elected the GPL
38	* Version 2 license, then the option applies only if the new code is
39	* made subject to such option by the copyright holder.
40	*/
41
42	package org.netbeans.lib.lexer.test.join;
43
44	import java.util.List;
45	import org.netbeans.api.lexer.Language;
46	import org.netbeans.api.lexer.LanguagePath;
47	import org.netbeans.api.lexer.PartType;
48	import org.netbeans.lib.lexer.lang.TestJoinSectionsTopTokenId;
49	import org.netbeans.lib.lexer.lang.TestTokenId;
50	import org.netbeans.api.lexer.TokenChange;
51	import org.netbeans.api.lexer.TokenHierarchy;
52	import org.netbeans.api.lexer.TokenHierarchyEvent;
53	import org.netbeans.api.lexer.TokenHierarchyListener;
54	import org.netbeans.api.lexer.TokenSequence;
55	import org.netbeans.junit.NbTestCase;
56	import org.netbeans.lib.lexer.lang.TestJoinSectionsTextTokenId;
57	import org.netbeans.lib.lexer.test.LexerTestUtilities;
58	import org.netbeans.lib.lexer.lang.TestPlainTokenId;
59	import org.netbeans.lib.lexer.test.ModificationTextDocument;
60	import org.netbeans.spi.lexer.LanguageEmbedding;
61
62	/**
63	* Test several simple lexer impls.
64	*
65	* <p>
66	* Top lexer recognizes TestJoinSectionsTopTokenId.TAG (text v zobacich)
67	* and TestJoinSectionsTopTokenId.TEXT (everything else).
68	*
69	* TestJoinSectionsTopTokenId.TAG is branched into
70	* TestJoinSectionsTextTokenId.BRACES "{...}"
71	* and TestJoinSectionsTextTokenId.TEXT (everything else)
72	*
73	* </p>
74	*
75	* @author mmetelka
76	*/
77	public class JoinSectionsMod2Test extends NbTestCase {
78
79	public JoinSectionsMod2Test(String testName) {
80	super(testName);
81	}
82
83	protected void setUp() throws java.lang.Exception {
84	}
85
86	protected void tearDown() throws java.lang.Exception {
87	}
88
89	public void testMove() throws Exception { // TokenSequence.move() and moveIndex()
90	// 000000000011111111112222222222
91	// 012345678901234567890123456789
92	String text = "a{b<cd>e}f<gh>i{}{j<kl>m}n<o>p<q>r";
93	ModificationTextDocument doc = new ModificationTextDocument();
94	doc.insertString(0, text, null);
95	doc.putProperty(Language.class,TestJoinSectionsTopTokenId.language());
96
97	TokenHierarchy<?> hi = TokenHierarchy.get(doc);
98	TokenSequence<?> ts = hi.tokenSequence();
99	assertTrue(ts.moveNext());
100	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "a{b", 0);
101	assertTrue(ts.moveNext());
102	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<cd>", 3);
103	assertTrue(ts.moveNext());
104	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "e}f", 7);
105	assertTrue(ts.moveNext());
106	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<gh>", 10);
107	assertTrue(ts.moveNext());
108	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "i{}{j", 14);
109	assertTrue(ts.moveNext());
110	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<kl>", 19);
111	assertTrue(ts.moveNext());
112	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "m}n", 23);
113	assertTrue(ts.moveNext());
114	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<o>", -1);
115	assertTrue(ts.moveNext());
116	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "p", -1);
117	assertTrue(ts.moveNext());
118	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TAG, "<q>", -1);
119	assertTrue(ts.moveNext());
120	LexerTestUtilities.assertTokenEquals(ts,TestJoinSectionsTopTokenId.TEXT, "r", -1);
121	assertFalse(ts.moveNext());
122
123	// Check regular TS.embedded()
124	ts.moveStart();
125	assertTrue(ts.moveNext()); // Over "a{b"
126	TokenSequence<?> tse = ts.embedded();
127	assertTrue(tse.moveNext());
128	LexerTestUtilities.assertTokenEquals(tse,TestJoinSectionsTextTokenId.TEXT, "a", 0);
129	assertTrue(tse.moveNext());
130	LexerTestUtilities.assertTokenEquals(tse,TestJoinSectionsTextTokenId.BRACES, "{b", 1);
131	assertEquals(tse.token().partType(), PartType.START);
132	assertFalse(tse.moveNext());
133
134	assertTrue(ts.moveNext()); // Over "<cd>"
135	assertTrue(ts.moveNext()); // Over "e}f"
136	TokenSequence<?> tse2 = ts.embedded();
137	assertTrue(tse2.moveNext());
138	LexerTestUtilities.assertTokenEquals(tse2,TestJoinSectionsTextTokenId.BRACES, "e}", 7);
139	assertEquals(tse2.token().partType(), PartType.END);
140	assertTrue(tse2.moveNext());
141	LexerTestUtilities.assertTokenEquals(tse2,TestJoinSectionsTextTokenId.TEXT, "f", 9);
142	assertEquals(tse2.token().partType(), PartType.START);
143	assertFalse(tse2.moveNext());
144
145	assertTrue(ts.moveNext()); // Over "<gh>"
146	assertTrue(ts.moveNext()); // Over "i{}{j"
147	TokenSequence<?> tse3 = ts.embedded();
148	assertTrue(tse3.moveNext());
149	LexerTestUtilities.assertTokenEquals(tse3,TestJoinSectionsTextTokenId.TEXT, "i", 14);
150	assertEquals(tse3.token().partType(), PartType.END);
151	assertTrue(tse3.moveNext());
152	LexerTestUtilities.assertTokenEquals(tse3,TestJoinSectionsTextTokenId.BRACES, "{}", 15);
153	assertEquals(tse3.token().partType(), PartType.COMPLETE);
154	assertTrue(tse3.moveNext());
155	LexerTestUtilities.assertTokenEquals(tse3,TestJoinSectionsTextTokenId.BRACES, "{j", 17);
156	assertEquals(tse3.token().partType(), PartType.START);
157	assertFalse(tse3.moveNext());
158
159
160	// Check TS.embeddedJoin()
161	TokenSequence<?> tsej = ts.embeddedJoined();
162	assertEquals(2, tsej.index());
163	assertTrue(tsej.moveNext());
164	int o = tsej.offset();
165	LexerTestUtilities.assertTokenEquals(tsej,TestJoinSectionsTextTokenId.TEXT, "fi", 9);
166	assertEquals(9, tsej.token().offset(null)); // Assert also token.offset() besides TS.offset()
167	assertEquals(tsej.token().partType(), PartType.COMPLETE);
168	assertTrue(tsej.moveNext());
169	LexerTestUtilities.assertTokenEquals(tsej,TestJoinSectionsTextTokenId.BRACES, "{}", 15);
170	assertEquals(tsej.token().partType(), PartType.COMPLETE);
171	assertTrue(tsej.moveNext());
172	LexerTestUtilities.assertTokenEquals(tsej,TestJoinSectionsTextTokenId.BRACES, "{jm}", 17);
173	assertEquals(tsej.token().partType(), PartType.COMPLETE);
174	// assertFalse(tsej.moveNext());
175
176	}
177
178	}