Attachment #37621 for bug #90796

This Bugzilla instance is a read-only archive of historic NetBeans bug reports. To report a bug in NetBeans please follow the project's instructions for reporting issues.

View | Details | Raw Unified | Return to bug 90796
Collapse All | Expand All




import org.netbeans.api.editor.settings.FontColorSettings;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.TokenChange;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenHierarchyEvent;
import org.netbeans.api.lexer.TokenHierarchyListener;

        synchronized (this) {
            version++;
        }
        
        TokenChange<? extends TokenId> tc = evt.tokenChange();
        int addedLenght = 0;
        int removedLength = 0;
        
        if (tc.addedTokenCount() > 0) {
            addedLenght = getTokensLength(tc.currentTokenSequence(), tc.index(), tc.addedTokenCount());
        }
        
        if (tc.removedTokenCount() > 0) {
            removedLength = getTokensLength(tc.removedTokenSequence(), 0, tc.removedTokenCount());
        }

        fireHighlightsChange(evt.affectedStartOffset(), evt.affectedEndOffset());
        int changeEnd;
        
        if (addedLenght == 0 && removedLength == 0) {
            // The real length couldn't be computed for some reason
            changeEnd = Integer.MAX_VALUE;
        } else {
            changeEnd = changeStart + Math.max(addedLenght, removedLength);
        }
        
        fireHighlightsChange(changeStart, changeEnd);
    }

    // ----------------------------------------------------------------------
    //  Private implementation
    // ----------------------------------------------------------------------

    private int getTokensLength(TokenSequence<? extends TokenId> seq, int startIdx, int tokenCount) {
        assert startIdx >= 0 && startIdx < seq.tokenCount() : 
            "Invalid startIdx: " + startIdx + ", sequence lenght: " + seq.tokenCount();
        assert tokenCount > 0 && startIdx + tokenCount <= seq.tokenCount() : 
            "Invalid tokenCount: " + tokenCount + ", startIdx: " + startIdx + 
            ", sequence lenght: " + seq.tokenCount();
            
        int startOffset = -1;
        int endOffset = -1;
        
        if (seq.moveIndex(startIdx)) {
            startOffset = seq.offset();
        }
        
        if (seq.moveIndex(startIdx + tokenCount - 1)) {
            endOffset = seq.offset() + seq.token().length();
        }

        return startOffset == -1 || endOffset == -1 ? 0 : endOffset - startOffset;
    }
    
    private final class HSImpl implements HighlightsSequence {
        
        private static final int S_NORMAL = 1;

                    case S_EMBEDDED_HEAD:
                        // The current token contains embedded language and we have processed it's head
                        TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
                        seq.moveStart();
                        if (seq.moveNext()) {
                            state = S_NORMAL;
                        } else {
                            throw new IllegalStateException("Invalid state");

                    // We have moved to the next normal token, so look what it is
                    TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
                    TokenSequence<? extends TokenId> embeddedSeq = seq.embedded();
                    while (embeddedSeq != null && embeddedSeq.moveNext()) {
                        sequences.add(sequences.size(), embeddedSeq);
                        if (embeddedSeq.offset() > seq.offset()) {
                            state = S_EMBEDDED_HEAD;

                    }
                    case S_EMBEDDED_TAIL: {
                        TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
                        seq.moveEnd();
                        if (seq.movePrevious()) {
                            return seq.offset() + seq.token().length();
                        } else {
                            throw new IllegalStateException("Invalid state");

                    }
                    case S_EMBEDDED_HEAD: {
                        TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
                        seq.moveStart();
                        if (seq.moveNext()) {
                            return seq.offset();
                        } else {
                            TokenSequence<? extends TokenId> embeddingSeq = sequences.get(sequences.size() - 2);

            } else {
                if (sequences.size() > 1) {
                    TokenSequence<? extends TokenId> embeddingSeq = sequences.get(sequences.size() - 2);
                    seq.moveEnd();
                    if (seq.movePrevious()) {
                        if ((seq.offset() + seq.token().length()) < (embeddingSeq.offset() + embeddingSeq.token().length())) {
                            return S_EMBEDDED_TAIL;
                        } else {

Lines 133-139 Link Here

(-)html/editor/lib/src/org/netbeans/editor/ext/html/HTMLSyntaxSupport.java (-1 / +2 lines)
133		133
134	// if the carret is after HTML tag ( after char '>' ), ship inside the tag	134	// if the carret is after HTML tag ( after char '>' ), ship inside the tag
135	if (token.id() == HTMLTokenId.TAG_CLOSE_SYMBOL) {	135	if (token.id() == HTMLTokenId.TAG_CLOSE_SYMBOL) {
136	if(ts.moveIndex(ts.index() - 2)) {	136	ts.moveIndex(ts.index() - 2);
		137	if(ts.moveNext()) {
137	token = ts.token();	138	token = ts.token();
138	}	139	}
139	}	140	}




            int openLtNum = 0;
            JavaTokenId lastNonWhitespaceTokenId = null;
            TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());
            ts.move(expEndPos);
            while (ts.moveNext()) {
            do {
                if (ts.offset() >= offset) {
                    break;
                }

                    default:
                        lastNonWhitespaceTokenId = ts.token().id();
                }
            }
            if (!afterDot) {
                if (expEndPos <= offset)
                    insideExpression(env, new TreePath(path, fa.getExpression()));

        
        private TokenSequence<JavaTokenId> findLastNonWhitespaceToken(Env env, int startPos, int endPos) {
            TokenSequence<JavaTokenId> ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language());
            if (ts.move(endPos) == 0) // When right at the token end
                ts.movePrevious(); // move to previous token
            else
                ts.moveNext(); // otherwise move to the token that "contains" the offset
            do {
                int offset = ts.offset();
                if (offset < startPos)

            if (upToOffset) {
                TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());
                if (ts.moveNext()) { //check for empty file
                    if (ts.move(offset) == 0) // When right at the token end
                        ts.movePrevious(); // Move to previous token
                    else
                        ts.moveNext(); // otherwise move to the token that "contains" the offset
                    if (ts.offset() < offset && (ts.token().id() == JavaTokenId.IDENTIFIER || "keyword".equals(ts.token().id().primaryCategory()))) { //TODO: Use isKeyword(...) when available
                        prefix = ts.token().toString().substring(0, offset - ts.offset());
                        offset = ts.offset();

Lines 98-103 Link Here

(-)java/editor/src/org/netbeans/modules/editor/java/Utilities.java (+2 lines)
98	return false;	98	return false;
99	if (!ts.moveNext() \|\| ts.move(offset) == 0)	99	if (!ts.moveNext() \|\| ts.move(offset) == 0)
100	return true;	100	return true;
		101	ts.moveNext(); // Move to the next token after move(offset)
101	switch(ts.token().id()) {	102	switch(ts.token().id()) {
102	case DOUBLE_LITERAL:	103	case DOUBLE_LITERAL:
103	if (ts.token().text().charAt(0) == '.')	104	if (ts.token().text().charAt(0) == '.')
Lines 127-132 Link Here
127	TokenSequence<? extends TokenId> ts = hierarchy.tokenSequence();	128	TokenSequence<? extends TokenId> ts = hierarchy.tokenSequence();
128	while(ts != null && ts.moveNext()) {	129	while(ts != null && ts.moveNext()) {
129	ts.move(offset);	130	ts.move(offset);
		131	ts.moveNext();
130	if (ts.language() == JavaTokenId.language())	132	if (ts.language() == JavaTokenId.language())
131	return (TokenSequence<JavaTokenId>)ts;	133	return (TokenSequence<JavaTokenId>)ts;
132	ts = ts.embedded();	134	ts = ts.embedded();




        TokenSequence<JavaTokenId> ts = info.getTokenHierarchy().tokenSequence();
        
        ts.move(offset);
        if (ts.moveNext()) {
            Token t = ts.token();

            if (t.id() == JavaTokenId.DOT) {
            ts.moveNext();
            t = ts.token();
        } else {
            if (t.id() == JavaTokenId.LT) {
                ts.moveNext();
                t = ts.token();
            } else {
                if (t.id() == JavaTokenId.LT) {
                    ts.moveNext();
                    t = ts.token();
                }
            }

            if (t.id() == JavaTokenId.IDENTIFIER) {
                return ts.offsetToken();
            }
        }
        
        if (t.id() == JavaTokenId.IDENTIFIER) {
            return ts.offsetToken();
        }
        
        return null;
    }
    

            
            int diff = ts.move((int) getPrefferedPosition(info, d));
            
            if (ts.moveNext() && diff >= 0 && diff < ts.token().length()) {
                Token<JavaTokenId> t = ts.token();
                
                if (t.id() == JavaTokenId.IDENTIFIER) {

Lines 158-164 Link Here

(-)java/source/src/org/netbeans/api/java/source/TreeUtilities.java (-1 / +2 lines)
158	return path;	158	return path;
159		159
160	TokenSequence<JavaTokenId> tokenList = tokensFor(path.getLeaf(), sourcePositions);	160	TokenSequence<JavaTokenId> tokenList = tokensFor(path.getLeaf(), sourcePositions);
161	if (tokenList.moveLast() && tokenList.offset() < pos) {	161	tokenList.moveEnd();
		162	if (tokenList.movePrevious() && tokenList.offset() < pos) {
162	switch (tokenList.token().id()) {	163	switch (tokenList.token().id()) {
163	case GTGTGT:	164	case GTGTGT:
164	case GTGT:	165	case GTGT:




        // skip the section when printing anonymous class
        if (anonClass == false) {
        tokenSequence.move(oldT.pos);
        tokenSequence.moveNext(); // First skip as move() does not position to token directly
        tokenSequence.moveNext();
        insertHint = TokenUtilities.moveNext(tokenSequence, tokenSequence.offset());
        localPointer = diffModifiers(oldT.mods, newT.mods, oldT, localPointer);

            // there was not any parameter in original tree.
            int startOffset = oldT.restype != null ? oldT.restype.getStartPosition() : oldT.getStartPosition();
            
            tokenSequence.move(startOffset);
            TokenUtilities.moveFwdToToken(tokenSequence, startOffset, JavaTokenId.RPAREN);
            posHint = tokenSequence.offset();
        } else {

            // now check, that there is a whitespace. It is not mandatory, we
            // have to ensure. If whitespace is not present, take body beginning
            tokenSequence.move(posHint);
            tokenSequence.moveNext();
            if (tokenSequence.token().id() != JavaTokenId.WHITESPACE) {
                ++posHint;
            }

            ResultItem<JCTree> item = result[j];
            switch (item.operation) {
                case MODIFY: {
                    // perhaps I shouldn't support this!
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

                    assert startOffset != -1 && endOffset != -1 : "Invalid offset!";
                    //printer.print(origText.substring(lastOldPos, startOffset));
                    //append(Diff.delete(startOffset, endOffset));
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

                    break;
                }
                case NOCHANGE: {
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

            return -1;
        }
        tokenSequence.moveIndex(tokenIndex);
        tokenSequence.moveNext();
        return tokenSequence.offset();
    }
    




                int treeEnd = (int) positions.getEndPosition(compilationUnit, item);
                
                seq.move(treeStart);
                seq.moveNext();
                int startIndex = seq.index();
                
                // go back to opening/closing curly, semicolon or other

                seq.moveNext();
                int start = seq.index();
                seq.move(treeEnd);
                // seq.index() set (no seq.moveNext() necessary)
                matrix[i++] = new int[] { start, startIndex, seq.index() };
                if (i == size) {
                    seq.move(treeEnd);
                    // seq.index() set (no seq.moveNext() necessary)
                    matrix[i][0] = seq.index();
                }
            }

            // to decide.
            if (tokenIndex == -1) return -1;
            seq.moveIndex(tokenIndex);
            seq.moveNext();
            return index == 0 ? goAfterLastNewLine(seq) : goAfterFirstNewLine(seq);
        }


        @Override()
        public int[] getPositions(int index) {
            int tokenIndex = matrix[index][0];
            if (tokenIndex != -1) {
                seq.moveIndex(tokenIndex);
                seq.moveNext();
            }
            int begin = goAfterLastNewLine(seq);
            if (matrix[index][2] != -1) {
                seq.moveIndex(matrix[index][2]);
                seq.moveNext();
            }
            int end = goAfterFirstNewLine(seq);
            return new int [] { begin, end };
        }

                if (treeEnd < 0) continue;
                
                seq.move(treeStart);
                seq.moveNext();
                int startIndex = seq.index();
                // go back to opening/closing curly, semicolon or other
                // token java-compiler important token.

            // to decide.
            if (tokenIndex == -1) return -1;
            seq.moveIndex(tokenIndex);
            seq.moveNext();
            return goAfterFirstNewLine(seq);
        }
        

        
        public int[] getPositions(int index) {
            int begin = getInsertPos(index);
            if (matrix[index][4] != -1) {
                seq.moveIndex(matrix[index][4]);
                seq.moveNext();
            }
            int end = goAfterFirstNewLine(seq);
            return new int [] { begin, end };
        }

            // to decide.
            if (tokenIndex == -1) return -1;
            seq.moveIndex(tokenIndex);
            seq.moveNext();
            int off = goAfterFirstNewLine(seq);
            return off;
        }

        
        public int[] getPositions(int index) {
            int begin = getInsertPos(index);
            if (matrix[index][4] != -1) {
                seq.moveIndex(matrix[index][4]);
                seq.moveNext();
            }
            int end = goAfterFirstNewLine(seq);
            return new int [] { begin, end };
        }

    // todo (#pf): remove - used for debugging reasons, doesn't do good job
    public void tablePrint(int[][] matrix, TokenSequence seq) {
        for (int i = 0; i < matrix.length; i++) {
            for (int j = 0; j < matrix[i].length; j++) {
                int item = matrix[i][j];
                String s = "(nothing)";
                if (item > -1) {
                    seq.moveIndex(item);
                    seq.moveNext();
                    s = "'" + seq.token().text();
                }
                s += "'                                           ";
    //            System.err.print(s.substring(0, 25));
                System.err.print(item + "\t");
            }
            System.err.println("");
        }
    }





     */
    static int moveNext(TokenSequence<JavaTokenId> tokenSequence, final int pos) {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        JavaTokenId type = tokenSequence.token().id();
        while (JavaTokenId.WHITESPACE.equals(type) ||
               JavaTokenId.BLOCK_COMMENT.equals(type) ||

            JavaTokenId id)
    {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        while (!id.equals(tokenSequence.token().id())) {
            if (!tokenSequence.moveNext())
                return -1;

            JavaTokenId id)
    {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        while (!id.equals(tokenSequence.token().id())) {
            if (!tokenSequence.movePrevious())
                return -1;

            String text)
    {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        while (!text.equals(tokenSequence.token().text())) {
            if (!tokenSequence.moveNext())
                return -1;

            String text)
    {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        while (!text.equals(tokenSequence.token().text())) {
            if (!tokenSequence.movePrevious())
                return -1;




            ResultItem<JCTree> item = result[j];
            switch (item.operation) {
                case MODIFY: {
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

                    int endOffset = toOff(s.tail(j) || s.next(j) ? matrix[i+1][2] : matrix[i][4]);
                    assert startOffset != -1 && endOffset != -1 : "Invalid offset!";
                    append(Diff.delete(startOffset, endOffset));
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

                    break;
                }
                case NOCHANGE: {
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();




        TokenHierarchy hi = TokenHierarchy.get(document);
        TokenSequence ts = hi.tokenSequence();
        // If necessary move ts to the requested offset
        ts.move(offset);
        while (ts.moveNext()) {
            Token t = ts.token();
            if (t.id() == ...) { ... }
            if (TokenUtilities.equals(t.text(), "mytext")) { ... }
            if (ts.offset() == ...) { ... }
                if (ts.offset() == ...) { ... }

            // Possibly retrieve embedded token sequence
            TokenSequence embedded = ts.embedded();
            if (embedded != null) { // Token has a valid language embedding
                ...
            }
            } while (ts.moveNext());
        }
    } finally {
        document.readUnlock();

Lines 1-4 Link Here

(-)lexer/manifest.mf (-1 / +1 lines)
1	OpenIDE-Module: org.netbeans.modules.lexer/2	1	OpenIDE-Module: org.netbeans.modules.lexer/2
2	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/lexer/Bundle.properties	2	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/lexer/Bundle.properties
3	OpenIDE-Module-Specification-Version: 1.12.0	3	OpenIDE-Module-Specification-Version: 1.13.0
4	OpenIDE-Module-Recommends: org.netbeans.spi.lexer.LanguageProvider	4	OpenIDE-Module-Recommends: org.netbeans.spi.lexer.LanguageProvider




<!-- ACTUAL CHANGES BEGIN HERE: -->

  <changes>
  <change id="api.tokensequence.move">
      <api name="api"/>
      <summary>Changed TokenSequence.move() and moveIndex() use</summary>
      <version major="1" minor="13"/>
      <date day="16" month="1" year="2007"/>
      <author login="mmetelka"/>
      <compatibility source="incompatible" deletion="yes" addition="yes" modification="yes"/>
      <description>
        <p>
            Changed the <code>TokenSequence.move()</code> to position <b>before</b> the particular token
            that "contains" the token (or after the last token if the given offset
            is too high). Additional <code>moveNext()</code> is necessary
            to actually move to the next token.
            <br/>
            <code>TokenSequence.moveIndex()</code> was modified in a similar way
            (see javadocs).
            <br/>
            <code>TokenSequence.moveFirst()</code> and <code>moveLast()</code>
            were replaced by <code>moveStart()</code> that positionins before
            the first token and by <code>moveEnd()</code> that positions
            after the last token.
            <code>
        </p>
      </description>
  </change>

  <change id="api.lexer.release">
      <api name="api"/>
      <summary>Added Lexer.release()</summary>




        if (active) {
            pastSequences = new Stack<TokenSequence>();
            tokenSequence = hi.tokenSequence();
            tokenSequence.move(startOffset);
            if (tokenSequence.moveNext()) {
                updateTokenEndOffsetAndColoring(startOffset);
            } else { // no tokens
                active = false;

                        break;
                    }
                } else {
                    embed.move(offset);
                    if (embed.moveNext()) {
                        pastSequences.push(tokenSequence);
                        tokenSequence = embed;
                    } else {




        assertEquals("Wrong language", "text/x-simple-plain", lang.mimeType());
        
        for(int i = 0; i < seq.tokenCount(); i++) {
            seq.moveIndex(i);
            assertTrue(seq.moveNext());
            Token token = seq.token();
            
            if (token.id() == SimplePlainTokenId.WORD) {

                assertNotNull("Can't find language of the embedded sequence", embeddedLang);
                assertEquals("Wrong language of the embedded sequence", "text/x-simple-char", embeddedLang.mimeType());
                
                embeddedSeq.moveStart();
                assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveNext());
                assertEquals("Wrong startSkipLength", 1, embeddedSeq.offset() - seq.offset());
                
                embeddedSeq.moveEnd();
                assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.movePrevious());
                assertEquals("Wrong endSkipLength", 2, 
                    (seq.offset() + seq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length()));
            }
        }




 * A typical use is a forward iteration through the tokens:
 * <pre>
 *   TokenSequence ts = tokenHierarchy.tokenSequence();
 *   // Possible positioning by ts.move(offset);
 *   while (ts.moveNext()) {
 *       Token t = ts.token();
 *       if (t.id() == ...) { ... }

    
    private AbstractToken<T> token; // 16 bytes
    
    private int tokenIndex; // 20 bytes
    
    /**
     * Offset in the input at which the current token is located

    /**
     * Get instance of current token to which this token sequence points to.
     * <br/>
     * A typical iteration usage:
     * <pre>
     *   TokenSequence ts = tokenHierarchy.tokenSequence();
     *   // Possible positioning by ts.move(offset);
     *   while (ts.moveNext()) {
     *       Token t = ts.token();
     *       if (t.id() == ...) { ... }
     *       if (TokenUtilities.equals(t.text(), "mytext")) { ... }
     *       if (ts.offset() == ...) { ... }
     *   }
     * </pre>
     *
     * <p>
     * The returned token instance may be flyweight
     * (returns true from {@link Token#isFlyweight()})
     * which means that its {@link Token#offset(TokenHierarchy)} will return -1.

     * The token instance should not be held across the input source modifications.
     * </p>
     *
     * @return token instance to which this token sequence is currently positioned
     *  or null if this token sequence is not positioned to any token which may
     *  happen after TS creation or after use of {@link #move(int)} or {@link moveIndex(int)}.
     * 
     * @see #offsetToken()
     * @throws IllegalStateException if this token sequence was not positioned
     *  to any token yet.
     */
    public Token<T> token() {
        checkToken();
        return token;
    }
    

     *  to any token yet.
     */
    public Token<T> offsetToken() {
        checkTokenNotNull();
        if (token.isFlyweight()) {
            token = tokenList.replaceFlyToken(tokenIndex, token, offset());
        }

     *  to any token yet.
     */
    public int offset() {
        checkTokenNotNull();
        if (tokenOffset == -1) {
            tokenOffset = tokenList.tokenOffset(tokenIndex);
        }

    }
    
    /**
     * Get an index of token to which (or before which) this TS is currently positioned.
     * <br/>
     * Zero is returned after TS creation.
     *
     * @return &gt;=0 index of the current token.
     *  if this token sequence is initially located in front of the first token.
     */
    public int index() {
        return tokenIndex;

     *  to any token yet.
     */
    public TokenSequence<? extends TokenId> embedded() {
        checkTokenNotNull();
        return embeddedImpl(null);
    }
    

     * has a language embedding.
     */
    public <ET extends TokenId> TokenSequence<ET> embedded(Language<ET> embeddedLanguage) {
        checkTokenNotNull();
        return embeddedImpl(embeddedLanguage);
    }


     */
    public boolean createEmbedding(Language<? extends TokenId> embeddedLanguage,
    int startSkipLength, int endSkipLength, boolean joinSections) {
        checkTokenNotNull();
        return EmbeddingContainer.createEmbedding(tokenList, tokenIndex,
                embeddedLanguage, startSkipLength, endSkipLength, joinSections);
    }

    /**
     * Move to the next token in this token sequence.
     * <br/>
     * This method can be used in case when TS is positioned before a token
     * (after its creation or after use of {@link #move(int)} or {@link #moveIndex(int)})
     * or in case the TS already points to a concrete token <code>{@link #token()} != null</code>
     * - in such case it moves to next token and {@link #index()} gets increased by one.
     * <br/>
     * The next token may not necessarily start at the offset where
     * the previous token ends (there may be gaps between tokens
     * caused by token filtering).
     *
     * @return true if the sequence was successfully moved to the next token
     *  or false if it was not moved before there are no more tokens
     *  in the forward direction.
     * @throws ConcurrentModificationException if this token sequence
     *  is no longer valid because of an underlying mutable input source modification.
     */
    public boolean moveNext() {
        checkModCount();
        if (token != null) // Token already fetched
            tokenIndex++;
        Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(tokenIndex);
        if (tokenOrEmbeddingContainer != null) {
            AbstractToken origToken = token;
            token = LexerUtilsConstants.token(tokenOrEmbeddingContainer);
            // If origToken == null then the right offset might already be pre-computed from move()
            if (tokenOffset != -1) {
                if (origToken != null) {
                    // If the token list is continuous or the fetched token
                    // is flyweight (there cannot be a gap before flyweight token)
                    // the original offset can be just increased
                    // by the original token's length.
                    if (tokenList.isContinuous() || token.isFlyweight()) {
                        tokenOffset += origToken.length(); // advance by previous token's length
                    } else // Offset must be recomputed
                        tokenOffset = -1; // mark the offset to be recomputed
                } else // Not valid token previously
                    tokenOffset = -1;
            }
            return true;
        }
        if (token != null) // Unsuccessful move from existing token
            tokenIndex--;
        return false;
    }

    /**
     * Move to a previous token in this token sequence.
     * <br/>
     * This method can be used in case when TS is positioned after a token
     * (after its creation or after use of {@link #move(int)} or {@link #moveIndex(int)})
     * or in case the TS already points to a concrete token <code>{@link #token()} != null</code>.
     * <br/>
     * In both cases {@link #index()} gets decreased by one.
     * <br/>
     * The next token may not necessarily end at the offset where
     * the present token starts (there may be gaps between tokens
     * caused by use of a token id filter).
     *
     * @return true if the sequence was successfully moved to the previous token
     *  or false if it was not moved because there are no more tokens
     *  in the backward direction.
     * @throws ConcurrentModificationException if this token sequence
     *  is no longer valid because of an underlying mutable input source modification.

        if (tokenIndex > 0) {
            AbstractToken origToken = token;
            tokenIndex--;
            token = LexerUtilsConstants.token(tokenList.tokenOrEmbeddingContainer(tokenIndex));
            if (tokenOffset != -1) {
                // If the token list is continuous or the original token
                // is flyweight (there cannot be a gap before flyweight token)

                } else { // mark the offset to be computed upon call to offset()
                    tokenOffset = -1;
                }
            } else {
                tokenOffset = -1; // mark the offset to be computed upon call to offset()
            }
            return true;


    }

    /**
     * Move TS to be positioned before a token with the given index.
     * <br/>
     * Example: Fetching tokens starting at the given index:
     * <pre>
     * TokenSequence ts = tokenHierarchy.tokenSequence();
     * ts.moveIndex(index);
     * while (ts.moveNext()) {
     *     Token t = ts.token();
     *     ...
     * }
     * </pre>
     *
     * @param index index of the token to which this sequence
     *   should be positioned.
     *   <br/>
     *   If <code>index >= {@link #tokenCount()}</code>
     *   then the TS will be positioned after the last token.
     *   <br/>
     *   If <code>index < 0</code>
     *   then the TS will be positioned before first token (at index 0).
     * @return difference between requested index and the index to which TS
     *   is really set.
     * @throws ConcurrentModificationException if this token sequence
     *  is no longer valid because of an underlying mutable input source modification.
     */
    public int moveIndex(int index) {
        checkModCount();
        if (index >= 0) {
            Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index);
            if (tokenOrEmbeddingContainer != null) { // enough tokens
                resetTokenIndex(index);
            } else // Token at the requested index does not exist - leave orig. index
                resetTokenIndex(tokenCount());
        } else // index < 0
            resetTokenIndex(0);
        return index - tokenIndex;
    }
    
    /**
     * Move the token sequence to be positioned before the first token.
     * <br/>
     * This is equivalent to <code>moveIndex(0)</code>.
     */
    public void moveStart() {
        moveIndex(0);
    }
    
    /**
     * Move the token sequence to be positioned after the last token.
     * <br/>
     * This is equivalent to <code>moveIndex(tokenCount())</code>.
     */
    public void moveEnd() {
        moveIndex(tokenCount());
    }
    
    /**
     * Move the token sequence to be positioned before the token
     * that "contains" the requested offset (the offset is at the begining
     * or inside of the token).
     * <br/>
     * Typical usage pattern:
     * <pre>
     * TokenSequence ts = ...
     * ts.move(offset);
     * while (ts.moveNext()) {
     *     Token t = ts.token();
     *     ...
     * }
     * </pre>
     * 
     * If the offset is too big then the token sequence will be positioned
     * behind the last token.
     * <p>
     * If token filtering is used there may be gaps that are not covered
     * by any tokens and if the offset is contained in such gap then
     * the token sequence will be positioned before the token that follows the gap.
     * </p>
     *
     * Example:
     * <pre>
     *   int diff = tokenSequence.move(targetOffset);
     *   // diff equals to (targetOffset - tokenSequence.token().offset())
     *   if (diff >= 0 && diff < tokenSequence.token().length()) {
     *       // Within token bounds - tokenSequence.token() starts at or "contains" targetOffset
     *
     *   } else if (diff == Integer.MAX_VALUE) {
     *       // No tokens in the token sequence at all.
     *       // Token sequence is not positioned to any token.
     *
     *   } else {
     *       // 1. diff >= tokenSequence.token().length()
     *       //   a) targetOffset is above the end of the last token in the sequence.
     *       //     Token sequence is positioned to the last token in the sequence.
     *       //   b) there are text areas not covered by any tokens
     *       //     due to skipped tokens (skipTokenIds was used
     *       //     in TokenHierarchy.create()) and the targetOffset points to such gap.
     *       //     Token sequence is positioned to the preceding token.
     *       // 
     *       // 2. diff < 0
     *       //   a) targetOffset < 0
     *       //   b) targetOffset >= 0 but there is a text area
     *       //     at the begining that is not covered by any tokens
     *       //     (skipTokenIds was used in TokenHierarchy.create())
     *       //     Token sequence is positioned to the first token in the sequence.
     *   }
     * </pre>
     * 
     *
     * @param offset absolute offset in the input to which
     *  the token sequence should be moved.
     *
     * @return difference between the reqeuested offset
     *  and the start offset of the token
     *  before which the the token sequence gets positioned.
     *  <br/>
     *  If the TS is positioned after the last token the difference is computed
     *  against the end offset of the last token.
     * 
     * @throws ConcurrentModificationException if this token sequence
     *  is no longer valid because of an underlying mutable input source modification.

            if (tokenList.tokenOrEmbeddingContainer(0) == null) { // really no tokens at all
                // In this case the token sequence could not be positioned yet
                // so no need to reset "index" or other vars
                resetTokenIndex(0);
                return offset;
            }
            // Re-get the present token count (could be created a chunk of tokens at once)
            tokenCount = tokenList.tokenCountCurrent();

                    tokenLength = t.length();
                    tokenCount++;

                } else { // no more tokens => position after last token
                    resetTokenIndex(tokenCount);
                    tokenOffset = prevTokenOffset + tokenLength; // May assign the token's offset in advance
                    return offset - tokenOffset;
                }
            }
            resetTokenIndex(tokenCount - 1);
            tokenOffset = prevTokenOffset; // May assign the token's offset in advance
            tokenOffset = prevTokenOffset;
            assignToken();
            return offset - prevTokenOffset;
        }
        

                high = mid - 1;
            } else {
                // Token starting exactly at offset found
                resetTokenIndex(mid);
                tokenOffset = midStartOffset;
                assignToken();
                return 0; // right at the token begining
            }
        }
        
        // Not found exactly and high + 1 == low => high < low
        // BTW there may be gaps between tokens; if offset is in gap then position to higher token
        if (high >= 0) { // could be -1
            AbstractToken t = LexerUtilsConstants.token(tokenList, high);
            prevTokenOffset = tokenList.tokenOffset(high);
            // If gaps allowed check whether the token at "high" contains the offset
            if (!tokenList.isContinuous() && offset > prevTokenOffset + t.length()) {
                // Offset in the gap above the "high" token
                high++;
                prevTokenOffset += t.length();
            }
        } else { // at least one token exists => use token at index 0
            high = 0;
            prevTokenOffset = tokenList.tokenOffset(0); // result may differ from 0
        }
        resetTokenIndex(high);
        tokenIndex = high;
        tokenOffset = prevTokenOffset;
        assignToken();
        return offset - prevTokenOffset;
    }

    /**
     * Move to the first token in this token sequence.
     *
     * @return true if the sequence was positioned on the first token
     *  or false if there are no tokens in the sequence.
     */
    public boolean moveFirst() {
        return moveIndex(0);
    }
    
    /**
     * Move to the last token in this token sequence.
     *
     * @return true if the sequence was positioned on the last token
     *  or false if there are no tokens in the sequence.
     */
    public boolean moveLast() {
        return moveIndex(tokenCount() - 1); // Can be -1 but handled in move(index)
    }

    /**
     * Return total count of tokens in this sequence.
     * <br>
     * <b>Note:</b> Calling this method will lead

        return parentTokenIndexes;
    }

    private void resetTokenIndex(int index) {
        // Position to the given index e.g. by move() and moveIndex()
        tokenIndex = index;
        token = null;
        tokenOffset = -1;
        assignToken(tokenList.tokenOrEmbeddingContainer(tokenIndex));
    }

    private void checkTokenNotNull() {
        if (token == null) {
            throw new IllegalStateException(
                "No token fetched by moveNext() from token sequence yet: index=" + tokenIndex




            sb.append(": ");
            AbstractToken token = token(tokenOrEmbeddingContainer);
            sb.append(token.dumpInfo(tokenHierarchy));
            int la = tokenList.lookahead(i);
            if (la != 0) {
                sb.append(", la=");
                sb.append(la);
            }
            Object state= tokenList.state(i);
            if (state != null) {
                sb.append(", s=");
                sb.append(state);
            }
            sb.append('\n');
        }
        return sb;

Lines 92-98 Link Here

(-)lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java (-1 / +1 lines)
92	if (limitStartOffset > 0) {	92	if (limitStartOffset > 0) {
93	int diff = move(limitStartOffset);	93	int diff = move(limitStartOffset);
94	if (diff != Integer.MAX_VALUE) { // some tokens exist	94	if (diff != Integer.MAX_VALUE) { // some tokens exist
95	if (diff >= lastToken.length()) {	95	if (diff >= lastToken.length()) { // lastToken initialized in move()
96	lastTokenIndex++;	96	lastTokenIndex++;
97	Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(lastTokenIndex);	97	Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(lastTokenIndex);
98	if (tokenOrEmbeddingContainer != null &&	98	if (tokenOrEmbeddingContainer != null &&

Lines 48-60 Link Here

(-)lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java (-41 / +129 lines)
48	TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());	48	TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());
49	TokenSequence<? extends TokenId> ts = hi.tokenSequence();	49	TokenSequence<? extends TokenId> ts = hi.tokenSequence();
50		50
51	// Fail if no "move*" method called yet	51	assertNull(ts.token());
52	try {
53	ts.token(); // should throw exception
54	fail("IllegalStateException expecxted");
55	} catch (IllegalStateException e) {
56	// expected
57	}
58		52
59	assertTrue(ts.moveNext());	53	assertTrue(ts.moveNext());
60	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	54	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
Lines 69-88 Link Here
69	assertEquals(ts.tokenCount(), 3);	63	assertEquals(ts.tokenCount(), 3);
70		64
71	assertEquals(0, ts.move(0));	65	assertEquals(0, ts.move(0));
		66	assertNull(ts.token());
		67	assertTrue(ts.moveNext());
72	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	68	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
73	assertEquals(2, ts.move(2));	69	assertEquals(2, ts.move(2));
		70	assertNull(ts.token());
		71	assertTrue(ts.moveNext());
74	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	72	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
75	assertEquals(-1, ts.move(-1));	73	assertEquals(-1, ts.move(-1));
		74	assertNull(ts.token());
		75	assertTrue(ts.moveNext());
76	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	76	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
77	assertEquals(0, ts.move(3));	77	assertEquals(0, ts.move(3));
		78	assertNull(ts.token());
		79	assertTrue(ts.moveNext());
78	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);	80	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
79	assertEquals(0, ts.move(4));	81	assertEquals(0, ts.move(4));
		82	assertNull(ts.token());
		83	assertTrue(ts.moveNext());
80	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);	84	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
81	assertEquals(1, ts.move(5));	85	assertEquals(1, ts.move(5));
		86	assertNull(ts.token());
		87	assertTrue(ts.moveNext());
82	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);	88	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
83	assertEquals(5, ts.move(9));	89	assertEquals(1, ts.move(9));
		90	assertNull(ts.token());
		91	assertFalse(ts.moveNext());
		92	assertTrue(ts.movePrevious());
84	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);	93	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
85	assertEquals(96, ts.move(100));	94	assertEquals(92, ts.move(100));
		95	assertNull(ts.token());
		96	assertFalse(ts.moveNext());
		97	assertTrue(ts.movePrevious());
86	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);	98	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
87		99
88		100
Lines 107-112 Link Here
107	sub = ts.subSequence(3, 3);	119	sub = ts.subSequence(3, 3);
108	assertFalse(sub.moveNext());	120	assertFalse(sub.moveNext());
109	}	121	}
		122
		123	public void testMoveNextPrevious() {
		124	String text = "abc+defg-";
		125	TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());
		126	TokenSequence<? extends TokenId> ts = hi.tokenSequence();
		127	assertEquals(0, ts.move(4));
		128	assertNull(ts.token());
		129	//assertEquals(4, ts.offset());
		130	assertTrue(ts.movePrevious());
		131	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
		132
		133	ts = hi.tokenSequence();
		134	ts.move(5);
		135	assertTrue(ts.moveNext());
		136	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
		137	assertTrue(ts.movePrevious());
		138	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
		139
		140	// Move single char before token's end
		141	ts = hi.tokenSequence();
		142	ts.move(7);
		143	assertTrue(ts.moveNext());
		144	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
		145	assertTrue(ts.movePrevious());
		146	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
		147
		148	// Move past all tokens
		149	ts = hi.tokenSequence();
		150	ts.move(text.length());
		151	assertTrue(ts.movePrevious());
		152	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", text.length() - 1);
		153
		154	// Move at the begining
		155	ts = hi.tokenSequence();
		156	ts.move(0);
		157	assertFalse(ts.movePrevious());
		158	assertTrue(ts.moveNext());
		159	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
		160	}
		161
		162	public void testMoveIndex() {
		163	String text = "abc+defg-";
		164	TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());
		165	TokenSequence<? extends TokenId> ts = hi.tokenSequence();
		166	assertEquals(0, ts.index());
		167	assertNull(ts.token());
		168	//assertEquals(4, ts.offset());
		169	assertTrue(ts.moveNext());
		170	assertEquals(0, ts.offset());
		171	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
		172	assertTrue(ts.moveNext());
		173	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
		174	assertEquals(3, ts.offset());
		175	ts.move(2);
		176	assertEquals(0, ts.index());
		177	assertTrue(ts.moveNext());
		178	assertEquals(0, ts.offset());
		179	assertEquals(0, ts.index());
		180	ts.moveIndex(1);
		181	assertEquals(1, ts.index());
		182	assertTrue(ts.moveNext());
		183	assertEquals(1, ts.index());
		184	assertEquals(3, ts.offset());
		185	}
110		186
111	public void testMoveSkipTokens() {	187	public void testMoveSkipTokens() {
112	String text = "-abc+defg--hi";	188	String text = "-abc+defg--hi";
Lines 122-133 Link Here
122	TokenSequence<? extends TokenId> ts = hi.tokenSequence();	198	TokenSequence<? extends TokenId> ts = hi.tokenSequence();
123		199
124	// Fail if no "move*" method called yet	200	// Fail if no "move*" method called yet
125	try {	201	assertNull(ts.token());
126	ts.token(); // should throw exception
127	fail("IllegalStateException expecxted");
128	} catch (IllegalStateException e) {
129	// expected
130	}
131		202
132	assertTrue(ts.moveNext());	203	assertTrue(ts.moveNext());
133	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	204	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
Lines 137-146 Link Here
137	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	208	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
138	assertFalse(ts.moveNext());	209	assertFalse(ts.moveNext());
139	assertEquals(3, ts.tokenCount());	210	assertEquals(3, ts.tokenCount());
140	assertTrue(ts.moveFirst());	211	assertEquals(0, ts.moveIndex(0));
		212	assertTrue(ts.moveNext());
141	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	213	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
142	assertTrue(ts.moveNext());	214	assertTrue(ts.moveNext());
143	assertTrue(ts.moveLast());	215	assertEquals(0, ts.moveIndex(ts.tokenCount()));
		216	assertTrue(ts.movePrevious());
144	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	217	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
145	assertFalse(ts.moveNext());	218	assertFalse(ts.moveNext());
146		219
Lines 151-166 Link Here
151	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	224	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
152		225
153	assertEquals(-1, ts.move(0)); // below filtered-out token	226	assertEquals(-1, ts.move(0)); // below filtered-out token
		227	assertNull(ts.token());
		228	assertTrue(ts.moveNext());
154	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	229	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
155	assertEquals(0, ts.move(1));	230	assertEquals(0, ts.move(1));
		231	assertNull(ts.token());
		232	assertTrue(ts.moveNext());
156	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	233	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
157	assertEquals(1, ts.move(2));	234	assertEquals(1, ts.move(2));
		235	assertNull(ts.token());
		236	assertTrue(ts.moveNext());
158	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	237	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
159	assertEquals(3, ts.move(4));	238	assertEquals(3, ts.move(4));
		239	assertNull(ts.token());
		240	assertTrue(ts.moveNext());
160	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	241	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
161	assertEquals(0, ts.move(5));	242	assertEquals(0, ts.move(5));
		243	assertNull(ts.token());
		244	assertTrue(ts.moveNext());
162	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 5);	245	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 5);
163	assertEquals(1, ts.move(12));	246	assertEquals(1, ts.move(12));
		247	assertNull(ts.token());
		248	assertTrue(ts.moveNext());
164	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	249	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
165		250
166		251
Lines 172-219 Link Here
172	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);	257	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
173	assertFalse(sub.moveNext());	258	assertFalse(sub.moveNext());
174		259
175	// Test moveFirst() and moveLast() on subsequence	260	// Test moves to first and last tokens on subsequence
176	sub = ts.subSequence(1, 6);	261	sub = ts.subSequence(1, 6);
177	assertTrue(sub.moveLast());	262	assertEquals(0, sub.moveIndex(sub.tokenCount()));
		263	assertEquals(2, sub.tokenCount()); // "abc" and "defg" (others filtered out
		264	assertTrue(sub.movePrevious());
178	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);	265	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
179	assertTrue(sub.moveFirst());	266	assertEquals(0, sub.moveIndex(0));
		267	assertTrue(sub.moveNext());
180	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "abc", 1);	268	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "abc", 1);
181		269
182	sub = ts.subSequence(4);	270	sub = ts.subSequence(4);
183	assertTrue(sub.moveNext());	271	assertTrue(sub.moveNext());
184	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);	272	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
185	assertTrue(sub.moveNext());	273	assertTrue(sub.moveNext());
186	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	274	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
187	assertFalse(sub.moveNext());	275	assertFalse(sub.moveNext());
188		276
189	sub = ts.subSequence(12, 15);	277	sub = ts.subSequence(12, 15);
190	assertTrue(sub.moveNext());	278	assertTrue(sub.moveNext());
191	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	279	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
192		280
193	sub = ts.subSequence(12, 15);	281	sub = ts.subSequence(12, 15);
194	assertEquals(-2, sub.move(9));	282	assertEquals(-2, sub.move(9));
195	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	283	assertNull(sub.token());
		284	assertTrue(sub.moveNext());
		285	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
196		286
197	sub = ts.subSequence(13, 15);	287	sub = ts.subSequence(13, 15);
198	assertFalse(sub.moveNext());	288	assertFalse(sub.moveNext());
199	assertFalse(sub.moveFirst());	289	assertEquals(0, sub.moveIndex(0));
200	assertFalse(sub.moveLast());	290	assertNull(sub.token());
		291	assertFalse(sub.moveNext());
		292	assertEquals(0, sub.moveIndex(sub.tokenCount()));
		293	assertNull(sub.token());
201	assertEquals(0, sub.tokenCount());	294	assertEquals(0, sub.tokenCount());
202		295
203	sub = ts.subSequence(-3, 1);	296	sub = ts.subSequence(-3, 1);
204	assertFalse(sub.moveNext());	297	assertFalse(sub.moveNext());
205		298
206	sub = ts.subSequence(13, 15);	299	sub = ts.subSequence(13, 15);
207	assertEquals(Integer.MAX_VALUE, sub.move(5));	300	assertEquals(5, sub.move(5));
208		301
209	sub = ts.subSequence(-3, 1);	302	sub = ts.subSequence(-3, 1);
210	assertEquals(Integer.MAX_VALUE, sub.move(1));	303	assertEquals(1, sub.move(1));
211		304
212	// Reversed bounds => should be empty token sequence	305	// Reversed bounds => should be empty token sequence
213	sub = ts.subSequence(6, 1);	306	sub = ts.subSequence(6, 1);
214	assertFalse(sub.moveNext());	307	assertFalse(sub.moveNext());
215	sub = ts.subSequence(6, 1);	308	sub = ts.subSequence(6, 1);
216	assertEquals(Integer.MAX_VALUE, sub.move(6));	309	assertEquals(6, sub.move(6));
217	}	310	}
218		311
219	public void DtestMoveSkipTokens2() throws IOException {	312	public void DtestMoveSkipTokens2() throws IOException {
Lines 244-275 Link Here
244		337
245	// Expect no tokens	338	// Expect no tokens
246	assertFalse(ts.moveNext());	339	assertFalse(ts.moveNext());
247	// Exception if ts.token() called	340	assertNull(ts.token());
248	try {
249	ts.token(); // should throw exception
250	fail("IllegalStateException expecxted");
251	} catch (IllegalStateException e) {
252	// expected
253	}
254		341
255	assertEquals(ts.move(0), Integer.MAX_VALUE);	342	assertEquals(0, ts.move(0));
256	assertEquals(ts.move(10), Integer.MAX_VALUE);	343	assertEquals(10, ts.move(10));
257		344
258	// Test subsequences	345	// Test subsequences
259	TokenSequence<? extends TokenId> sub = ts.subSequence(1, 6);	346	TokenSequence<? extends TokenId> sub = ts.subSequence(1, 6);
260	assertFalse(sub.moveNext());	347	assertFalse(sub.moveNext());
261	sub = ts.subSequence(1, 6);	348	sub = ts.subSequence(1, 6);
262	assertEquals(Integer.MAX_VALUE, sub.move(1));	349	assertEquals(1, sub.move(1));
263		350
264	sub = ts.subSequence(0);	351	sub = ts.subSequence(0);
265	assertFalse(sub.moveNext());	352	assertFalse(sub.moveNext());
266	sub = ts.subSequence(0);	353	sub = ts.subSequence(0);
267	assertEquals(Integer.MAX_VALUE, sub.move(0));	354	assertEquals(0, sub.move(0));
268		355
269	sub = ts.subSequence(1);	356	sub = ts.subSequence(1);
270	assertFalse(sub.moveNext());	357	assertFalse(sub.moveNext());
271	sub = ts.subSequence(1);	358	sub = ts.subSequence(1);
272	assertEquals(Integer.MAX_VALUE, sub.move(0));	359	assertEquals(0, sub.move(0));
273	}	360	}
274		361
275	public void testTokenSize() {	362	public void testTokenSize() {
Lines 284-290 Link Here
284	assertFalse(ts.moveNext());	371	assertFalse(ts.moveNext());
285		372
286	TokenList tokenList = LexerTestUtilities.tokenList(ts);	373	TokenList tokenList = LexerTestUtilities.tokenList(ts);
287	ts.moveIndex(0); // move to "abc"	374	ts.moveIndex(0); // move before "abc"
		375	assertTrue(ts.moveNext());
288	// Test DefaultToken size	376	// Test DefaultToken size
289	assertSame(DefaultToken.class, ts.token().getClass());	377	assertSame(DefaultToken.class, ts.token().getClass());
290	assertSize("Token instance too big", Collections.singletonList(ts.token()), 24,	378	assertSize("Token instance too big", Collections.singletonList(ts.token()), 24,




    public void testCachingE() {
        TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
        TokenSequence tokens = th.tokenSequence();
        tokens.moveStart();
        assertEquals(true, tokens.moveNext());
        
        TokenSequence embeddedA = tokens.embedded();
        assertNotNull("There should be an embedded language", embeddedA);

    public void testGCedE() {
        TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
        TokenSequence tokens = th.tokenSequence();
        tokens.moveStart();
        assertEquals(true, tokens.moveNext());
        
        TokenSequence embedded = tokens.embedded();
        assertNotNull("There should be an embedded language", embedded);

    public void testCacheRefreshedE() {
        TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
        TokenSequence tokens = th.tokenSequence();
        tokens.moveStart();
        assertEquals(true, tokens.moveNext());
        
        TokenSequence embeddedA = tokens.embedded();
        assertNotNull("There should be an embedded language", embeddedA);




    public static void assertTokenEquals(String message, TokenSequence<? extends TokenId> ts, TokenId id, String text, int offset) {
        message = messagePrefix(message);
        Token<? extends TokenId> t = ts.token();
        TestCase.assertNotNull("Token is null", t);
        TokenId tId = t.id();
        TestCase.assertEquals(message + "Invalid token.id()", id, tId);
        CharSequence tText = t.text();

        }
        TokenHierarchy<?> thBatch = TokenHierarchy.create(docText, language);
        boolean success = false;
        TokenSequence<?> batchTS = thBatch.tokenSequence();
        try {
            // Compare lookaheads and states as well
            assertTokenSequencesEqual(batchTS, thBatch,
                    thInc.tokenSequence(), thInc, true);
            success = true;
        } finally {
            if (!success) {
                // Go forward two tokens to have an extra tokens context
                batchTS.moveNext();
                batchTS.moveNext();
                System.err.println("BATCH token sequence dump:\n" + thBatch.tokenSequence());
                TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);
                if (lastHi != null) {
                    System.err.println("PREVIOUS batch token sequence dump:\n" + lastHi.tokenSequence());
                }
            }
        }





        // Check that all the non-fly tokens are mutable
        ts = snapshot1.tokenSequence();
        assertEquals(0, ts.moveIndex(0));
        assertTrue(ts.moveNext());
        
        LexerTestUtilities.assertTokenSequencesEqual(hi1.tokenSequence(), hi1,
                snapshot1.tokenSequence(), snapshot1, false);

        doc.insertString(4, "+", null);

        // Check that the snapshot token sequence can be further operated.
        assertEquals(0, ts.moveIndex(0));
        assertTrue(ts.moveNext());
        assertNotNull(ts.token());

        // Check that the tokens except '+' are live


        // Check that all the non-fly tokens are mutable
        ts = snapshot2.tokenSequence();
        assertEquals(0, ts.moveIndex(0));
        assertTrue(ts.moveNext());

        LexerTestUtilities.assertTokenSequencesEqual(hi2.tokenSequence(), hi2,
                snapshot2.tokenSequence(), snapshot2, false);

Lines 81-87 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java (-1 / +2 lines)
81	assertFalse(ts.moveNext());	81	assertFalse(ts.moveNext());
82		82
83	// Go back to block comment	83	// Go back to block comment
84	assertTrue(ts.moveIndex(commentIndex));	84	assertEquals(0, ts.moveIndex(commentIndex));
		85	assertTrue(ts.moveNext());
85	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.BLOCK_COMMENT, commentText, commentTextStartOffset);	86	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.BLOCK_COMMENT, commentText, commentTextStartOffset);
86		87
87	// Test embedded token sequence	88	// Test embedded token sequence




        
        // Check TokenSequence.move()
        int relOffset = ts.move(50); // past the end of all tokens
        assertEquals(relOffset, 50 - (offset + 3));
        assertTrue(ts.movePrevious());
        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset);

        relOffset = ts.move(6); // right at begining of "-"
        assertEquals(relOffset, 0);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", 6);

        relOffset = ts.move(-5); // to first token "abc"
        assertEquals(relOffset, -5);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);

        relOffset = ts.move(5); // to first token "abc"
        assertEquals(relOffset, 1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "uv", 4);



Lines 1-3 Link Here

(-)openide/text/src/org/openide/text/CloneableEditorSupport.java (+1 lines)
		1
1	/*	2	/*
2	* The contents of this file are subject to the terms of the Common Development	3	* The contents of this file are subject to the terms of the Common Development
3	* and Distribution License (the License). You may not use this file except in	4	* and Distribution License (the License). You may not use this file except in

Lines 98-104 Link Here

(-)web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java (-1 / +1 lines)
98	TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);	98	TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);
99	TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); //get top level token sequence	99	TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); //get top level token sequence
100		100
101	if(!tokenSequence.moveFirst()) {	101	if(!tokenSequence.moveNext()) {
102	return ; //no tokens in token sequence	102	return ; //no tokens in token sequence
103	}	103	}
104		104




         * e.g. <tag   |attr=
         * 
         */
        while (tokenSequence.moveNext()) {
            tokenSequence.moveNext();
            token = tokenSequence.token();
            tokenOffset = tokenSequence.offset();
            if (!isWSTag(token) || tagStartLine != Utilities.getLineOffset(doc, tokenOffset)) {
                if (!isWSTag(token) && tagStartLine == Utilities.getLineOffset(doc, tokenOffset)){
                    return tokenOffset - Utilities.getRowIndent(doc, tokenOffset) - Utilities.getRowStart(doc, tokenOffset);
                }
                break;
            }
        }
        
        
        return getShiftWidth(); // default;
    }
    

            int indentsWithinTags[] = new int[lastLine + 1];
            
            TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
            tokenSequence.moveStart();
            boolean thereAreMoreTokens = tokenSequence.moveNext();
            
            
            if (tokenSequence != null){
                // calc line indents - pass 1

        tokenSequence.move(offset);
        int currentOffset;
        
        while (tokenSequence.moveNext()) {
            currentOffset = tokenSequence.offset();
            
            if (isClosingTag(tokenHierarchy, currentOffset)){
                return currentOffset;
            }
            
        }

        return -1;
    }

    protected Token getTokenAtOffset(TokenHierarchy tokenHierarchy, int tagTokenOffset){
        TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
        
        if (tokenSequence != null) {
            tokenSequence.move(tagTokenOffset);
            if (tokenSequence.moveNext())
                return tokenSequence.token();
        }
        
        return null;

Return to bug 90796

Lines 1101-1109 Link Here

(-)java/editor/src/org/netbeans/modules/editor/java/JavaCompletionProvider.java (-8 / +11 lines)
1101	int openLtNum = 0;	1101	int openLtNum = 0;
1102	JavaTokenId lastNonWhitespaceTokenId = null;	1102	JavaTokenId lastNonWhitespaceTokenId = null;
1103	TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());	1103	TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());
1104	if (ts.move(expEndPos) > 0)	1104	ts.move(expEndPos);
1105	ts.moveNext();	1105	while (ts.moveNext()) {
1106	do {
1107	if (ts.offset() >= offset) {	1106	if (ts.offset() >= offset) {
1108	break;	1107	break;
1109	}	1108	}
Lines 1136-1142 Link Here
1136	default:	1135	default:
1137	lastNonWhitespaceTokenId = ts.token().id();	1136	lastNonWhitespaceTokenId = ts.token().id();
1138	}	1137	}
1139	} while (ts.moveNext());	1138	}
1140	if (!afterDot) {	1139	if (!afterDot) {
1141	if (expEndPos <= offset)	1140	if (expEndPos <= offset)
1142	insideExpression(env, new TreePath(path, fa.getExpression()));	1141	insideExpression(env, new TreePath(path, fa.getExpression()));
Lines 3403-3410 Link Here
3403		3402
3404	private TokenSequence<JavaTokenId> findLastNonWhitespaceToken(Env env, int startPos, int endPos) {	3403	private TokenSequence<JavaTokenId> findLastNonWhitespaceToken(Env env, int startPos, int endPos) {
3405	TokenSequence<JavaTokenId> ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language());	3404	TokenSequence<JavaTokenId> ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language());
3406	if (ts.move(endPos) == 0)	3405	if (ts.move(endPos) == 0) // When right at the token end
3407	ts.movePrevious();	3406	ts.movePrevious(); // move to previous token
		3407	else
		3408	ts.moveNext(); // otherwise move to the token that "contains" the offset
3408	do {	3409	do {
3409	int offset = ts.offset();	3410	int offset = ts.offset();
3410	if (offset < startPos)	3411	if (offset < startPos)
Lines 3541-3548 Link Here
3541	if (upToOffset) {	3542	if (upToOffset) {
3542	TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());	3543	TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());
3543	if (ts.moveNext()) { //check for empty file	3544	if (ts.moveNext()) { //check for empty file
3544	if (ts.move(offset) == 0)	3545	if (ts.move(offset) == 0) // When right at the token end
3545	ts.movePrevious();	3546	ts.movePrevious(); // Move to previous token
		3547	else
		3548	ts.moveNext(); // otherwise move to the token that "contains" the offset
3546	if (ts.offset() < offset && (ts.token().id() == JavaTokenId.IDENTIFIER \|\| "keyword".equals(ts.token().id().primaryCategory()))) { //TODO: Use isKeyword(...) when available	3549	if (ts.offset() < offset && (ts.token().id() == JavaTokenId.IDENTIFIER \|\| "keyword".equals(ts.token().id().primaryCategory()))) { //TODO: Use isKeyword(...) when available
3547	prefix = ts.token().toString().substring(0, offset - ts.offset());	3550	prefix = ts.token().toString().substring(0, offset - ts.offset());
3548	offset = ts.offset();	3551	offset = ts.offset();

Lines 316-321 Link Here

(-)java/source/src/org/netbeans/modules/java/source/save/CasualDiff.java (-5 / +10 lines)
316	// skip the section when printing anonymous class	316	// skip the section when printing anonymous class
317	if (anonClass == false) {	317	if (anonClass == false) {
318	tokenSequence.move(oldT.pos);	318	tokenSequence.move(oldT.pos);
		319	tokenSequence.moveNext(); // First skip as move() does not position to token directly
319	tokenSequence.moveNext();	320	tokenSequence.moveNext();
320	insertHint = TokenUtilities.moveNext(tokenSequence, tokenSequence.offset());	321	insertHint = TokenUtilities.moveNext(tokenSequence, tokenSequence.offset());
321	localPointer = diffModifiers(oldT.mods, newT.mods, oldT, localPointer);	322	localPointer = diffModifiers(oldT.mods, newT.mods, oldT, localPointer);
Lines 508-514 Link Here
508	// there was not any parameter in original tree.	509	// there was not any parameter in original tree.
509	int startOffset = oldT.restype != null ? oldT.restype.getStartPosition() : oldT.getStartPosition();	510	int startOffset = oldT.restype != null ? oldT.restype.getStartPosition() : oldT.getStartPosition();
510		511
511	tokenSequence.move(startOffset);
512	TokenUtilities.moveFwdToToken(tokenSequence, startOffset, JavaTokenId.RPAREN);	512	TokenUtilities.moveFwdToToken(tokenSequence, startOffset, JavaTokenId.RPAREN);
513	posHint = tokenSequence.offset();	513	posHint = tokenSequence.offset();
514	} else {	514	} else {
Lines 533-538 Link Here
533	// now check, that there is a whitespace. It is not mandatory, we	533	// now check, that there is a whitespace. It is not mandatory, we
534	// have to ensure. If whitespace is not present, take body beginning	534	// have to ensure. If whitespace is not present, take body beginning
535	tokenSequence.move(posHint);	535	tokenSequence.move(posHint);
		536	tokenSequence.moveNext();
536	if (tokenSequence.token().id() != JavaTokenId.WHITESPACE) {	537	if (tokenSequence.token().id() != JavaTokenId.WHITESPACE) {
537	++posHint;	538	++posHint;
538	}	539	}
Lines 1441-1448 Link Here
1441	ResultItem<JCTree> item = result[j];	1442	ResultItem<JCTree> item = result[j];
1442	switch (item.operation) {	1443	switch (item.operation) {
1443	case MODIFY: {	1444	case MODIFY: {
1444	// perhaps I shouldn't support this!	1445	// perhaps I shouldn't support this!
1445	if (tokenSequence.moveIndex(matrix[i][4])) {	1446	tokenSequence.moveIndex(matrix[i][4]);
		1447	if (tokenSequence.moveNext()) {
1446	testPos = tokenSequence.offset();	1448	testPos = tokenSequence.offset();
1447	if (JavaTokenId.COMMA == tokenSequence.token().id())	1449	if (JavaTokenId.COMMA == tokenSequence.token().id())
1448	testPos += JavaTokenId.COMMA.fixedText().length();	1450	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1488-1494 Link Here
1488	assert startOffset != -1 && endOffset != -1 : "Invalid offset!";	1490	assert startOffset != -1 && endOffset != -1 : "Invalid offset!";
1489	//printer.print(origText.substring(lastOldPos, startOffset));	1491	//printer.print(origText.substring(lastOldPos, startOffset));
1490	//append(Diff.delete(startOffset, endOffset));	1492	//append(Diff.delete(startOffset, endOffset));
1491	if (tokenSequence.moveIndex(matrix[i][4])) {	1493	tokenSequence.moveIndex(matrix[i][4]);
		1494	if (tokenSequence.moveNext()) {
1492	testPos = tokenSequence.offset();	1495	testPos = tokenSequence.offset();
1493	if (JavaTokenId.COMMA == tokenSequence.token().id())	1496	if (JavaTokenId.COMMA == tokenSequence.token().id())
1494	testPos += JavaTokenId.COMMA.fixedText().length();	1497	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1502-1508 Link Here
1502	break;	1505	break;
1503	}	1506	}
1504	case NOCHANGE: {	1507	case NOCHANGE: {
1505	if (tokenSequence.moveIndex(matrix[i][4])) {	1508	tokenSequence.moveIndex(matrix[i][4]);
		1509	if (tokenSequence.moveNext()) {
1506	testPos = tokenSequence.offset();	1510	testPos = tokenSequence.offset();
1507	if (JavaTokenId.COMMA == tokenSequence.token().id())	1511	if (JavaTokenId.COMMA == tokenSequence.token().id())
1508	testPos += JavaTokenId.COMMA.fixedText().length();	1512	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1521-1526 Link Here
1521	return -1;	1525	return -1;
1522	}	1526	}
1523	tokenSequence.moveIndex(tokenIndex);	1527	tokenSequence.moveIndex(tokenIndex);
		1528	tokenSequence.moveNext();
1524	return tokenSequence.offset();	1529	return tokenSequence.offset();
1525	}	1530	}
1526		1531

Lines 1247-1253 Link Here

(-)java/source/src/org/netbeans/modules/java/source/save/TreeDiff.java (-3 / +6 lines)
1247	ResultItem<JCTree> item = result[j];	1247	ResultItem<JCTree> item = result[j];
1248	switch (item.operation) {	1248	switch (item.operation) {
1249	case MODIFY: {	1249	case MODIFY: {
1250	if (tokenSequence.moveIndex(matrix[i][4])) {	1250	tokenSequence.moveIndex(matrix[i][4]);
		1251	if (tokenSequence.moveNext()) {
1251	testPos = tokenSequence.offset();	1252	testPos = tokenSequence.offset();
1252	if (JavaTokenId.COMMA == tokenSequence.token().id())	1253	if (JavaTokenId.COMMA == tokenSequence.token().id())
1253	testPos += JavaTokenId.COMMA.fixedText().length();	1254	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1283-1289 Link Here
1283	int endOffset = toOff(s.tail(j) \|\| s.next(j) ? matrix[i+1][2] : matrix[i][4]);	1284	int endOffset = toOff(s.tail(j) \|\| s.next(j) ? matrix[i+1][2] : matrix[i][4]);
1284	assert startOffset != -1 && endOffset != -1 : "Invalid offset!";	1285	assert startOffset != -1 && endOffset != -1 : "Invalid offset!";
1285	append(Diff.delete(startOffset, endOffset));	1286	append(Diff.delete(startOffset, endOffset));
1286	if (tokenSequence.moveIndex(matrix[i][4])) {	1287	tokenSequence.moveIndex(matrix[i][4]);
		1288	if (tokenSequence.moveNext()) {
1287	testPos = tokenSequence.offset();	1289	testPos = tokenSequence.offset();
1288	if (JavaTokenId.COMMA == tokenSequence.token().id())	1290	if (JavaTokenId.COMMA == tokenSequence.token().id())
1289	testPos += JavaTokenId.COMMA.fixedText().length();	1291	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1293-1299 Link Here
1293	break;	1295	break;
1294	}	1296	}
1295	case NOCHANGE: {	1297	case NOCHANGE: {
1296	if (tokenSequence.moveIndex(matrix[i][4])) {	1298	tokenSequence.moveIndex(matrix[i][4]);
		1299	if (tokenSequence.moveNext()) {
1297	testPos = tokenSequence.offset();	1300	testPos = tokenSequence.offset();
1298	if (JavaTokenId.COMMA == tokenSequence.token().id())	1301	if (JavaTokenId.COMMA == tokenSequence.token().id())
1299	testPos += JavaTokenId.COMMA.fixedText().length();	1302	testPos += JavaTokenId.COMMA.fixedText().length();

Lines 35-41 Link Here

(-)editor/lib2/src/org/netbeans/modules/editor/lib2/highlighting/SyntaxHighlighting.java (-50 / +10 lines)
35	import org.netbeans.api.editor.settings.FontColorSettings;	35	import org.netbeans.api.editor.settings.FontColorSettings;
36	import org.netbeans.api.lexer.Language;	36	import org.netbeans.api.lexer.Language;
37	import org.netbeans.api.lexer.LanguagePath;	37	import org.netbeans.api.lexer.LanguagePath;
38	import org.netbeans.api.lexer.TokenChange;
39	import org.netbeans.api.lexer.TokenHierarchy;	38	import org.netbeans.api.lexer.TokenHierarchy;
40	import org.netbeans.api.lexer.TokenHierarchyEvent;	39	import org.netbeans.api.lexer.TokenHierarchyEvent;
41	import org.netbeans.api.lexer.TokenHierarchyListener;	40	import org.netbeans.api.lexer.TokenHierarchyListener;
Lines 98-154 Link Here
98	synchronized (this) {	97	synchronized (this) {
99	version++;	98	version++;
100	}	99	}
101
102	TokenChange<? extends TokenId> tc = evt.tokenChange();
103	int addedLenght = 0;
104	int removedLength = 0;
105
106	if (tc.addedTokenCount() > 0) {
107	addedLenght = getTokensLength(tc.currentTokenSequence(), tc.index(), tc.addedTokenCount());
108	}
109
110	if (tc.removedTokenCount() > 0) {
111	removedLength = getTokensLength(tc.removedTokenSequence(), 0, tc.removedTokenCount());
112	}
113		100
114	int changeStart = tc.offset();	101	fireHighlightsChange(evt.affectedStartOffset(), evt.affectedEndOffset());
115	int changeEnd;
116
117	if (addedLenght == 0 && removedLength == 0) {
118	// The real length couldn't be computed for some reason
119	changeEnd = Integer.MAX_VALUE;
120	} else {
121	changeEnd = changeStart + Math.max(addedLenght, removedLength);
122	}
123
124	fireHighlightsChange(changeStart, changeEnd);
125	}	102	}
126		103
127	// ----------------------------------------------------------------------	104	// ----------------------------------------------------------------------
128	// Private implementation	105	// Private implementation
129	// ----------------------------------------------------------------------	106	// ----------------------------------------------------------------------
130		107
131	private int getTokensLength(TokenSequence<? extends TokenId> seq, int startIdx, int tokenCount) {
132	assert startIdx >= 0 && startIdx < seq.tokenCount() :
133	"Invalid startIdx: " + startIdx + ", sequence lenght: " + seq.tokenCount();
134	assert tokenCount > 0 && startIdx + tokenCount <= seq.tokenCount() :
135	"Invalid tokenCount: " + tokenCount + ", startIdx: " + startIdx +
136	", sequence lenght: " + seq.tokenCount();
137
138	int startOffset = -1;
139	int endOffset = -1;
140
141	if (seq.moveIndex(startIdx)) {
142	startOffset = seq.offset();
143	}
144
145	if (seq.moveIndex(startIdx + tokenCount - 1)) {
146	endOffset = seq.offset() + seq.token().length();
147	}
148
149	return startOffset == -1 \|\| endOffset == -1 ? 0 : endOffset - startOffset;
150	}
151
152	private final class HSImpl implements HighlightsSequence {	108	private final class HSImpl implements HighlightsSequence {
153		109
154	private static final int S_NORMAL = 1;	110	private static final int S_NORMAL = 1;
Lines 192-198 Link Here
192	case S_EMBEDDED_HEAD:	148	case S_EMBEDDED_HEAD:
193	// The current token contains embedded language and we have processed it's head	149	// The current token contains embedded language and we have processed it's head
194	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);	150	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
195	if (seq.moveFirst()) {	151	seq.moveStart();
		152	if (seq.moveNext()) {
196	state = S_NORMAL;	153	state = S_NORMAL;
197	} else {	154	} else {
198	throw new IllegalStateException("Invalid state");	155	throw new IllegalStateException("Invalid state");
Lines 217-223 Link Here
217	// We have moved to the next normal token, so look what it is	174	// We have moved to the next normal token, so look what it is
218	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);	175	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
219	TokenSequence<? extends TokenId> embeddedSeq = seq.embedded();	176	TokenSequence<? extends TokenId> embeddedSeq = seq.embedded();
220	while (embeddedSeq != null && embeddedSeq.moveFirst()) {	177	while (embeddedSeq != null && embeddedSeq.moveNext()) {
221	sequences.add(sequences.size(), embeddedSeq);	178	sequences.add(sequences.size(), embeddedSeq);
222	if (embeddedSeq.offset() > seq.offset()) {	179	if (embeddedSeq.offset() > seq.offset()) {
223	state = S_EMBEDDED_HEAD;	180	state = S_EMBEDDED_HEAD;
Lines 254-260 Link Here
254	}	211	}
255	case S_EMBEDDED_TAIL: {	212	case S_EMBEDDED_TAIL: {
256	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);	213	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
257	if (seq.moveLast()) {	214	seq.moveEnd();
		215	if (seq.movePrevious()) {
258	return seq.offset() + seq.token().length();	216	return seq.offset() + seq.token().length();
259	} else {	217	} else {
260	throw new IllegalStateException("Invalid state");	218	throw new IllegalStateException("Invalid state");
Lines 284-290 Link Here
284	}	242	}
285	case S_EMBEDDED_HEAD: {	243	case S_EMBEDDED_HEAD: {
286	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);	244	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
287	if (seq.moveFirst()) {	245	seq.moveStart();
		246	if (seq.moveNext()) {
288	return seq.offset();	247	return seq.offset();
289	} else {	248	} else {
290	TokenSequence<? extends TokenId> embeddingSeq = sequences.get(sequences.size() - 2);	249	TokenSequence<? extends TokenId> embeddingSeq = sequences.get(sequences.size() - 2);
Lines 443-449 Link Here
443	} else {	402	} else {
444	if (sequences.size() > 1) {	403	if (sequences.size() > 1) {
445	TokenSequence<? extends TokenId> embeddingSeq = sequences.get(sequences.size() - 2);	404	TokenSequence<? extends TokenId> embeddingSeq = sequences.get(sequences.size() - 2);
446	if (seq.moveLast()) {	405	seq.moveEnd();
		406	if (seq.movePrevious()) {
447	if ((seq.offset() + seq.token().length()) < (embeddingSeq.offset() + embeddingSeq.token().length())) {	407	if ((seq.offset() + seq.token().length()) < (embeddingSeq.offset() + embeddingSeq.token().length())) {
448	return S_EMBEDDED_TAIL;	408	return S_EMBEDDED_TAIL;
449	} else {	409	} else {

Lines 175-197 Link Here

(-)java/hints/src/org/netbeans/modules/java/hints/JavaHintsProvider.java (-14 / +14 lines)
175	TokenSequence<JavaTokenId> ts = info.getTokenHierarchy().tokenSequence();	175	TokenSequence<JavaTokenId> ts = info.getTokenHierarchy().tokenSequence();
176		176
177	ts.move(offset);	177	ts.move(offset);
178		178	if (ts.moveNext()) {
179	Token t = ts.token();	179	Token t = ts.token();
180		180
181	if (t.id() == JavaTokenId.DOT) {	181	if (t.id() == JavaTokenId.DOT) {
182	ts.moveNext();
183	t = ts.token();
184	} else {
185	if (t.id() == JavaTokenId.LT) {
186	ts.moveNext();	182	ts.moveNext();
187	t = ts.token();	183	t = ts.token();
		184	} else {
		185	if (t.id() == JavaTokenId.LT) {
		186	ts.moveNext();
		187	t = ts.token();
		188	}
		189	}
		190
		191	if (t.id() == JavaTokenId.IDENTIFIER) {
		192	return ts.offsetToken();
188	}	193	}
189	}	194	}
190
191	if (t.id() == JavaTokenId.IDENTIFIER) {
192	return ts.offsetToken();
193	}
194
195	return null;	195	return null;
196	}	196	}
197		197
Lines 256-262 Link Here
256		256
257	int diff = ts.move((int) getPrefferedPosition(info, d));	257	int diff = ts.move((int) getPrefferedPosition(info, d));
258		258
259	if (diff >= 0 && diff < ts.token().length()) {	259	if (ts.moveNext() && diff >= 0 && diff < ts.token().length()) {
260	Token<JavaTokenId> t = ts.token();	260	Token<JavaTokenId> t = ts.token();
261		261
262	if (t.id() == JavaTokenId.IDENTIFIER) {	262	if (t.id() == JavaTokenId.IDENTIFIER) {

Lines 139-144 Link Here

(-)java/source/src/org/netbeans/modules/java/source/save/PositionEstimator.java (-6 / +28 lines)
139	int treeEnd = (int) positions.getEndPosition(compilationUnit, item);	139	int treeEnd = (int) positions.getEndPosition(compilationUnit, item);
140		140
141	seq.move(treeStart);	141	seq.move(treeStart);
		142	seq.moveNext();
142	int startIndex = seq.index();	143	int startIndex = seq.index();
143		144
144	// go back to opening/closing curly, semicolon or other	145	// go back to opening/closing curly, semicolon or other
Lines 147-155 Link Here
147	seq.moveNext();	148	seq.moveNext();
148	int start = seq.index();	149	int start = seq.index();
149	seq.move(treeEnd);	150	seq.move(treeEnd);
		151	// seq.index() set (no seq.moveNext() necessary)
150	matrix[i++] = new int[] { start, startIndex, seq.index() };	152	matrix[i++] = new int[] { start, startIndex, seq.index() };
151	if (i == size) {	153	if (i == size) {
152	seq.move(treeEnd);	154	seq.move(treeEnd);
		155	// seq.index() set (no seq.moveNext() necessary)
153	matrix[i][0] = seq.index();	156	matrix[i][0] = seq.index();
154	}	157	}
155	}	158	}
Lines 163-168 Link Here
163	// to decide.	166	// to decide.
164	if (tokenIndex == -1) return -1;	167	if (tokenIndex == -1) return -1;
165	seq.moveIndex(tokenIndex);	168	seq.moveIndex(tokenIndex);
		169	seq.moveNext();
166	return index == 0 ? goAfterLastNewLine(seq) : goAfterFirstNewLine(seq);	170	return index == 0 ? goAfterLastNewLine(seq) : goAfterFirstNewLine(seq);
167	}	171	}
168		172
Lines 209-217 Link Here
209	@Override()	213	@Override()
210	public int[] getPositions(int index) {	214	public int[] getPositions(int index) {
211	int tokenIndex = matrix[index][0];	215	int tokenIndex = matrix[index][0];
212	seq.moveIndex(tokenIndex);	216	if (tokenIndex != -1) {
		217	seq.moveIndex(tokenIndex);
		218	seq.moveNext();
		219	}
213	int begin = goAfterLastNewLine(seq);	220	int begin = goAfterLastNewLine(seq);
214	seq.moveIndex(matrix[index][2]);	221	if (matrix[index][2] != -1) {
		222	seq.moveIndex(matrix[index][2]);
		223	seq.moveNext();
		224	}
215	int end = goAfterFirstNewLine(seq);	225	int end = goAfterFirstNewLine(seq);
216	return new int [] { begin, end };	226	return new int [] { begin, end };
217	}	227	}
Lines 262-267 Link Here
262	if (treeEnd < 0) continue;	272	if (treeEnd < 0) continue;
263		273
264	seq.move(treeStart);	274	seq.move(treeStart);
		275	seq.moveNext();
265	int startIndex = seq.index();	276	int startIndex = seq.index();
266	// go back to opening/closing curly, semicolon or other	277	// go back to opening/closing curly, semicolon or other
267	// token java-compiler important token.	278	// token java-compiler important token.
Lines 285-290 Link Here
285	// to decide.	296	// to decide.
286	if (tokenIndex == -1) return -1;	297	if (tokenIndex == -1) return -1;
287	seq.moveIndex(tokenIndex);	298	seq.moveIndex(tokenIndex);
		299	seq.moveNext();
288	return goAfterFirstNewLine(seq);	300	return goAfterFirstNewLine(seq);
289	}	301	}
290		302
Lines 296-302 Link Here
296		308
297	public int[] getPositions(int index) {	309	public int[] getPositions(int index) {
298	int begin = getInsertPos(index);	310	int begin = getInsertPos(index);
299	seq.moveIndex(matrix[index][4]);	311	if (matrix[index][4] != -1) {
		312	seq.moveIndex(matrix[index][4]);
		313	seq.moveNext();
		314	}
300	int end = goAfterFirstNewLine(seq);	315	int end = goAfterFirstNewLine(seq);
301	return new int [] { begin, end };	316	return new int [] { begin, end };
302	}	317	}
Lines 360-365 Link Here
360	// to decide.	375	// to decide.
361	if (tokenIndex == -1) return -1;	376	if (tokenIndex == -1) return -1;
362	seq.moveIndex(tokenIndex);	377	seq.moveIndex(tokenIndex);
		378	seq.moveNext();
363	int off = goAfterFirstNewLine(seq);	379	int off = goAfterFirstNewLine(seq);
364	return off;	380	return off;
365	}	381	}
Lines 372-378 Link Here
372		388
373	public int[] getPositions(int index) {	389	public int[] getPositions(int index) {
374	int begin = getInsertPos(index);	390	int begin = getInsertPos(index);
375	seq.moveIndex(matrix[index][4]);	391	if (matrix[index][4] != -1) {
		392	seq.moveIndex(matrix[index][4]);
		393	seq.moveNext();
		394	}
376	int end = goAfterFirstNewLine(seq);	395	int end = goAfterFirstNewLine(seq);
377	return new int [] { begin, end };	396	return new int [] { begin, end };
378	}	397	}
Lines 497-512 Link Here
497	// todo (#pf): remove - used for debugging reasons, doesn't do good job	516	// todo (#pf): remove - used for debugging reasons, doesn't do good job
498	public void tablePrint(int[][] matrix, TokenSequence seq) {	517	public void tablePrint(int[][] matrix, TokenSequence seq) {
499	for (int i = 0; i < matrix.length; i++) {	518	for (int i = 0; i < matrix.length; i++) {
500	for (int j = 0; j < 5; j++) {	519	for (int j = 0; j < matrix[i].length; j++) {
501	int item = matrix[i][j];	520	int item = matrix[i][j];
502	String s = "(nothing)";	521	String s = "(nothing)";
503	if (item > -1) {	522	if (item > -1) {
504	seq.moveIndex(item);	523	seq.moveIndex(item);
		524	seq.moveNext();
505	s = "'" + seq.token().text();	525	s = "'" + seq.token().text();
506	}	526	}
507	s += "' ";	527	s += "' ";
508	System.err.print(s.substring(0, 25));	528	// System.err.print(s.substring(0, 25));
		529	System.err.print(item + "\t");
509	}	530	}
		531	System.err.println("");
510	}	532	}
511	}	533	}
512		534

Lines 65-70 Link Here

(-)java/source/src/org/netbeans/modules/java/source/save/TokenUtilities.java (+5 lines)
65	*/	65	*/
66	static int moveNext(TokenSequence<JavaTokenId> tokenSequence, final int pos) {	66	static int moveNext(TokenSequence<JavaTokenId> tokenSequence, final int pos) {
67	tokenSequence.move(pos);	67	tokenSequence.move(pos);
		68	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
68	JavaTokenId type = tokenSequence.token().id();	69	JavaTokenId type = tokenSequence.token().id();
69	while (JavaTokenId.WHITESPACE.equals(type) \|\|	70	while (JavaTokenId.WHITESPACE.equals(type) \|\|
70	JavaTokenId.BLOCK_COMMENT.equals(type) \|\|	71	JavaTokenId.BLOCK_COMMENT.equals(type) \|\|
Lines 82-87 Link Here
82	JavaTokenId id)	83	JavaTokenId id)
83	{	84	{
84	tokenSequence.move(pos);	85	tokenSequence.move(pos);
		86	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
85	while (!id.equals(tokenSequence.token().id())) {	87	while (!id.equals(tokenSequence.token().id())) {
86	if (!tokenSequence.moveNext())	88	if (!tokenSequence.moveNext())
87	return -1;	89	return -1;
Lines 94-99 Link Here
94	JavaTokenId id)	96	JavaTokenId id)
95	{	97	{
96	tokenSequence.move(pos);	98	tokenSequence.move(pos);
		99	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
97	while (!id.equals(tokenSequence.token().id())) {	100	while (!id.equals(tokenSequence.token().id())) {
98	if (!tokenSequence.movePrevious())	101	if (!tokenSequence.movePrevious())
99	return -1;	102	return -1;
Lines 106-111 Link Here
106	String text)	109	String text)
107	{	110	{
108	tokenSequence.move(pos);	111	tokenSequence.move(pos);
		112	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
109	while (!text.equals(tokenSequence.token().text())) {	113	while (!text.equals(tokenSequence.token().text())) {
110	if (!tokenSequence.moveNext())	114	if (!tokenSequence.moveNext())
111	return -1;	115	return -1;
Lines 118-123 Link Here
118	String text)	122	String text)
119	{	123	{
120	tokenSequence.move(pos);	124	tokenSequence.move(pos);
		125	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
121	while (!text.equals(tokenSequence.token().text())) {	126	while (!text.equals(tokenSequence.token().text())) {
122	if (!tokenSequence.movePrevious())	127	if (!tokenSequence.movePrevious())
123	return -1;	128	return -1;

Lines 300-319 Link Here

(-)lexer/arch.xml (-13 / +11 lines)
300	TokenHierarchy hi = TokenHierarchy.get(document);	300	TokenHierarchy hi = TokenHierarchy.get(document);
301	TokenSequence ts = hi.tokenSequence();	301	TokenSequence ts = hi.tokenSequence();
302	// If necessary move ts to the requested offset	302	// If necessary move ts to the requested offset
303	int diff = ts.move(offset);	303	ts.move(offset);
304	if (diff != Integer.MAX_VALUE)) { // Some tokens exist	304	while (ts.moveNext()) {
305	do {	305	Token t = ts.token();
306	Token t = ts.token();	306	if (t.id() == ...) { ... }
307	if (t.id() == ...) { ... }	307	if (TokenUtilities.equals(t.text(), "mytext")) { ... }
308	if (TokenUtilities.equals(t.text(), "mytext")) { ... }	308	if (ts.offset() == ...) { ... }
309	if (ts.offset() == ...) { ... }
310		309
311	// Possibly retrieve embedded token sequence	310	// Possibly retrieve embedded token sequence
312	TokenSequence embedded = ts.embedded();	311	TokenSequence embedded = ts.embedded();
313	if (embedded != null) { // Token has a valid language embedding	312	if (embedded != null) { // Token has a valid language embedding
314	...	313	...
315	}	314	}
316	} while (ts.moveNext());
317	}	315	}
318	} finally {	316	} finally {
319	document.readUnlock();	317	document.readUnlock();

Lines 91-96 Link Here

(-)lexer/api/apichanges.xml (+26 lines)
91	<!-- ACTUAL CHANGES BEGIN HERE: -->	91	<!-- ACTUAL CHANGES BEGIN HERE: -->
92		92
93	<changes>	93	<changes>
		94	<change id="api.tokensequence.move">
		95	<api name="api"/>
		96	<summary>Changed TokenSequence.move() and moveIndex() use</summary>
		97	<version major="1" minor="13"/>
		98	<date day="16" month="1" year="2007"/>
		99	<author login="mmetelka"/>
		100	<compatibility source="incompatible" deletion="yes" addition="yes" modification="yes"/>
		101	<description>
		102	<p>
		103	Changed the <code>TokenSequence.move()</code> to position <b>before</b> the particular token
		104	that "contains" the token (or after the last token if the given offset
		105	is too high). Additional <code>moveNext()</code> is necessary
		106	to actually move to the next token.
		107	<br/>
		108	<code>TokenSequence.moveIndex()</code> was modified in a similar way
		109	(see javadocs).
		110	<br/>
		111	<code>TokenSequence.moveFirst()</code> and <code>moveLast()</code>
		112	were replaced by <code>moveStart()</code> that positionins before
		113	the first token and by <code>moveEnd()</code> that positions
		114	after the last token.
		115	<code>
		116	</p>
		117	</description>
		118	</change>
		119
94	<change id="api.lexer.release">	120	<change id="api.lexer.release">
95	<api name="api"/>	121	<api name="api"/>
96	<summary>Added Lexer.release()</summary>	122	<summary>Added Lexer.release()</summary>

Lines 108-115 Link Here

(-)lexer/editorbridge/src/org/netbeans/modules/lexer/editorbridge/LexerLayer.java (-3 / +4 lines)
108	if (active) {	108	if (active) {
109	pastSequences = new Stack<TokenSequence>();	109	pastSequences = new Stack<TokenSequence>();
110	tokenSequence = hi.tokenSequence();	110	tokenSequence = hi.tokenSequence();
111	int relOffset = tokenSequence.move(startOffset);	111	tokenSequence.move(startOffset);
112	if (relOffset != Integer.MAX_VALUE) {	112	if (tokenSequence.moveNext()) {
113	updateTokenEndOffsetAndColoring(startOffset);	113	updateTokenEndOffsetAndColoring(startOffset);
114	} else { // no tokens	114	} else { // no tokens
115	active = false;	115	active = false;
Lines 178-184 Link Here
178	break;	178	break;
179	}	179	}
180	} else {	180	} else {
181	if (embed.move(offset) != Integer.MAX_VALUE) {	181	embed.move(offset);
		182	if (embed.moveNext()) {
182	pastSequences.push(tokenSequence);	183	pastSequences.push(tokenSequence);
183	tokenSequence = embed;	184	tokenSequence = embed;
184	} else {	185	} else {

Lines 75-81 Link Here

(-)lexer/nbbridge/test/unit/src/org/netbeans/modules/lexer/nbbridge/test/MimeLookupLanguageProviderTest.java (-5 / +8 lines)
75	assertEquals("Wrong language", "text/x-simple-plain", lang.mimeType());	75	assertEquals("Wrong language", "text/x-simple-plain", lang.mimeType());
76		76
77	for(int i = 0; i < seq.tokenCount(); i++) {	77	for(int i = 0; i < seq.tokenCount(); i++) {
78	seq.move(i);	78	seq.moveIndex(i);
		79	assertTrue(seq.moveNext());
79	Token token = seq.token();	80	Token token = seq.token();
80		81
81	if (token.id() == SimplePlainTokenId.WORD) {	82	if (token.id() == SimplePlainTokenId.WORD) {
Lines 86-96 Link Here
86	assertNotNull("Can't find language of the embedded sequence", embeddedLang);	87	assertNotNull("Can't find language of the embedded sequence", embeddedLang);
87	assertEquals("Wrong language of the embedded sequence", "text/x-simple-char", embeddedLang.mimeType());	88	assertEquals("Wrong language of the embedded sequence", "text/x-simple-char", embeddedLang.mimeType());
88		89
89	assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveFirst());	90	embeddedSeq.moveStart();
90	assertEquals("Wrong startSkipLenght", 1, embeddedSeq.offset() - seq.offset());	91	assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveNext());
		92	assertEquals("Wrong startSkipLength", 1, embeddedSeq.offset() - seq.offset());
91		93
92	assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.moveLast());	94	embeddedSeq.moveEnd();
93	assertEquals("Wrong endSkipLenght", 2,	95	assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.movePrevious());
		96	assertEquals("Wrong endSkipLength", 2,
94	(seq.offset() + seq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length()));	97	(seq.offset() + seq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length()));
95	}	98	}
96	}	99	}

Lines 237-242 Link Here

(-)lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java (+10 lines)
237	sb.append(": ");	237	sb.append(": ");
238	AbstractToken token = token(tokenOrEmbeddingContainer);	238	AbstractToken token = token(tokenOrEmbeddingContainer);
239	sb.append(token.dumpInfo(tokenHierarchy));	239	sb.append(token.dumpInfo(tokenHierarchy));
		240	int la = tokenList.lookahead(i);
		241	if (la != 0) {
		242	sb.append(", la=");
		243	sb.append(la);
		244	}
		245	Object state= tokenList.state(i);
		246	if (state != null) {
		247	sb.append(", s=");
		248	sb.append(state);
		249	}
240	sb.append('\n');	250	sb.append('\n');
241	}	251	}
242	return sb;	252	return sb;

Lines 130-136 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java (-3 / +6 lines)
130	public void testCachingE() {	130	public void testCachingE() {
131	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());	131	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
132	TokenSequence tokens = th.tokenSequence();	132	TokenSequence tokens = th.tokenSequence();
133	tokens.moveFirst();	133	tokens.moveStart();
		134	assertEquals(true, tokens.moveNext());
134		135
135	TokenSequence embeddedA = tokens.embedded();	136	TokenSequence embeddedA = tokens.embedded();
136	assertNotNull("There should be an embedded language", embeddedA);	137	assertNotNull("There should be an embedded language", embeddedA);
Lines 144-150 Link Here
144	public void testGCedE() {	145	public void testGCedE() {
145	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());	146	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
146	TokenSequence tokens = th.tokenSequence();	147	TokenSequence tokens = th.tokenSequence();
147	tokens.moveFirst();	148	tokens.moveStart();
		149	assertEquals(true, tokens.moveNext());
148		150
149	TokenSequence embedded = tokens.embedded();	151	TokenSequence embedded = tokens.embedded();
150	assertNotNull("There should be an embedded language", embedded);	152	assertNotNull("There should be an embedded language", embedded);
Lines 164-170 Link Here
164	public void testCacheRefreshedE() {	166	public void testCacheRefreshedE() {
165	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());	167	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
166	TokenSequence tokens = th.tokenSequence();	168	TokenSequence tokens = th.tokenSequence();
167	tokens.moveFirst();	169	tokens.moveStart();
		170	assertEquals(true, tokens.moveNext());
168		171
169	TokenSequence embeddedA = tokens.embedded();	172	TokenSequence embeddedA = tokens.embedded();
170	assertNotNull("There should be an embedded language", embeddedA);	173	assertNotNull("There should be an embedded language", embeddedA);

Lines 70-75 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java (-2 / +7 lines)
70	public static void assertTokenEquals(String message, TokenSequence<? extends TokenId> ts, TokenId id, String text, int offset) {	70	public static void assertTokenEquals(String message, TokenSequence<? extends TokenId> ts, TokenId id, String text, int offset) {
71	message = messagePrefix(message);	71	message = messagePrefix(message);
72	Token<? extends TokenId> t = ts.token();	72	Token<? extends TokenId> t = ts.token();
		73	TestCase.assertNotNull("Token is null", t);
73	TokenId tId = t.id();	74	TokenId tId = t.id();
74	TestCase.assertEquals(message + "Invalid token.id()", id, tId);	75	TestCase.assertEquals(message + "Invalid token.id()", id, tId);
75	CharSequence tText = t.text();	76	CharSequence tText = t.text();
Lines 312-328 Link Here
312	}	313	}
313	TokenHierarchy<?> thBatch = TokenHierarchy.create(docText, language);	314	TokenHierarchy<?> thBatch = TokenHierarchy.create(docText, language);
314	boolean success = false;	315	boolean success = false;
		316	TokenSequence<?> batchTS = thBatch.tokenSequence();
315	try {	317	try {
316	// Compare lookaheads and states as well	318	// Compare lookaheads and states as well
317	assertTokenSequencesEqual(thBatch.tokenSequence(), thBatch,	319	assertTokenSequencesEqual(batchTS, thBatch,
318	thInc.tokenSequence(), thInc, true);	320	thInc.tokenSequence(), thInc, true);
319	success = true;	321	success = true;
320	} finally {	322	} finally {
321	if (!success) {	323	if (!success) {
		324	// Go forward two tokens to have an extra tokens context
		325	batchTS.moveNext();
		326	batchTS.moveNext();
322	System.err.println("BATCH token sequence dump:\n" + thBatch.tokenSequence());	327	System.err.println("BATCH token sequence dump:\n" + thBatch.tokenSequence());
323	TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);	328	TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);
324	if (lastHi != null) {	329	if (lastHi != null) {
325	System.err.println("PREVIOUS token sequence dump:\n" + lastHi.tokenSequence());	330	System.err.println("PREVIOUS batch token sequence dump:\n" + lastHi.tokenSequence());
326	}	331	}
327	}	332	}
328	}	333	}

Lines 76-82 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java (-3 / +6 lines)
76		76
77	// Check that all the non-fly tokens are mutable	77	// Check that all the non-fly tokens are mutable
78	ts = snapshot1.tokenSequence();	78	ts = snapshot1.tokenSequence();
79	assertTrue(ts.moveIndex(0));	79	assertEquals(0, ts.moveIndex(0));
		80	assertTrue(ts.moveNext());
80		81
81	LexerTestUtilities.assertTokenSequencesEqual(hi1.tokenSequence(), hi1,	82	LexerTestUtilities.assertTokenSequencesEqual(hi1.tokenSequence(), hi1,
82	snapshot1.tokenSequence(), snapshot1, false);	83	snapshot1.tokenSequence(), snapshot1, false);
Lines 84-90 Link Here
84	doc.insertString(4, "+", null);	85	doc.insertString(4, "+", null);
85		86
86	// Check that the snapshot token sequence can be further operated.	87	// Check that the snapshot token sequence can be further operated.
87	assertTrue(ts.moveIndex(0));	88	assertEquals(0, ts.moveIndex(0));
		89	assertTrue(ts.moveNext());
88	assertNotNull(ts.token());	90	assertNotNull(ts.token());
89		91
90	// Check that the tokens except '+' are live	92	// Check that the tokens except '+' are live
Lines 101-107 Link Here
101		103
102	// Check that all the non-fly tokens are mutable	104	// Check that all the non-fly tokens are mutable
103	ts = snapshot2.tokenSequence();	105	ts = snapshot2.tokenSequence();
104	assertTrue(ts.moveIndex(0));	106	assertEquals(0, ts.moveIndex(0));
		107	assertTrue(ts.moveNext());
105		108
106	LexerTestUtilities.assertTokenSequencesEqual(hi2.tokenSequence(), hi2,	109	LexerTestUtilities.assertTokenSequencesEqual(hi2.tokenSequence(), hi2,
107	snapshot2.tokenSequence(), snapshot2, false);	110	snapshot2.tokenSequence(), snapshot2, false);

Lines 90-108 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java (-1 / +5 lines)
90		90
91	// Check TokenSequence.move()	91	// Check TokenSequence.move()
92	int relOffset = ts.move(50); // past the end of all tokens	92	int relOffset = ts.move(50); // past the end of all tokens
93	assertEquals(relOffset, 50 - offset);	93	assertEquals(relOffset, 50 - (offset + 3));
		94	assertTrue(ts.movePrevious());
94	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset);	95	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset);
95		96
96	relOffset = ts.move(6); // right at begining of "-"	97	relOffset = ts.move(6); // right at begining of "-"
97	assertEquals(relOffset, 0);	98	assertEquals(relOffset, 0);
		99	assertTrue(ts.moveNext());
98	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", 6);	100	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", 6);
99		101
100	relOffset = ts.move(-5); // to first token "abc"	102	relOffset = ts.move(-5); // to first token "abc"
101	assertEquals(relOffset, -5);	103	assertEquals(relOffset, -5);
		104	assertTrue(ts.moveNext());
102	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	105	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
103		106
104	relOffset = ts.move(5); // to first token "abc"	107	relOffset = ts.move(5); // to first token "abc"
105	assertEquals(relOffset, 1);	108	assertEquals(relOffset, 1);
		109	assertTrue(ts.moveNext());
106	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "uv", 4);	110	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "uv", 4);
107		111
108		112

Lines 86-102 Link Here

(-)xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java (-13 / +18 lines)
86	* e.g. <tag \|attr=	86	* e.g. <tag \|attr=
87	*	87	*
88	*/	88	*/
89	do {	89	while (tokenSequence.moveNext()) {
90	tokenSequence.moveNext();
91	token = tokenSequence.token();	90	token = tokenSequence.token();
92	tokenOffset = token.offset(tokenHierarchy);	91	tokenOffset = tokenSequence.offset();
93	} while (token != null && isWSTag(token)	92	if (!isWSTag(token) \|\| tagStartLine != Utilities.getLineOffset(doc, tokenOffset)) {
94	&& tagStartLine == Utilities.getLineOffset(doc, tokenOffset));	93	if (!isWSTag(token) && tagStartLine == Utilities.getLineOffset(doc, tokenOffset)){
95		94	return tokenOffset - Utilities.getRowIndent(doc, tokenOffset) - Utilities.getRowStart(doc, tokenOffset);
96	if (!isWSTag(token) && tagStartLine == Utilities.getLineOffset(doc, tokenOffset)){	95	}
97	return tokenOffset - Utilities.getRowIndent(doc, tokenOffset) - Utilities.getRowStart(doc, tokenOffset);	96	break;
		97	}
98	}	98	}
99		99
		100
100	return getShiftWidth(); // default;	101	return getShiftWidth(); // default;
101	}	102	}
102		103
Lines 122-128 Link Here
122	int indentsWithinTags[] = new int[lastLine + 1];	123	int indentsWithinTags[] = new int[lastLine + 1];
123		124
124	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();	125	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
125	boolean thereAreMoreTokens = tokenSequence.moveFirst();	126	tokenSequence.moveStart();
		127	boolean thereAreMoreTokens = tokenSequence.moveNext();
		128
126		129
127	if (tokenSequence != null){	130	if (tokenSequence != null){
128	// calc line indents - pass 1	131	// calc line indents - pass 1
Lines 435-448 Link Here
435	tokenSequence.move(offset);	438	tokenSequence.move(offset);
436	int currentOffset;	439	int currentOffset;
437		440
438	do{	441	while (tokenSequence.moveNext()) {
439	currentOffset = tokenSequence.offset();	442	currentOffset = tokenSequence.offset();
440		443
441	if (isClosingTag(tokenHierarchy, currentOffset)){	444	if (isClosingTag(tokenHierarchy, currentOffset)){
442	return currentOffset;	445	return currentOffset;
443	}	446	}
444		447
445	} while (tokenSequence.moveNext());	448	}
446		449
447	return -1;	450	return -1;
448	}	451	}
Lines 459-466 Link Here
459	protected Token getTokenAtOffset(TokenHierarchy tokenHierarchy, int tagTokenOffset){	462	protected Token getTokenAtOffset(TokenHierarchy tokenHierarchy, int tagTokenOffset){
460	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();	463	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
461		464
462	if (tokenSequence != null && tokenSequence.move(tagTokenOffset) != Integer.MAX_VALUE){	465	if (tokenSequence != null) {
463	return tokenSequence.token();	466	tokenSequence.move(tagTokenOffset);
		467	if (tokenSequence.moveNext())
		468	return tokenSequence.token();
464	}	469	}
465		470
466	return null;	471	return null;