Attachment #37475 for bug #90796

This Bugzilla instance is a read-only archive of historic NetBeans bug reports. To report a bug in NetBeans please follow the project's instructions for reporting issues.

View | Details | Raw Unified | Return to bug 90796
Collapse All | Expand All

Lines 133-139 Link Here

(-)html/editor/lib/src/org/netbeans/editor/ext/html/HTMLSyntaxSupport.java (-1 / +2 lines)
133		133
134	// if the carret is after HTML tag ( after char '>' ), ship inside the tag	134	// if the carret is after HTML tag ( after char '>' ), ship inside the tag
135	if (token.id() == HTMLTokenId.TAG_CLOSE_SYMBOL) {	135	if (token.id() == HTMLTokenId.TAG_CLOSE_SYMBOL) {
136	if(ts.moveIndex(ts.index() - 2)) {	136	ts.moveIndex(ts.index() - 2);
		137	if(ts.moveNext()) {
137	token = ts.token();	138	token = ts.token();
138	}	139	}
139	}	140	}




            int openLtNum = 0;
            JavaTokenId lastNonWhitespaceTokenId = null;
            TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());
            ts.move(expEndPos);
            while (ts.moveNext()) {
            do {
                if (ts.offset() >= offset) {
                    break;
                }

                    default:
                        lastNonWhitespaceTokenId = ts.token().id();
                }
            }
            if (!afterDot) {
                if (expEndPos <= offset)
                    insideExpression(env, new TreePath(path, fa.getExpression()));

        
        private TokenSequence<JavaTokenId> findLastNonWhitespaceToken(Env env, int startPos, int endPos) {
            TokenSequence<JavaTokenId> ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language());
            if (ts.move(endPos) == 0) // When right at the token end
                ts.movePrevious(); // move to previous token
            else
                ts.moveNext(); // otherwise move to the token that "contains" the offset
            do {
                int offset = ts.offset();
                if (offset < startPos)

            if (upToOffset) {
                TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());
                if (ts.moveNext()) { //check for empty file
                    if (ts.move(offset) == 0) // When right at the token end
                        ts.movePrevious(); // Move to previous token
                    else
                        ts.moveNext(); // otherwise move to the token that "contains" the offset
                    if (ts.offset() < offset && (ts.token().id() == JavaTokenId.IDENTIFIER || "keyword".equals(ts.token().id().primaryCategory()))) { //TODO: Use isKeyword(...) when available
                        prefix = ts.token().toString().substring(0, offset - ts.offset());
                        offset = ts.offset();

Lines 98-103 Link Here

(-)java/editor/src/org/netbeans/modules/editor/java/Utilities.java (+2 lines)
98	return false;	98	return false;
99	if (!ts.moveNext() \|\| ts.move(offset) == 0)	99	if (!ts.moveNext() \|\| ts.move(offset) == 0)
100	return true;	100	return true;
		101	ts.moveNext();
101	switch(ts.token().id()) {	102	switch(ts.token().id()) {
102	case DOUBLE_LITERAL:	103	case DOUBLE_LITERAL:
103	if (ts.token().text().charAt(0) == '.')	104	if (ts.token().text().charAt(0) == '.')
Lines 127-132 Link Here
127	TokenSequence<? extends TokenId> ts = hierarchy.tokenSequence();	128	TokenSequence<? extends TokenId> ts = hierarchy.tokenSequence();
128	while(ts != null && ts.moveNext()) {	129	while(ts != null && ts.moveNext()) {
129	ts.move(offset);	130	ts.move(offset);
		131	ts.moveNext();
130	if (ts.language() == JavaTokenId.language())	132	if (ts.language() == JavaTokenId.language())
131	return (TokenSequence<JavaTokenId>)ts;	133	return (TokenSequence<JavaTokenId>)ts;
132	ts = ts.embedded();	134	ts = ts.embedded();




        TokenSequence<JavaTokenId> ts = info.getTokenHierarchy().tokenSequence();
        
        ts.move(offset);
        if (ts.moveNext()) {
            Token t = ts.token();

            if (t.id() == JavaTokenId.DOT) {
            ts.moveNext();
            t = ts.token();
        } else {
            if (t.id() == JavaTokenId.LT) {
                ts.moveNext();
                t = ts.token();
            } else {
                if (t.id() == JavaTokenId.LT) {
                    ts.moveNext();
                    t = ts.token();
                }
            }

            if (t.id() == JavaTokenId.IDENTIFIER) {
                return ts.offsetToken();
            }
        }
        
        if (t.id() == JavaTokenId.IDENTIFIER) {
            return ts.offsetToken();
        }
        
        return null;
    }
    

Lines 158-164 Link Here

(-)java/source/src/org/netbeans/api/java/source/TreeUtilities.java (-1 / +2 lines)
158	return path;	158	return path;
159		159
160	TokenSequence<JavaTokenId> tokenList = tokensFor(path.getLeaf(), sourcePositions);	160	TokenSequence<JavaTokenId> tokenList = tokensFor(path.getLeaf(), sourcePositions);
161	if (tokenList.moveLast() && tokenList.offset() < pos) {	161	tokenList.moveEnd();
		162	if (tokenList.movePrevious() && tokenList.offset() < pos) {
162	switch (tokenList.token().id()) {	163	switch (tokenList.token().id()) {
163	case GTGTGT:	164	case GTGTGT:
164	case GTGT:	165	case GTGT:




        // skip the section when printing anonymous class
        if (anonClass == false) {
        tokenSequence.move(oldT.pos);
        tokenSequence.moveNext(); // First skip as move() does not position to token directly
        tokenSequence.moveNext();
        insertHint = TokenUtilities.moveNext(tokenSequence, tokenSequence.offset());
        localPointer = diffModifiers(oldT.mods, newT.mods, oldT, localPointer);

            // there was not any parameter in original tree.
            int startOffset = oldT.restype != null ? oldT.restype.getStartPosition() : oldT.getStartPosition();
            
            // Extra move()? - performed in moveFwdToToken()
            tokenSequence.move(startOffset);
            TokenUtilities.moveFwdToToken(tokenSequence, startOffset, JavaTokenId.RPAREN);
            posHint = tokenSequence.offset();

            // now check, that there is a whitespace. It is not mandatory, we
            // have to ensure. If whitespace is not present, take body beginning
            tokenSequence.move(posHint);
            tokenSequence.moveNext();
            if (tokenSequence.token().id() != JavaTokenId.WHITESPACE) {
                ++posHint;
            }

            ResultItem<JCTree> item = result[j];
            switch (item.operation) {
                case MODIFY: {
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

                    assert startOffset != -1 && endOffset != -1 : "Invalid offset!";
                    //printer.print(origText.substring(lastOldPos, startOffset));
                    //append(Diff.delete(startOffset, endOffset));
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

                    break;
                }
                case NOCHANGE: {
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();




                int treeEnd = (int) positions.getEndPosition(compilationUnit, item);
                
                seq.move(treeStart);
                // seq.index() set (no seq.moveNext() necessary)
                int startIndex = seq.index();
                
                // go back to opening/closing curly, semicolon or other

                seq.moveNext();
                int start = seq.index();
                seq.move(treeEnd);
                // seq.index() set (no seq.moveNext() necessary)
                matrix[i++] = new int[] { start, startIndex, seq.index() };
                if (i == size) {
                    seq.move(treeEnd);
                    // seq.index() set (no seq.moveNext() necessary)
                    matrix[i][0] = seq.index();
                }
            }

            // to decide.
            if (tokenIndex == -1) return -1;
            seq.moveIndex(tokenIndex);
            seq.moveNext();
            return index == 0 ? goAfterLastNewLine(seq) : goAfterFirstNewLine(seq);
        }


        public int[] getPositions(int index) {
            int tokenIndex = matrix[index][0];
            seq.moveIndex(tokenIndex);
            seq.moveNext();
            int begin = goAfterLastNewLine(seq);
            seq.moveIndex(matrix[index][2]);
            seq.moveNext();
            int end = goAfterFirstNewLine(seq);
            return new int [] { begin, end };
        }

            // to decide.
            if (tokenIndex == -1) return -1;
            seq.moveIndex(tokenIndex);
            seq.moveNext();
            return goAfterFirstNewLine(seq);
        }
        

        public int[] getPositions(int index) {
            int begin = getInsertPos(index);
            seq.moveIndex(matrix[index][4]);
            seq.moveNext();
            int end = goAfterFirstNewLine(seq);
            return new int [] { begin, end };
        }

            // to decide.
            if (tokenIndex == -1) return -1;
            seq.moveIndex(tokenIndex);
            seq.moveNext();
            int off = goAfterFirstNewLine(seq);
            return off;
        }

        public int[] getPositions(int index) {
            int begin = getInsertPos(index);
            seq.moveIndex(matrix[index][4]);
            seq.moveNext();
            int end = goAfterFirstNewLine(seq);
            return new int [] { begin, end };
        }

                String s = "(nothing)";
                if (item > -1) {
                    seq.moveIndex(item);
                    seq.moveNext();
                    s = "'" + seq.token().text();
                }
                s += "'                                           ";




     */
    static int moveNext(TokenSequence<JavaTokenId> tokenSequence, final int pos) {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        JavaTokenId type = tokenSequence.token().id();
        while (JavaTokenId.WHITESPACE.equals(type) ||
               JavaTokenId.BLOCK_COMMENT.equals(type) ||

            JavaTokenId id)
    {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        while (!id.equals(tokenSequence.token().id())) {
            if (!tokenSequence.moveNext())
                return -1;

            JavaTokenId id)
    {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        while (!id.equals(tokenSequence.token().id())) {
            if (!tokenSequence.movePrevious())
                return -1;

            String text)
    {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        while (!text.equals(tokenSequence.token().text())) {
            if (!tokenSequence.moveNext())
                return -1;

            String text)
    {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        while (!text.equals(tokenSequence.token().text())) {
            if (!tokenSequence.movePrevious())
                return -1;




            ResultItem<JCTree> item = result[j];
            switch (item.operation) {
                case MODIFY: {
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

                    int endOffset = toOff(s.tail(j) || s.next(j) ? matrix[i+1][2] : matrix[i][4]);
                    assert startOffset != -1 && endOffset != -1 : "Invalid offset!";
                    append(Diff.delete(startOffset, endOffset));
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

                    break;
                }
                case NOCHANGE: {
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();




        TokenHierarchy hi = TokenHierarchy.get(document);
        TokenSequence ts = hi.tokenSequence();
        // If necessary move ts to the requested offset
        ts.move(offset);
        while (ts.moveNext()) {
            Token t = ts.token();
            if (t.id() == ...) { ... }
            if (TokenUtilities.equals(t.text(), "mytext")) { ... }
            if (ts.offset() == ...) { ... }
                if (ts.offset() == ...) { ... }

            // Possibly retrieve embedded token sequence
            TokenSequence embedded = ts.embedded();
            if (embedded != null) { // Token has a valid language embedding
                ...
            }
            } while (ts.moveNext());
        }
    } finally {
        document.readUnlock();

Lines 1-4 Link Here

(-)lexer/manifest.mf (-1 / +1 lines)
1	OpenIDE-Module: org.netbeans.modules.lexer/2	1	OpenIDE-Module: org.netbeans.modules.lexer/2
2	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/lexer/Bundle.properties	2	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/lexer/Bundle.properties
3	OpenIDE-Module-Specification-Version: 1.12.0	3	OpenIDE-Module-Specification-Version: 1.13.0
4	OpenIDE-Module-Recommends: org.netbeans.spi.lexer.LanguageProvider	4	OpenIDE-Module-Recommends: org.netbeans.spi.lexer.LanguageProvider




<!-- ACTUAL CHANGES BEGIN HERE: -->

  <changes>
  <change id="api.tokensequence.move">
      <api name="api"/>
      <summary>Changed TokenSequence.move() and moveIndex() use</summary>
      <version major="1" minor="13"/>
      <date day="16" month="1" year="2007"/>
      <author login="mmetelka"/>
      <compatibility source="incompatible" deletion="yes" addition="yes" modification="yes"/>
      <description>
        <p>
            Changed the <code>TokenSequence.move()</code> to position <b>before</b> the particular token
            that "contains" the token (or after the last token if the given offset
            is too high). Additional <code>moveNext()</code> is necessary
            to actually move to the next token.
            <br/>
            <code>TokenSequence.moveIndex()</code> was modified in a similar way
            (see javadocs).
            <br/>
            <code>TokenSequence.moveFirst()</code> and <code>moveLast()</code>
            were replaced by <code>moveStart()</code> that positionins before
            the first token and by <code>moveEnd()</code> that positions
            after the last token.
            <code>
        </p>
      </description>
  </change>

  <change id="api.lexer.release">
      <api name="api"/>
      <summary>Added Lexer.release()</summary>




        if (active) {
            pastSequences = new Stack<TokenSequence>();
            tokenSequence = hi.tokenSequence();
            tokenSequence.move(startOffset);
            if (tokenSequence.moveNext()) {
                updateTokenEndOffsetAndColoring(startOffset);
            } else { // no tokens
                active = false;

                        break;
                    }
                } else {
                    embed.move(offset);
                    if (embed.moveNext()) {
                        pastSequences.push(tokenSequence);
                        tokenSequence = embed;
                    } else {




        assertEquals("Wrong language", "text/x-simple-plain", lang.mimeType());
        
        for(int i = 0; i < seq.tokenCount(); i++) {
            seq.moveIndex(i);
            assertTrue(seq.moveNext());
            Token token = seq.token();
            
            if (token.id() == SimplePlainTokenId.WORD) {

                assertNotNull("Can't find language of the embedded sequence", embeddedLang);
                assertEquals("Wrong language of the embedded sequence", "text/x-simple-char", embeddedLang.mimeType());
                
                embeddedSeq.moveStart();
                assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveNext());
                assertEquals("Wrong startSkipLength", 1, embeddedSeq.offset() - seq.offset());
                
                embeddedSeq.moveEnd();
                assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.movePrevious());
                assertEquals("Wrong endSkipLength", 2, 
                    (seq.offset() + seq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length()));
            }
        }




 * A typical use is a forward iteration through the tokens:
 * <pre>
 *   TokenSequence ts = tokenHierarchy.tokenSequence();
 *   // Possible positioning by ts.move(offset);
 *   while (ts.moveNext()) {
 *       Token t = ts.token();
 *       if (t.id() == ...) { ... }

    
    private AbstractToken<T> token; // 16 bytes
    
    private int tokenIndex; // 20 bytes
    
    /**
     * Offset in the input at which the current token is located

    /**
     * Get instance of current token to which this token sequence points to.
     * <br/>
     * A typical iteration usage:
     * <pre>
     *   TokenSequence ts = tokenHierarchy.tokenSequence();
     *   // Possible positioning by ts.move(offset);
     *   while (ts.moveNext()) {
     *       Token t = ts.token();
     *       if (t.id() == ...) { ... }
     *       if (TokenUtilities.equals(t.text(), "mytext")) { ... }
     *       if (ts.offset() == ...) { ... }
     *   }
     * </pre>
     *
     * <p>
     * The returned token instance may be flyweight
     * (returns true from {@link Token#isFlyweight()})
     * which means that its {@link Token#offset(TokenHierarchy)} will return -1.

     * The token instance should not be held across the input source modifications.
     * </p>
     *
     * @return token instance to which this token sequence is currently positioned
     *  or null if this token sequence is not positioned to any token which may
     *  happen after TS creation or after use of {@link #move(int)} or {@link moveIndex(int)}.
     * 
     * @see #offsetToken()
     * @throws IllegalStateException if this token sequence was not positioned
     *  to any token yet.
     */
    public Token<T> token() {
        checkToken();
        return token;
    }
    

     *  to any token yet.
     */
    public Token<T> offsetToken() {
        checkTokenNotNull();
        if (token.isFlyweight()) {
            token = tokenList.replaceFlyToken(tokenIndex, token, offset());
        }

     *  to any token yet.
     */
    public int offset() {
        checkTokenNotNull();
        if (tokenOffset == -1) {
            tokenOffset = tokenList.tokenOffset(tokenIndex);
        }

    }
    
    /**
     * Get an index of token to which (or before which) this TS is currently positioned.
     * <br/>
     * Zero is returned after TS creation.
     *
     * @return &gt;=0 index of the current token.
     *  if this token sequence is initially located in front of the first token.
     */
    public int index() {
        return tokenIndex;

     *  to any token yet.
     */
    public TokenSequence<? extends TokenId> embedded() {
        checkTokenNotNull();
        return embeddedImpl(null);
    }
    

     * has a language embedding.
     */
    public <ET extends TokenId> TokenSequence<ET> embedded(Language<ET> embeddedLanguage) {
        checkTokenNotNull();
        return embeddedImpl(embeddedLanguage);
    }


     */
    public boolean createEmbedding(Language<? extends TokenId> embeddedLanguage,
    int startSkipLength, int endSkipLength, boolean joinSections) {
        checkTokenNotNull();
        return EmbeddingContainer.createEmbedding(tokenList, tokenIndex,
                embeddedLanguage, startSkipLength, endSkipLength, joinSections);
    }

    /**
     * Move to the next token in this token sequence.
     * <br/>
     * This method can be used in case when TS is positioned before a token
     * (after its creation or after use of {@link #move(int)} or {@link #moveIndex(int)})
     * or in case the TS already points to a concrete token <code>{@link #token()} != null</code>
     * - in such case it moves to next token and {@link #index()} gets increased by one.
     * <br/>
     * The next token may not necessarily start at the offset where
     * the previous token ends (there may be gaps between tokens
     * caused by token filtering).
     *
     * @return true if the sequence was successfully moved to the next token
     *  or false if it was not moved before there are no more tokens
     *  in the forward direction.
     * @throws ConcurrentModificationException if this token sequence
     *  is no longer valid because of an underlying mutable input source modification.
     */
    public boolean moveNext() {
        checkModCount();
        if (token != null) // Token already fetched
            tokenIndex++;
        Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(tokenIndex);
        if (tokenOrEmbeddingContainer != null) {
            AbstractToken origToken = token;
            token = LexerUtilsConstants.token(tokenOrEmbeddingContainer);
            // If origToken == null then the right offset might already be pre-computed from move()
            if (tokenOffset != -1) {
                if (origToken != null) {
                    // If the token list is continuous or the fetched token
                    // is flyweight (there cannot be a gap before flyweight token)
                    // the original offset can be just increased
                    // by the original token's length.
                    if (tokenList.isContinuous() || token.isFlyweight()) {
                        tokenOffset += origToken.length(); // advance by previous token's length
                    } else // Offset must be recomputed
                        tokenOffset = -1; // mark the offset to be recomputed
                } else // Not valid token previously
                    tokenOffset = -1;
            }
            return true;
        }
        if (token != null) // Unsuccessful move from existing token
            tokenIndex--;
        return false;
    }

    /**
     * Move to a previous token in this token sequence.
     * <br/>
     * This method can be used in case when TS is positioned after a token
     * (after its creation or after use of {@link #move(int)} or {@link #moveIndex(int)})
     * or in case the TS already points to a concrete token <code>{@link #token()} != null</code>.
     * <br/>
     * In both cases {@link #index()} gets decreased by one.
     * <br/>
     * The next token may not necessarily end at the offset where
     * the present token starts (there may be gaps between tokens
     * caused by use of a token id filter).
     *
     * @return true if the sequence was successfully moved to the previous token
     *  or false if it was not moved because there are no more tokens
     *  in the backward direction.
     * @throws ConcurrentModificationException if this token sequence
     *  is no longer valid because of an underlying mutable input source modification.

        if (tokenIndex > 0) {
            AbstractToken origToken = token;
            tokenIndex--;
            token = LexerUtilsConstants.token(tokenList.tokenOrEmbeddingContainer(tokenIndex));
            if (tokenOffset != -1) {
                // If the token list is continuous or the original token
                // is flyweight (there cannot be a gap before flyweight token)

                } else { // mark the offset to be computed upon call to offset()
                    tokenOffset = -1;
                }
            } else {
                tokenOffset = -1; // mark the offset to be computed upon call to offset()
            }
            return true;


    }

    /**
     * Move TS to be positioned before a token with the given index.
     * <br/>
     * Example: Fetching tokens starting at the given index:
     * <pre>
     * TokenSequence ts = tokenHierarchy.tokenSequence();
     * ts.moveIndex(index);
     * while (ts.moveNext()) {
     *     Token t = ts.token();
     *     ...
     * }
     * </pre>
     *
     * @param index index of the token to which this sequence
     *   should be positioned.
     *   <br/>
     *   If <code>index >= {@link #tokenCount()}</code>
     *   then the TS will be positioned after the last token.
     *   <br/>
     *   If <code>index < 0</code>
     *   then the TS will be positioned before first token (at index 0).
     * @return difference between requested index and the index to which TS
     *   is really set.
     * @throws ConcurrentModificationException if this token sequence
     *  is no longer valid because of an underlying mutable input source modification.
     */
    public int moveIndex(int index) {
        checkModCount();
        if (index >= 0) {
            Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index);
            if (tokenOrEmbeddingContainer != null) { // enough tokens
                resetTokenIndex(index);
            } else // Token at the requested index does not exist - leave orig. index
                resetTokenIndex(tokenCount());
        } else // index < 0
            resetTokenIndex(0);
        return index - tokenIndex;
    }
    
    /**
     * Move the token sequence to be positioned before the first token.
     * <br/>
     * This is equivalent to <code>moveIndex(0)</code>.
     */
    public void moveStart() {
        moveIndex(0);
    }
    
    /**
     * Move the token sequence to be positioned after the last token.
     * <br/>
     * This is equivalent to <code>moveIndex(tokenCount())</code>.
     */
    public void moveEnd() {
        moveIndex(tokenCount());
    }
    
    /**
     * Move the token sequence to be positioned before the token
     * that "contains" the requested offset (the offset is at the begining
     * or inside of the token).
     * <br/>
     * Typical usage pattern:
     * <pre>
     * TokenSequence ts = ...
     * ts.move(offset);
     * while (ts.moveNext()) {
     *     Token t = ts.token();
     *     ...
     * }
     * </pre>
     * 
     * If the offset is too big then the token sequence will be positioned
     * behind the last token.
     * <p>
     * If token filtering is used there may be gaps that are not covered
     * by any tokens and if the offset is contained in such gap then
     * the token sequence will be positioned before the token that follows the gap.
     * </p>
     *
     * Example:
     * <pre>
     *   int diff = tokenSequence.move(targetOffset);
     *   // diff equals to (targetOffset - tokenSequence.token().offset())
     *   if (diff >= 0 && diff < tokenSequence.token().length()) {
     *       // Within token bounds - tokenSequence.token() starts at or "contains" targetOffset
     *
     *   } else if (diff == Integer.MAX_VALUE) {
     *       // No tokens in the token sequence at all.
     *       // Token sequence is not positioned to any token.
     *
     *   } else {
     *       // 1. diff >= tokenSequence.token().length()
     *       //   a) targetOffset is above the end of the last token in the sequence.
     *       //     Token sequence is positioned to the last token in the sequence.
     *       //   b) there are text areas not covered by any tokens
     *       //     due to skipped tokens (skipTokenIds was used
     *       //     in TokenHierarchy.create()) and the targetOffset points to such gap.
     *       //     Token sequence is positioned to the preceding token.
     *       // 
     *       // 2. diff < 0
     *       //   a) targetOffset < 0
     *       //   b) targetOffset >= 0 but there is a text area
     *       //     at the begining that is not covered by any tokens
     *       //     (skipTokenIds was used in TokenHierarchy.create())
     *       //     Token sequence is positioned to the first token in the sequence.
     *   }
     * </pre>
     * 
     *
     * @param offset absolute offset in the input to which
     *  the token sequence should be moved.
     *
     * @return difference between the reqeuested offset
     *  and the start offset of the token
     *  before which the the token sequence gets positioned.
     *  <br/>
     *  If the TS is positioned after the last token the difference is computed
     *  against the end offset of the last token.
     * 
     * @throws ConcurrentModificationException if this token sequence
     *  is no longer valid because of an underlying mutable input source modification.

            if (tokenList.tokenOrEmbeddingContainer(0) == null) { // really no tokens at all
                // In this case the token sequence could not be positioned yet
                // so no need to reset "index" or other vars
                resetTokenIndex(0);
                return offset;
            }
            // Re-get the present token count (could be created a chunk of tokens at once)
            tokenCount = tokenList.tokenCountCurrent();

                    tokenLength = t.length();
                    tokenCount++;

                } else { // no more tokens => position after last token
                    resetTokenIndex(tokenCount);
                    tokenOffset = prevTokenOffset + tokenLength; // May assign the token's offset in advance
                    return offset - tokenOffset;
                }
            }
            resetTokenIndex(tokenCount - 1);
            tokenOffset = prevTokenOffset; // May assign the token's offset in advance
            tokenOffset = prevTokenOffset;
            assignToken();
            return offset - prevTokenOffset;
        }
        

                high = mid - 1;
            } else {
                // Token starting exactly at offset found
                resetTokenIndex(mid);
                tokenOffset = midStartOffset;
                assignToken();
                return 0; // right at the token begining
            }
        }
        
        // Not found exactly and high + 1 == low => high < low
        // BTW there may be gaps between tokens; if offset is in gap then position to higher token
        if (high >= 0) { // could be -1
            AbstractToken t = LexerUtilsConstants.token(tokenList, high);
            prevTokenOffset = tokenList.tokenOffset(high);
            // If gaps allowed check whether the token at "high" contains the offset
            if (!tokenList.isContinuous() && offset > prevTokenOffset + t.length()) {
                // Offset in the gap above the "high" token
                high++;
                prevTokenOffset += t.length();
            }
        } else { // at least one token exists => use token at index 0
            high = 0;
            prevTokenOffset = tokenList.tokenOffset(0); // result may differ from 0
        }
        resetTokenIndex(high);
        tokenIndex = high;
        tokenOffset = prevTokenOffset;
        assignToken();
        return offset - prevTokenOffset;
    }

    /**
     * Move to the first token in this token sequence.
     *
     * @return true if the sequence was positioned on the first token
     *  or false if there are no tokens in the sequence.
     */
    public boolean moveFirst() {
        return moveIndex(0);
    }
    
    /**
     * Move to the last token in this token sequence.
     *
     * @return true if the sequence was positioned on the last token
     *  or false if there are no tokens in the sequence.
     */
    public boolean moveLast() {
        return moveIndex(tokenCount() - 1); // Can be -1 but handled in move(index)
    }

    /**
     * Return total count of tokens in this sequence.
     * <br>
     * <b>Note:</b> Calling this method will lead

        return parentTokenIndexes;
    }

    private void resetTokenIndex(int index) {
        // Position to the given index e.g. by move() and moveIndex()
        tokenIndex = index;
        token = null;
        tokenOffset = -1;
        assignToken(tokenList.tokenOrEmbeddingContainer(tokenIndex));
    }

    private void checkTokenNotNull() {
        if (token == null) {
            throw new IllegalStateException(
                "No token fetched by moveNext() from token sequence yet: index=" + tokenIndex




            sb.append(": ");
            AbstractToken token = token(tokenOrEmbeddingContainer);
            sb.append(token.dumpInfo(tokenHierarchy));
            int la = tokenList.lookahead(i);
            if (la != 0) {
                sb.append(", la=");
                sb.append(la);
            }
            Object state= tokenList.state(i);
            if (state != null) {
                sb.append(", s=");
                sb.append(state);
            }
            sb.append('\n');
        }
        return sb;

Lines 92-98 Link Here

(-)lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java (-1 / +1 lines)
92	if (limitStartOffset > 0) {	92	if (limitStartOffset > 0) {
93	int diff = move(limitStartOffset);	93	int diff = move(limitStartOffset);
94	if (diff != Integer.MAX_VALUE) { // some tokens exist	94	if (diff != Integer.MAX_VALUE) { // some tokens exist
95	if (diff >= lastToken.length()) {	95	if (diff >= lastToken.length()) { // lastToken initialized in move()
96	lastTokenIndex++;	96	lastTokenIndex++;
97	Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(lastTokenIndex);	97	Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(lastTokenIndex);
98	if (tokenOrEmbeddingContainer != null &&	98	if (tokenOrEmbeddingContainer != null &&

Lines 48-60 Link Here

(-)lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java (-41 / +65 lines)
48	TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());	48	TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());
49	TokenSequence<? extends TokenId> ts = hi.tokenSequence();	49	TokenSequence<? extends TokenId> ts = hi.tokenSequence();
50		50
51	// Fail if no "move*" method called yet	51	assertNull(ts.token());
52	try {
53	ts.token(); // should throw exception
54	fail("IllegalStateException expecxted");
55	} catch (IllegalStateException e) {
56	// expected
57	}
58		52
59	assertTrue(ts.moveNext());	53	assertTrue(ts.moveNext());
60	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	54	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
Lines 69-88 Link Here
69	assertEquals(ts.tokenCount(), 3);	63	assertEquals(ts.tokenCount(), 3);
70		64
71	assertEquals(0, ts.move(0));	65	assertEquals(0, ts.move(0));
		66	assertNull(ts.token());
		67	assertTrue(ts.moveNext());
72	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	68	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
73	assertEquals(2, ts.move(2));	69	assertEquals(2, ts.move(2));
		70	assertNull(ts.token());
		71	assertTrue(ts.moveNext());
74	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	72	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
75	assertEquals(-1, ts.move(-1));	73	assertEquals(-1, ts.move(-1));
		74	assertNull(ts.token());
		75	assertTrue(ts.moveNext());
76	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	76	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
77	assertEquals(0, ts.move(3));	77	assertEquals(0, ts.move(3));
		78	assertNull(ts.token());
		79	assertTrue(ts.moveNext());
78	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);	80	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
79	assertEquals(0, ts.move(4));	81	assertEquals(0, ts.move(4));
		82	assertNull(ts.token());
		83	assertTrue(ts.moveNext());
80	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);	84	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
81	assertEquals(1, ts.move(5));	85	assertEquals(1, ts.move(5));
		86	assertNull(ts.token());
		87	assertTrue(ts.moveNext());
82	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);	88	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
83	assertEquals(5, ts.move(9));	89	assertEquals(1, ts.move(9));
		90	assertNull(ts.token());
		91	assertFalse(ts.moveNext());
		92	assertTrue(ts.movePrevious());
84	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);	93	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
85	assertEquals(96, ts.move(100));	94	assertEquals(92, ts.move(100));
		95	assertNull(ts.token());
		96	assertFalse(ts.moveNext());
		97	assertTrue(ts.movePrevious());
86	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);	98	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
87		99
88		100
Lines 122-133 Link Here
122	TokenSequence<? extends TokenId> ts = hi.tokenSequence();	134	TokenSequence<? extends TokenId> ts = hi.tokenSequence();
123		135
124	// Fail if no "move*" method called yet	136	// Fail if no "move*" method called yet
125	try {	137	assertNull(ts.token());
126	ts.token(); // should throw exception
127	fail("IllegalStateException expecxted");
128	} catch (IllegalStateException e) {
129	// expected
130	}
131		138
132	assertTrue(ts.moveNext());	139	assertTrue(ts.moveNext());
133	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	140	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
Lines 137-146 Link Here
137	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	144	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
138	assertFalse(ts.moveNext());	145	assertFalse(ts.moveNext());
139	assertEquals(3, ts.tokenCount());	146	assertEquals(3, ts.tokenCount());
140	assertTrue(ts.moveFirst());	147	assertEquals(0, ts.moveIndex(0));
		148	assertTrue(ts.moveNext());
141	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	149	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
142	assertTrue(ts.moveNext());	150	assertTrue(ts.moveNext());
143	assertTrue(ts.moveLast());	151	assertEquals(0, ts.moveIndex(ts.tokenCount()));
		152	assertTrue(ts.movePrevious());
144	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	153	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
145	assertFalse(ts.moveNext());	154	assertFalse(ts.moveNext());
146		155
Lines 151-166 Link Here
151	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	160	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
152		161
153	assertEquals(-1, ts.move(0)); // below filtered-out token	162	assertEquals(-1, ts.move(0)); // below filtered-out token
		163	assertNull(ts.token());
		164	assertTrue(ts.moveNext());
154	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	165	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
155	assertEquals(0, ts.move(1));	166	assertEquals(0, ts.move(1));
		167	assertNull(ts.token());
		168	assertTrue(ts.moveNext());
156	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	169	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
157	assertEquals(1, ts.move(2));	170	assertEquals(1, ts.move(2));
		171	assertNull(ts.token());
		172	assertTrue(ts.moveNext());
158	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	173	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
159	assertEquals(3, ts.move(4));	174	assertEquals(3, ts.move(4));
		175	assertNull(ts.token());
		176	assertTrue(ts.moveNext());
160	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	177	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
161	assertEquals(0, ts.move(5));	178	assertEquals(0, ts.move(5));
		179	assertNull(ts.token());
		180	assertTrue(ts.moveNext());
162	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 5);	181	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 5);
163	assertEquals(1, ts.move(12));	182	assertEquals(1, ts.move(12));
		183	assertNull(ts.token());
		184	assertTrue(ts.moveNext());
164	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	185	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
165		186
166		187
Lines 172-219 Link Here
172	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);	193	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
173	assertFalse(sub.moveNext());	194	assertFalse(sub.moveNext());
174		195
175	// Test moveFirst() and moveLast() on subsequence	196	// Test moves to first and last tokens on subsequence
176	sub = ts.subSequence(1, 6);	197	sub = ts.subSequence(1, 6);
177	assertTrue(sub.moveLast());	198	assertEquals(0, sub.moveIndex(sub.tokenCount()));
		199	assertEquals(2, sub.tokenCount()); // "abc" and "defg" (others filtered out
		200	assertTrue(sub.movePrevious());
178	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);	201	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
179	assertTrue(sub.moveFirst());	202	assertEquals(0, sub.moveIndex(0));
		203	assertTrue(sub.moveNext());
180	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "abc", 1);	204	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "abc", 1);
181		205
182	sub = ts.subSequence(4);	206	sub = ts.subSequence(4);
183	assertTrue(sub.moveNext());	207	assertTrue(sub.moveNext());
184	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);	208	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
185	assertTrue(sub.moveNext());	209	assertTrue(sub.moveNext());
186	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	210	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
187	assertFalse(sub.moveNext());	211	assertFalse(sub.moveNext());
188		212
189	sub = ts.subSequence(12, 15);	213	sub = ts.subSequence(12, 15);
190	assertTrue(sub.moveNext());	214	assertTrue(sub.moveNext());
191	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	215	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
192		216
193	sub = ts.subSequence(12, 15);	217	sub = ts.subSequence(12, 15);
194	assertEquals(-2, sub.move(9));	218	assertEquals(-2, sub.move(9));
195	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	219	assertNull(sub.token());
		220	assertTrue(sub.moveNext());
		221	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
196		222
197	sub = ts.subSequence(13, 15);	223	sub = ts.subSequence(13, 15);
198	assertFalse(sub.moveNext());	224	assertFalse(sub.moveNext());
199	assertFalse(sub.moveFirst());	225	assertEquals(0, sub.moveIndex(0));
200	assertFalse(sub.moveLast());	226	assertNull(sub.token());
		227	assertFalse(sub.moveNext());
		228	assertEquals(0, sub.moveIndex(sub.tokenCount()));
		229	assertNull(sub.token());
201	assertEquals(0, sub.tokenCount());	230	assertEquals(0, sub.tokenCount());
202		231
203	sub = ts.subSequence(-3, 1);	232	sub = ts.subSequence(-3, 1);
204	assertFalse(sub.moveNext());	233	assertFalse(sub.moveNext());
205		234
206	sub = ts.subSequence(13, 15);	235	sub = ts.subSequence(13, 15);
207	assertEquals(Integer.MAX_VALUE, sub.move(5));	236	assertEquals(5, sub.move(5));
208		237
209	sub = ts.subSequence(-3, 1);	238	sub = ts.subSequence(-3, 1);
210	assertEquals(Integer.MAX_VALUE, sub.move(1));	239	assertEquals(1, sub.move(1));
211		240
212	// Reversed bounds => should be empty token sequence	241	// Reversed bounds => should be empty token sequence
213	sub = ts.subSequence(6, 1);	242	sub = ts.subSequence(6, 1);
214	assertFalse(sub.moveNext());	243	assertFalse(sub.moveNext());
215	sub = ts.subSequence(6, 1);	244	sub = ts.subSequence(6, 1);
216	assertEquals(Integer.MAX_VALUE, sub.move(6));	245	assertEquals(6, sub.move(6));
217	}	246	}
218		247
219	public void DtestMoveSkipTokens2() throws IOException {	248	public void DtestMoveSkipTokens2() throws IOException {
Lines 244-275 Link Here
244		273
245	// Expect no tokens	274	// Expect no tokens
246	assertFalse(ts.moveNext());	275	assertFalse(ts.moveNext());
247	// Exception if ts.token() called	276	assertNull(ts.token());
248	try {
249	ts.token(); // should throw exception
250	fail("IllegalStateException expecxted");
251	} catch (IllegalStateException e) {
252	// expected
253	}
254		277
255	assertEquals(ts.move(0), Integer.MAX_VALUE);	278	assertEquals(0, ts.move(0));
256	assertEquals(ts.move(10), Integer.MAX_VALUE);	279	assertEquals(10, ts.move(10));
257		280
258	// Test subsequences	281	// Test subsequences
259	TokenSequence<? extends TokenId> sub = ts.subSequence(1, 6);	282	TokenSequence<? extends TokenId> sub = ts.subSequence(1, 6);
260	assertFalse(sub.moveNext());	283	assertFalse(sub.moveNext());
261	sub = ts.subSequence(1, 6);	284	sub = ts.subSequence(1, 6);
262	assertEquals(Integer.MAX_VALUE, sub.move(1));	285	assertEquals(1, sub.move(1));
263		286
264	sub = ts.subSequence(0);	287	sub = ts.subSequence(0);
265	assertFalse(sub.moveNext());	288	assertFalse(sub.moveNext());
266	sub = ts.subSequence(0);	289	sub = ts.subSequence(0);
267	assertEquals(Integer.MAX_VALUE, sub.move(0));	290	assertEquals(0, sub.move(0));
268		291
269	sub = ts.subSequence(1);	292	sub = ts.subSequence(1);
270	assertFalse(sub.moveNext());	293	assertFalse(sub.moveNext());
271	sub = ts.subSequence(1);	294	sub = ts.subSequence(1);
272	assertEquals(Integer.MAX_VALUE, sub.move(0));	295	assertEquals(0, sub.move(0));
273	}	296	}
274		297
275	public void testTokenSize() {	298	public void testTokenSize() {
Lines 284-290 Link Here
284	assertFalse(ts.moveNext());	307	assertFalse(ts.moveNext());
285		308
286	TokenList tokenList = LexerTestUtilities.tokenList(ts);	309	TokenList tokenList = LexerTestUtilities.tokenList(ts);
287	ts.moveIndex(0); // move to "abc"	310	ts.moveIndex(0); // move before "abc"
		311	assertTrue(ts.moveNext());
288	// Test DefaultToken size	312	// Test DefaultToken size
289	assertSame(DefaultToken.class, ts.token().getClass());	313	assertSame(DefaultToken.class, ts.token().getClass());
290	assertSize("Token instance too big", Collections.singletonList(ts.token()), 24,	314	assertSize("Token instance too big", Collections.singletonList(ts.token()), 24,




    public void testCachingE() {
        TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
        TokenSequence tokens = th.tokenSequence();
        tokens.moveStart();
        assertEquals(true, tokens.moveNext());
        
        TokenSequence embeddedA = tokens.embedded();
        assertNotNull("There should be an embedded language", embeddedA);

    public void testGCedE() {
        TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
        TokenSequence tokens = th.tokenSequence();
        tokens.moveStart();
        assertEquals(true, tokens.moveNext());
        
        TokenSequence embedded = tokens.embedded();
        assertNotNull("There should be an embedded language", embedded);

    public void testCacheRefreshedE() {
        TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
        TokenSequence tokens = th.tokenSequence();
        tokens.moveStart();
        assertEquals(true, tokens.moveNext());
        
        TokenSequence embeddedA = tokens.embedded();
        assertNotNull("There should be an embedded language", embeddedA);




    public static void assertTokenEquals(String message, TokenSequence<? extends TokenId> ts, TokenId id, String text, int offset) {
        message = messagePrefix(message);
        Token<? extends TokenId> t = ts.token();
        TestCase.assertNotNull("Token is null", t);
        TokenId tId = t.id();
        TestCase.assertEquals(message + "Invalid token.id()", id, tId);
        CharSequence tText = t.text();

        }
        TokenHierarchy<?> thBatch = TokenHierarchy.create(docText, language);
        boolean success = false;
        TokenSequence<?> batchTS = thBatch.tokenSequence();
        try {
            // Compare lookaheads and states as well
            assertTokenSequencesEqual(batchTS, thBatch,
                    thInc.tokenSequence(), thInc, true);
            success = true;
        } finally {
            if (!success) {
                // Go forward two tokens to have an extra tokens context
                batchTS.moveNext();
                batchTS.moveNext();
                System.err.println("BATCH token sequence dump:\n" + thBatch.tokenSequence());
                TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);
                if (lastHi != null) {
                    System.err.println("PREVIOUS batch token sequence dump:\n" + lastHi.tokenSequence());
                }
            }
        }





        // Check that all the non-fly tokens are mutable
        ts = snapshot1.tokenSequence();
        assertEquals(0, ts.moveIndex(0));
        assertTrue(ts.moveNext());
        
        LexerTestUtilities.assertTokenSequencesEqual(hi1.tokenSequence(), hi1,
                snapshot1.tokenSequence(), snapshot1, false);

        doc.insertString(4, "+", null);

        // Check that the snapshot token sequence can be further operated.
        assertEquals(0, ts.moveIndex(0));
        assertTrue(ts.moveNext());
        assertNotNull(ts.token());

        // Check that the tokens except '+' are live


        // Check that all the non-fly tokens are mutable
        ts = snapshot2.tokenSequence();
        assertEquals(0, ts.moveIndex(0));
        assertTrue(ts.moveNext());

        LexerTestUtilities.assertTokenSequencesEqual(hi2.tokenSequence(), hi2,
                snapshot2.tokenSequence(), snapshot2, false);

Lines 81-87 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java (-1 / +2 lines)
81	assertFalse(ts.moveNext());	81	assertFalse(ts.moveNext());
82		82
83	// Go back to block comment	83	// Go back to block comment
84	assertTrue(ts.moveIndex(commentIndex));	84	assertEquals(0, ts.moveIndex(commentIndex));
		85	assertTrue(ts.moveNext());
85	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.BLOCK_COMMENT, commentText, commentTextStartOffset);	86	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.BLOCK_COMMENT, commentText, commentTextStartOffset);
86		87
87	// Test embedded token sequence	88	// Test embedded token sequence




        
        // Check TokenSequence.move()
        int relOffset = ts.move(50); // past the end of all tokens
        assertEquals(relOffset, 50 - (offset + 3));
        assertTrue(ts.movePrevious());
        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset);

        relOffset = ts.move(6); // right at begining of "-"
        assertEquals(relOffset, 0);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", 6);

        relOffset = ts.move(-5); // to first token "abc"
        assertEquals(relOffset, -5);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);

        relOffset = ts.move(5); // to first token "abc"
        assertEquals(relOffset, 1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "uv", 4);



Lines 1-3 Link Here

(-)openide/text/src/org/openide/text/CloneableEditorSupport.java (+1 lines)
		1
1	/*	2	/*
2	* The contents of this file are subject to the terms of the Common Development	3	* The contents of this file are subject to the terms of the Common Development
3	* and Distribution License (the License). You may not use this file except in	4	* and Distribution License (the License). You may not use this file except in

Lines 97-103 Link Here

(-)web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java (-1 / +1 lines)
97	TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);	97	TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);
98	TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); //get top level token sequence	98	TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); //get top level token sequence
99		99
100	if(!tokenSequence.moveFirst()) {	100	if(!tokenSequence.moveNext()) {
101	return ; //no tokens in token sequence	101	return ; //no tokens in token sequence
102	}	102	}
103		103




         * e.g. <tag   |attr=
         * 
         */
        while (tokenSequence.moveNext()) {
            tokenSequence.moveNext();
            token = tokenSequence.token();
            tokenOffset = tokenSequence.offset();
            if (!isWSTag(token) || tagStartLine != Utilities.getLineOffset(doc, tokenOffset)) {
                if (!isWSTag(token) && tagStartLine == Utilities.getLineOffset(doc, tokenOffset)){
                    return tokenOffset - Utilities.getRowIndent(doc, tokenOffset) - Utilities.getRowStart(doc, tokenOffset);
                }
                break;
            }
        }
        
        
        return getShiftWidth(); // default;
    }
    

            int indentsWithinTags[] = new int[lastLine + 1];
            
            TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
            tokenSequence.moveStart();
            boolean thereAreMoreTokens = tokenSequence.moveNext();
            
            
            if (tokenSequence != null){
                // calc line indents - pass 1

        tokenSequence.move(offset);
        int currentOffset;
        
        while (tokenSequence.moveNext()) {
            currentOffset = tokenSequence.offset();
            
            if (isClosingTag(tokenHierarchy, currentOffset)){
                return currentOffset;
            }
            
        }

        return -1;
    }

    protected Token getTokenAtOffset(TokenHierarchy tokenHierarchy, int tagTokenOffset){
        TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
        
        if (tokenSequence != null) {
            tokenSequence.move(tagTokenOffset);
            if (tokenSequence.moveNext())
                return tokenSequence.token();
        }
        
        return null;

Return to bug 90796

Lines 1101-1109 Link Here

(-)java/editor/src/org/netbeans/modules/editor/java/JavaCompletionProvider.java (-8 / +11 lines)
1101	int openLtNum = 0;	1101	int openLtNum = 0;
1102	JavaTokenId lastNonWhitespaceTokenId = null;	1102	JavaTokenId lastNonWhitespaceTokenId = null;
1103	TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());	1103	TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());
1104	if (ts.move(expEndPos) > 0)	1104	ts.move(expEndPos);
1105	ts.moveNext();	1105	while (ts.moveNext()) {
1106	do {
1107	if (ts.offset() >= offset) {	1106	if (ts.offset() >= offset) {
1108	break;	1107	break;
1109	}	1108	}
Lines 1136-1142 Link Here
1136	default:	1135	default:
1137	lastNonWhitespaceTokenId = ts.token().id();	1136	lastNonWhitespaceTokenId = ts.token().id();
1138	}	1137	}
1139	} while (ts.moveNext());	1138	}
1140	if (!afterDot) {	1139	if (!afterDot) {
1141	if (expEndPos <= offset)	1140	if (expEndPos <= offset)
1142	insideExpression(env, new TreePath(path, fa.getExpression()));	1141	insideExpression(env, new TreePath(path, fa.getExpression()));
Lines 3403-3410 Link Here
3403		3402
3404	private TokenSequence<JavaTokenId> findLastNonWhitespaceToken(Env env, int startPos, int endPos) {	3403	private TokenSequence<JavaTokenId> findLastNonWhitespaceToken(Env env, int startPos, int endPos) {
3405	TokenSequence<JavaTokenId> ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language());	3404	TokenSequence<JavaTokenId> ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language());
3406	if (ts.move(endPos) == 0)	3405	if (ts.move(endPos) == 0) // When right at the token end
3407	ts.movePrevious();	3406	ts.movePrevious(); // move to previous token
		3407	else
		3408	ts.moveNext(); // otherwise move to the token that "contains" the offset
3408	do {	3409	do {
3409	int offset = ts.offset();	3410	int offset = ts.offset();
3410	if (offset < startPos)	3411	if (offset < startPos)
Lines 3541-3548 Link Here
3541	if (upToOffset) {	3542	if (upToOffset) {
3542	TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());	3543	TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());
3543	if (ts.moveNext()) { //check for empty file	3544	if (ts.moveNext()) { //check for empty file
3544	if (ts.move(offset) == 0)	3545	if (ts.move(offset) == 0) // When right at the token end
3545	ts.movePrevious();	3546	ts.movePrevious(); // Move to previous token
		3547	else
		3548	ts.moveNext(); // otherwise move to the token that "contains" the offset
3546	if (ts.offset() < offset && (ts.token().id() == JavaTokenId.IDENTIFIER \|\| "keyword".equals(ts.token().id().primaryCategory()))) { //TODO: Use isKeyword(...) when available	3549	if (ts.offset() < offset && (ts.token().id() == JavaTokenId.IDENTIFIER \|\| "keyword".equals(ts.token().id().primaryCategory()))) { //TODO: Use isKeyword(...) when available
3547	prefix = ts.token().toString().substring(0, offset - ts.offset());	3550	prefix = ts.token().toString().substring(0, offset - ts.offset());
3548	offset = ts.offset();	3551	offset = ts.offset();

Lines 317-322 Link Here

(-)java/source/src/org/netbeans/modules/java/source/save/CasualDiff.java (-3 / +9 lines)
317	// skip the section when printing anonymous class	317	// skip the section when printing anonymous class
318	if (anonClass == false) {	318	if (anonClass == false) {
319	tokenSequence.move(oldT.pos);	319	tokenSequence.move(oldT.pos);
		320	tokenSequence.moveNext(); // First skip as move() does not position to token directly
320	tokenSequence.moveNext();	321	tokenSequence.moveNext();
321	insertHint = TokenUtilities.moveNext(tokenSequence, tokenSequence.offset());	322	insertHint = TokenUtilities.moveNext(tokenSequence, tokenSequence.offset());
322	localPointer = diffModifiers(oldT.mods, newT.mods, oldT, localPointer);	323	localPointer = diffModifiers(oldT.mods, newT.mods, oldT, localPointer);
Lines 510-515 Link Here
510	// there was not any parameter in original tree.	511	// there was not any parameter in original tree.
511	int startOffset = oldT.restype != null ? oldT.restype.getStartPosition() : oldT.getStartPosition();	512	int startOffset = oldT.restype != null ? oldT.restype.getStartPosition() : oldT.getStartPosition();
512		513
		514	// Extra move()? - performed in moveFwdToToken()
513	tokenSequence.move(startOffset);	515	tokenSequence.move(startOffset);
514	TokenUtilities.moveFwdToToken(tokenSequence, startOffset, JavaTokenId.RPAREN);	516	TokenUtilities.moveFwdToToken(tokenSequence, startOffset, JavaTokenId.RPAREN);
515	posHint = tokenSequence.offset();	517	posHint = tokenSequence.offset();
Lines 529-534 Link Here
529	// now check, that there is a whitespace. It is not mandatory, we	531	// now check, that there is a whitespace. It is not mandatory, we
530	// have to ensure. If whitespace is not present, take body beginning	532	// have to ensure. If whitespace is not present, take body beginning
531	tokenSequence.move(posHint);	533	tokenSequence.move(posHint);
		534	tokenSequence.moveNext();
532	if (tokenSequence.token().id() != JavaTokenId.WHITESPACE) {	535	if (tokenSequence.token().id() != JavaTokenId.WHITESPACE) {
533	++posHint;	536	++posHint;
534	}	537	}
Lines 1411-1417 Link Here
1411	ResultItem<JCTree> item = result[j];	1414	ResultItem<JCTree> item = result[j];
1412	switch (item.operation) {	1415	switch (item.operation) {
1413	case MODIFY: {	1416	case MODIFY: {
1414	if (tokenSequence.moveIndex(matrix[i][4])) {	1417	tokenSequence.moveIndex(matrix[i][4]);
		1418	if (tokenSequence.moveNext()) {
1415	testPos = tokenSequence.offset();	1419	testPos = tokenSequence.offset();
1416	if (JavaTokenId.COMMA == tokenSequence.token().id())	1420	if (JavaTokenId.COMMA == tokenSequence.token().id())
1417	testPos += JavaTokenId.COMMA.fixedText().length();	1421	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1454-1460 Link Here
1454	assert startOffset != -1 && endOffset != -1 : "Invalid offset!";	1458	assert startOffset != -1 && endOffset != -1 : "Invalid offset!";
1455	//printer.print(origText.substring(lastOldPos, startOffset));	1459	//printer.print(origText.substring(lastOldPos, startOffset));
1456	//append(Diff.delete(startOffset, endOffset));	1460	//append(Diff.delete(startOffset, endOffset));
1457	if (tokenSequence.moveIndex(matrix[i][4])) {	1461	tokenSequence.moveIndex(matrix[i][4]);
		1462	if (tokenSequence.moveNext()) {
1458	testPos = tokenSequence.offset();	1463	testPos = tokenSequence.offset();
1459	if (JavaTokenId.COMMA == tokenSequence.token().id())	1464	if (JavaTokenId.COMMA == tokenSequence.token().id())
1460	testPos += JavaTokenId.COMMA.fixedText().length();	1465	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1468-1474 Link Here
1468	break;	1473	break;
1469	}	1474	}
1470	case NOCHANGE: {	1475	case NOCHANGE: {
1471	if (tokenSequence.moveIndex(matrix[i][4])) {	1476	tokenSequence.moveIndex(matrix[i][4]);
		1477	if (tokenSequence.moveNext()) {
1472	testPos = tokenSequence.offset();	1478	testPos = tokenSequence.offset();
1473	if (JavaTokenId.COMMA == tokenSequence.token().id())	1479	if (JavaTokenId.COMMA == tokenSequence.token().id())
1474	testPos += JavaTokenId.COMMA.fixedText().length();	1480	testPos += JavaTokenId.COMMA.fixedText().length();

Lines 1247-1253 Link Here

(-)java/source/src/org/netbeans/modules/java/source/save/TreeDiff.java (-3 / +6 lines)
1247	ResultItem<JCTree> item = result[j];	1247	ResultItem<JCTree> item = result[j];
1248	switch (item.operation) {	1248	switch (item.operation) {
1249	case MODIFY: {	1249	case MODIFY: {
1250	if (tokenSequence.moveIndex(matrix[i][4])) {	1250	tokenSequence.moveIndex(matrix[i][4]);
		1251	if (tokenSequence.moveNext()) {
1251	testPos = tokenSequence.offset();	1252	testPos = tokenSequence.offset();
1252	if (JavaTokenId.COMMA == tokenSequence.token().id())	1253	if (JavaTokenId.COMMA == tokenSequence.token().id())
1253	testPos += JavaTokenId.COMMA.fixedText().length();	1254	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1283-1289 Link Here
1283	int endOffset = toOff(s.tail(j) \|\| s.next(j) ? matrix[i+1][2] : matrix[i][4]);	1284	int endOffset = toOff(s.tail(j) \|\| s.next(j) ? matrix[i+1][2] : matrix[i][4]);
1284	assert startOffset != -1 && endOffset != -1 : "Invalid offset!";	1285	assert startOffset != -1 && endOffset != -1 : "Invalid offset!";
1285	append(Diff.delete(startOffset, endOffset));	1286	append(Diff.delete(startOffset, endOffset));
1286	if (tokenSequence.moveIndex(matrix[i][4])) {	1287	tokenSequence.moveIndex(matrix[i][4]);
		1288	if (tokenSequence.moveNext()) {
1287	testPos = tokenSequence.offset();	1289	testPos = tokenSequence.offset();
1288	if (JavaTokenId.COMMA == tokenSequence.token().id())	1290	if (JavaTokenId.COMMA == tokenSequence.token().id())
1289	testPos += JavaTokenId.COMMA.fixedText().length();	1291	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1293-1299 Link Here
1293	break;	1295	break;
1294	}	1296	}
1295	case NOCHANGE: {	1297	case NOCHANGE: {
1296	if (tokenSequence.moveIndex(matrix[i][4])) {	1298	tokenSequence.moveIndex(matrix[i][4]);
		1299	if (tokenSequence.moveNext()) {
1297	testPos = tokenSequence.offset();	1300	testPos = tokenSequence.offset();
1298	if (JavaTokenId.COMMA == tokenSequence.token().id())	1301	if (JavaTokenId.COMMA == tokenSequence.token().id())
1299	testPos += JavaTokenId.COMMA.fixedText().length();	1302	testPos += JavaTokenId.COMMA.fixedText().length();

Lines 175-197 Link Here

(-)java/hints/src/org/netbeans/modules/java/hints/JavaHintsProvider.java (-13 / +13 lines)
175	TokenSequence<JavaTokenId> ts = info.getTokenHierarchy().tokenSequence();	175	TokenSequence<JavaTokenId> ts = info.getTokenHierarchy().tokenSequence();
176		176
177	ts.move(offset);	177	ts.move(offset);
178		178	if (ts.moveNext()) {
179	Token t = ts.token();	179	Token t = ts.token();
180		180
181	if (t.id() == JavaTokenId.DOT) {	181	if (t.id() == JavaTokenId.DOT) {
182	ts.moveNext();
183	t = ts.token();
184	} else {
185	if (t.id() == JavaTokenId.LT) {
186	ts.moveNext();	182	ts.moveNext();
187	t = ts.token();	183	t = ts.token();
		184	} else {
		185	if (t.id() == JavaTokenId.LT) {
		186	ts.moveNext();
		187	t = ts.token();
		188	}
		189	}
		190
		191	if (t.id() == JavaTokenId.IDENTIFIER) {
		192	return ts.offsetToken();
188	}	193	}
189	}	194	}
190
191	if (t.id() == JavaTokenId.IDENTIFIER) {
192	return ts.offsetToken();
193	}
194
195	return null;	195	return null;
196	}	196	}
197		197

Lines 139-144 Link Here

(-)java/source/src/org/netbeans/modules/java/source/save/PositionEstimator.java (+11 lines)
139	int treeEnd = (int) positions.getEndPosition(compilationUnit, item);	139	int treeEnd = (int) positions.getEndPosition(compilationUnit, item);
140		140
141	seq.move(treeStart);	141	seq.move(treeStart);
		142	// seq.index() set (no seq.moveNext() necessary)
142	int startIndex = seq.index();	143	int startIndex = seq.index();
143		144
144	// go back to opening/closing curly, semicolon or other	145	// go back to opening/closing curly, semicolon or other
Lines 147-155 Link Here
147	seq.moveNext();	148	seq.moveNext();
148	int start = seq.index();	149	int start = seq.index();
149	seq.move(treeEnd);	150	seq.move(treeEnd);
		151	// seq.index() set (no seq.moveNext() necessary)
150	matrix[i++] = new int[] { start, startIndex, seq.index() };	152	matrix[i++] = new int[] { start, startIndex, seq.index() };
151	if (i == size) {	153	if (i == size) {
152	seq.move(treeEnd);	154	seq.move(treeEnd);
		155	// seq.index() set (no seq.moveNext() necessary)
153	matrix[i][0] = seq.index();	156	matrix[i][0] = seq.index();
154	}	157	}
155	}	158	}
Lines 163-168 Link Here
163	// to decide.	166	// to decide.
164	if (tokenIndex == -1) return -1;	167	if (tokenIndex == -1) return -1;
165	seq.moveIndex(tokenIndex);	168	seq.moveIndex(tokenIndex);
		169	seq.moveNext();
166	return index == 0 ? goAfterLastNewLine(seq) : goAfterFirstNewLine(seq);	170	return index == 0 ? goAfterLastNewLine(seq) : goAfterFirstNewLine(seq);
167	}	171	}
168		172
Lines 210-217 Link Here
210	public int[] getPositions(int index) {	214	public int[] getPositions(int index) {
211	int tokenIndex = matrix[index][0];	215	int tokenIndex = matrix[index][0];
212	seq.moveIndex(tokenIndex);	216	seq.moveIndex(tokenIndex);
		217	seq.moveNext();
213	int begin = goAfterLastNewLine(seq);	218	int begin = goAfterLastNewLine(seq);
214	seq.moveIndex(matrix[index][2]);	219	seq.moveIndex(matrix[index][2]);
		220	seq.moveNext();
215	int end = goAfterFirstNewLine(seq);	221	int end = goAfterFirstNewLine(seq);
216	return new int [] { begin, end };	222	return new int [] { begin, end };
217	}	223	}
Lines 285-290 Link Here
285	// to decide.	291	// to decide.
286	if (tokenIndex == -1) return -1;	292	if (tokenIndex == -1) return -1;
287	seq.moveIndex(tokenIndex);	293	seq.moveIndex(tokenIndex);
		294	seq.moveNext();
288	return goAfterFirstNewLine(seq);	295	return goAfterFirstNewLine(seq);
289	}	296	}
290		297
Lines 297-302 Link Here
297	public int[] getPositions(int index) {	304	public int[] getPositions(int index) {
298	int begin = getInsertPos(index);	305	int begin = getInsertPos(index);
299	seq.moveIndex(matrix[index][4]);	306	seq.moveIndex(matrix[index][4]);
		307	seq.moveNext();
300	int end = goAfterFirstNewLine(seq);	308	int end = goAfterFirstNewLine(seq);
301	return new int [] { begin, end };	309	return new int [] { begin, end };
302	}	310	}
Lines 360-365 Link Here
360	// to decide.	368	// to decide.
361	if (tokenIndex == -1) return -1;	369	if (tokenIndex == -1) return -1;
362	seq.moveIndex(tokenIndex);	370	seq.moveIndex(tokenIndex);
		371	seq.moveNext();
363	int off = goAfterFirstNewLine(seq);	372	int off = goAfterFirstNewLine(seq);
364	return off;	373	return off;
365	}	374	}
Lines 373-378 Link Here
373	public int[] getPositions(int index) {	382	public int[] getPositions(int index) {
374	int begin = getInsertPos(index);	383	int begin = getInsertPos(index);
375	seq.moveIndex(matrix[index][4]);	384	seq.moveIndex(matrix[index][4]);
		385	seq.moveNext();
376	int end = goAfterFirstNewLine(seq);	386	int end = goAfterFirstNewLine(seq);
377	return new int [] { begin, end };	387	return new int [] { begin, end };
378	}	388	}
Lines 502-507 Link Here
502	String s = "(nothing)";	512	String s = "(nothing)";
503	if (item > -1) {	513	if (item > -1) {
504	seq.moveIndex(item);	514	seq.moveIndex(item);
		515	seq.moveNext();
505	s = "'" + seq.token().text();	516	s = "'" + seq.token().text();
506	}	517	}
507	s += "' ";	518	s += "' ";

Lines 65-70 Link Here

(-)java/source/src/org/netbeans/modules/java/source/save/TokenUtilities.java (+5 lines)
65	*/	65	*/
66	static int moveNext(TokenSequence<JavaTokenId> tokenSequence, final int pos) {	66	static int moveNext(TokenSequence<JavaTokenId> tokenSequence, final int pos) {
67	tokenSequence.move(pos);	67	tokenSequence.move(pos);
		68	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
68	JavaTokenId type = tokenSequence.token().id();	69	JavaTokenId type = tokenSequence.token().id();
69	while (JavaTokenId.WHITESPACE.equals(type) \|\|	70	while (JavaTokenId.WHITESPACE.equals(type) \|\|
70	JavaTokenId.BLOCK_COMMENT.equals(type) \|\|	71	JavaTokenId.BLOCK_COMMENT.equals(type) \|\|
Lines 82-87 Link Here
82	JavaTokenId id)	83	JavaTokenId id)
83	{	84	{
84	tokenSequence.move(pos);	85	tokenSequence.move(pos);
		86	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
85	while (!id.equals(tokenSequence.token().id())) {	87	while (!id.equals(tokenSequence.token().id())) {
86	if (!tokenSequence.moveNext())	88	if (!tokenSequence.moveNext())
87	return -1;	89	return -1;
Lines 94-99 Link Here
94	JavaTokenId id)	96	JavaTokenId id)
95	{	97	{
96	tokenSequence.move(pos);	98	tokenSequence.move(pos);
		99	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
97	while (!id.equals(tokenSequence.token().id())) {	100	while (!id.equals(tokenSequence.token().id())) {
98	if (!tokenSequence.movePrevious())	101	if (!tokenSequence.movePrevious())
99	return -1;	102	return -1;
Lines 106-111 Link Here
106	String text)	109	String text)
107	{	110	{
108	tokenSequence.move(pos);	111	tokenSequence.move(pos);
		112	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
109	while (!text.equals(tokenSequence.token().text())) {	113	while (!text.equals(tokenSequence.token().text())) {
110	if (!tokenSequence.moveNext())	114	if (!tokenSequence.moveNext())
111	return -1;	115	return -1;
Lines 118-123 Link Here
118	String text)	122	String text)
119	{	123	{
120	tokenSequence.move(pos);	124	tokenSequence.move(pos);
		125	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
121	while (!text.equals(tokenSequence.token().text())) {	126	while (!text.equals(tokenSequence.token().text())) {
122	if (!tokenSequence.movePrevious())	127	if (!tokenSequence.movePrevious())
123	return -1;	128	return -1;

Lines 300-319 Link Here

(-)lexer/arch.xml (-13 / +11 lines)
300	TokenHierarchy hi = TokenHierarchy.get(document);	300	TokenHierarchy hi = TokenHierarchy.get(document);
301	TokenSequence ts = hi.tokenSequence();	301	TokenSequence ts = hi.tokenSequence();
302	// If necessary move ts to the requested offset	302	// If necessary move ts to the requested offset
303	int diff = ts.move(offset);	303	ts.move(offset);
304	if (diff != Integer.MAX_VALUE)) { // Some tokens exist	304	while (ts.moveNext()) {
305	do {	305	Token t = ts.token();
306	Token t = ts.token();	306	if (t.id() == ...) { ... }
307	if (t.id() == ...) { ... }	307	if (TokenUtilities.equals(t.text(), "mytext")) { ... }
308	if (TokenUtilities.equals(t.text(), "mytext")) { ... }	308	if (ts.offset() == ...) { ... }
309	if (ts.offset() == ...) { ... }
310		309
311	// Possibly retrieve embedded token sequence	310	// Possibly retrieve embedded token sequence
312	TokenSequence embedded = ts.embedded();	311	TokenSequence embedded = ts.embedded();
313	if (embedded != null) { // Token has a valid language embedding	312	if (embedded != null) { // Token has a valid language embedding
314	...	313	...
315	}	314	}
316	} while (ts.moveNext());
317	}	315	}
318	} finally {	316	} finally {
319	document.readUnlock();	317	document.readUnlock();

Lines 91-96 Link Here

(-)lexer/api/apichanges.xml (+26 lines)
91	<!-- ACTUAL CHANGES BEGIN HERE: -->	91	<!-- ACTUAL CHANGES BEGIN HERE: -->
92		92
93	<changes>	93	<changes>
		94	<change id="api.tokensequence.move">
		95	<api name="api"/>
		96	<summary>Changed TokenSequence.move() and moveIndex() use</summary>
		97	<version major="1" minor="13"/>
		98	<date day="16" month="1" year="2007"/>
		99	<author login="mmetelka"/>
		100	<compatibility source="incompatible" deletion="yes" addition="yes" modification="yes"/>
		101	<description>
		102	<p>
		103	Changed the <code>TokenSequence.move()</code> to position <b>before</b> the particular token
		104	that "contains" the token (or after the last token if the given offset
		105	is too high). Additional <code>moveNext()</code> is necessary
		106	to actually move to the next token.
		107	<br/>
		108	<code>TokenSequence.moveIndex()</code> was modified in a similar way
		109	(see javadocs).
		110	<br/>
		111	<code>TokenSequence.moveFirst()</code> and <code>moveLast()</code>
		112	were replaced by <code>moveStart()</code> that positionins before
		113	the first token and by <code>moveEnd()</code> that positions
		114	after the last token.
		115	<code>
		116	</p>
		117	</description>
		118	</change>
		119
94	<change id="api.lexer.release">	120	<change id="api.lexer.release">
95	<api name="api"/>	121	<api name="api"/>
96	<summary>Added Lexer.release()</summary>	122	<summary>Added Lexer.release()</summary>

Lines 108-115 Link Here

(-)lexer/editorbridge/src/org/netbeans/modules/lexer/editorbridge/LexerLayer.java (-3 / +4 lines)
108	if (active) {	108	if (active) {
109	pastSequences = new Stack<TokenSequence>();	109	pastSequences = new Stack<TokenSequence>();
110	tokenSequence = hi.tokenSequence();	110	tokenSequence = hi.tokenSequence();
111	int relOffset = tokenSequence.move(startOffset);	111	tokenSequence.move(startOffset);
112	if (relOffset != Integer.MAX_VALUE) {	112	if (tokenSequence.moveNext()) {
113	updateTokenEndOffsetAndColoring(startOffset);	113	updateTokenEndOffsetAndColoring(startOffset);
114	} else { // no tokens	114	} else { // no tokens
115	active = false;	115	active = false;
Lines 178-184 Link Here
178	break;	178	break;
179	}	179	}
180	} else {	180	} else {
181	if (embed.move(offset) != Integer.MAX_VALUE) {	181	embed.move(offset);
		182	if (embed.moveNext()) {
182	pastSequences.push(tokenSequence);	183	pastSequences.push(tokenSequence);
183	tokenSequence = embed;	184	tokenSequence = embed;
184	} else {	185	} else {

Lines 75-81 Link Here

(-)lexer/nbbridge/test/unit/src/org/netbeans/modules/lexer/nbbridge/test/MimeLookupLanguageProviderTest.java (-5 / +8 lines)
75	assertEquals("Wrong language", "text/x-simple-plain", lang.mimeType());	75	assertEquals("Wrong language", "text/x-simple-plain", lang.mimeType());
76		76
77	for(int i = 0; i < seq.tokenCount(); i++) {	77	for(int i = 0; i < seq.tokenCount(); i++) {
78	seq.move(i);	78	seq.moveIndex(i);
		79	assertTrue(seq.moveNext());
79	Token token = seq.token();	80	Token token = seq.token();
80		81
81	if (token.id() == SimplePlainTokenId.WORD) {	82	if (token.id() == SimplePlainTokenId.WORD) {
Lines 86-96 Link Here
86	assertNotNull("Can't find language of the embedded sequence", embeddedLang);	87	assertNotNull("Can't find language of the embedded sequence", embeddedLang);
87	assertEquals("Wrong language of the embedded sequence", "text/x-simple-char", embeddedLang.mimeType());	88	assertEquals("Wrong language of the embedded sequence", "text/x-simple-char", embeddedLang.mimeType());
88		89
89	assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveFirst());	90	embeddedSeq.moveStart();
90	assertEquals("Wrong startSkipLenght", 1, embeddedSeq.offset() - seq.offset());	91	assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveNext());
		92	assertEquals("Wrong startSkipLength", 1, embeddedSeq.offset() - seq.offset());
91		93
92	assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.moveLast());	94	embeddedSeq.moveEnd();
93	assertEquals("Wrong endSkipLenght", 2,	95	assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.movePrevious());
		96	assertEquals("Wrong endSkipLength", 2,
94	(seq.offset() + seq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length()));	97	(seq.offset() + seq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length()));
95	}	98	}
96	}	99	}

Lines 237-242 Link Here

(-)lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java (+10 lines)
237	sb.append(": ");	237	sb.append(": ");
238	AbstractToken token = token(tokenOrEmbeddingContainer);	238	AbstractToken token = token(tokenOrEmbeddingContainer);
239	sb.append(token.dumpInfo(tokenHierarchy));	239	sb.append(token.dumpInfo(tokenHierarchy));
		240	int la = tokenList.lookahead(i);
		241	if (la != 0) {
		242	sb.append(", la=");
		243	sb.append(la);
		244	}
		245	Object state= tokenList.state(i);
		246	if (state != null) {
		247	sb.append(", s=");
		248	sb.append(state);
		249	}
240	sb.append('\n');	250	sb.append('\n');
241	}	251	}
242	return sb;	252	return sb;

Lines 130-136 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java (-3 / +6 lines)
130	public void testCachingE() {	130	public void testCachingE() {
131	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());	131	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
132	TokenSequence tokens = th.tokenSequence();	132	TokenSequence tokens = th.tokenSequence();
133	tokens.moveFirst();	133	tokens.moveStart();
		134	assertEquals(true, tokens.moveNext());
134		135
135	TokenSequence embeddedA = tokens.embedded();	136	TokenSequence embeddedA = tokens.embedded();
136	assertNotNull("There should be an embedded language", embeddedA);	137	assertNotNull("There should be an embedded language", embeddedA);
Lines 144-150 Link Here
144	public void testGCedE() {	145	public void testGCedE() {
145	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());	146	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
146	TokenSequence tokens = th.tokenSequence();	147	TokenSequence tokens = th.tokenSequence();
147	tokens.moveFirst();	148	tokens.moveStart();
		149	assertEquals(true, tokens.moveNext());
148		150
149	TokenSequence embedded = tokens.embedded();	151	TokenSequence embedded = tokens.embedded();
150	assertNotNull("There should be an embedded language", embedded);	152	assertNotNull("There should be an embedded language", embedded);
Lines 164-170 Link Here
164	public void testCacheRefreshedE() {	166	public void testCacheRefreshedE() {
165	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());	167	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
166	TokenSequence tokens = th.tokenSequence();	168	TokenSequence tokens = th.tokenSequence();
167	tokens.moveFirst();	169	tokens.moveStart();
		170	assertEquals(true, tokens.moveNext());
168		171
169	TokenSequence embeddedA = tokens.embedded();	172	TokenSequence embeddedA = tokens.embedded();
170	assertNotNull("There should be an embedded language", embeddedA);	173	assertNotNull("There should be an embedded language", embeddedA);

Lines 70-75 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java (-2 / +7 lines)
70	public static void assertTokenEquals(String message, TokenSequence<? extends TokenId> ts, TokenId id, String text, int offset) {	70	public static void assertTokenEquals(String message, TokenSequence<? extends TokenId> ts, TokenId id, String text, int offset) {
71	message = messagePrefix(message);	71	message = messagePrefix(message);
72	Token<? extends TokenId> t = ts.token();	72	Token<? extends TokenId> t = ts.token();
		73	TestCase.assertNotNull("Token is null", t);
73	TokenId tId = t.id();	74	TokenId tId = t.id();
74	TestCase.assertEquals(message + "Invalid token.id()", id, tId);	75	TestCase.assertEquals(message + "Invalid token.id()", id, tId);
75	CharSequence tText = t.text();	76	CharSequence tText = t.text();
Lines 312-328 Link Here
312	}	313	}
313	TokenHierarchy<?> thBatch = TokenHierarchy.create(docText, language);	314	TokenHierarchy<?> thBatch = TokenHierarchy.create(docText, language);
314	boolean success = false;	315	boolean success = false;
		316	TokenSequence<?> batchTS = thBatch.tokenSequence();
315	try {	317	try {
316	// Compare lookaheads and states as well	318	// Compare lookaheads and states as well
317	assertTokenSequencesEqual(thBatch.tokenSequence(), thBatch,	319	assertTokenSequencesEqual(batchTS, thBatch,
318	thInc.tokenSequence(), thInc, true);	320	thInc.tokenSequence(), thInc, true);
319	success = true;	321	success = true;
320	} finally {	322	} finally {
321	if (!success) {	323	if (!success) {
		324	// Go forward two tokens to have an extra tokens context
		325	batchTS.moveNext();
		326	batchTS.moveNext();
322	System.err.println("BATCH token sequence dump:\n" + thBatch.tokenSequence());	327	System.err.println("BATCH token sequence dump:\n" + thBatch.tokenSequence());
323	TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);	328	TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);
324	if (lastHi != null) {	329	if (lastHi != null) {
325	System.err.println("PREVIOUS token sequence dump:\n" + lastHi.tokenSequence());	330	System.err.println("PREVIOUS batch token sequence dump:\n" + lastHi.tokenSequence());
326	}	331	}
327	}	332	}
328	}	333	}

Lines 76-82 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java (-3 / +6 lines)
76		76
77	// Check that all the non-fly tokens are mutable	77	// Check that all the non-fly tokens are mutable
78	ts = snapshot1.tokenSequence();	78	ts = snapshot1.tokenSequence();
79	assertTrue(ts.moveIndex(0));	79	assertEquals(0, ts.moveIndex(0));
		80	assertTrue(ts.moveNext());
80		81
81	LexerTestUtilities.assertTokenSequencesEqual(hi1.tokenSequence(), hi1,	82	LexerTestUtilities.assertTokenSequencesEqual(hi1.tokenSequence(), hi1,
82	snapshot1.tokenSequence(), snapshot1, false);	83	snapshot1.tokenSequence(), snapshot1, false);
Lines 84-90 Link Here
84	doc.insertString(4, "+", null);	85	doc.insertString(4, "+", null);
85		86
86	// Check that the snapshot token sequence can be further operated.	87	// Check that the snapshot token sequence can be further operated.
87	assertTrue(ts.moveIndex(0));	88	assertEquals(0, ts.moveIndex(0));
		89	assertTrue(ts.moveNext());
88	assertNotNull(ts.token());	90	assertNotNull(ts.token());
89		91
90	// Check that the tokens except '+' are live	92	// Check that the tokens except '+' are live
Lines 101-107 Link Here
101		103
102	// Check that all the non-fly tokens are mutable	104	// Check that all the non-fly tokens are mutable
103	ts = snapshot2.tokenSequence();	105	ts = snapshot2.tokenSequence();
104	assertTrue(ts.moveIndex(0));	106	assertEquals(0, ts.moveIndex(0));
		107	assertTrue(ts.moveNext());
105		108
106	LexerTestUtilities.assertTokenSequencesEqual(hi2.tokenSequence(), hi2,	109	LexerTestUtilities.assertTokenSequencesEqual(hi2.tokenSequence(), hi2,
107	snapshot2.tokenSequence(), snapshot2, false);	110	snapshot2.tokenSequence(), snapshot2, false);

Lines 90-108 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java (-1 / +5 lines)
90		90
91	// Check TokenSequence.move()	91	// Check TokenSequence.move()
92	int relOffset = ts.move(50); // past the end of all tokens	92	int relOffset = ts.move(50); // past the end of all tokens
93	assertEquals(relOffset, 50 - offset);	93	assertEquals(relOffset, 50 - (offset + 3));
		94	assertTrue(ts.movePrevious());
94	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset);	95	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset);
95		96
96	relOffset = ts.move(6); // right at begining of "-"	97	relOffset = ts.move(6); // right at begining of "-"
97	assertEquals(relOffset, 0);	98	assertEquals(relOffset, 0);
		99	assertTrue(ts.moveNext());
98	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", 6);	100	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", 6);
99		101
100	relOffset = ts.move(-5); // to first token "abc"	102	relOffset = ts.move(-5); // to first token "abc"
101	assertEquals(relOffset, -5);	103	assertEquals(relOffset, -5);
		104	assertTrue(ts.moveNext());
102	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	105	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
103		106
104	relOffset = ts.move(5); // to first token "abc"	107	relOffset = ts.move(5); // to first token "abc"
105	assertEquals(relOffset, 1);	108	assertEquals(relOffset, 1);
		109	assertTrue(ts.moveNext());
106	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "uv", 4);	110	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "uv", 4);
107		111
108		112

Lines 86-102 Link Here

(-)xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java (-13 / +18 lines)
86	* e.g. <tag \|attr=	86	* e.g. <tag \|attr=
87	*	87	*
88	*/	88	*/
89	do {	89	while (tokenSequence.moveNext()) {
90	tokenSequence.moveNext();
91	token = tokenSequence.token();	90	token = tokenSequence.token();
92	tokenOffset = token.offset(tokenHierarchy);	91	tokenOffset = tokenSequence.offset();
93	} while (token != null && isWSTag(token)	92	if (!isWSTag(token) \|\| tagStartLine != Utilities.getLineOffset(doc, tokenOffset)) {
94	&& tagStartLine == Utilities.getLineOffset(doc, tokenOffset));	93	if (!isWSTag(token) && tagStartLine == Utilities.getLineOffset(doc, tokenOffset)){
95		94	return tokenOffset - Utilities.getRowIndent(doc, tokenOffset) - Utilities.getRowStart(doc, tokenOffset);
96	if (!isWSTag(token) && tagStartLine == Utilities.getLineOffset(doc, tokenOffset)){	95	}
97	return tokenOffset - Utilities.getRowIndent(doc, tokenOffset) - Utilities.getRowStart(doc, tokenOffset);	96	break;
		97	}
98	}	98	}
99		99
		100
100	return getShiftWidth(); // default;	101	return getShiftWidth(); // default;
101	}	102	}
102		103
Lines 122-128 Link Here
122	int indentsWithinTags[] = new int[lastLine + 1];	123	int indentsWithinTags[] = new int[lastLine + 1];
123		124
124	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();	125	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
125	boolean thereAreMoreTokens = tokenSequence.moveFirst();	126	tokenSequence.moveStart();
		127	boolean thereAreMoreTokens = tokenSequence.moveNext();
		128
126		129
127	if (tokenSequence != null){	130	if (tokenSequence != null){
128	// calc line indents - pass 1	131	// calc line indents - pass 1
Lines 435-448 Link Here
435	tokenSequence.move(offset);	438	tokenSequence.move(offset);
436	int currentOffset;	439	int currentOffset;
437		440
438	do{	441	while (tokenSequence.moveNext()) {
439	currentOffset = tokenSequence.offset();	442	currentOffset = tokenSequence.offset();
440		443
441	if (isClosingTag(tokenHierarchy, currentOffset)){	444	if (isClosingTag(tokenHierarchy, currentOffset)){
442	return currentOffset;	445	return currentOffset;
443	}	446	}
444		447
445	} while (tokenSequence.moveNext());	448	}
446		449
447	return -1;	450	return -1;
448	}	451	}
Lines 459-466 Link Here
459	protected Token getTokenAtOffset(TokenHierarchy tokenHierarchy, int tagTokenOffset){	462	protected Token getTokenAtOffset(TokenHierarchy tokenHierarchy, int tagTokenOffset){
460	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();	463	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
461		464
462	if (tokenSequence != null && tokenSequence.move(tagTokenOffset) != Integer.MAX_VALUE){	465	if (tokenSequence != null) {
463	return tokenSequence.token();	466	tokenSequence.move(tagTokenOffset);
		467	if (tokenSequence.moveNext())
		468	return tokenSequence.token();
464	}	469	}
465		470
466	return null;	471	return null;