Attachment #37742 for bug #90796

This Bugzilla instance is a read-only archive of historic NetBeans bug reports. To report a bug in NetBeans please follow the project's instructions for reporting issues.

View | Details | Raw Unified | Return to bug 90796
Collapse All | Expand All




import org.netbeans.api.editor.settings.FontColorSettings;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.TokenChange;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenHierarchyEvent;
import org.netbeans.api.lexer.TokenHierarchyListener;

        synchronized (this) {
            version++;
        }
        
        TokenChange<? extends TokenId> tc = evt.tokenChange();
        int addedLenght = 0;
        int removedLength = 0;
        
        if (tc.addedTokenCount() > 0) {
            addedLenght = getTokensLength(tc.currentTokenSequence(), tc.index(), tc.addedTokenCount());
        }
        
        if (tc.removedTokenCount() > 0) {
            removedLength = getTokensLength(tc.removedTokenSequence(), 0, tc.removedTokenCount());
        }

        fireHighlightsChange(evt.affectedStartOffset(), evt.affectedEndOffset());
        int changeEnd;
        
        if (addedLenght == 0 && removedLength == 0) {
            // The real length couldn't be computed for some reason
            changeEnd = Integer.MAX_VALUE;
        } else {
            changeEnd = changeStart + Math.max(addedLenght, removedLength);
        }
        
        fireHighlightsChange(changeStart, changeEnd);
    }

    // ----------------------------------------------------------------------
    //  Private implementation
    // ----------------------------------------------------------------------

    private int getTokensLength(TokenSequence<? extends TokenId> seq, int startIdx, int tokenCount) {
        assert startIdx >= 0 && startIdx < seq.tokenCount() : 
            "Invalid startIdx: " + startIdx + ", sequence lenght: " + seq.tokenCount();
        assert tokenCount > 0 && startIdx + tokenCount <= seq.tokenCount() : 
            "Invalid tokenCount: " + tokenCount + ", startIdx: " + startIdx + 
            ", sequence lenght: " + seq.tokenCount();
            
        int startOffset = -1;
        int endOffset = -1;
        
        if (seq.moveIndex(startIdx)) {
            startOffset = seq.offset();
        }
        
        if (seq.moveIndex(startIdx + tokenCount - 1)) {
            endOffset = seq.offset() + seq.token().length();
        }

        return startOffset == -1 || endOffset == -1 ? 0 : endOffset - startOffset;
    }
    
    private final class HSImpl implements HighlightsSequence {
        
        private static final int S_NORMAL = 1;

                    case S_EMBEDDED_HEAD:
                        // The current token contains embedded language and we have processed it's head
                        TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
                        seq.moveStart();
                        if (seq.moveNext()) {
                            state = S_NORMAL;
                        } else {
                            throw new IllegalStateException("Invalid state");

                    // We have moved to the next normal token, so look what it is
                    TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
                    TokenSequence<? extends TokenId> embeddedSeq = seq.embedded();
                    while (embeddedSeq != null && embeddedSeq.moveNext()) {
                        sequences.add(sequences.size(), embeddedSeq);
                        if (embeddedSeq.offset() > seq.offset()) {
                            state = S_EMBEDDED_HEAD;

                    }
                    case S_EMBEDDED_TAIL: {
                        TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
                        seq.moveEnd();
                        if (seq.movePrevious()) {
                            return seq.offset() + seq.token().length();
                        } else {
                            throw new IllegalStateException("Invalid state");

                    }
                    case S_EMBEDDED_HEAD: {
                        TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
                        seq.moveStart();
                        if (seq.moveNext()) {
                            return seq.offset();
                        } else {
                            TokenSequence<? extends TokenId> embeddingSeq = sequences.get(sequences.size() - 2);

            } else {
                if (sequences.size() > 1) {
                    TokenSequence<? extends TokenId> embeddingSeq = sequences.get(sequences.size() - 2);
                    seq.moveEnd();
                    if (seq.movePrevious()) {
                        if ((seq.offset() + seq.token().length()) < (embeddingSeq.offset() + embeddingSeq.token().length())) {
                            return S_EMBEDDED_TAIL;
                        } else {




            if(ts == null) {
                //HTML language is not top level one
                ts = hi.tokenSequence();
                ts.move(offset);
                if (ts.moveNext() || ts.movePrevious()) {
                    return null; //no token found
                } else {
                    ts = ts.embedded(HTMLTokenId.language());
                } else { // no tokens
                    return null;
                }
            }
            

            }
            
            int diff = ts.move(offset);
            if(!ts.moveNext() && !ts.movePrevious()) {
                return null; //no token found
            }
            
            Token item = ts.token();
            
            // are we inside token or between tokens
            boolean inside = ts.offset() < offset;
            
            if(!inside) { //use the previous token
                if(ts.movePrevious()) {

                        //there are some attributes or an end of the tag
                        
                        ts.move(offset);
                        ts.moveNext();
                        Token t = ts.token();
                        
                        //test if next token is a whitespace and the next a tag token or an attribute token

                    if( tag == null ) return null; // unknown tag
                    
                    ts.move(item.offset(hi));
                    ts.moveNext();
                    Token argItem = ts.token();
                    while(argItem.id() != HTMLTokenId.ARGUMENT && ts.movePrevious()) {
                        argItem = ts.token();

Lines 51-56 Link Here

(-)html/editor/lib/src/org/netbeans/editor/ext/html/HTMLLexerFormatter.java (+2 lines)
51	if (position >= 0) {	51	if (position >= 0) {
52	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();	52	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
53	tokenSequence.move(position);	53	tokenSequence.move(position);
		54	tokenSequence.moveNext();
54	Token token = tokenSequence.token();	55	Token token = tokenSequence.token();
55		56
56	if (token.id() == HTMLTokenId.TAG_CLOSE_SYMBOL &&	57	if (token.id() == HTMLTokenId.TAG_CLOSE_SYMBOL &&
Lines 72-77 Link Here
72	@Override protected int getTagEndOffset(TokenHierarchy tokenHierarchy, int tagStartOffset){	73	@Override protected int getTagEndOffset(TokenHierarchy tokenHierarchy, int tagStartOffset){
73	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();	74	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
74	tokenSequence.move(tagStartOffset);	75	tokenSequence.move(tagStartOffset);
		76	tokenSequence.moveNext();
75	boolean thereAreMoreTokens = true;	77	boolean thereAreMoreTokens = true;
76		78
77	while (thereAreMoreTokens && tokenSequence.token().id() != HTMLTokenId.TAG_CLOSE_SYMBOL){	79	while (thereAreMoreTokens && tokenSequence.token().id() != HTMLTokenId.TAG_CLOSE_SYMBOL){




                return null;
            }
            
            ts.move(offset);
            if(!ts.moveNext() && !ts.movePrevious()) {
                return null; //no token found
            }
            

            
            // if the carret is after HTML tag ( after char '>' ), ship inside the tag
            if (token.id() == HTMLTokenId.TAG_CLOSE_SYMBOL) {
                ts.moveIndex(ts.index() - 2);
                if(ts.moveNext()) {
                    token = ts.token();
                }
            }

                                    ts.movePrevious();
                                    start = ts.token().offset(hi);
                                    ts.moveIndex(ts.index() + 2);
                                    ts.moveNext();
                                    Token tok = ts.token();
                                    end = tok.offset(hi)+ (tok.id() == HTMLTokenId.TAG_CLOSE_SYMBOL ? tok.text().length() : 0);
                                    return new int[] {start, end};

            }
            
            ts.move(offset); //reset the token sequence to the original position
            ts.moveNext();
            token = ts.token();
            
            //match html comments

                } while(ts.moveNext());
            }finally{
                ts.moveIndex(tsIndex); //backup the TokenSequence position
                ts.moveNext();
            }
        } else {
            //ts is rewinded out of tokens

                        return COMPLETION_POST_REFRESH;
                    }
                    
                    ts.move(dotPos-1);
                    if(ts.moveNext() || ts.movePrevious()) {
                        if(ts.token().id() == HTMLTokenId.WS) {
                            return COMPLETION_POPUP;
                        }

            //HTML language is not top level one
            ts = hi.tokenSequence();
            int diff = ts.move(offset);
            if(!ts.moveNext() && !ts.movePrevious()) {
                return null; //no token found
            } else {
                ts = ts.embedded(HTMLTokenId.language());




                return null;
            }
            
            ts.move(offset);
            if(!ts.moveNext() && !ts.movePrevious()) return null; //no token found
            
            Token item = ts.token();
            
            int beginning = ts.offset();
            
            if( item.id() == HTMLTokenId.CHARACTER ) {
                do {
                    item = ts.token();
                    beginning = ts.offset();
                } while(item.id() == HTMLTokenId.CHARACTER && ts.movePrevious());
                
                // now item is either HTMLSyntax.VALUE or we're in text, or at BOF

            
            if( item.id() == HTMLTokenId.TEXT ) {
                do {
                    beginning = ts.offset();
                } while ( ts.movePrevious() && (ts.token().id() == HTMLTokenId.TEXT || ts.token().id() == HTMLTokenId.CHARACTER));
                
                return getNextElement(  beginning ); // from start of Commment

            return null;
        }
        
        ts.move(offset);
        if (!ts.moveNext())
        if (diff >= ts.token().length() || diff == java.lang.Integer.MAX_VALUE)
            return null;
        org.netbeans.api.lexer.Token item = ts.token();
        int lastOffset = getTokenEnd(hi, item);

        if(ts == null) {
            //HTML language is not top level one
            ts = hi.tokenSequence();
            ts.move(offset);
            if(!ts.moveNext() && !ts.movePrevious()) {
                return null; //no token found
            } else {
                ts = ts.embedded(HTMLTokenId.language());

Lines 165-172 Link Here

(-)html/editor/src/org/netbeans/modules/editor/html/HTMLCompletionProvider.java (-2 / +3 lines)
165	TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);	165	TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);
166	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();	166	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
167		167
168	int diff = tokenSequence.move(caretOffset == 0 ? 0 : caretOffset - 1);	168	tokenSequence.move(caretOffset == 0 ? 0 : caretOffset - 1);
169	if(diff >= tokenSequence.token().length() \|\| diff == Integer.MAX_VALUE) return; //no token found	169	if (!tokenSequence.moveNext())
		170	return;
170		171
171	Token tokenItem = tokenSequence.token();	172	Token tokenItem = tokenSequence.token();
172	if(tokenItem.id() == HTMLTokenId.TEXT && !tokenItem.text().toString().startsWith("<") && !tokenItem.text().toString().startsWith("&")) {	173	if(tokenItem.id() == HTMLTokenId.TEXT && !tokenItem.text().toString().startsWith("<") && !tokenItem.text().toString().startsWith("&")) {




            int openLtNum = 0;
            JavaTokenId lastNonWhitespaceTokenId = null;
            TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());
            ts.move(expEndPos);
            while (ts.moveNext()) {
            do {
                if (ts.offset() >= offset) {
                    break;
                }

                    default:
                        lastNonWhitespaceTokenId = ts.token().id();
                }
            }
            if (!afterDot) {
                if (expEndPos <= offset)
                    insideExpression(env, new TreePath(path, fa.getExpression()));

        
        private TokenSequence<JavaTokenId> findLastNonWhitespaceToken(Env env, int startPos, int endPos) {
            TokenSequence<JavaTokenId> ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language());
            if (ts.move(endPos) == 0) // When right at the token end
                ts.movePrevious(); // move to previous token
            else
                ts.moveNext(); // otherwise move to the token that "contains" the offset
            do {
                int offset = ts.offset();
                if (offset < startPos)

            if (upToOffset) {
                TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());
                if (ts.moveNext()) { //check for empty file
                    if (ts.move(offset) == 0) // When right at the token end
                        ts.movePrevious(); // Move to previous token
                    else
                        ts.moveNext(); // otherwise move to the token that "contains" the offset
                    if (ts.offset() < offset && (ts.token().id() == JavaTokenId.IDENTIFIER || "keyword".equals(ts.token().id().primaryCategory()))) { //TODO: Use isKeyword(...) when available
                        prefix = ts.token().toString().substring(0, offset - ts.offset());
                        offset = ts.offset();

Lines 98-103 Link Here

(-)java/editor/src/org/netbeans/modules/editor/java/Utilities.java (+2 lines)
98	return false;	98	return false;
99	if (!ts.moveNext() \|\| ts.move(offset) == 0)	99	if (!ts.moveNext() \|\| ts.move(offset) == 0)
100	return true;	100	return true;
		101	ts.moveNext(); // Move to the next token after move(offset)
101	switch(ts.token().id()) {	102	switch(ts.token().id()) {
102	case DOUBLE_LITERAL:	103	case DOUBLE_LITERAL:
103	if (ts.token().text().charAt(0) == '.')	104	if (ts.token().text().charAt(0) == '.')
Lines 127-132 Link Here
127	TokenSequence<? extends TokenId> ts = hierarchy.tokenSequence();	128	TokenSequence<? extends TokenId> ts = hierarchy.tokenSequence();
128	while(ts != null && ts.moveNext()) {	129	while(ts != null && ts.moveNext()) {
129	ts.move(offset);	130	ts.move(offset);
		131	ts.moveNext();
130	if (ts.language() == JavaTokenId.language())	132	if (ts.language() == JavaTokenId.language())
131	return (TokenSequence<JavaTokenId>)ts;	133	return (TokenSequence<JavaTokenId>)ts;
132	ts = ts.embedded();	134	ts = ts.embedded();




        TokenSequence<JavaTokenId> ts = info.getTokenHierarchy().tokenSequence();
        
        ts.move(offset);
        if (ts.moveNext()) {
            Token t = ts.token();

            if (t.id() == JavaTokenId.DOT) {
            ts.moveNext();
            t = ts.token();
        } else {
            if (t.id() == JavaTokenId.LT) {
                ts.moveNext();
                t = ts.token();
            } else {
                if (t.id() == JavaTokenId.LT) {
                    ts.moveNext();
                    t = ts.token();
                }
            }

            if (t.id() == JavaTokenId.IDENTIFIER) {
                return ts.offsetToken();
            }
        }
        
        if (t.id() == JavaTokenId.IDENTIFIER) {
            return ts.offsetToken();
        }
        
        return null;
    }
    

            
            int diff = ts.move((int) getPrefferedPosition(info, d));
            
            if (ts.moveNext() && diff >= 0 && diff < ts.token().length()) {
                Token<JavaTokenId> t = ts.token();
                
                if (t.id() == JavaTokenId.IDENTIFIER) {

Lines 158-164 Link Here

(-)java/source/src/org/netbeans/api/java/source/TreeUtilities.java (-1 / +2 lines)
158	return path;	158	return path;
159		159
160	TokenSequence<JavaTokenId> tokenList = tokensFor(path.getLeaf(), sourcePositions);	160	TokenSequence<JavaTokenId> tokenList = tokensFor(path.getLeaf(), sourcePositions);
161	if (tokenList.moveLast() && tokenList.offset() < pos) {	161	tokenList.moveEnd();
		162	if (tokenList.movePrevious() && tokenList.offset() < pos) {
162	switch (tokenList.token().id()) {	163	switch (tokenList.token().id()) {
163	case GTGTGT:	164	case GTGTGT:
164	case GTGT:	165	case GTGT:




        // skip the section when printing anonymous class
        if (anonClass == false) {
        tokenSequence.move(oldT.pos);
        tokenSequence.moveNext(); // First skip as move() does not position to token directly
        tokenSequence.moveNext();
        insertHint = TokenUtilities.moveNext(tokenSequence, tokenSequence.offset());
        localPointer = diffModifiers(oldT.mods, newT.mods, oldT, localPointer);

            // there was not any parameter in original tree.
            int startOffset = oldT.restype != null ? oldT.restype.getStartPosition() : oldT.getStartPosition();
            
            tokenSequence.move(startOffset);
            TokenUtilities.moveFwdToToken(tokenSequence, startOffset, JavaTokenId.RPAREN);
            posHint = tokenSequence.offset();
        } else {

            // now check, that there is a whitespace. It is not mandatory, we
            // have to ensure. If whitespace is not present, take body beginning
            tokenSequence.move(posHint);
            tokenSequence.moveNext();
            if (tokenSequence.token().id() != JavaTokenId.WHITESPACE) {
                ++posHint;
            }

            ResultItem<JCTree> item = result[j];
            switch (item.operation) {
                case MODIFY: {
                    // perhaps I shouldn't support this!
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

                    assert startOffset != -1 && endOffset != -1 : "Invalid offset!";
                    //printer.print(origText.substring(lastOldPos, startOffset));
                    //append(Diff.delete(startOffset, endOffset));
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

                    break;
                }
                case NOCHANGE: {
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

            return -1;
        }
        tokenSequence.moveIndex(tokenIndex);
        tokenSequence.moveNext();
        return tokenSequence.offset();
    }
    




                int treeEnd = (int) positions.getEndPosition(compilationUnit, item);
                
                seq.move(treeStart);
                seq.moveNext();
                int startIndex = seq.index();
                
                // go back to opening/closing curly, semicolon or other

                seq.moveNext();
                int start = seq.index();
                seq.move(treeEnd);
                // seq.index() set (no seq.moveNext() necessary)
                matrix[i++] = new int[] { start, startIndex, seq.index() };
                if (i == size) {
                    seq.move(treeEnd);
                    // seq.index() set (no seq.moveNext() necessary)
                    matrix[i][0] = seq.index();
                }
            }

            // to decide.
            if (tokenIndex == -1) return -1;
            seq.moveIndex(tokenIndex);
            seq.moveNext();
            return index == 0 ? goAfterLastNewLine(seq) : goAfterFirstNewLine(seq);
        }


        @Override()
        public int[] getPositions(int index) {
            int tokenIndex = matrix[index][0];
            if (tokenIndex != -1) {
                seq.moveIndex(tokenIndex);
                seq.moveNext();
            }
            int begin = goAfterLastNewLine(seq);
            if (matrix[index][2] != -1) {
                seq.moveIndex(matrix[index][2]);
                seq.moveNext();
            }
            int end = goAfterFirstNewLine(seq);
            return new int [] { begin, end };
        }

                if (treeEnd < 0) continue;
                
                seq.move(treeStart);
                seq.moveNext();
                int startIndex = seq.index();
                // go back to opening/closing curly, semicolon or other
                // token java-compiler important token.

            // to decide.
            if (tokenIndex == -1) return -1;
            seq.moveIndex(tokenIndex);
            seq.moveNext();
            return goAfterFirstNewLine(seq);
        }
        

        
        public int[] getPositions(int index) {
            int begin = getInsertPos(index);
            if (matrix[index][4] != -1) {
                seq.moveIndex(matrix[index][4]);
                seq.moveNext();
            }
            int end = goAfterFirstNewLine(seq);
            return new int [] { begin, end };
        }

            // to decide.
            if (tokenIndex == -1) return -1;
            seq.moveIndex(tokenIndex);
            seq.moveNext();
            int off = goAfterFirstNewLine(seq);
            return off;
        }

        
        public int[] getPositions(int index) {
            int begin = getInsertPos(index);
            if (matrix[index][4] != -1) {
                seq.moveIndex(matrix[index][4]);
                seq.moveNext();
            }
            int end = goAfterFirstNewLine(seq);
            return new int [] { begin, end };
        }

    // todo (#pf): remove - used for debugging reasons, doesn't do good job
    public void tablePrint(int[][] matrix, TokenSequence seq) {
        for (int i = 0; i < matrix.length; i++) {
            for (int j = 0; j < matrix[i].length; j++) {
                int item = matrix[i][j];
                String s = "(nothing)";
                if (item > -1) {
                    seq.moveIndex(item);
                    seq.moveNext();
                    s = "'" + seq.token().text();
                }
                s += "'                                           ";
    //            System.err.print(s.substring(0, 25));
                System.err.print(item + "\t");
            }
            System.err.println("");
        }
    }





     */
    static int moveNext(TokenSequence<JavaTokenId> tokenSequence, final int pos) {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        JavaTokenId type = tokenSequence.token().id();
        while (JavaTokenId.WHITESPACE.equals(type) ||
               JavaTokenId.BLOCK_COMMENT.equals(type) ||

            JavaTokenId id)
    {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        while (!id.equals(tokenSequence.token().id())) {
            if (!tokenSequence.moveNext())
                return -1;

            JavaTokenId id)
    {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        while (!id.equals(tokenSequence.token().id())) {
            if (!tokenSequence.movePrevious())
                return -1;

            String text)
    {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        while (!text.equals(tokenSequence.token().text())) {
            if (!tokenSequence.moveNext())
                return -1;

            String text)
    {
        tokenSequence.move(pos);
        tokenSequence.moveNext(); // Assumes the pos is located within input bounds
        while (!text.equals(tokenSequence.token().text())) {
            if (!tokenSequence.movePrevious())
                return -1;




            ResultItem<JCTree> item = result[j];
            switch (item.operation) {
                case MODIFY: {
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

                    int endOffset = toOff(s.tail(j) || s.next(j) ? matrix[i+1][2] : matrix[i][4]);
                    assert startOffset != -1 && endOffset != -1 : "Invalid offset!";
                    append(Diff.delete(startOffset, endOffset));
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();

                    break;
                }
                case NOCHANGE: {
                    tokenSequence.moveIndex(matrix[i][4]);
                    if (tokenSequence.moveNext()) {
                        testPos = tokenSequence.offset();
                        if (JavaTokenId.COMMA == tokenSequence.token().id())
                            testPos += JavaTokenId.COMMA.fixedText().length();




        TokenHierarchy hi = TokenHierarchy.get(document);
        TokenSequence ts = hi.tokenSequence();
        // If necessary move ts to the requested offset
        ts.move(offset);
        while (ts.moveNext()) {
            Token t = ts.token();
            if (t.id() == ...) { ... }
            if (TokenUtilities.equals(t.text(), "mytext")) { ... }
            if (ts.offset() == ...) { ... }
                if (ts.offset() == ...) { ... }

            // Possibly retrieve embedded token sequence
            TokenSequence embedded = ts.embedded();
            if (embedded != null) { // Token has a valid language embedding
                ...
            }
            } while (ts.moveNext());
        }
    } finally {
        document.readUnlock();

Lines 1-4 Link Here

(-)lexer/manifest.mf (-1 / +1 lines)
1	OpenIDE-Module: org.netbeans.modules.lexer/2	1	OpenIDE-Module: org.netbeans.modules.lexer/2
2	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/lexer/Bundle.properties	2	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/lexer/Bundle.properties
3	OpenIDE-Module-Specification-Version: 1.12.0	3	OpenIDE-Module-Specification-Version: 1.13.0
4	OpenIDE-Module-Recommends: org.netbeans.spi.lexer.LanguageProvider	4	OpenIDE-Module-Recommends: org.netbeans.spi.lexer.LanguageProvider




<!-- ACTUAL CHANGES BEGIN HERE: -->

  <changes>
  <change id="api.tokensequence.move.semantics">
      <api name="api"/>
      <summary>Changed TokenSequence.move() and moveIndex() use</summary>
      <version major="1" minor="13"/>
      <date day="16" month="1" year="2007"/>
      <author login="mmetelka"/>
      <compatibility source="incompatible" deletion="yes" addition="yes" modification="yes"/>
      <description>
        <p>
            Changed the <code>TokenSequence.move()</code> to position <b>before</b> the particular token
            that "contains" the offset (or after the last token if the given offset
            is too high). Additional <code>moveNext()</code> is necessary
            to actually move to the next token.
            <br/>
            <code>TokenSequence.moveIndex()</code> was modified in a similar way
            (see javadocs).
            <br/>
            <code>TokenSequence.moveFirst()</code> and <code>moveLast()</code>
            were replaced by <code>moveStart()</code> that positionins before
            the first token and by <code>moveEnd()</code> that positions
            after the last token.
            <br/>
            <code>TokenSequence.isEmpty()</code> added to check whether there are
            no tokens in the TS.
        </p>
      </description>
  </change>

  <change id="api.lexer.release">
      <api name="api"/>
      <summary>Added Lexer.release()</summary>




        if (active) {
            pastSequences = new Stack<TokenSequence>();
            tokenSequence = hi.tokenSequence();
            tokenSequence.move(startOffset);
            if (tokenSequence.moveNext()) {
                updateTokenEndOffsetAndColoring(startOffset);
            } else { // no tokens
                active = false;

                        break;
                    }
                } else {
                    embed.move(offset);
                    if (embed.moveNext()) {
                        pastSequences.push(tokenSequence);
                        tokenSequence = embed;
                    } else {




        assertEquals("Wrong language", "text/x-simple-plain", lang.mimeType());
        
        for(int i = 0; i < seq.tokenCount(); i++) {
            seq.moveIndex(i);
            assertTrue(seq.moveNext());
            Token token = seq.token();
            
            if (token.id() == SimplePlainTokenId.WORD) {

                assertNotNull("Can't find language of the embedded sequence", embeddedLang);
                assertEquals("Wrong language of the embedded sequence", "text/x-simple-char", embeddedLang.mimeType());
                
                embeddedSeq.moveStart();
                assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveNext());
                assertEquals("Wrong startSkipLength", 1, embeddedSeq.offset() - seq.offset());
                
                embeddedSeq.moveEnd();
                assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.movePrevious());
                assertEquals("Wrong endSkipLength", 2, 
                    (seq.offset() + seq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length()));
            }
        }

Lines 149-154 Link Here

(-)lexer/src/org/netbeans/api/lexer/Token.java (+3 lines)
149	* Get the offset at which this token is present in the input	149	* Get the offset at which this token is present in the input
150	* or <code>-1</code> if this token is flyweight (and therefore does not store offset).	150	* or <code>-1</code> if this token is flyweight (and therefore does not store offset).
151	* <br/>	151	* <br/>
		152	* <b>Note:</b> Use of {@link TokenSequence#offset()} is usually preferred over
		153	* this method.
		154	* <br/>
152	* For flyweight tokens the real offset of the token may only be determined	155	* For flyweight tokens the real offset of the token may only be determined
153	* by doing {@link TokenSequence#offset()} when positioned on the particular	156	* by doing {@link TokenSequence#offset()} when positioned on the particular
154	* flyweight token.	157	* flyweight token.




import org.netbeans.lib.lexer.token.AbstractToken;

/**
 * Token sequence allows to iterate between tokens
 * of a token hierarchy.
 * and by index/offset positioning.
 * <br/>
 * Token sequence for top-level language of a token hierarchy
 * may be obtained by {@link TokenHierarchy#tokenSequence()}.
 * 
 * <p>
 * Use of token sequence is a two-step operation:
 * <ol>
 *   <li>
 *     Position token sequence before token that should first be retrieved
 *     (or behind desired token when iterating backwards).
 *     <br/>
 *     One of the following ways may be used:
 *     <ul>
 *       <li> {@link #move(int)} positions TS before token that either starts
 *           at the given offset or "contains" it.
 *       </li>
 *       <li> {@link #moveIndex(int)} positions TS before n-th token in the underlying
 *           token list.
 *       </li>
 *       <li> {@link #moveStart()} positions TS before the first token. </li>
 *       <li> {@link #moveEnd()} positions TS behind the last token. </li>
 *       <li> Do nothing - TS is positioned before the first token automatically by default. </li>
 *     </ul>
 *     Token sequence will always be positioned between tokens
 *     when using one of the operations above
 *     ({@link #token()} will return <code>null</code> to signal between-tokens location).
 *     <br/>
 *   </li>
 * 
 *   <li>
 *     Start iterating through the tokens in forward/backward direction
 *     by using {@link #moveNext()} or {@link #movePrevious()}.
 *     <br/>
 *     If <code>moveNext()</code> or <code>movePrevious()</code> returned
 *     <code>true</code> then TS is positioned
 *     over a concrete token retrievable by {@link #token()}.
 *     <br/>
 *     Its offset can be retrieved by {@link #offset()}.
 *   </li>
 * </ol>
 * </p>
 * 
 * <p>
 * An example of forward iteration through the tokens:
 * <pre>
 *   TokenSequence ts = tokenHierarchy.tokenSequence();
 *   // Possible positioning by ts.move(offset) or ts.moveIndex(index)
 *   while (ts.moveNext()) {
 *       Token t = ts.token();
 *       if (t.id() == ...) { ... }

 *       if (ts.offset() == ...) { ... }
 *   }
 * </pre>
 * </p>
 * Token sequence provides correct offset information
 * for the token to which the sequence is positioned
 * (some tokens may be flyweight and do not hold the offset by themselves).
 *
 * <p>
 * This class should be used by a single thread only.

    
    private AbstractToken<T> token; // 16 bytes
    
    private int tokenIndex; // 20 bytes
    
    /**
     * Offset in the input at which the current token is located

    }

    /**
     * Get token to which this token sequence points to or null
     * if TS is positioned between tokens
     * ({@link #moveNext()} or {@link #movePrevious()} were not called yet).
     * <br/>
     * A typical iteration usage:
     * <pre>
     *   TokenSequence ts = tokenHierarchy.tokenSequence();
     *   // Possible positioning by ts.move(offset) or ts.moveIndex(index)
     *   while (ts.moveNext()) {
     *       Token t = ts.token();
     *       if (t.id() == ...) { ... }
     *       if (TokenUtilities.equals(t.text(), "mytext")) { ... }
     *       if (ts.offset() == ...) { ... }
     *   }
     * </pre>
     *
     * <p>
     * The returned token instance may be flyweight
     * ({@link Token#isFlyweight()} returns true)
     * which means that its {@link Token#offset(TokenHierarchy)} will return -1.
     * <br/>
     * To find a correct offset use {@link #offset()}.
     * <br/>
     * Or if its necessary to revert to a regular non-flyweigt token
     * the {@link #offsetToken()} may be used.
     * </p>
     *
     * <p>

     * The token instance should not be held across the input source modifications.
     * </p>
     *
     * @return token instance to which this token sequence is currently positioned
     *  or null if this token sequence is not positioned to any token which may
     *  happen after TS creation or after use of {@link #move(int)} or {@link moveIndex(int)}.
     * 
     * @see #offsetToken()
     * @throws IllegalStateException if this token sequence was not positioned
     *  to any token yet.
     */
    public Token<T> token() {
        checkToken();
        return token;
    }
    

     * <br/>
     * If the current token is flyweight then this method replaces it
     * with the corresponding non-flyweight token which it then returns.
     * <br/>
     * Subsequent calls to {@link #token()} will also return this non-flyweight token.
     *
     * <p>
     * This method may be handy if the token instance is referenced in a standalone way

     * to get the appropriate offset from the token itself
     * later when a token sequence will not be available.
     * </p>
     * @throws IllegalStateException if {@link #token()} returns null.
     *  to any token yet.
     */
    public Token<T> offsetToken() {
        checkTokenNotNull();
        if (token.isFlyweight()) {
            token = tokenList.replaceFlyToken(tokenIndex, token, offset());
        }

     * best performance with a constant time complexity.
     *
     * @return &gt;=0 absolute offset of the current token in the underlying input.
     * @throws IllegalStateException if {@link #token()} returns null.
     *  to any token yet.
     */
    public int offset() {
        checkTokenNotNull();
        if (tokenOffset == -1) {
            tokenOffset = tokenList.tokenOffset(tokenIndex);
        }

    }
    
    /**
     * Get an index of token to which (or before which) this TS is currently positioned.
     * <br/>
     * <p>
     * Initially or after {@link #move(int)} or {@link #moveIndex(int)}
     * token sequence is positioned between tokens:
     * <pre>
     *          Token[0]   Token[1]   ...   Token[n]
     *        ^          ^                ^
     * Index: 0          1                n
     * </pre>
     * </p>
     * 
     * <p>
     * After use of {@link #moveNext()} or {@link #movePrevious()}
     * the token sequence is positioned over one of the actual tokens:
     * <pre>
     *          Token[0]   Token[1]   ...   Token[n]
     *             ^          ^                ^
     * Index:      0          1                n
     * </pre>
     * </p>
     * 
     * @return &gt;=0 index of token to which (or before which) this TS is currently positioned.
     */
    public int index() {
        return tokenIndex;

     * or <code>LanguageProvider</code>).
     *
     * @return embedded sequence or null if no embedding exists for this token.
     * @throws IllegalStateException if {@link #token()} returns null.
     *  to any token yet.
     */
    public TokenSequence<? extends TokenId> embedded() {
        checkTokenNotNull();
        return embeddedImpl(null);
    }
    

     * Get embedded token sequence if the token
     * to which this token sequence is currently positioned
     * has a language embedding.
     * 
     * @throws IllegalStateException if {@link #token()} returns null.
     */
    public <ET extends TokenId> TokenSequence<ET> embedded(Language<ET> embeddedLanguage) {
        checkTokenNotNull();
        return embeddedImpl(embeddedLanguage);
    }

    /**
     * Create language embedding without joining of the embedded sections.
     *
     * @throws IllegalStateException if {@link #token()} returns null.
     * @see #createEmbedding(Language, int, int, boolean)
     */
    public boolean createEmbedding(Language<? extends TokenId> embeddedLanguage,

     *  Only the embedded sections with the same language path can be joined.
     * @return true if the embedding was created successfully or false if an embedding
     *  with the given language already exists for this token.
     * @throws IllegalStateException if {@link #token()} returns null.
     */
    public boolean createEmbedding(Language<? extends TokenId> embeddedLanguage,
    int startSkipLength, int endSkipLength, boolean joinSections) {
        checkTokenNotNull();
        return EmbeddingContainer.createEmbedding(tokenList, tokenIndex,
                embeddedLanguage, startSkipLength, endSkipLength, joinSections);
    }

    /**
     * Move to the next token in this token sequence.
     * 
     * <p>
     * The next token may not necessarily start at the offset where
     * the previous token ends (there may be gaps between tokens
     * caused by token filtering). {@link #offset()} should be used
     * for offset retrieval.
     * </p>
     *
     * @return true if the sequence was successfully moved to the next token
     *  or false if it was not moved before there are no more tokens
     *  in the forward direction.
     * @throws ConcurrentModificationException if this token sequence
     *  is no longer valid because of an underlying mutable input source modification.
     */
    public boolean moveNext() {
        checkModCount();
        if (token != null) // Token already fetched
            tokenIndex++;
        Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(tokenIndex);
        if (tokenOrEmbeddingContainer != null) {
            AbstractToken origToken = token;
            token = LexerUtilsConstants.token(tokenOrEmbeddingContainer);
            // If origToken == null then the right offset might already be pre-computed from move()
            if (tokenOffset != -1) {
                if (origToken != null) {
                    // If the token list is continuous or the fetched token
                    // is flyweight (there cannot be a gap before flyweight token)
                    // the original offset can be just increased
                    // by the original token's length.
                    if (tokenList.isContinuous() || token.isFlyweight()) {
                        tokenOffset += origToken.length(); // advance by previous token's length
                    } else // Offset must be recomputed
                        tokenOffset = -1; // mark the offset to be recomputed
                } else // Not valid token previously
                    tokenOffset = -1;
            }
            return true;
        }
        if (token != null) // Unsuccessful move from existing token
            tokenIndex--;
        return false;
    }

    /**
     * Move to a previous token in this token sequence.
     *
     * <p>
     * The previous token may not necessarily end at the offset where
     * the previous token started (there may be gaps between tokens
     * caused by token filtering). {@link #offset()} should be used
     * for offset retrieval.
     * </p>
     *
     * @return true if the sequence was successfully moved to the previous token
     *  or false if it was not moved because there are no more tokens
     *  in the backward direction.
     * @throws ConcurrentModificationException if this token sequence
     *  is no longer valid because of an underlying mutable input source modification.

        if (tokenIndex > 0) {
            AbstractToken origToken = token;
            tokenIndex--;
            token = LexerUtilsConstants.token(tokenList.tokenOrEmbeddingContainer(tokenIndex));
            if (tokenOffset != -1) {
                // If the token list is continuous or the original token
                // is flyweight (there cannot be a gap before flyweight token)

                } else { // mark the offset to be computed upon call to offset()
                    tokenOffset = -1;
                }
            } else {
                tokenOffset = -1; // mark the offset to be computed upon call to offset()
            }
            return true;


    }

    /**
     * Position token sequence between <code>index-1</code>
     * and <code>index</code> tokens.
     * <br/>
     * TS will be positioned in the following way:
     * <pre>
     *          Token[0]   ...   Token[index-1]   Token[index] ...
     *        ^                ^                ^
     * Index: 0             index-1           index
     * </pre>
     * 
     * <p>
     * Subsequent {@link #moveNext()} or {@link #movePrevious()} is needed to fetch
     * a concrete token in the desired direction.
     * <br/>
     * Subsequent {@link #moveNext()} will position TS over <code>Token[index]</code>
     * (or {@link #movePrevious()} will position TS over <code>Token[index-1]</code>)
     * so that <code>{@link #token()} != null</code>.
     *
     * @param index index of the token to which this sequence
     *   should be positioned.
     *   <br/>
     *   If <code>index >= {@link #tokenCount()}</code>
     *   then the TS will be positioned to {@link #tokenCount()}.
     *   <br/>
     *   If <code>index < 0</code> then the TS will be positioned to index 0.
     * 
     * @return difference between requested index and the index to which TS
     *   is really set.
     * @throws ConcurrentModificationException if this token sequence
     *  is no longer valid because of an underlying mutable input source modification.
     */
    public int moveIndex(int index) {
        checkModCount();
        if (index >= 0) {
            Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index);
            if (tokenOrEmbeddingContainer != null) { // enough tokens
                resetTokenIndex(index);
            } else // Token at the requested index does not exist - leave orig. index
                resetTokenIndex(tokenCount());
        } else // index < 0
            resetTokenIndex(0);
        return index - tokenIndex;

        } else // Token at the requested index does not exist - leave orig. index
            return false;
    }
    
    /**
     * Move the token sequence to be positioned before the first token.
     * <br/>
     * This is equivalent to <code>moveIndex(0)</code>.
     */
    public void moveStart() {
        moveIndex(0);
    }
    
    /**
     * Move the token sequence to be positioned behind the last token.
     * <br/>
     * This is equivalent to <code>moveIndex(tokenCount())</code>.
     */
    public void moveEnd() {
        moveIndex(tokenCount());
    }
    
    /**
     * Move token sequence to be positioned between <code>index-1</code>
     * and <code>index</code> tokens where Token[index] either starts at offset
     * or "contains" the offset.
     * <br/>
     * <pre>
     *        +----------+-----+----------------+--------------+------
     *        | Token[0] | ... | Token[index-1] | Token[index] | ...
     *        | "public" | ... | "static"       | "int"        | ...
     *        +----------+-----+----------------+--------------+------
     *        ^                ^                ^
     * Index: 0             index-1           index
     * Offset:                                  ---^ (if offset points to 'i','n' or 't')
     *       // Token sequence is not positioned to any token.
     *
     *   } else {
     *       // 1. diff >= tokenSequence.token().length()
     *       //   a) targetOffset is above the end of the last token in the sequence.
     *       //     Token sequence is positioned to the last token in the sequence.
     *       //   b) there are text areas not covered by any tokens
     *       //     due to skipped tokens (skipTokenIds was used
     *       //     in TokenHierarchy.create()) and the targetOffset points to such gap.
     *       //     Token sequence is positioned to the preceding token.
     *       // 
     *       // 2. diff < 0
     *       //   a) targetOffset < 0
     *       //   b) targetOffset >= 0 but there is a text area
     *       //     at the begining that is not covered by any tokens
     *       //     (skipTokenIds was used in TokenHierarchy.create())
     *       //     Token sequence is positioned to the first token in the sequence.
     *   }
     * </pre>
     * 
     * <p>
     * Subsequent {@link #moveNext()} or {@link #movePrevious()} is needed to fetch
     * a concrete token.
     * <br/>
     * If the offset is too big then the token sequence will be positioned
     * behind the last token.
     * </p>
     * 
     * <p>
     * If token filtering is used there may be gaps that are not covered
     * by any tokens and if the offset is contained in such gap then
     * the token sequence will be positioned before the token that follows the gap.
     * </p>
     *
     * @param offset absolute offset in the input to which
     *  the token sequence should be moved.
     *
     * @param offset absolute offset to which the token sequence should be moved.
     * @return difference between the reqeuested offset
     *  and the start offset of the token
     *  before which the the token sequence gets positioned.
     *  <br>
     *  Returns {@link Integer#MAX_VALUE} if there are no tokens in the sequence.
     *  In such case there is no active token.
     * 
     * @throws ConcurrentModificationException if this token sequence
     *  is no longer valid because of an underlying mutable input source modification.

            if (tokenList.tokenOrEmbeddingContainer(0) == null) { // really no tokens at all
                // In this case the token sequence could not be positioned yet
                // so no need to reset "index" or other vars
                resetTokenIndex(0);
                return offset;
            }
            // Re-get the present token count (could be created a chunk of tokens at once)
            tokenCount = tokenList.tokenCountCurrent();

                    tokenLength = t.length();
                    tokenCount++;

                } else { // no more tokens => position behind last token
                    resetTokenIndex(tokenCount);
                    tokenOffset = prevTokenOffset + tokenLength; // May assign the token's offset in advance
                    return offset - tokenOffset;
                }
            }
            resetTokenIndex(tokenCount - 1);
            tokenOffset = prevTokenOffset; // May assign the token's offset in advance
            tokenOffset = prevTokenOffset;
            assignToken();
            return offset - prevTokenOffset;
        }
        

                high = mid - 1;
            } else {
                // Token starting exactly at offset found
                resetTokenIndex(mid);
                tokenOffset = midStartOffset;
                assignToken();
                return 0; // right at the token begining
            }
        }
        
        // Not found exactly and high + 1 == low => high < low
        // BTW there may be gaps between tokens; if offset is in gap then position to higher token
        if (high >= 0) { // could be -1
            AbstractToken t = LexerUtilsConstants.token(tokenList, high);
            prevTokenOffset = tokenList.tokenOffset(high);
            // If gaps allowed check whether the token at "high" contains the offset
            if (!tokenList.isContinuous() && offset > prevTokenOffset + t.length()) {
                // Offset in the gap above the "high" token
                high++;
                prevTokenOffset += t.length();
            }
        } else { // at least one token exists => use token at index 0
            high = 0;
            prevTokenOffset = tokenList.tokenOffset(0); // result may differ from 0
        }
        resetTokenIndex(high);
        tokenIndex = high;
        tokenOffset = prevTokenOffset;
        assignToken();
        return offset - prevTokenOffset;
    }

    /**
     * Move to the first token in this token sequence.
     *
     * @return true if the sequence was positioned on the first token
     *  or false if there are no tokens in the sequence.
     */
    public boolean moveFirst() {
        return moveIndex(0);
    }
    
    /**
     * Check whether this TS contains zero tokens.
     * <br/>
     * This check is strongly preferred over <code>tokenCount() == 0</code>.
     * 
     * @see #tokenCount()
     */
    public boolean isEmpty() {
        return (tokenIndex == 0 && tokenList.tokenOrEmbeddingContainer(0) == null);
    }

    /**

        return parentTokenIndexes;
    }

    private void resetTokenIndex(int index) {
        // Position to the given index e.g. by move() and moveIndex()
        tokenIndex = index;
        token = null;
        tokenOffset = -1;
        assignToken(tokenList.tokenOrEmbeddingContainer(tokenIndex));
    }

    private void checkTokenNotNull() {
        if (token == null) {
            throw new IllegalStateException(
                "No token fetched by moveNext() from token sequence yet: index=" + tokenIndex




            sb.append(": ");
            AbstractToken token = token(tokenOrEmbeddingContainer);
            sb.append(token.dumpInfo(tokenHierarchy));
            int la = tokenList.lookahead(i);
            if (la != 0) {
                sb.append(", la=");
                sb.append(la);
            }
            Object state= tokenList.state(i);
            if (state != null) {
                sb.append(", s=");
                sb.append(state);
            }
            sb.append('\n');
        }
        return sb;

Lines 92-98 Link Here

(-)lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java (-1 / +1 lines)
92	if (limitStartOffset > 0) {	92	if (limitStartOffset > 0) {
93	int diff = move(limitStartOffset);	93	int diff = move(limitStartOffset);
94	if (diff != Integer.MAX_VALUE) { // some tokens exist	94	if (diff != Integer.MAX_VALUE) { // some tokens exist
95	if (diff >= lastToken.length()) {	95	if (diff >= lastToken.length()) { // lastToken initialized in move()
96	lastTokenIndex++;	96	lastTokenIndex++;
97	Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(lastTokenIndex);	97	Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(lastTokenIndex);
98	if (tokenOrEmbeddingContainer != null &&	98	if (tokenOrEmbeddingContainer != null &&

Lines 48-60 Link Here

(-)lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java (-41 / +129 lines)
48	TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());	48	TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());
49	TokenSequence<? extends TokenId> ts = hi.tokenSequence();	49	TokenSequence<? extends TokenId> ts = hi.tokenSequence();
50		50
51	// Fail if no "move*" method called yet	51	assertNull(ts.token());
52	try {
53	ts.token(); // should throw exception
54	fail("IllegalStateException expecxted");
55	} catch (IllegalStateException e) {
56	// expected
57	}
58		52
59	assertTrue(ts.moveNext());	53	assertTrue(ts.moveNext());
60	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	54	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
Lines 69-88 Link Here
69	assertEquals(ts.tokenCount(), 3);	63	assertEquals(ts.tokenCount(), 3);
70		64
71	assertEquals(0, ts.move(0));	65	assertEquals(0, ts.move(0));
		66	assertNull(ts.token());
		67	assertTrue(ts.moveNext());
72	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	68	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
73	assertEquals(2, ts.move(2));	69	assertEquals(2, ts.move(2));
		70	assertNull(ts.token());
		71	assertTrue(ts.moveNext());
74	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	72	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
75	assertEquals(-1, ts.move(-1));	73	assertEquals(-1, ts.move(-1));
		74	assertNull(ts.token());
		75	assertTrue(ts.moveNext());
76	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	76	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
77	assertEquals(0, ts.move(3));	77	assertEquals(0, ts.move(3));
		78	assertNull(ts.token());
		79	assertTrue(ts.moveNext());
78	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);	80	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
79	assertEquals(0, ts.move(4));	81	assertEquals(0, ts.move(4));
		82	assertNull(ts.token());
		83	assertTrue(ts.moveNext());
80	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);	84	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
81	assertEquals(1, ts.move(5));	85	assertEquals(1, ts.move(5));
		86	assertNull(ts.token());
		87	assertTrue(ts.moveNext());
82	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);	88	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
83	assertEquals(5, ts.move(9));	89	assertEquals(1, ts.move(9));
		90	assertNull(ts.token());
		91	assertFalse(ts.moveNext());
		92	assertTrue(ts.movePrevious());
84	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);	93	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
85	assertEquals(96, ts.move(100));	94	assertEquals(92, ts.move(100));
		95	assertNull(ts.token());
		96	assertFalse(ts.moveNext());
		97	assertTrue(ts.movePrevious());
86	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);	98	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
87		99
88		100
Lines 107-112 Link Here
107	sub = ts.subSequence(3, 3);	119	sub = ts.subSequence(3, 3);
108	assertFalse(sub.moveNext());	120	assertFalse(sub.moveNext());
109	}	121	}
		122
		123	public void testMoveNextPrevious() {
		124	String text = "abc+defg-";
		125	TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());
		126	TokenSequence<? extends TokenId> ts = hi.tokenSequence();
		127	assertEquals(0, ts.move(4));
		128	assertNull(ts.token());
		129	//assertEquals(4, ts.offset());
		130	assertTrue(ts.movePrevious());
		131	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
		132
		133	ts = hi.tokenSequence();
		134	ts.move(5);
		135	assertTrue(ts.moveNext());
		136	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
		137	assertTrue(ts.movePrevious());
		138	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
		139
		140	// Move single char before token's end
		141	ts = hi.tokenSequence();
		142	ts.move(7);
		143	assertTrue(ts.moveNext());
		144	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
		145	assertTrue(ts.movePrevious());
		146	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
		147
		148	// Move past all tokens
		149	ts = hi.tokenSequence();
		150	ts.move(text.length());
		151	assertTrue(ts.movePrevious());
		152	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", text.length() - 1);
		153
		154	// Move at the begining
		155	ts = hi.tokenSequence();
		156	ts.move(0);
		157	assertFalse(ts.movePrevious());
		158	assertTrue(ts.moveNext());
		159	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
		160	}
		161
		162	public void testMoveIndex() {
		163	String text = "abc+defg-";
		164	TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());
		165	TokenSequence<? extends TokenId> ts = hi.tokenSequence();
		166	assertEquals(0, ts.index());
		167	assertNull(ts.token());
		168	//assertEquals(4, ts.offset());
		169	assertTrue(ts.moveNext());
		170	assertEquals(0, ts.offset());
		171	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
		172	assertTrue(ts.moveNext());
		173	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
		174	assertEquals(3, ts.offset());
		175	ts.move(2);
		176	assertEquals(0, ts.index());
		177	assertTrue(ts.moveNext());
		178	assertEquals(0, ts.offset());
		179	assertEquals(0, ts.index());
		180	ts.moveIndex(1);
		181	assertEquals(1, ts.index());
		182	assertTrue(ts.moveNext());
		183	assertEquals(1, ts.index());
		184	assertEquals(3, ts.offset());
		185	}
110		186
111	public void testMoveSkipTokens() {	187	public void testMoveSkipTokens() {
112	String text = "-abc+defg--hi";	188	String text = "-abc+defg--hi";
Lines 122-133 Link Here
122	TokenSequence<? extends TokenId> ts = hi.tokenSequence();	198	TokenSequence<? extends TokenId> ts = hi.tokenSequence();
123		199
124	// Fail if no "move*" method called yet	200	// Fail if no "move*" method called yet
125	try {	201	assertNull(ts.token());
126	ts.token(); // should throw exception
127	fail("IllegalStateException expecxted");
128	} catch (IllegalStateException e) {
129	// expected
130	}
131		202
132	assertTrue(ts.moveNext());	203	assertTrue(ts.moveNext());
133	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	204	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
Lines 137-146 Link Here
137	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	208	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
138	assertFalse(ts.moveNext());	209	assertFalse(ts.moveNext());
139	assertEquals(3, ts.tokenCount());	210	assertEquals(3, ts.tokenCount());
140	assertTrue(ts.moveFirst());	211	assertEquals(0, ts.moveIndex(0));
		212	assertTrue(ts.moveNext());
141	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	213	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
142	assertTrue(ts.moveNext());	214	assertTrue(ts.moveNext());
143	assertTrue(ts.moveLast());	215	assertEquals(0, ts.moveIndex(ts.tokenCount()));
		216	assertTrue(ts.movePrevious());
144	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	217	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
145	assertFalse(ts.moveNext());	218	assertFalse(ts.moveNext());
146		219
Lines 151-166 Link Here
151	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	224	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
152		225
153	assertEquals(-1, ts.move(0)); // below filtered-out token	226	assertEquals(-1, ts.move(0)); // below filtered-out token
		227	assertNull(ts.token());
		228	assertTrue(ts.moveNext());
154	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	229	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
155	assertEquals(0, ts.move(1));	230	assertEquals(0, ts.move(1));
		231	assertNull(ts.token());
		232	assertTrue(ts.moveNext());
156	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	233	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
157	assertEquals(1, ts.move(2));	234	assertEquals(1, ts.move(2));
		235	assertNull(ts.token());
		236	assertTrue(ts.moveNext());
158	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	237	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
159	assertEquals(3, ts.move(4));	238	assertEquals(3, ts.move(4));
		239	assertNull(ts.token());
		240	assertTrue(ts.moveNext());
160	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);	241	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
161	assertEquals(0, ts.move(5));	242	assertEquals(0, ts.move(5));
		243	assertNull(ts.token());
		244	assertTrue(ts.moveNext());
162	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 5);	245	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 5);
163	assertEquals(1, ts.move(12));	246	assertEquals(1, ts.move(12));
		247	assertNull(ts.token());
		248	assertTrue(ts.moveNext());
164	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	249	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
165		250
166		251
Lines 172-219 Link Here
172	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);	257	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
173	assertFalse(sub.moveNext());	258	assertFalse(sub.moveNext());
174		259
175	// Test moveFirst() and moveLast() on subsequence	260	// Test moves to first and last tokens on subsequence
176	sub = ts.subSequence(1, 6);	261	sub = ts.subSequence(1, 6);
177	assertTrue(sub.moveLast());	262	assertEquals(0, sub.moveIndex(sub.tokenCount()));
		263	assertEquals(2, sub.tokenCount()); // "abc" and "defg" (others filtered out
		264	assertTrue(sub.movePrevious());
178	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);	265	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
179	assertTrue(sub.moveFirst());	266	assertEquals(0, sub.moveIndex(0));
		267	assertTrue(sub.moveNext());
180	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "abc", 1);	268	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "abc", 1);
181		269
182	sub = ts.subSequence(4);	270	sub = ts.subSequence(4);
183	assertTrue(sub.moveNext());	271	assertTrue(sub.moveNext());
184	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);	272	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
185	assertTrue(sub.moveNext());	273	assertTrue(sub.moveNext());
186	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	274	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
187	assertFalse(sub.moveNext());	275	assertFalse(sub.moveNext());
188		276
189	sub = ts.subSequence(12, 15);	277	sub = ts.subSequence(12, 15);
190	assertTrue(sub.moveNext());	278	assertTrue(sub.moveNext());
191	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	279	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
192		280
193	sub = ts.subSequence(12, 15);	281	sub = ts.subSequence(12, 15);
194	assertEquals(-2, sub.move(9));	282	assertEquals(-2, sub.move(9));
195	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);	283	assertNull(sub.token());
		284	assertTrue(sub.moveNext());
		285	LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
196		286
197	sub = ts.subSequence(13, 15);	287	sub = ts.subSequence(13, 15);
198	assertFalse(sub.moveNext());	288	assertFalse(sub.moveNext());
199	assertFalse(sub.moveFirst());	289	assertEquals(0, sub.moveIndex(0));
200	assertFalse(sub.moveLast());	290	assertNull(sub.token());
		291	assertFalse(sub.moveNext());
		292	assertEquals(0, sub.moveIndex(sub.tokenCount()));
		293	assertNull(sub.token());
201	assertEquals(0, sub.tokenCount());	294	assertEquals(0, sub.tokenCount());
202		295
203	sub = ts.subSequence(-3, 1);	296	sub = ts.subSequence(-3, 1);
204	assertFalse(sub.moveNext());	297	assertFalse(sub.moveNext());
205		298
206	sub = ts.subSequence(13, 15);	299	sub = ts.subSequence(13, 15);
207	assertEquals(Integer.MAX_VALUE, sub.move(5));	300	assertEquals(5, sub.move(5));
208		301
209	sub = ts.subSequence(-3, 1);	302	sub = ts.subSequence(-3, 1);
210	assertEquals(Integer.MAX_VALUE, sub.move(1));	303	assertEquals(1, sub.move(1));
211		304
212	// Reversed bounds => should be empty token sequence	305	// Reversed bounds => should be empty token sequence
213	sub = ts.subSequence(6, 1);	306	sub = ts.subSequence(6, 1);
214	assertFalse(sub.moveNext());	307	assertFalse(sub.moveNext());
215	sub = ts.subSequence(6, 1);	308	sub = ts.subSequence(6, 1);
216	assertEquals(Integer.MAX_VALUE, sub.move(6));	309	assertEquals(6, sub.move(6));
217	}	310	}
218		311
219	public void DtestMoveSkipTokens2() throws IOException {	312	public void DtestMoveSkipTokens2() throws IOException {
Lines 244-275 Link Here
244		337
245	// Expect no tokens	338	// Expect no tokens
246	assertFalse(ts.moveNext());	339	assertFalse(ts.moveNext());
247	// Exception if ts.token() called	340	assertNull(ts.token());
248	try {
249	ts.token(); // should throw exception
250	fail("IllegalStateException expecxted");
251	} catch (IllegalStateException e) {
252	// expected
253	}
254		341
255	assertEquals(ts.move(0), Integer.MAX_VALUE);	342	assertEquals(0, ts.move(0));
256	assertEquals(ts.move(10), Integer.MAX_VALUE);	343	assertEquals(10, ts.move(10));
257		344
258	// Test subsequences	345	// Test subsequences
259	TokenSequence<? extends TokenId> sub = ts.subSequence(1, 6);	346	TokenSequence<? extends TokenId> sub = ts.subSequence(1, 6);
260	assertFalse(sub.moveNext());	347	assertFalse(sub.moveNext());
261	sub = ts.subSequence(1, 6);	348	sub = ts.subSequence(1, 6);
262	assertEquals(Integer.MAX_VALUE, sub.move(1));	349	assertEquals(1, sub.move(1));
263		350
264	sub = ts.subSequence(0);	351	sub = ts.subSequence(0);
265	assertFalse(sub.moveNext());	352	assertFalse(sub.moveNext());
266	sub = ts.subSequence(0);	353	sub = ts.subSequence(0);
267	assertEquals(Integer.MAX_VALUE, sub.move(0));	354	assertEquals(0, sub.move(0));
268		355
269	sub = ts.subSequence(1);	356	sub = ts.subSequence(1);
270	assertFalse(sub.moveNext());	357	assertFalse(sub.moveNext());
271	sub = ts.subSequence(1);	358	sub = ts.subSequence(1);
272	assertEquals(Integer.MAX_VALUE, sub.move(0));	359	assertEquals(0, sub.move(0));
273	}	360	}
274		361
275	public void testTokenSize() {	362	public void testTokenSize() {
Lines 284-290 Link Here
284	assertFalse(ts.moveNext());	371	assertFalse(ts.moveNext());
285		372
286	TokenList tokenList = LexerTestUtilities.tokenList(ts);	373	TokenList tokenList = LexerTestUtilities.tokenList(ts);
287	ts.moveIndex(0); // move to "abc"	374	ts.moveIndex(0); // move before "abc"
		375	assertTrue(ts.moveNext());
288	// Test DefaultToken size	376	// Test DefaultToken size
289	assertSame(DefaultToken.class, ts.token().getClass());	377	assertSame(DefaultToken.class, ts.token().getClass());
290	assertSize("Token instance too big", Collections.singletonList(ts.token()), 24,	378	assertSize("Token instance too big", Collections.singletonList(ts.token()), 24,




    public void testCachingE() {
        TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
        TokenSequence tokens = th.tokenSequence();
        tokens.moveStart();
        assertEquals(true, tokens.moveNext());
        
        TokenSequence embeddedA = tokens.embedded();
        assertNotNull("There should be an embedded language", embeddedA);

    public void testGCedE() {
        TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
        TokenSequence tokens = th.tokenSequence();
        tokens.moveStart();
        assertEquals(true, tokens.moveNext());
        
        TokenSequence embedded = tokens.embedded();
        assertNotNull("There should be an embedded language", embedded);

    public void testCacheRefreshedE() {
        TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
        TokenSequence tokens = th.tokenSequence();
        tokens.moveStart();
        assertEquals(true, tokens.moveNext());
        
        TokenSequence embeddedA = tokens.embedded();
        assertNotNull("There should be an embedded language", embeddedA);




    public static void assertTokenEquals(String message, TokenSequence<? extends TokenId> ts, TokenId id, String text, int offset) {
        message = messagePrefix(message);
        Token<? extends TokenId> t = ts.token();
        TestCase.assertNotNull("Token is null", t);
        TokenId tId = t.id();
        TestCase.assertEquals(message + "Invalid token.id()", id, tId);
        CharSequence tText = t.text();

        }
        TokenHierarchy<?> thBatch = TokenHierarchy.create(docText, language);
        boolean success = false;
        TokenSequence<?> batchTS = thBatch.tokenSequence();
        try {
            // Compare lookaheads and states as well
            assertTokenSequencesEqual(batchTS, thBatch,
                    thInc.tokenSequence(), thInc, true);
            success = true;
        } finally {
            if (!success) {
                // Go forward two tokens to have an extra tokens context
                batchTS.moveNext();
                batchTS.moveNext();
                System.err.println("BATCH token sequence dump:\n" + thBatch.tokenSequence());
                TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);
                if (lastHi != null) {
                    System.err.println("PREVIOUS batch token sequence dump:\n" + lastHi.tokenSequence());
                }
            }
        }





        // Check that all the non-fly tokens are mutable
        ts = snapshot1.tokenSequence();
        assertEquals(0, ts.moveIndex(0));
        assertTrue(ts.moveNext());
        
        LexerTestUtilities.assertTokenSequencesEqual(hi1.tokenSequence(), hi1,
                snapshot1.tokenSequence(), snapshot1, false);

        doc.insertString(4, "+", null);

        // Check that the snapshot token sequence can be further operated.
        assertEquals(0, ts.moveIndex(0));
        assertTrue(ts.moveNext());
        assertNotNull(ts.token());

        // Check that the tokens except '+' are live


        // Check that all the non-fly tokens are mutable
        ts = snapshot2.tokenSequence();
        assertEquals(0, ts.moveIndex(0));
        assertTrue(ts.moveNext());

        LexerTestUtilities.assertTokenSequencesEqual(hi2.tokenSequence(), hi2,
                snapshot2.tokenSequence(), snapshot2, false);

Lines 81-87 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java (-1 / +2 lines)
81	assertFalse(ts.moveNext());	81	assertFalse(ts.moveNext());
82		82
83	// Go back to block comment	83	// Go back to block comment
84	assertTrue(ts.moveIndex(commentIndex));	84	assertEquals(0, ts.moveIndex(commentIndex));
		85	assertTrue(ts.moveNext());
85	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.BLOCK_COMMENT, commentText, commentTextStartOffset);	86	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.BLOCK_COMMENT, commentText, commentTextStartOffset);
86		87
87	// Test embedded token sequence	88	// Test embedded token sequence




        
        // Check TokenSequence.move()
        int relOffset = ts.move(50); // past the end of all tokens
        assertEquals(relOffset, 50 - (offset + 3));
        assertTrue(ts.movePrevious());
        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset);

        relOffset = ts.move(6); // right at begining of "-"
        assertEquals(relOffset, 0);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", 6);

        relOffset = ts.move(-5); // to first token "abc"
        assertEquals(relOffset, -5);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);

        relOffset = ts.move(5); // to first token "abc"
        assertEquals(relOffset, 1);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "uv", 4);






            
            TokenHierarchy tokenHierarchy = TokenHierarchy.get(bdoc);
            TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
            tokenSequence.move(offset);
            if (!tokenSequence.moveNext() && !tokenSequence.movePrevious()) {
                return false; //no token found
            }
            Token token = tokenSequence.token();

            
            // is it a bean in EL?
            tokenSequence.move(offset); //reset tokenSequence
            tokenSequence.moveNext();
            TokenSequence elTokenSequence = tokenSequence.embedded(ELTokenId.language());
            if (elTokenSequence != null){
                ELExpression exp = new ELExpression((JspSyntaxSupport)bdoc.getSyntaxSupport());
                elTokenSequence.move(offset);
                elTokenSequence.moveNext();
                
                if (elTokenSequence.token().id() == ELTokenId.DOT){
                    return false;

        
        TokenHierarchy tokenHierarchy = TokenHierarchy.get(bdoc);
        TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
        tokenSequence.move(offset);
        if (!tokenSequence.moveNext() && !tokenSequence.movePrevious()) {
            return null; //no token found
        }
        Token token = tokenSequence.token();

            if (elTokenSequence != null){
                ELExpression exp = new ELExpression((JspSyntaxSupport)bdoc.getSyntaxSupport());
                elTokenSequence.move(offset);
                elTokenSequence.moveNext();
                int elEnd = elTokenSequence.offset() + elTokenSequence.token().length();
                int res = exp.parse(elEnd);
                

            
            TokenHierarchy tokenHierarchy = TokenHierarchy.get(bdoc);
            TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
            tokenSequence.move(offset);
            if (!tokenSequence.moveNext() && !tokenSequence.movePrevious()) {
                return; //no token found
            }
            Token token = tokenSequence.token();

                ELExpression exp = new ELExpression((JspSyntaxSupport)bdoc.getSyntaxSupport());
                
                elTokenSequence.move(offset);
                elTokenSequence.moveNext();
                int elEnd = elTokenSequence.offset() + elTokenSequence.token().length();
                int res = exp.parse(elEnd);
                if (res == ELExpression.EL_START ){

            }
            
            tokenSequence.move(offset);//reset tokenSequence
            tokenSequence.moveNext();
            FileObject fObj = getTagFile(tokenSequence, jspSup);
            if ( fObj != null)
                openInEditor(fObj);

        TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
        
        while (index > 0){
            tokenSequence.move(index);
            if (!tokenSequence.moveNext() && !tokenSequence.movePrevious()) {
                return; //no token found
            }
            Token token = tokenSequence.token();

Lines 108-114 Link Here

(-)web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JspParserErrorAnnotation.java (-1 / +2 lines)
108	}	108	}
109	TokenHierarchy tokenHierarchy = TokenHierarchy.get(document);	109	TokenHierarchy tokenHierarchy = TokenHierarchy.get(document);
110	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();	110	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
111	if(tokenSequence.move(offset - 1) == Integer.MAX_VALUE) {	111	tokenSequence.move(offset - 1);
		112	if (!tokenSequence.moveNext() && !tokenSequence.movePrevious()) {
112	//no token	113	//no token
113	textOnLine(docline);	114	textOnLine(docline);
114	return ;	115	return ;

Lines 2024-2030 Link Here

(-)web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JspSyntaxSupport.java (-1 / +1 lines)
2024	//HTML language is not top level one	2024	//HTML language is not top level one
2025	ts = hi.tokenSequence();	2025	ts = hi.tokenSequence();
2026	int diff = ts.move(offset);	2026	int diff = ts.move(offset);
2027	if(diff == Integer.MAX_VALUE) {	2027	if(!ts.moveNext() && !ts.movePrevious()) {
2028	return null; //no token found	2028	return null; //no token found
2029	} else {	2029	} else {
2030	ts = ts.embedded(language);	2030	ts = ts.embedded(language);

Lines 98-104 Link Here

(-)web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java (-1 / +1 lines)
98	TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);	98	TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);
99	TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); //get top level token sequence	99	TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); //get top level token sequence
100		100
101	if(!tokenSequence.moveFirst()) {	101	if(!tokenSequence.moveNext()) {
102	return ; //no tokens in token sequence	102	return ; //no tokens in token sequence
103	}	103	}
104		104

Lines 110-115 Link Here

(-)web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/ELExpression.java (-1 / +3 lines)
110		110
111	TokenSequence jspTokenSequence = hi.tokenSequence();	111	TokenSequence jspTokenSequence = hi.tokenSequence();
112	jspTokenSequence.move(offset);	112	jspTokenSequence.move(offset);
		113	jspTokenSequence.moveNext();
113	Token jspToken = jspTokenSequence.token();	114	Token jspToken = jspTokenSequence.token();
114	isDefferedExecution = jspToken.text().toString().startsWith("#{"); //NOI18N	115	isDefferedExecution = jspToken.text().toString().startsWith("#{"); //NOI18N
115		116
Lines 119-125 Link Here
119	}	120	}
120	}	121	}
121		122
122	if(ts.move(offset) == Integer.MAX_VALUE) {	123	ts.move(offset);
		124	if (!ts.moveNext() && !ts.movePrevious()) {
123	return NOT_EL;	125	return NOT_EL;
124	}	126	}
125		127

Lines 70-76 Link Here

(-)web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/JavaJSPCompletionProvider.java (-1 / +2 lines)
70	TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);	70	TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);
71	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();	71	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
72		72
73	if(tokenSequence.move(offset) != Integer.MAX_VALUE) {	73	tokenSequence.move(offset);
		74	if (tokenSequence.moveNext() \|\| tokenSequence.movePrevious()) {
74	Object tokenID = tokenSequence.token().id();	75	Object tokenID = tokenSequence.token().id();
75	if (tokenID == JspTokenId.SCRIPTLET){	76	if (tokenID == JspTokenId.SCRIPTLET){
76	return true;	77	return true;

Lines 203-210 Link Here

(-)web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/JspCompletionProvider.java (-2 / +2 lines)
203	TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);	203	TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);
204	TokenSequence tokenSequence = JspSyntaxSupport.tokenSequence(tokenHierarchy, HTMLTokenId.language(), adjustedOffset);	204	TokenSequence tokenSequence = JspSyntaxSupport.tokenSequence(tokenHierarchy, HTMLTokenId.language(), adjustedOffset);
205	if(tokenSequence != null) {	205	if(tokenSequence != null) {
206	int diff = tokenSequence.move(adjustedOffset);	206	tokenSequence.move(adjustedOffset);
207	if(diff >= tokenSequence.token().length() \|\| diff == Integer.MAX_VALUE) {	207	if (!tokenSequence.moveNext() && !tokenSequence.movePrevious()) {
208	return; //no token found	208	return; //no token found
209	}	209	}
210		210




         * e.g. <tag   |attr=
         * 
         */
        while (tokenSequence.moveNext()) {
            tokenSequence.moveNext();
            token = tokenSequence.token();
            tokenOffset = tokenSequence.offset();
            if (!isWSTag(token) || tagStartLine != Utilities.getLineOffset(doc, tokenOffset)) {
                if (!isWSTag(token) && tagStartLine == Utilities.getLineOffset(doc, tokenOffset)){
                    return tokenOffset - Utilities.getRowIndent(doc, tokenOffset) - Utilities.getRowStart(doc, tokenOffset);
                }
                break;
            }
        }
        
        
        return getShiftWidth(); // default;
    }
    

            int indentsWithinTags[] = new int[lastLine + 1];
            
            TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
            tokenSequence.moveStart();
            boolean thereAreMoreTokens = tokenSequence.moveNext();
            
            
            if (tokenSequence != null){
                // calc line indents - pass 1

        tokenSequence.move(offset);
        int currentOffset;
        
        while (tokenSequence.moveNext()) {
            currentOffset = tokenSequence.offset();
            
            if (isClosingTag(tokenHierarchy, currentOffset)){
                return currentOffset;
            }
            
        }

        return -1;
    }

    protected Token getTokenAtOffset(TokenHierarchy tokenHierarchy, int tagTokenOffset){
        TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
        
        if (tokenSequence != null) {
            tokenSequence.move(tagTokenOffset);
            if (tokenSequence.moveNext())
                return tokenSequence.token();
        }
        
        return null;

Return to bug 90796

Lines 1116-1124 Link Here

(-)java/editor/src/org/netbeans/modules/editor/java/JavaCompletionProvider.java (-8 / +11 lines)
1116	int openLtNum = 0;	1116	int openLtNum = 0;
1117	JavaTokenId lastNonWhitespaceTokenId = null;	1117	JavaTokenId lastNonWhitespaceTokenId = null;
1118	TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());	1118	TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());
1119	if (ts.move(expEndPos) > 0)	1119	ts.move(expEndPos);
1120	ts.moveNext();	1120	while (ts.moveNext()) {
1121	do {
1122	if (ts.offset() >= offset) {	1121	if (ts.offset() >= offset) {
1123	break;	1122	break;
1124	}	1123	}
Lines 1151-1157 Link Here
1151	default:	1150	default:
1152	lastNonWhitespaceTokenId = ts.token().id();	1151	lastNonWhitespaceTokenId = ts.token().id();
1153	}	1152	}
1154	} while (ts.moveNext());	1153	}
1155	if (!afterDot) {	1154	if (!afterDot) {
1156	if (expEndPos <= offset)	1155	if (expEndPos <= offset)
1157	insideExpression(env, new TreePath(path, fa.getExpression()));	1156	insideExpression(env, new TreePath(path, fa.getExpression()));
Lines 3426-3433 Link Here
3426		3425
3427	private TokenSequence<JavaTokenId> findLastNonWhitespaceToken(Env env, int startPos, int endPos) {	3426	private TokenSequence<JavaTokenId> findLastNonWhitespaceToken(Env env, int startPos, int endPos) {
3428	TokenSequence<JavaTokenId> ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language());	3427	TokenSequence<JavaTokenId> ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language());
3429	if (ts.move(endPos) == 0)	3428	if (ts.move(endPos) == 0) // When right at the token end
3430	ts.movePrevious();	3429	ts.movePrevious(); // move to previous token
		3430	else
		3431	ts.moveNext(); // otherwise move to the token that "contains" the offset
3431	do {	3432	do {
3432	int offset = ts.offset();	3433	int offset = ts.offset();
3433	if (offset < startPos)	3434	if (offset < startPos)
Lines 3564-3571 Link Here
3564	if (upToOffset) {	3565	if (upToOffset) {
3565	TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());	3566	TokenSequence<JavaTokenId> ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language());
3566	if (ts.moveNext()) { //check for empty file	3567	if (ts.moveNext()) { //check for empty file
3567	if (ts.move(offset) == 0)	3568	if (ts.move(offset) == 0) // When right at the token end
3568	ts.movePrevious();	3569	ts.movePrevious(); // Move to previous token
		3570	else
		3571	ts.moveNext(); // otherwise move to the token that "contains" the offset
3569	if (ts.offset() < offset && (ts.token().id() == JavaTokenId.IDENTIFIER \|\| "keyword".equals(ts.token().id().primaryCategory()))) { //TODO: Use isKeyword(...) when available	3572	if (ts.offset() < offset && (ts.token().id() == JavaTokenId.IDENTIFIER \|\| "keyword".equals(ts.token().id().primaryCategory()))) { //TODO: Use isKeyword(...) when available
3570	prefix = ts.token().toString().substring(0, offset - ts.offset());	3573	prefix = ts.token().toString().substring(0, offset - ts.offset());
3571	offset = ts.offset();	3574	offset = ts.offset();

Lines 316-321 Link Here

(-)java/source/src/org/netbeans/modules/java/source/save/CasualDiff.java (-5 / +10 lines)
316	// skip the section when printing anonymous class	316	// skip the section when printing anonymous class
317	if (anonClass == false) {	317	if (anonClass == false) {
318	tokenSequence.move(oldT.pos);	318	tokenSequence.move(oldT.pos);
		319	tokenSequence.moveNext(); // First skip as move() does not position to token directly
319	tokenSequence.moveNext();	320	tokenSequence.moveNext();
320	insertHint = TokenUtilities.moveNext(tokenSequence, tokenSequence.offset());	321	insertHint = TokenUtilities.moveNext(tokenSequence, tokenSequence.offset());
321	localPointer = diffModifiers(oldT.mods, newT.mods, oldT, localPointer);	322	localPointer = diffModifiers(oldT.mods, newT.mods, oldT, localPointer);
Lines 508-514 Link Here
508	// there was not any parameter in original tree.	509	// there was not any parameter in original tree.
509	int startOffset = oldT.restype != null ? oldT.restype.getStartPosition() : oldT.getStartPosition();	510	int startOffset = oldT.restype != null ? oldT.restype.getStartPosition() : oldT.getStartPosition();
510		511
511	tokenSequence.move(startOffset);
512	TokenUtilities.moveFwdToToken(tokenSequence, startOffset, JavaTokenId.RPAREN);	512	TokenUtilities.moveFwdToToken(tokenSequence, startOffset, JavaTokenId.RPAREN);
513	posHint = tokenSequence.offset();	513	posHint = tokenSequence.offset();
514	} else {	514	} else {
Lines 533-538 Link Here
533	// now check, that there is a whitespace. It is not mandatory, we	533	// now check, that there is a whitespace. It is not mandatory, we
534	// have to ensure. If whitespace is not present, take body beginning	534	// have to ensure. If whitespace is not present, take body beginning
535	tokenSequence.move(posHint);	535	tokenSequence.move(posHint);
		536	tokenSequence.moveNext();
536	if (tokenSequence.token().id() != JavaTokenId.WHITESPACE) {	537	if (tokenSequence.token().id() != JavaTokenId.WHITESPACE) {
537	++posHint;	538	++posHint;
538	}	539	}
Lines 1441-1448 Link Here
1441	ResultItem<JCTree> item = result[j];	1442	ResultItem<JCTree> item = result[j];
1442	switch (item.operation) {	1443	switch (item.operation) {
1443	case MODIFY: {	1444	case MODIFY: {
1444	// perhaps I shouldn't support this!	1445	// perhaps I shouldn't support this!
1445	if (tokenSequence.moveIndex(matrix[i][4])) {	1446	tokenSequence.moveIndex(matrix[i][4]);
		1447	if (tokenSequence.moveNext()) {
1446	testPos = tokenSequence.offset();	1448	testPos = tokenSequence.offset();
1447	if (JavaTokenId.COMMA == tokenSequence.token().id())	1449	if (JavaTokenId.COMMA == tokenSequence.token().id())
1448	testPos += JavaTokenId.COMMA.fixedText().length();	1450	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1488-1494 Link Here
1488	assert startOffset != -1 && endOffset != -1 : "Invalid offset!";	1490	assert startOffset != -1 && endOffset != -1 : "Invalid offset!";
1489	//printer.print(origText.substring(lastOldPos, startOffset));	1491	//printer.print(origText.substring(lastOldPos, startOffset));
1490	//append(Diff.delete(startOffset, endOffset));	1492	//append(Diff.delete(startOffset, endOffset));
1491	if (tokenSequence.moveIndex(matrix[i][4])) {	1493	tokenSequence.moveIndex(matrix[i][4]);
		1494	if (tokenSequence.moveNext()) {
1492	testPos = tokenSequence.offset();	1495	testPos = tokenSequence.offset();
1493	if (JavaTokenId.COMMA == tokenSequence.token().id())	1496	if (JavaTokenId.COMMA == tokenSequence.token().id())
1494	testPos += JavaTokenId.COMMA.fixedText().length();	1497	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1502-1508 Link Here
1502	break;	1505	break;
1503	}	1506	}
1504	case NOCHANGE: {	1507	case NOCHANGE: {
1505	if (tokenSequence.moveIndex(matrix[i][4])) {	1508	tokenSequence.moveIndex(matrix[i][4]);
		1509	if (tokenSequence.moveNext()) {
1506	testPos = tokenSequence.offset();	1510	testPos = tokenSequence.offset();
1507	if (JavaTokenId.COMMA == tokenSequence.token().id())	1511	if (JavaTokenId.COMMA == tokenSequence.token().id())
1508	testPos += JavaTokenId.COMMA.fixedText().length();	1512	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1521-1526 Link Here
1521	return -1;	1525	return -1;
1522	}	1526	}
1523	tokenSequence.moveIndex(tokenIndex);	1527	tokenSequence.moveIndex(tokenIndex);
		1528	tokenSequence.moveNext();
1524	return tokenSequence.offset();	1529	return tokenSequence.offset();
1525	}	1530	}
1526		1531

Lines 1247-1253 Link Here

(-)java/source/src/org/netbeans/modules/java/source/save/TreeDiff.java (-3 / +6 lines)
1247	ResultItem<JCTree> item = result[j];	1247	ResultItem<JCTree> item = result[j];
1248	switch (item.operation) {	1248	switch (item.operation) {
1249	case MODIFY: {	1249	case MODIFY: {
1250	if (tokenSequence.moveIndex(matrix[i][4])) {	1250	tokenSequence.moveIndex(matrix[i][4]);
		1251	if (tokenSequence.moveNext()) {
1251	testPos = tokenSequence.offset();	1252	testPos = tokenSequence.offset();
1252	if (JavaTokenId.COMMA == tokenSequence.token().id())	1253	if (JavaTokenId.COMMA == tokenSequence.token().id())
1253	testPos += JavaTokenId.COMMA.fixedText().length();	1254	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1283-1289 Link Here
1283	int endOffset = toOff(s.tail(j) \|\| s.next(j) ? matrix[i+1][2] : matrix[i][4]);	1284	int endOffset = toOff(s.tail(j) \|\| s.next(j) ? matrix[i+1][2] : matrix[i][4]);
1284	assert startOffset != -1 && endOffset != -1 : "Invalid offset!";	1285	assert startOffset != -1 && endOffset != -1 : "Invalid offset!";
1285	append(Diff.delete(startOffset, endOffset));	1286	append(Diff.delete(startOffset, endOffset));
1286	if (tokenSequence.moveIndex(matrix[i][4])) {	1287	tokenSequence.moveIndex(matrix[i][4]);
		1288	if (tokenSequence.moveNext()) {
1287	testPos = tokenSequence.offset();	1289	testPos = tokenSequence.offset();
1288	if (JavaTokenId.COMMA == tokenSequence.token().id())	1290	if (JavaTokenId.COMMA == tokenSequence.token().id())
1289	testPos += JavaTokenId.COMMA.fixedText().length();	1291	testPos += JavaTokenId.COMMA.fixedText().length();
Lines 1293-1299 Link Here
1293	break;	1295	break;
1294	}	1296	}
1295	case NOCHANGE: {	1297	case NOCHANGE: {
1296	if (tokenSequence.moveIndex(matrix[i][4])) {	1298	tokenSequence.moveIndex(matrix[i][4]);
		1299	if (tokenSequence.moveNext()) {
1297	testPos = tokenSequence.offset();	1300	testPos = tokenSequence.offset();
1298	if (JavaTokenId.COMMA == tokenSequence.token().id())	1301	if (JavaTokenId.COMMA == tokenSequence.token().id())
1299	testPos += JavaTokenId.COMMA.fixedText().length();	1302	testPos += JavaTokenId.COMMA.fixedText().length();

Lines 36-42 Link Here

(-)editor/lib2/src/org/netbeans/modules/editor/lib2/highlighting/SyntaxHighlighting.java (-50 / +10 lines)
36	import org.netbeans.api.editor.settings.FontColorSettings;	36	import org.netbeans.api.editor.settings.FontColorSettings;
37	import org.netbeans.api.lexer.Language;	37	import org.netbeans.api.lexer.Language;
38	import org.netbeans.api.lexer.LanguagePath;	38	import org.netbeans.api.lexer.LanguagePath;
39	import org.netbeans.api.lexer.TokenChange;
40	import org.netbeans.api.lexer.TokenHierarchy;	39	import org.netbeans.api.lexer.TokenHierarchy;
41	import org.netbeans.api.lexer.TokenHierarchyEvent;	40	import org.netbeans.api.lexer.TokenHierarchyEvent;
42	import org.netbeans.api.lexer.TokenHierarchyListener;	41	import org.netbeans.api.lexer.TokenHierarchyListener;
Lines 102-158 Link Here
102	synchronized (this) {	101	synchronized (this) {
103	version++;	102	version++;
104	}	103	}
105
106	TokenChange<? extends TokenId> tc = evt.tokenChange();
107	int addedLenght = 0;
108	int removedLength = 0;
109
110	if (tc.addedTokenCount() > 0) {
111	addedLenght = getTokensLength(tc.currentTokenSequence(), tc.index(), tc.addedTokenCount());
112	}
113
114	if (tc.removedTokenCount() > 0) {
115	removedLength = getTokensLength(tc.removedTokenSequence(), 0, tc.removedTokenCount());
116	}
117		104
118	int changeStart = tc.offset();	105	fireHighlightsChange(evt.affectedStartOffset(), evt.affectedEndOffset());
119	int changeEnd;
120
121	if (addedLenght == 0 && removedLength == 0) {
122	// The real length couldn't be computed for some reason
123	changeEnd = Integer.MAX_VALUE;
124	} else {
125	changeEnd = changeStart + Math.max(addedLenght, removedLength);
126	}
127
128	fireHighlightsChange(changeStart, changeEnd);
129	}	106	}
130		107
131	// ----------------------------------------------------------------------	108	// ----------------------------------------------------------------------
132	// Private implementation	109	// Private implementation
133	// ----------------------------------------------------------------------	110	// ----------------------------------------------------------------------
134		111
135	private int getTokensLength(TokenSequence<? extends TokenId> seq, int startIdx, int tokenCount) {
136	assert startIdx >= 0 && startIdx < seq.tokenCount() :
137	"Invalid startIdx: " + startIdx + ", sequence lenght: " + seq.tokenCount();
138	assert tokenCount > 0 && startIdx + tokenCount <= seq.tokenCount() :
139	"Invalid tokenCount: " + tokenCount + ", startIdx: " + startIdx +
140	", sequence lenght: " + seq.tokenCount();
141
142	int startOffset = -1;
143	int endOffset = -1;
144
145	if (seq.moveIndex(startIdx)) {
146	startOffset = seq.offset();
147	}
148
149	if (seq.moveIndex(startIdx + tokenCount - 1)) {
150	endOffset = seq.offset() + seq.token().length();
151	}
152
153	return startOffset == -1 \|\| endOffset == -1 ? 0 : endOffset - startOffset;
154	}
155
156	private final class HSImpl implements HighlightsSequence {	112	private final class HSImpl implements HighlightsSequence {
157		113
158	private static final int S_NORMAL = 1;	114	private static final int S_NORMAL = 1;
Lines 196-202 Link Here
196	case S_EMBEDDED_HEAD:	152	case S_EMBEDDED_HEAD:
197	// The current token contains embedded language and we have processed it's head	153	// The current token contains embedded language and we have processed it's head
198	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);	154	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
199	if (seq.moveFirst()) {	155	seq.moveStart();
		156	if (seq.moveNext()) {
200	state = S_NORMAL;	157	state = S_NORMAL;
201	} else {	158	} else {
202	throw new IllegalStateException("Invalid state");	159	throw new IllegalStateException("Invalid state");
Lines 221-227 Link Here
221	// We have moved to the next normal token, so look what it is	178	// We have moved to the next normal token, so look what it is
222	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);	179	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
223	TokenSequence<? extends TokenId> embeddedSeq = seq.embedded();	180	TokenSequence<? extends TokenId> embeddedSeq = seq.embedded();
224	while (embeddedSeq != null && embeddedSeq.moveFirst()) {	181	while (embeddedSeq != null && embeddedSeq.moveNext()) {
225	sequences.add(sequences.size(), embeddedSeq);	182	sequences.add(sequences.size(), embeddedSeq);
226	if (embeddedSeq.offset() > seq.offset()) {	183	if (embeddedSeq.offset() > seq.offset()) {
227	state = S_EMBEDDED_HEAD;	184	state = S_EMBEDDED_HEAD;
Lines 258-264 Link Here
258	}	215	}
259	case S_EMBEDDED_TAIL: {	216	case S_EMBEDDED_TAIL: {
260	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);	217	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
261	if (seq.moveLast()) {	218	seq.moveEnd();
		219	if (seq.movePrevious()) {
262	return seq.offset() + seq.token().length();	220	return seq.offset() + seq.token().length();
263	} else {	221	} else {
264	throw new IllegalStateException("Invalid state");	222	throw new IllegalStateException("Invalid state");
Lines 288-294 Link Here
288	}	246	}
289	case S_EMBEDDED_HEAD: {	247	case S_EMBEDDED_HEAD: {
290	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);	248	TokenSequence<? extends TokenId> seq = sequences.get(sequences.size() - 1);
291	if (seq.moveFirst()) {	249	seq.moveStart();
		250	if (seq.moveNext()) {
292	return seq.offset();	251	return seq.offset();
293	} else {	252	} else {
294	TokenSequence<? extends TokenId> embeddingSeq = sequences.get(sequences.size() - 2);	253	TokenSequence<? extends TokenId> embeddingSeq = sequences.get(sequences.size() - 2);
Lines 447-453 Link Here
447	} else {	406	} else {
448	if (sequences.size() > 1) {	407	if (sequences.size() > 1) {
449	TokenSequence<? extends TokenId> embeddingSeq = sequences.get(sequences.size() - 2);	408	TokenSequence<? extends TokenId> embeddingSeq = sequences.get(sequences.size() - 2);
450	if (seq.moveLast()) {	409	seq.moveEnd();
		410	if (seq.movePrevious()) {
451	if ((seq.offset() + seq.token().length()) < (embeddingSeq.offset() + embeddingSeq.token().length())) {	411	if ((seq.offset() + seq.token().length()) < (embeddingSeq.offset() + embeddingSeq.token().length())) {
452	return S_EMBEDDED_TAIL;	412	return S_EMBEDDED_TAIL;
453	} else {	413	} else {

Lines 106-116 Link Here

(-)html/editor/lib/src/org/netbeans/editor/ext/html/HTMLCompletionQuery.java (-6 / +8 lines)
106	if(ts == null) {	106	if(ts == null) {
107	//HTML language is not top level one	107	//HTML language is not top level one
108	ts = hi.tokenSequence();	108	ts = hi.tokenSequence();
109	int diff = ts.move(offset);	109	ts.move(offset);
110	if(diff == Integer.MAX_VALUE) {	110	if (ts.moveNext() \|\| ts.movePrevious()) {
111	return null; //no token found
112	} else {
113	ts = ts.embedded(HTMLTokenId.language());	111	ts = ts.embedded(HTMLTokenId.language());
		112	} else { // no tokens
		113	return null;
114	}	114	}
115	}	115	}
116		116
Lines 120-133 Link Here
120	}	120	}
121		121
122	int diff = ts.move(offset);	122	int diff = ts.move(offset);
123	if(diff == Integer.MAX_VALUE) {	123	if(!ts.moveNext() && !ts.movePrevious()) {
124	return null; //no token found	124	return null; //no token found
125	}	125	}
126		126
127	Token item = ts.token();	127	Token item = ts.token();
128		128
129	// are we inside token or between tokens	129	// are we inside token or between tokens
130	boolean inside = item.offset(hi) < offset;	130	boolean inside = ts.offset() < offset;
131		131
132	if(!inside) { //use the previous token	132	if(!inside) { //use the previous token
133	if(ts.movePrevious()) {	133	if(ts.movePrevious()) {
Lines 192-197 Link Here
192	//there are some attributes or an end of the tag	192	//there are some attributes or an end of the tag
193		193
194	ts.move(offset);	194	ts.move(offset);
		195	ts.moveNext();
195	Token t = ts.token();	196	Token t = ts.token();
196		197
197	//test if next token is a whitespace and the next a tag token or an attribute token	198	//test if next token is a whitespace and the next a tag token or an attribute token
Lines 324-329 Link Here
324	if( tag == null ) return null; // unknown tag	325	if( tag == null ) return null; // unknown tag
325		326
326	ts.move(item.offset(hi));	327	ts.move(item.offset(hi));
		328	ts.moveNext();
327	Token argItem = ts.token();	329	Token argItem = ts.token();
328	while(argItem.id() != HTMLTokenId.ARGUMENT && ts.movePrevious()) {	330	while(argItem.id() != HTMLTokenId.ARGUMENT && ts.movePrevious()) {
329	argItem = ts.token();	331	argItem = ts.token();

Lines 175-197 Link Here

(-)java/hints/src/org/netbeans/modules/java/hints/JavaHintsProvider.java (-14 / +14 lines)
175	TokenSequence<JavaTokenId> ts = info.getTokenHierarchy().tokenSequence();	175	TokenSequence<JavaTokenId> ts = info.getTokenHierarchy().tokenSequence();
176		176
177	ts.move(offset);	177	ts.move(offset);
178		178	if (ts.moveNext()) {
179	Token t = ts.token();	179	Token t = ts.token();
180		180
181	if (t.id() == JavaTokenId.DOT) {	181	if (t.id() == JavaTokenId.DOT) {
182	ts.moveNext();
183	t = ts.token();
184	} else {
185	if (t.id() == JavaTokenId.LT) {
186	ts.moveNext();	182	ts.moveNext();
187	t = ts.token();	183	t = ts.token();
		184	} else {
		185	if (t.id() == JavaTokenId.LT) {
		186	ts.moveNext();
		187	t = ts.token();
		188	}
		189	}
		190
		191	if (t.id() == JavaTokenId.IDENTIFIER) {
		192	return ts.offsetToken();
188	}	193	}
189	}	194	}
190
191	if (t.id() == JavaTokenId.IDENTIFIER) {
192	return ts.offsetToken();
193	}
194
195	return null;	195	return null;
196	}	196	}
197		197
Lines 256-262 Link Here
256		256
257	int diff = ts.move((int) getPrefferedPosition(info, d));	257	int diff = ts.move((int) getPrefferedPosition(info, d));
258		258
259	if (diff >= 0 && diff < ts.token().length()) {	259	if (ts.moveNext() && diff >= 0 && diff < ts.token().length()) {
260	Token<JavaTokenId> t = ts.token();	260	Token<JavaTokenId> t = ts.token();
261		261
262	if (t.id() == JavaTokenId.IDENTIFIER) {	262	if (t.id() == JavaTokenId.IDENTIFIER) {

Lines 139-144 Link Here

(-)java/source/src/org/netbeans/modules/java/source/save/PositionEstimator.java (-6 / +28 lines)
139	int treeEnd = (int) positions.getEndPosition(compilationUnit, item);	139	int treeEnd = (int) positions.getEndPosition(compilationUnit, item);
140		140
141	seq.move(treeStart);	141	seq.move(treeStart);
		142	seq.moveNext();
142	int startIndex = seq.index();	143	int startIndex = seq.index();
143		144
144	// go back to opening/closing curly, semicolon or other	145	// go back to opening/closing curly, semicolon or other
Lines 147-155 Link Here
147	seq.moveNext();	148	seq.moveNext();
148	int start = seq.index();	149	int start = seq.index();
149	seq.move(treeEnd);	150	seq.move(treeEnd);
		151	// seq.index() set (no seq.moveNext() necessary)
150	matrix[i++] = new int[] { start, startIndex, seq.index() };	152	matrix[i++] = new int[] { start, startIndex, seq.index() };
151	if (i == size) {	153	if (i == size) {
152	seq.move(treeEnd);	154	seq.move(treeEnd);
		155	// seq.index() set (no seq.moveNext() necessary)
153	matrix[i][0] = seq.index();	156	matrix[i][0] = seq.index();
154	}	157	}
155	}	158	}
Lines 163-168 Link Here
163	// to decide.	166	// to decide.
164	if (tokenIndex == -1) return -1;	167	if (tokenIndex == -1) return -1;
165	seq.moveIndex(tokenIndex);	168	seq.moveIndex(tokenIndex);
		169	seq.moveNext();
166	return index == 0 ? goAfterLastNewLine(seq) : goAfterFirstNewLine(seq);	170	return index == 0 ? goAfterLastNewLine(seq) : goAfterFirstNewLine(seq);
167	}	171	}
168		172
Lines 209-217 Link Here
209	@Override()	213	@Override()
210	public int[] getPositions(int index) {	214	public int[] getPositions(int index) {
211	int tokenIndex = matrix[index][0];	215	int tokenIndex = matrix[index][0];
212	seq.moveIndex(tokenIndex);	216	if (tokenIndex != -1) {
		217	seq.moveIndex(tokenIndex);
		218	seq.moveNext();
		219	}
213	int begin = goAfterLastNewLine(seq);	220	int begin = goAfterLastNewLine(seq);
214	seq.moveIndex(matrix[index][2]);	221	if (matrix[index][2] != -1) {
		222	seq.moveIndex(matrix[index][2]);
		223	seq.moveNext();
		224	}
215	int end = goAfterFirstNewLine(seq);	225	int end = goAfterFirstNewLine(seq);
216	return new int [] { begin, end };	226	return new int [] { begin, end };
217	}	227	}
Lines 262-267 Link Here
262	if (treeEnd < 0) continue;	272	if (treeEnd < 0) continue;
263		273
264	seq.move(treeStart);	274	seq.move(treeStart);
		275	seq.moveNext();
265	int startIndex = seq.index();	276	int startIndex = seq.index();
266	// go back to opening/closing curly, semicolon or other	277	// go back to opening/closing curly, semicolon or other
267	// token java-compiler important token.	278	// token java-compiler important token.
Lines 285-290 Link Here
285	// to decide.	296	// to decide.
286	if (tokenIndex == -1) return -1;	297	if (tokenIndex == -1) return -1;
287	seq.moveIndex(tokenIndex);	298	seq.moveIndex(tokenIndex);
		299	seq.moveNext();
288	return goAfterFirstNewLine(seq);	300	return goAfterFirstNewLine(seq);
289	}	301	}
290		302
Lines 296-302 Link Here
296		308
297	public int[] getPositions(int index) {	309	public int[] getPositions(int index) {
298	int begin = getInsertPos(index);	310	int begin = getInsertPos(index);
299	seq.moveIndex(matrix[index][4]);	311	if (matrix[index][4] != -1) {
		312	seq.moveIndex(matrix[index][4]);
		313	seq.moveNext();
		314	}
300	int end = goAfterFirstNewLine(seq);	315	int end = goAfterFirstNewLine(seq);
301	return new int [] { begin, end };	316	return new int [] { begin, end };
302	}	317	}
Lines 360-365 Link Here
360	// to decide.	375	// to decide.
361	if (tokenIndex == -1) return -1;	376	if (tokenIndex == -1) return -1;
362	seq.moveIndex(tokenIndex);	377	seq.moveIndex(tokenIndex);
		378	seq.moveNext();
363	int off = goAfterFirstNewLine(seq);	379	int off = goAfterFirstNewLine(seq);
364	return off;	380	return off;
365	}	381	}
Lines 372-378 Link Here
372		388
373	public int[] getPositions(int index) {	389	public int[] getPositions(int index) {
374	int begin = getInsertPos(index);	390	int begin = getInsertPos(index);
375	seq.moveIndex(matrix[index][4]);	391	if (matrix[index][4] != -1) {
		392	seq.moveIndex(matrix[index][4]);
		393	seq.moveNext();
		394	}
376	int end = goAfterFirstNewLine(seq);	395	int end = goAfterFirstNewLine(seq);
377	return new int [] { begin, end };	396	return new int [] { begin, end };
378	}	397	}
Lines 497-512 Link Here
497	// todo (#pf): remove - used for debugging reasons, doesn't do good job	516	// todo (#pf): remove - used for debugging reasons, doesn't do good job
498	public void tablePrint(int[][] matrix, TokenSequence seq) {	517	public void tablePrint(int[][] matrix, TokenSequence seq) {
499	for (int i = 0; i < matrix.length; i++) {	518	for (int i = 0; i < matrix.length; i++) {
500	for (int j = 0; j < 5; j++) {	519	for (int j = 0; j < matrix[i].length; j++) {
501	int item = matrix[i][j];	520	int item = matrix[i][j];
502	String s = "(nothing)";	521	String s = "(nothing)";
503	if (item > -1) {	522	if (item > -1) {
504	seq.moveIndex(item);	523	seq.moveIndex(item);
		524	seq.moveNext();
505	s = "'" + seq.token().text();	525	s = "'" + seq.token().text();
506	}	526	}
507	s += "' ";	527	s += "' ";
508	System.err.print(s.substring(0, 25));	528	// System.err.print(s.substring(0, 25));
		529	System.err.print(item + "\t");
509	}	530	}
		531	System.err.println("");
510	}	532	}
511	}	533	}
512		534

Lines 65-70 Link Here

(-)java/source/src/org/netbeans/modules/java/source/save/TokenUtilities.java (+5 lines)
65	*/	65	*/
66	static int moveNext(TokenSequence<JavaTokenId> tokenSequence, final int pos) {	66	static int moveNext(TokenSequence<JavaTokenId> tokenSequence, final int pos) {
67	tokenSequence.move(pos);	67	tokenSequence.move(pos);
		68	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
68	JavaTokenId type = tokenSequence.token().id();	69	JavaTokenId type = tokenSequence.token().id();
69	while (JavaTokenId.WHITESPACE.equals(type) \|\|	70	while (JavaTokenId.WHITESPACE.equals(type) \|\|
70	JavaTokenId.BLOCK_COMMENT.equals(type) \|\|	71	JavaTokenId.BLOCK_COMMENT.equals(type) \|\|
Lines 82-87 Link Here
82	JavaTokenId id)	83	JavaTokenId id)
83	{	84	{
84	tokenSequence.move(pos);	85	tokenSequence.move(pos);
		86	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
85	while (!id.equals(tokenSequence.token().id())) {	87	while (!id.equals(tokenSequence.token().id())) {
86	if (!tokenSequence.moveNext())	88	if (!tokenSequence.moveNext())
87	return -1;	89	return -1;
Lines 94-99 Link Here
94	JavaTokenId id)	96	JavaTokenId id)
95	{	97	{
96	tokenSequence.move(pos);	98	tokenSequence.move(pos);
		99	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
97	while (!id.equals(tokenSequence.token().id())) {	100	while (!id.equals(tokenSequence.token().id())) {
98	if (!tokenSequence.movePrevious())	101	if (!tokenSequence.movePrevious())
99	return -1;	102	return -1;
Lines 106-111 Link Here
106	String text)	109	String text)
107	{	110	{
108	tokenSequence.move(pos);	111	tokenSequence.move(pos);
		112	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
109	while (!text.equals(tokenSequence.token().text())) {	113	while (!text.equals(tokenSequence.token().text())) {
110	if (!tokenSequence.moveNext())	114	if (!tokenSequence.moveNext())
111	return -1;	115	return -1;
Lines 118-123 Link Here
118	String text)	122	String text)
119	{	123	{
120	tokenSequence.move(pos);	124	tokenSequence.move(pos);
		125	tokenSequence.moveNext(); // Assumes the pos is located within input bounds
121	while (!text.equals(tokenSequence.token().text())) {	126	while (!text.equals(tokenSequence.token().text())) {
122	if (!tokenSequence.movePrevious())	127	if (!tokenSequence.movePrevious())
123	return -1;	128	return -1;

Lines 300-319 Link Here

(-)lexer/arch.xml (-13 / +11 lines)
300	TokenHierarchy hi = TokenHierarchy.get(document);	300	TokenHierarchy hi = TokenHierarchy.get(document);
301	TokenSequence ts = hi.tokenSequence();	301	TokenSequence ts = hi.tokenSequence();
302	// If necessary move ts to the requested offset	302	// If necessary move ts to the requested offset
303	int diff = ts.move(offset);	303	ts.move(offset);
304	if (diff != Integer.MAX_VALUE)) { // Some tokens exist	304	while (ts.moveNext()) {
305	do {	305	Token t = ts.token();
306	Token t = ts.token();	306	if (t.id() == ...) { ... }
307	if (t.id() == ...) { ... }	307	if (TokenUtilities.equals(t.text(), "mytext")) { ... }
308	if (TokenUtilities.equals(t.text(), "mytext")) { ... }	308	if (ts.offset() == ...) { ... }
309	if (ts.offset() == ...) { ... }
310		309
311	// Possibly retrieve embedded token sequence	310	// Possibly retrieve embedded token sequence
312	TokenSequence embedded = ts.embedded();	311	TokenSequence embedded = ts.embedded();
313	if (embedded != null) { // Token has a valid language embedding	312	if (embedded != null) { // Token has a valid language embedding
314	...	313	...
315	}	314	}
316	} while (ts.moveNext());
317	}	315	}
318	} finally {	316	} finally {
319	document.readUnlock();	317	document.readUnlock();

Lines 91-96 Link Here

(-)lexer/api/apichanges.xml (+28 lines)
91	<!-- ACTUAL CHANGES BEGIN HERE: -->	91	<!-- ACTUAL CHANGES BEGIN HERE: -->
92		92
93	<changes>	93	<changes>
		94	<change id="api.tokensequence.move.semantics">
		95	<api name="api"/>
		96	<summary>Changed TokenSequence.move() and moveIndex() use</summary>
		97	<version major="1" minor="13"/>
		98	<date day="16" month="1" year="2007"/>
		99	<author login="mmetelka"/>
		100	<compatibility source="incompatible" deletion="yes" addition="yes" modification="yes"/>
		101	<description>
		102	<p>
		103	Changed the <code>TokenSequence.move()</code> to position <b>before</b> the particular token
		104	that "contains" the offset (or after the last token if the given offset
		105	is too high). Additional <code>moveNext()</code> is necessary
		106	to actually move to the next token.
		107	<br/>
		108	<code>TokenSequence.moveIndex()</code> was modified in a similar way
		109	(see javadocs).
		110	<br/>
		111	<code>TokenSequence.moveFirst()</code> and <code>moveLast()</code>
		112	were replaced by <code>moveStart()</code> that positionins before
		113	the first token and by <code>moveEnd()</code> that positions
		114	after the last token.
		115	<br/>
		116	<code>TokenSequence.isEmpty()</code> added to check whether there are
		117	no tokens in the TS.
		118	</p>
		119	</description>
		120	</change>
		121
94	<change id="api.lexer.release">	122	<change id="api.lexer.release">
95	<api name="api"/>	123	<api name="api"/>
96	<summary>Added Lexer.release()</summary>	124	<summary>Added Lexer.release()</summary>

Lines 108-115 Link Here

(-)lexer/editorbridge/src/org/netbeans/modules/lexer/editorbridge/LexerLayer.java (-3 / +4 lines)
108	if (active) {	108	if (active) {
109	pastSequences = new Stack<TokenSequence>();	109	pastSequences = new Stack<TokenSequence>();
110	tokenSequence = hi.tokenSequence();	110	tokenSequence = hi.tokenSequence();
111	int relOffset = tokenSequence.move(startOffset);	111	tokenSequence.move(startOffset);
112	if (relOffset != Integer.MAX_VALUE) {	112	if (tokenSequence.moveNext()) {
113	updateTokenEndOffsetAndColoring(startOffset);	113	updateTokenEndOffsetAndColoring(startOffset);
114	} else { // no tokens	114	} else { // no tokens
115	active = false;	115	active = false;
Lines 178-184 Link Here
178	break;	178	break;
179	}	179	}
180	} else {	180	} else {
181	if (embed.move(offset) != Integer.MAX_VALUE) {	181	embed.move(offset);
		182	if (embed.moveNext()) {
182	pastSequences.push(tokenSequence);	183	pastSequences.push(tokenSequence);
183	tokenSequence = embed;	184	tokenSequence = embed;
184	} else {	185	} else {

Lines 75-81 Link Here

(-)lexer/nbbridge/test/unit/src/org/netbeans/modules/lexer/nbbridge/test/MimeLookupLanguageProviderTest.java (-5 / +8 lines)
75	assertEquals("Wrong language", "text/x-simple-plain", lang.mimeType());	75	assertEquals("Wrong language", "text/x-simple-plain", lang.mimeType());
76		76
77	for(int i = 0; i < seq.tokenCount(); i++) {	77	for(int i = 0; i < seq.tokenCount(); i++) {
78	seq.move(i);	78	seq.moveIndex(i);
		79	assertTrue(seq.moveNext());
79	Token token = seq.token();	80	Token token = seq.token();
80		81
81	if (token.id() == SimplePlainTokenId.WORD) {	82	if (token.id() == SimplePlainTokenId.WORD) {
Lines 86-96 Link Here
86	assertNotNull("Can't find language of the embedded sequence", embeddedLang);	87	assertNotNull("Can't find language of the embedded sequence", embeddedLang);
87	assertEquals("Wrong language of the embedded sequence", "text/x-simple-char", embeddedLang.mimeType());	88	assertEquals("Wrong language of the embedded sequence", "text/x-simple-char", embeddedLang.mimeType());
88		89
89	assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveFirst());	90	embeddedSeq.moveStart();
90	assertEquals("Wrong startSkipLenght", 1, embeddedSeq.offset() - seq.offset());	91	assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveNext());
		92	assertEquals("Wrong startSkipLength", 1, embeddedSeq.offset() - seq.offset());
91		93
92	assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.moveLast());	94	embeddedSeq.moveEnd();
93	assertEquals("Wrong endSkipLenght", 2,	95	assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.movePrevious());
		96	assertEquals("Wrong endSkipLength", 2,
94	(seq.offset() + seq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length()));	97	(seq.offset() + seq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length()));
95	}	98	}
96	}	99	}

Lines 237-242 Link Here

(-)lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java (+10 lines)
237	sb.append(": ");	237	sb.append(": ");
238	AbstractToken token = token(tokenOrEmbeddingContainer);	238	AbstractToken token = token(tokenOrEmbeddingContainer);
239	sb.append(token.dumpInfo(tokenHierarchy));	239	sb.append(token.dumpInfo(tokenHierarchy));
		240	int la = tokenList.lookahead(i);
		241	if (la != 0) {
		242	sb.append(", la=");
		243	sb.append(la);
		244	}
		245	Object state= tokenList.state(i);
		246	if (state != null) {
		247	sb.append(", s=");
		248	sb.append(state);
		249	}
240	sb.append('\n');	250	sb.append('\n');
241	}	251	}
242	return sb;	252	return sb;

Lines 130-136 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java (-3 / +6 lines)
130	public void testCachingE() {	130	public void testCachingE() {
131	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());	131	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
132	TokenSequence tokens = th.tokenSequence();	132	TokenSequence tokens = th.tokenSequence();
133	tokens.moveFirst();	133	tokens.moveStart();
		134	assertEquals(true, tokens.moveNext());
134		135
135	TokenSequence embeddedA = tokens.embedded();	136	TokenSequence embeddedA = tokens.embedded();
136	assertNotNull("There should be an embedded language", embeddedA);	137	assertNotNull("There should be an embedded language", embeddedA);
Lines 144-150 Link Here
144	public void testGCedE() {	145	public void testGCedE() {
145	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());	146	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
146	TokenSequence tokens = th.tokenSequence();	147	TokenSequence tokens = th.tokenSequence();
147	tokens.moveFirst();	148	tokens.moveStart();
		149	assertEquals(true, tokens.moveNext());
148		150
149	TokenSequence embedded = tokens.embedded();	151	TokenSequence embedded = tokens.embedded();
150	assertNotNull("There should be an embedded language", embedded);	152	assertNotNull("There should be an embedded language", embedded);
Lines 164-170 Link Here
164	public void testCacheRefreshedE() {	166	public void testCacheRefreshedE() {
165	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());	167	TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
166	TokenSequence tokens = th.tokenSequence();	168	TokenSequence tokens = th.tokenSequence();
167	tokens.moveFirst();	169	tokens.moveStart();
		170	assertEquals(true, tokens.moveNext());
168		171
169	TokenSequence embeddedA = tokens.embedded();	172	TokenSequence embeddedA = tokens.embedded();
170	assertNotNull("There should be an embedded language", embeddedA);	173	assertNotNull("There should be an embedded language", embeddedA);

Lines 70-75 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java (-2 / +7 lines)
70	public static void assertTokenEquals(String message, TokenSequence<? extends TokenId> ts, TokenId id, String text, int offset) {	70	public static void assertTokenEquals(String message, TokenSequence<? extends TokenId> ts, TokenId id, String text, int offset) {
71	message = messagePrefix(message);	71	message = messagePrefix(message);
72	Token<? extends TokenId> t = ts.token();	72	Token<? extends TokenId> t = ts.token();
		73	TestCase.assertNotNull("Token is null", t);
73	TokenId tId = t.id();	74	TokenId tId = t.id();
74	TestCase.assertEquals(message + "Invalid token.id()", id, tId);	75	TestCase.assertEquals(message + "Invalid token.id()", id, tId);
75	CharSequence tText = t.text();	76	CharSequence tText = t.text();
Lines 312-328 Link Here
312	}	313	}
313	TokenHierarchy<?> thBatch = TokenHierarchy.create(docText, language);	314	TokenHierarchy<?> thBatch = TokenHierarchy.create(docText, language);
314	boolean success = false;	315	boolean success = false;
		316	TokenSequence<?> batchTS = thBatch.tokenSequence();
315	try {	317	try {
316	// Compare lookaheads and states as well	318	// Compare lookaheads and states as well
317	assertTokenSequencesEqual(thBatch.tokenSequence(), thBatch,	319	assertTokenSequencesEqual(batchTS, thBatch,
318	thInc.tokenSequence(), thInc, true);	320	thInc.tokenSequence(), thInc, true);
319	success = true;	321	success = true;
320	} finally {	322	} finally {
321	if (!success) {	323	if (!success) {
		324	// Go forward two tokens to have an extra tokens context
		325	batchTS.moveNext();
		326	batchTS.moveNext();
322	System.err.println("BATCH token sequence dump:\n" + thBatch.tokenSequence());	327	System.err.println("BATCH token sequence dump:\n" + thBatch.tokenSequence());
323	TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);	328	TokenHierarchy<?> lastHi = (TokenHierarchy<?>)doc.getProperty(LAST_TOKEN_HIERARCHY);
324	if (lastHi != null) {	329	if (lastHi != null) {
325	System.err.println("PREVIOUS token sequence dump:\n" + lastHi.tokenSequence());	330	System.err.println("PREVIOUS batch token sequence dump:\n" + lastHi.tokenSequence());
326	}	331	}
327	}	332	}
328	}	333	}

Lines 76-82 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java (-3 / +6 lines)
76		76
77	// Check that all the non-fly tokens are mutable	77	// Check that all the non-fly tokens are mutable
78	ts = snapshot1.tokenSequence();	78	ts = snapshot1.tokenSequence();
79	assertTrue(ts.moveIndex(0));	79	assertEquals(0, ts.moveIndex(0));
		80	assertTrue(ts.moveNext());
80		81
81	LexerTestUtilities.assertTokenSequencesEqual(hi1.tokenSequence(), hi1,	82	LexerTestUtilities.assertTokenSequencesEqual(hi1.tokenSequence(), hi1,
82	snapshot1.tokenSequence(), snapshot1, false);	83	snapshot1.tokenSequence(), snapshot1, false);
Lines 84-90 Link Here
84	doc.insertString(4, "+", null);	85	doc.insertString(4, "+", null);
85		86
86	// Check that the snapshot token sequence can be further operated.	87	// Check that the snapshot token sequence can be further operated.
87	assertTrue(ts.moveIndex(0));	88	assertEquals(0, ts.moveIndex(0));
		89	assertTrue(ts.moveNext());
88	assertNotNull(ts.token());	90	assertNotNull(ts.token());
89		91
90	// Check that the tokens except '+' are live	92	// Check that the tokens except '+' are live
Lines 101-107 Link Here
101		103
102	// Check that all the non-fly tokens are mutable	104	// Check that all the non-fly tokens are mutable
103	ts = snapshot2.tokenSequence();	105	ts = snapshot2.tokenSequence();
104	assertTrue(ts.moveIndex(0));	106	assertEquals(0, ts.moveIndex(0));
		107	assertTrue(ts.moveNext());
105		108
106	LexerTestUtilities.assertTokenSequencesEqual(hi2.tokenSequence(), hi2,	109	LexerTestUtilities.assertTokenSequencesEqual(hi2.tokenSequence(), hi2,
107	snapshot2.tokenSequence(), snapshot2, false);	110	snapshot2.tokenSequence(), snapshot2, false);

Lines 90-108 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java (-1 / +5 lines)
90		90
91	// Check TokenSequence.move()	91	// Check TokenSequence.move()
92	int relOffset = ts.move(50); // past the end of all tokens	92	int relOffset = ts.move(50); // past the end of all tokens
93	assertEquals(relOffset, 50 - offset);	93	assertEquals(relOffset, 50 - (offset + 3));
		94	assertTrue(ts.movePrevious());
94	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset);	95	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset);
95		96
96	relOffset = ts.move(6); // right at begining of "-"	97	relOffset = ts.move(6); // right at begining of "-"
97	assertEquals(relOffset, 0);	98	assertEquals(relOffset, 0);
		99	assertTrue(ts.moveNext());
98	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", 6);	100	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", 6);
99		101
100	relOffset = ts.move(-5); // to first token "abc"	102	relOffset = ts.move(-5); // to first token "abc"
101	assertEquals(relOffset, -5);	103	assertEquals(relOffset, -5);
		104	assertTrue(ts.moveNext());
102	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);	105	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
103		106
104	relOffset = ts.move(5); // to first token "abc"	107	relOffset = ts.move(5); // to first token "abc"
105	assertEquals(relOffset, 1);	108	assertEquals(relOffset, 1);
		109	assertTrue(ts.moveNext());
106	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "uv", 4);	110	LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "uv", 4);
107		111
108		112

Lines 86-102 Link Here

(-)xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java (-13 / +18 lines)
86	* e.g. <tag \|attr=	86	* e.g. <tag \|attr=
87	*	87	*
88	*/	88	*/
89	do {	89	while (tokenSequence.moveNext()) {
90	tokenSequence.moveNext();
91	token = tokenSequence.token();	90	token = tokenSequence.token();
92	tokenOffset = token.offset(tokenHierarchy);	91	tokenOffset = tokenSequence.offset();
93	} while (token != null && isWSTag(token)	92	if (!isWSTag(token) \|\| tagStartLine != Utilities.getLineOffset(doc, tokenOffset)) {
94	&& tagStartLine == Utilities.getLineOffset(doc, tokenOffset));	93	if (!isWSTag(token) && tagStartLine == Utilities.getLineOffset(doc, tokenOffset)){
95		94	return tokenOffset - Utilities.getRowIndent(doc, tokenOffset) - Utilities.getRowStart(doc, tokenOffset);
96	if (!isWSTag(token) && tagStartLine == Utilities.getLineOffset(doc, tokenOffset)){	95	}
97	return tokenOffset - Utilities.getRowIndent(doc, tokenOffset) - Utilities.getRowStart(doc, tokenOffset);	96	break;
		97	}
98	}	98	}
99		99
		100
100	return getShiftWidth(); // default;	101	return getShiftWidth(); // default;
101	}	102	}
102		103
Lines 122-128 Link Here
122	int indentsWithinTags[] = new int[lastLine + 1];	123	int indentsWithinTags[] = new int[lastLine + 1];
123		124
124	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();	125	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
125	boolean thereAreMoreTokens = tokenSequence.moveFirst();	126	tokenSequence.moveStart();
		127	boolean thereAreMoreTokens = tokenSequence.moveNext();
		128
126		129
127	if (tokenSequence != null){	130	if (tokenSequence != null){
128	// calc line indents - pass 1	131	// calc line indents - pass 1
Lines 435-448 Link Here
435	tokenSequence.move(offset);	438	tokenSequence.move(offset);
436	int currentOffset;	439	int currentOffset;
437		440
438	do{	441	while (tokenSequence.moveNext()) {
439	currentOffset = tokenSequence.offset();	442	currentOffset = tokenSequence.offset();
440		443
441	if (isClosingTag(tokenHierarchy, currentOffset)){	444	if (isClosingTag(tokenHierarchy, currentOffset)){
442	return currentOffset;	445	return currentOffset;
443	}	446	}
444		447
445	} while (tokenSequence.moveNext());	448	}
446		449
447	return -1;	450	return -1;
448	}	451	}
Lines 459-466 Link Here
459	protected Token getTokenAtOffset(TokenHierarchy tokenHierarchy, int tagTokenOffset){	462	protected Token getTokenAtOffset(TokenHierarchy tokenHierarchy, int tagTokenOffset){
460	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();	463	TokenSequence tokenSequence = tokenHierarchy.tokenSequence();
461		464
462	if (tokenSequence != null && tokenSequence.move(tagTokenOffset) != Integer.MAX_VALUE){	465	if (tokenSequence != null) {
463	return tokenSequence.token();	466	tokenSequence.move(tagTokenOffset);
		467	if (tokenSequence.moveNext())
		468	return tokenSequence.token();
464	}	469	}
465		470
466	return null;	471	return null;