? libs/j2eeeditor/build ? mdr/jmiutils/nbproject/private/private.properties ? mdr/nbproject/private/private.properties Index: editor/lib2/src/org/netbeans/modules/editor/lib2/highlighting/SyntaxHighlighting.java =================================================================== RCS file: /cvs/editor/lib2/src/org/netbeans/modules/editor/lib2/highlighting/SyntaxHighlighting.java,v retrieving revision 1.6 diff -u -r1.6 SyntaxHighlighting.java --- editor/lib2/src/org/netbeans/modules/editor/lib2/highlighting/SyntaxHighlighting.java 26 Jan 2007 04:58:51 -0000 1.6 +++ editor/lib2/src/org/netbeans/modules/editor/lib2/highlighting/SyntaxHighlighting.java 26 Jan 2007 20:18:33 -0000 @@ -36,7 +36,6 @@ import org.netbeans.api.editor.settings.FontColorSettings; import org.netbeans.api.lexer.Language; import org.netbeans.api.lexer.LanguagePath; -import org.netbeans.api.lexer.TokenChange; import org.netbeans.api.lexer.TokenHierarchy; import org.netbeans.api.lexer.TokenHierarchyEvent; import org.netbeans.api.lexer.TokenHierarchyListener; @@ -102,57 +101,14 @@ synchronized (this) { version++; } - - TokenChange tc = evt.tokenChange(); - int addedLenght = 0; - int removedLength = 0; - - if (tc.addedTokenCount() > 0) { - addedLenght = getTokensLength(tc.currentTokenSequence(), tc.index(), tc.addedTokenCount()); - } - - if (tc.removedTokenCount() > 0) { - removedLength = getTokensLength(tc.removedTokenSequence(), 0, tc.removedTokenCount()); - } - int changeStart = tc.offset(); - int changeEnd; - - if (addedLenght == 0 && removedLength == 0) { - // The real length couldn't be computed for some reason - changeEnd = Integer.MAX_VALUE; - } else { - changeEnd = changeStart + Math.max(addedLenght, removedLength); - } - - fireHighlightsChange(changeStart, changeEnd); + fireHighlightsChange(evt.affectedStartOffset(), evt.affectedEndOffset()); } // ---------------------------------------------------------------------- // Private implementation // ---------------------------------------------------------------------- - private int getTokensLength(TokenSequence seq, int startIdx, int tokenCount) { - assert startIdx >= 0 && startIdx < seq.tokenCount() : - "Invalid startIdx: " + startIdx + ", sequence lenght: " + seq.tokenCount(); - assert tokenCount > 0 && startIdx + tokenCount <= seq.tokenCount() : - "Invalid tokenCount: " + tokenCount + ", startIdx: " + startIdx + - ", sequence lenght: " + seq.tokenCount(); - - int startOffset = -1; - int endOffset = -1; - - if (seq.moveIndex(startIdx)) { - startOffset = seq.offset(); - } - - if (seq.moveIndex(startIdx + tokenCount - 1)) { - endOffset = seq.offset() + seq.token().length(); - } - - return startOffset == -1 || endOffset == -1 ? 0 : endOffset - startOffset; - } - private final class HSImpl implements HighlightsSequence { private static final int S_NORMAL = 1; @@ -196,7 +152,8 @@ case S_EMBEDDED_HEAD: // The current token contains embedded language and we have processed it's head TokenSequence seq = sequences.get(sequences.size() - 1); - if (seq.moveFirst()) { + seq.moveStart(); + if (seq.moveNext()) { state = S_NORMAL; } else { throw new IllegalStateException("Invalid state"); @@ -221,7 +178,7 @@ // We have moved to the next normal token, so look what it is TokenSequence seq = sequences.get(sequences.size() - 1); TokenSequence embeddedSeq = seq.embedded(); - while (embeddedSeq != null && embeddedSeq.moveFirst()) { + while (embeddedSeq != null && embeddedSeq.moveNext()) { sequences.add(sequences.size(), embeddedSeq); if (embeddedSeq.offset() > seq.offset()) { state = S_EMBEDDED_HEAD; @@ -258,7 +215,8 @@ } case S_EMBEDDED_TAIL: { TokenSequence seq = sequences.get(sequences.size() - 1); - if (seq.moveLast()) { + seq.moveEnd(); + if (seq.movePrevious()) { return seq.offset() + seq.token().length(); } else { throw new IllegalStateException("Invalid state"); @@ -288,7 +246,8 @@ } case S_EMBEDDED_HEAD: { TokenSequence seq = sequences.get(sequences.size() - 1); - if (seq.moveFirst()) { + seq.moveStart(); + if (seq.moveNext()) { return seq.offset(); } else { TokenSequence embeddingSeq = sequences.get(sequences.size() - 2); @@ -447,7 +406,8 @@ } else { if (sequences.size() > 1) { TokenSequence embeddingSeq = sequences.get(sequences.size() - 2); - if (seq.moveLast()) { + seq.moveEnd(); + if (seq.movePrevious()) { if ((seq.offset() + seq.token().length()) < (embeddingSeq.offset() + embeddingSeq.token().length())) { return S_EMBEDDED_TAIL; } else { Index: html/editor/lib/src/org/netbeans/editor/ext/html/HTMLCompletionQuery.java =================================================================== RCS file: /cvs/html/editor/lib/src/org/netbeans/editor/ext/html/HTMLCompletionQuery.java,v retrieving revision 1.36 diff -u -r1.36 HTMLCompletionQuery.java --- html/editor/lib/src/org/netbeans/editor/ext/html/HTMLCompletionQuery.java 25 Jan 2007 15:04:23 -0000 1.36 +++ html/editor/lib/src/org/netbeans/editor/ext/html/HTMLCompletionQuery.java 26 Jan 2007 20:18:41 -0000 @@ -106,11 +106,11 @@ if(ts == null) { //HTML language is not top level one ts = hi.tokenSequence(); - int diff = ts.move(offset); - if(diff == Integer.MAX_VALUE) { - return null; //no token found - } else { + ts.move(offset); + if (ts.moveNext() || ts.movePrevious()) { ts = ts.embedded(HTMLTokenId.language()); + } else { // no tokens + return null; } } @@ -120,14 +120,14 @@ } int diff = ts.move(offset); - if(diff == Integer.MAX_VALUE) { + if(!ts.moveNext() && !ts.movePrevious()) { return null; //no token found } Token item = ts.token(); // are we inside token or between tokens - boolean inside = item.offset(hi) < offset; + boolean inside = ts.offset() < offset; if(!inside) { //use the previous token if(ts.movePrevious()) { @@ -192,6 +192,7 @@ //there are some attributes or an end of the tag ts.move(offset); + ts.moveNext(); Token t = ts.token(); //test if next token is a whitespace and the next a tag token or an attribute token @@ -324,6 +325,7 @@ if( tag == null ) return null; // unknown tag ts.move(item.offset(hi)); + ts.moveNext(); Token argItem = ts.token(); while(argItem.id() != HTMLTokenId.ARGUMENT && ts.movePrevious()) { argItem = ts.token(); Index: html/editor/lib/src/org/netbeans/editor/ext/html/HTMLLexerFormatter.java =================================================================== RCS file: /cvs/html/editor/lib/src/org/netbeans/editor/ext/html/HTMLLexerFormatter.java,v retrieving revision 1.1 diff -u -r1.1 HTMLLexerFormatter.java --- html/editor/lib/src/org/netbeans/editor/ext/html/HTMLLexerFormatter.java 2 Jan 2007 13:39:04 -0000 1.1 +++ html/editor/lib/src/org/netbeans/editor/ext/html/HTMLLexerFormatter.java 26 Jan 2007 20:18:41 -0000 @@ -51,6 +51,7 @@ if (position >= 0) { TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); tokenSequence.move(position); + tokenSequence.moveNext(); Token token = tokenSequence.token(); if (token.id() == HTMLTokenId.TAG_CLOSE_SYMBOL && @@ -72,6 +73,7 @@ @Override protected int getTagEndOffset(TokenHierarchy tokenHierarchy, int tagStartOffset){ TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); tokenSequence.move(tagStartOffset); + tokenSequence.moveNext(); boolean thereAreMoreTokens = true; while (thereAreMoreTokens && tokenSequence.token().id() != HTMLTokenId.TAG_CLOSE_SYMBOL){ Index: html/editor/lib/src/org/netbeans/editor/ext/html/HTMLSyntaxSupport.java =================================================================== RCS file: /cvs/html/editor/lib/src/org/netbeans/editor/ext/html/HTMLSyntaxSupport.java,v retrieving revision 1.33 diff -u -r1.33 HTMLSyntaxSupport.java --- html/editor/lib/src/org/netbeans/editor/ext/html/HTMLSyntaxSupport.java 25 Jan 2007 15:04:24 -0000 1.33 +++ html/editor/lib/src/org/netbeans/editor/ext/html/HTMLSyntaxSupport.java 26 Jan 2007 20:18:41 -0000 @@ -122,8 +122,8 @@ return null; } - int diff = ts.move(offset); - if(diff == Integer.MAX_VALUE) { + ts.move(offset); + if(!ts.moveNext() && !ts.movePrevious()) { return null; //no token found } @@ -131,7 +131,8 @@ // if the carret is after HTML tag ( after char '>' ), ship inside the tag if (token.id() == HTMLTokenId.TAG_CLOSE_SYMBOL) { - if(ts.moveIndex(ts.index() - 2)) { + ts.moveIndex(ts.index() - 2); + if(ts.moveNext()) { token = ts.token(); } } @@ -192,6 +193,7 @@ ts.movePrevious(); start = ts.token().offset(hi); ts.moveIndex(ts.index() + 2); + ts.moveNext(); Token tok = ts.token(); end = tok.offset(hi)+ (tok.id() == HTMLTokenId.TAG_CLOSE_SYMBOL ? tok.text().length() : 0); return new int[] {start, end}; @@ -251,6 +253,7 @@ } ts.move(offset); //reset the token sequence to the original position + ts.moveNext(); token = ts.token(); //match html comments @@ -323,6 +326,7 @@ } while(ts.moveNext()); }finally{ ts.moveIndex(tsIndex); //backup the TokenSequence position + ts.moveNext(); } } else { //ts is rewinded out of tokens @@ -437,8 +441,8 @@ return COMPLETION_POST_REFRESH; } - int diff = ts.move(dotPos-1); - if(diff != Integer.MAX_VALUE) { + ts.move(dotPos-1); + if(ts.moveNext() || ts.movePrevious()) { if(ts.token().id() == HTMLTokenId.WS) { return COMPLETION_POPUP; } @@ -489,7 +493,7 @@ //HTML language is not top level one ts = hi.tokenSequence(); int diff = ts.move(offset); - if(diff == Integer.MAX_VALUE) { + if(!ts.moveNext() && !ts.movePrevious()) { return null; //no token found } else { ts = ts.embedded(HTMLTokenId.language()); Index: html/editor/lib/src/org/netbeans/editor/ext/html/parser/SyntaxParser.java =================================================================== RCS file: /cvs/html/editor/lib/src/org/netbeans/editor/ext/html/parser/SyntaxParser.java,v retrieving revision 1.1 diff -u -r1.1 SyntaxParser.java --- html/editor/lib/src/org/netbeans/editor/ext/html/parser/SyntaxParser.java 25 Jan 2007 15:04:24 -0000 1.1 +++ html/editor/lib/src/org/netbeans/editor/ext/html/parser/SyntaxParser.java 26 Jan 2007 20:18:41 -0000 @@ -70,17 +70,17 @@ return null; } - int diff = ts.move(offset); - if(diff == Integer.MAX_VALUE) return null; //no token found + ts.move(offset); + if(!ts.moveNext() && !ts.movePrevious()) return null; //no token found Token item = ts.token(); - int beginning = item.offset(hi); + int beginning = ts.offset(); if( item.id() == HTMLTokenId.CHARACTER ) { do { item = ts.token(); - beginning = item.offset(hi); + beginning = ts.offset(); } while(item.id() == HTMLTokenId.CHARACTER && ts.movePrevious()); // now item is either HTMLSyntax.VALUE or we're in text, or at BOF @@ -97,7 +97,7 @@ if( item.id() == HTMLTokenId.TEXT ) { do { - beginning = ts.token().offset(hi); + beginning = ts.offset(); } while ( ts.movePrevious() && (ts.token().id() == HTMLTokenId.TEXT || ts.token().id() == HTMLTokenId.CHARACTER)); return getNextElement( beginning ); // from start of Commment @@ -158,9 +158,8 @@ return null; } - int diff = ts.move(offset); - - if (diff >= ts.token().length() || diff == java.lang.Integer.MAX_VALUE) + ts.move(offset); + if (!ts.moveNext()) return null; org.netbeans.api.lexer.Token item = ts.token(); int lastOffset = getTokenEnd(hi, item); @@ -393,8 +392,8 @@ if(ts == null) { //HTML language is not top level one ts = hi.tokenSequence(); - int diff = ts.move(offset); - if(diff == Integer.MAX_VALUE) { + ts.move(offset); + if(!ts.moveNext() && !ts.movePrevious()) { return null; //no token found } else { ts = ts.embedded(HTMLTokenId.language()); Index: html/editor/src/org/netbeans/modules/editor/html/HTMLCompletionProvider.java =================================================================== RCS file: /cvs/html/editor/src/org/netbeans/modules/editor/html/HTMLCompletionProvider.java,v retrieving revision 1.9 diff -u -r1.9 HTMLCompletionProvider.java --- html/editor/src/org/netbeans/modules/editor/html/HTMLCompletionProvider.java 5 Dec 2006 08:42:17 -0000 1.9 +++ html/editor/src/org/netbeans/modules/editor/html/HTMLCompletionProvider.java 26 Jan 2007 20:18:41 -0000 @@ -165,8 +165,9 @@ TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc); TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); - int diff = tokenSequence.move(caretOffset == 0 ? 0 : caretOffset - 1); - if(diff >= tokenSequence.token().length() || diff == Integer.MAX_VALUE) return; //no token found + tokenSequence.move(caretOffset == 0 ? 0 : caretOffset - 1); + if (!tokenSequence.moveNext()) + return; Token tokenItem = tokenSequence.token(); if(tokenItem.id() == HTMLTokenId.TEXT && !tokenItem.text().toString().startsWith("<") && !tokenItem.text().toString().startsWith("&")) { Index: java/editor/src/org/netbeans/modules/editor/java/JavaCompletionProvider.java =================================================================== RCS file: /cvs/java/editor/src/org/netbeans/modules/editor/java/JavaCompletionProvider.java,v retrieving revision 1.75 diff -u -r1.75 JavaCompletionProvider.java --- java/editor/src/org/netbeans/modules/editor/java/JavaCompletionProvider.java 23 Jan 2007 15:29:55 -0000 1.75 +++ java/editor/src/org/netbeans/modules/editor/java/JavaCompletionProvider.java 26 Jan 2007 20:19:16 -0000 @@ -1116,9 +1116,8 @@ int openLtNum = 0; JavaTokenId lastNonWhitespaceTokenId = null; TokenSequence ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language()); - if (ts.move(expEndPos) > 0) - ts.moveNext(); - do { + ts.move(expEndPos); + while (ts.moveNext()) { if (ts.offset() >= offset) { break; } @@ -1151,7 +1150,7 @@ default: lastNonWhitespaceTokenId = ts.token().id(); } - } while (ts.moveNext()); + } if (!afterDot) { if (expEndPos <= offset) insideExpression(env, new TreePath(path, fa.getExpression())); @@ -3426,8 +3425,10 @@ private TokenSequence findLastNonWhitespaceToken(Env env, int startPos, int endPos) { TokenSequence ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language()); - if (ts.move(endPos) == 0) - ts.movePrevious(); + if (ts.move(endPos) == 0) // When right at the token end + ts.movePrevious(); // move to previous token + else + ts.moveNext(); // otherwise move to the token that "contains" the offset do { int offset = ts.offset(); if (offset < startPos) @@ -3564,8 +3565,10 @@ if (upToOffset) { TokenSequence ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language()); if (ts.moveNext()) { //check for empty file - if (ts.move(offset) == 0) - ts.movePrevious(); + if (ts.move(offset) == 0) // When right at the token end + ts.movePrevious(); // Move to previous token + else + ts.moveNext(); // otherwise move to the token that "contains" the offset if (ts.offset() < offset && (ts.token().id() == JavaTokenId.IDENTIFIER || "keyword".equals(ts.token().id().primaryCategory()))) { //TODO: Use isKeyword(...) when available prefix = ts.token().toString().substring(0, offset - ts.offset()); offset = ts.offset(); Index: java/editor/src/org/netbeans/modules/editor/java/Utilities.java =================================================================== RCS file: /cvs/java/editor/src/org/netbeans/modules/editor/java/Utilities.java,v retrieving revision 1.13 diff -u -r1.13 Utilities.java --- java/editor/src/org/netbeans/modules/editor/java/Utilities.java 11 Jan 2007 10:55:50 -0000 1.13 +++ java/editor/src/org/netbeans/modules/editor/java/Utilities.java 26 Jan 2007 20:19:16 -0000 @@ -98,6 +98,7 @@ return false; if (!ts.moveNext() || ts.move(offset) == 0) return true; + ts.moveNext(); // Move to the next token after move(offset) switch(ts.token().id()) { case DOUBLE_LITERAL: if (ts.token().text().charAt(0) == '.') @@ -127,6 +128,7 @@ TokenSequence ts = hierarchy.tokenSequence(); while(ts != null && ts.moveNext()) { ts.move(offset); + ts.moveNext(); if (ts.language() == JavaTokenId.language()) return (TokenSequence)ts; ts = ts.embedded(); Index: java/hints/src/org/netbeans/modules/java/hints/JavaHintsProvider.java =================================================================== RCS file: /cvs/java/hints/src/org/netbeans/modules/java/hints/JavaHintsProvider.java,v retrieving revision 1.65 diff -u -r1.65 JavaHintsProvider.java --- java/hints/src/org/netbeans/modules/java/hints/JavaHintsProvider.java 16 Jan 2007 09:39:22 -0000 1.65 +++ java/hints/src/org/netbeans/modules/java/hints/JavaHintsProvider.java 26 Jan 2007 20:19:20 -0000 @@ -175,23 +175,23 @@ TokenSequence ts = info.getTokenHierarchy().tokenSequence(); ts.move(offset); - - Token t = ts.token(); - - if (t.id() == JavaTokenId.DOT) { - ts.moveNext(); - t = ts.token(); - } else { - if (t.id() == JavaTokenId.LT) { + if (ts.moveNext()) { + Token t = ts.token(); + + if (t.id() == JavaTokenId.DOT) { ts.moveNext(); t = ts.token(); + } else { + if (t.id() == JavaTokenId.LT) { + ts.moveNext(); + t = ts.token(); + } + } + + if (t.id() == JavaTokenId.IDENTIFIER) { + return ts.offsetToken(); } } - - if (t.id() == JavaTokenId.IDENTIFIER) { - return ts.offsetToken(); - } - return null; } @@ -256,7 +256,7 @@ int diff = ts.move((int) getPrefferedPosition(info, d)); - if (diff >= 0 && diff < ts.token().length()) { + if (ts.moveNext() && diff >= 0 && diff < ts.token().length()) { Token t = ts.token(); if (t.id() == JavaTokenId.IDENTIFIER) { Index: java/source/src/org/netbeans/api/java/source/TreeUtilities.java =================================================================== RCS file: /cvs/java/source/src/org/netbeans/api/java/source/TreeUtilities.java,v retrieving revision 1.13 diff -u -r1.13 TreeUtilities.java --- java/source/src/org/netbeans/api/java/source/TreeUtilities.java 11 Jan 2007 12:24:03 -0000 1.13 +++ java/source/src/org/netbeans/api/java/source/TreeUtilities.java 26 Jan 2007 20:19:26 -0000 @@ -158,7 +158,8 @@ return path; TokenSequence tokenList = tokensFor(path.getLeaf(), sourcePositions); - if (tokenList.moveLast() && tokenList.offset() < pos) { + tokenList.moveEnd(); + if (tokenList.movePrevious() && tokenList.offset() < pos) { switch (tokenList.token().id()) { case GTGTGT: case GTGT: Index: java/source/src/org/netbeans/modules/java/source/save/CasualDiff.java =================================================================== RCS file: /cvs/java/source/src/org/netbeans/modules/java/source/save/CasualDiff.java,v retrieving revision 1.51 diff -u -r1.51 CasualDiff.java --- java/source/src/org/netbeans/modules/java/source/save/CasualDiff.java 19 Jan 2007 15:12:32 -0000 1.51 +++ java/source/src/org/netbeans/modules/java/source/save/CasualDiff.java 26 Jan 2007 20:19:27 -0000 @@ -316,6 +316,7 @@ // skip the section when printing anonymous class if (anonClass == false) { tokenSequence.move(oldT.pos); + tokenSequence.moveNext(); // First skip as move() does not position to token directly tokenSequence.moveNext(); insertHint = TokenUtilities.moveNext(tokenSequence, tokenSequence.offset()); localPointer = diffModifiers(oldT.mods, newT.mods, oldT, localPointer); @@ -508,7 +509,6 @@ // there was not any parameter in original tree. int startOffset = oldT.restype != null ? oldT.restype.getStartPosition() : oldT.getStartPosition(); - tokenSequence.move(startOffset); TokenUtilities.moveFwdToToken(tokenSequence, startOffset, JavaTokenId.RPAREN); posHint = tokenSequence.offset(); } else { @@ -533,6 +533,7 @@ // now check, that there is a whitespace. It is not mandatory, we // have to ensure. If whitespace is not present, take body beginning tokenSequence.move(posHint); + tokenSequence.moveNext(); if (tokenSequence.token().id() != JavaTokenId.WHITESPACE) { ++posHint; } @@ -1441,8 +1442,9 @@ ResultItem item = result[j]; switch (item.operation) { case MODIFY: { - // perhaps I shouldn't support this! - if (tokenSequence.moveIndex(matrix[i][4])) { + // perhaps I shouldn't support this! + tokenSequence.moveIndex(matrix[i][4]); + if (tokenSequence.moveNext()) { testPos = tokenSequence.offset(); if (JavaTokenId.COMMA == tokenSequence.token().id()) testPos += JavaTokenId.COMMA.fixedText().length(); @@ -1488,7 +1490,8 @@ assert startOffset != -1 && endOffset != -1 : "Invalid offset!"; //printer.print(origText.substring(lastOldPos, startOffset)); //append(Diff.delete(startOffset, endOffset)); - if (tokenSequence.moveIndex(matrix[i][4])) { + tokenSequence.moveIndex(matrix[i][4]); + if (tokenSequence.moveNext()) { testPos = tokenSequence.offset(); if (JavaTokenId.COMMA == tokenSequence.token().id()) testPos += JavaTokenId.COMMA.fixedText().length(); @@ -1502,7 +1505,8 @@ break; } case NOCHANGE: { - if (tokenSequence.moveIndex(matrix[i][4])) { + tokenSequence.moveIndex(matrix[i][4]); + if (tokenSequence.moveNext()) { testPos = tokenSequence.offset(); if (JavaTokenId.COMMA == tokenSequence.token().id()) testPos += JavaTokenId.COMMA.fixedText().length(); @@ -1521,6 +1525,7 @@ return -1; } tokenSequence.moveIndex(tokenIndex); + tokenSequence.moveNext(); return tokenSequence.offset(); } Index: java/source/src/org/netbeans/modules/java/source/save/PositionEstimator.java =================================================================== RCS file: /cvs/java/source/src/org/netbeans/modules/java/source/save/PositionEstimator.java,v retrieving revision 1.2 diff -u -r1.2 PositionEstimator.java --- java/source/src/org/netbeans/modules/java/source/save/PositionEstimator.java 17 Dec 2006 16:37:47 -0000 1.2 +++ java/source/src/org/netbeans/modules/java/source/save/PositionEstimator.java 26 Jan 2007 20:19:27 -0000 @@ -139,6 +139,7 @@ int treeEnd = (int) positions.getEndPosition(compilationUnit, item); seq.move(treeStart); + seq.moveNext(); int startIndex = seq.index(); // go back to opening/closing curly, semicolon or other @@ -147,9 +148,11 @@ seq.moveNext(); int start = seq.index(); seq.move(treeEnd); + // seq.index() set (no seq.moveNext() necessary) matrix[i++] = new int[] { start, startIndex, seq.index() }; if (i == size) { seq.move(treeEnd); + // seq.index() set (no seq.moveNext() necessary) matrix[i][0] = seq.index(); } } @@ -163,6 +166,7 @@ // to decide. if (tokenIndex == -1) return -1; seq.moveIndex(tokenIndex); + seq.moveNext(); return index == 0 ? goAfterLastNewLine(seq) : goAfterFirstNewLine(seq); } @@ -209,9 +213,15 @@ @Override() public int[] getPositions(int index) { int tokenIndex = matrix[index][0]; - seq.moveIndex(tokenIndex); + if (tokenIndex != -1) { + seq.moveIndex(tokenIndex); + seq.moveNext(); + } int begin = goAfterLastNewLine(seq); - seq.moveIndex(matrix[index][2]); + if (matrix[index][2] != -1) { + seq.moveIndex(matrix[index][2]); + seq.moveNext(); + } int end = goAfterFirstNewLine(seq); return new int [] { begin, end }; } @@ -262,6 +272,7 @@ if (treeEnd < 0) continue; seq.move(treeStart); + seq.moveNext(); int startIndex = seq.index(); // go back to opening/closing curly, semicolon or other // token java-compiler important token. @@ -285,6 +296,7 @@ // to decide. if (tokenIndex == -1) return -1; seq.moveIndex(tokenIndex); + seq.moveNext(); return goAfterFirstNewLine(seq); } @@ -296,7 +308,10 @@ public int[] getPositions(int index) { int begin = getInsertPos(index); - seq.moveIndex(matrix[index][4]); + if (matrix[index][4] != -1) { + seq.moveIndex(matrix[index][4]); + seq.moveNext(); + } int end = goAfterFirstNewLine(seq); return new int [] { begin, end }; } @@ -360,6 +375,7 @@ // to decide. if (tokenIndex == -1) return -1; seq.moveIndex(tokenIndex); + seq.moveNext(); int off = goAfterFirstNewLine(seq); return off; } @@ -372,7 +388,10 @@ public int[] getPositions(int index) { int begin = getInsertPos(index); - seq.moveIndex(matrix[index][4]); + if (matrix[index][4] != -1) { + seq.moveIndex(matrix[index][4]); + seq.moveNext(); + } int end = goAfterFirstNewLine(seq); return new int [] { begin, end }; } @@ -497,16 +516,19 @@ // todo (#pf): remove - used for debugging reasons, doesn't do good job public void tablePrint(int[][] matrix, TokenSequence seq) { for (int i = 0; i < matrix.length; i++) { - for (int j = 0; j < 5; j++) { + for (int j = 0; j < matrix[i].length; j++) { int item = matrix[i][j]; String s = "(nothing)"; if (item > -1) { seq.moveIndex(item); + seq.moveNext(); s = "'" + seq.token().text(); } s += "' "; - System.err.print(s.substring(0, 25)); + // System.err.print(s.substring(0, 25)); + System.err.print(item + "\t"); } + System.err.println(""); } } Index: java/source/src/org/netbeans/modules/java/source/save/TokenUtilities.java =================================================================== RCS file: /cvs/java/source/src/org/netbeans/modules/java/source/save/TokenUtilities.java,v retrieving revision 1.1 diff -u -r1.1 TokenUtilities.java --- java/source/src/org/netbeans/modules/java/source/save/TokenUtilities.java 29 Nov 2006 21:54:48 -0000 1.1 +++ java/source/src/org/netbeans/modules/java/source/save/TokenUtilities.java 26 Jan 2007 20:19:27 -0000 @@ -65,6 +65,7 @@ */ static int moveNext(TokenSequence tokenSequence, final int pos) { tokenSequence.move(pos); + tokenSequence.moveNext(); // Assumes the pos is located within input bounds JavaTokenId type = tokenSequence.token().id(); while (JavaTokenId.WHITESPACE.equals(type) || JavaTokenId.BLOCK_COMMENT.equals(type) || @@ -82,6 +83,7 @@ JavaTokenId id) { tokenSequence.move(pos); + tokenSequence.moveNext(); // Assumes the pos is located within input bounds while (!id.equals(tokenSequence.token().id())) { if (!tokenSequence.moveNext()) return -1; @@ -94,6 +96,7 @@ JavaTokenId id) { tokenSequence.move(pos); + tokenSequence.moveNext(); // Assumes the pos is located within input bounds while (!id.equals(tokenSequence.token().id())) { if (!tokenSequence.movePrevious()) return -1; @@ -106,6 +109,7 @@ String text) { tokenSequence.move(pos); + tokenSequence.moveNext(); // Assumes the pos is located within input bounds while (!text.equals(tokenSequence.token().text())) { if (!tokenSequence.moveNext()) return -1; @@ -118,6 +122,7 @@ String text) { tokenSequence.move(pos); + tokenSequence.moveNext(); // Assumes the pos is located within input bounds while (!text.equals(tokenSequence.token().text())) { if (!tokenSequence.movePrevious()) return -1; Index: java/source/src/org/netbeans/modules/java/source/save/TreeDiff.java =================================================================== RCS file: /cvs/java/source/src/org/netbeans/modules/java/source/save/TreeDiff.java,v retrieving revision 1.6 diff -u -r1.6 TreeDiff.java --- java/source/src/org/netbeans/modules/java/source/save/TreeDiff.java 11 Jan 2007 12:24:36 -0000 1.6 +++ java/source/src/org/netbeans/modules/java/source/save/TreeDiff.java 26 Jan 2007 20:19:28 -0000 @@ -1247,7 +1247,8 @@ ResultItem item = result[j]; switch (item.operation) { case MODIFY: { - if (tokenSequence.moveIndex(matrix[i][4])) { + tokenSequence.moveIndex(matrix[i][4]); + if (tokenSequence.moveNext()) { testPos = tokenSequence.offset(); if (JavaTokenId.COMMA == tokenSequence.token().id()) testPos += JavaTokenId.COMMA.fixedText().length(); @@ -1283,7 +1284,8 @@ int endOffset = toOff(s.tail(j) || s.next(j) ? matrix[i+1][2] : matrix[i][4]); assert startOffset != -1 && endOffset != -1 : "Invalid offset!"; append(Diff.delete(startOffset, endOffset)); - if (tokenSequence.moveIndex(matrix[i][4])) { + tokenSequence.moveIndex(matrix[i][4]); + if (tokenSequence.moveNext()) { testPos = tokenSequence.offset(); if (JavaTokenId.COMMA == tokenSequence.token().id()) testPos += JavaTokenId.COMMA.fixedText().length(); @@ -1293,7 +1295,8 @@ break; } case NOCHANGE: { - if (tokenSequence.moveIndex(matrix[i][4])) { + tokenSequence.moveIndex(matrix[i][4]); + if (tokenSequence.moveNext()) { testPos = tokenSequence.offset(); if (JavaTokenId.COMMA == tokenSequence.token().id()) testPos += JavaTokenId.COMMA.fixedText().length(); Index: lexer/arch.xml =================================================================== RCS file: /cvs/lexer/arch.xml,v retrieving revision 1.8 diff -u -r1.8 arch.xml --- lexer/arch.xml 6 Dec 2006 15:26:46 -0000 1.8 +++ lexer/arch.xml 26 Jan 2007 20:19:42 -0000 @@ -300,20 +300,18 @@ TokenHierarchy hi = TokenHierarchy.get(document); TokenSequence ts = hi.tokenSequence(); // If necessary move ts to the requested offset - int diff = ts.move(offset); - if (diff != Integer.MAX_VALUE)) { // Some tokens exist - do { - Token t = ts.token(); - if (t.id() == ...) { ... } - if (TokenUtilities.equals(t.text(), "mytext")) { ... } - if (ts.offset() == ...) { ... } + ts.move(offset); + while (ts.moveNext()) { + Token t = ts.token(); + if (t.id() == ...) { ... } + if (TokenUtilities.equals(t.text(), "mytext")) { ... } + if (ts.offset() == ...) { ... } - // Possibly retrieve embedded token sequence - TokenSequence embedded = ts.embedded(); - if (embedded != null) { // Token has a valid language embedding - ... - } - } while (ts.moveNext()); + // Possibly retrieve embedded token sequence + TokenSequence embedded = ts.embedded(); + if (embedded != null) { // Token has a valid language embedding + ... + } } } finally { document.readUnlock(); Index: lexer/manifest.mf =================================================================== RCS file: /cvs/lexer/manifest.mf,v retrieving revision 1.13 diff -u -r1.13 manifest.mf --- lexer/manifest.mf 14 Dec 2006 13:03:01 -0000 1.13 +++ lexer/manifest.mf 26 Jan 2007 20:19:42 -0000 @@ -1,4 +1,4 @@ OpenIDE-Module: org.netbeans.modules.lexer/2 OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/lexer/Bundle.properties -OpenIDE-Module-Specification-Version: 1.12.0 +OpenIDE-Module-Specification-Version: 1.13.0 OpenIDE-Module-Recommends: org.netbeans.spi.lexer.LanguageProvider Index: lexer/api/apichanges.xml =================================================================== RCS file: /cvs/lexer/api/apichanges.xml,v retrieving revision 1.10 diff -u -r1.10 apichanges.xml --- lexer/api/apichanges.xml 14 Dec 2006 13:03:02 -0000 1.10 +++ lexer/api/apichanges.xml 26 Jan 2007 20:19:42 -0000 @@ -91,6 +91,34 @@ + + + Changed TokenSequence.move() and moveIndex() use + + + + + +

+ Changed the TokenSequence.move() to position before the particular token + that "contains" the offset (or after the last token if the given offset + is too high). Additional moveNext() is necessary + to actually move to the next token. +
+ TokenSequence.moveIndex() was modified in a similar way + (see javadocs). +
+ TokenSequence.moveFirst() and moveLast() + were replaced by moveStart() that positionins before + the first token and by moveEnd() that positions + after the last token. +
+ TokenSequence.isEmpty() added to check whether there are + no tokens in the TS. +

+ + + Added Lexer.release() Index: lexer/editorbridge/src/org/netbeans/modules/lexer/editorbridge/LexerLayer.java =================================================================== RCS file: /cvs/lexer/editorbridge/src/org/netbeans/modules/lexer/editorbridge/LexerLayer.java,v retrieving revision 1.9 diff -u -r1.9 LexerLayer.java --- lexer/editorbridge/src/org/netbeans/modules/lexer/editorbridge/LexerLayer.java 5 Jan 2007 03:30:36 -0000 1.9 +++ lexer/editorbridge/src/org/netbeans/modules/lexer/editorbridge/LexerLayer.java 26 Jan 2007 20:19:43 -0000 @@ -108,8 +108,8 @@ if (active) { pastSequences = new Stack(); tokenSequence = hi.tokenSequence(); - int relOffset = tokenSequence.move(startOffset); - if (relOffset != Integer.MAX_VALUE) { + tokenSequence.move(startOffset); + if (tokenSequence.moveNext()) { updateTokenEndOffsetAndColoring(startOffset); } else { // no tokens active = false; @@ -178,7 +178,8 @@ break; } } else { - if (embed.move(offset) != Integer.MAX_VALUE) { + embed.move(offset); + if (embed.moveNext()) { pastSequences.push(tokenSequence); tokenSequence = embed; } else { Index: lexer/nbbridge/test/unit/src/org/netbeans/modules/lexer/nbbridge/test/MimeLookupLanguageProviderTest.java =================================================================== RCS file: /cvs/lexer/nbbridge/test/unit/src/org/netbeans/modules/lexer/nbbridge/test/MimeLookupLanguageProviderTest.java,v retrieving revision 1.4 diff -u -r1.4 MimeLookupLanguageProviderTest.java --- lexer/nbbridge/test/unit/src/org/netbeans/modules/lexer/nbbridge/test/MimeLookupLanguageProviderTest.java 26 Oct 2006 20:45:21 -0000 1.4 +++ lexer/nbbridge/test/unit/src/org/netbeans/modules/lexer/nbbridge/test/MimeLookupLanguageProviderTest.java 26 Jan 2007 20:19:43 -0000 @@ -75,7 +75,8 @@ assertEquals("Wrong language", "text/x-simple-plain", lang.mimeType()); for(int i = 0; i < seq.tokenCount(); i++) { - seq.move(i); + seq.moveIndex(i); + assertTrue(seq.moveNext()); Token token = seq.token(); if (token.id() == SimplePlainTokenId.WORD) { @@ -86,11 +87,13 @@ assertNotNull("Can't find language of the embedded sequence", embeddedLang); assertEquals("Wrong language of the embedded sequence", "text/x-simple-char", embeddedLang.mimeType()); - assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveFirst()); - assertEquals("Wrong startSkipLenght", 1, embeddedSeq.offset() - seq.offset()); + embeddedSeq.moveStart(); + assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveNext()); + assertEquals("Wrong startSkipLength", 1, embeddedSeq.offset() - seq.offset()); - assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.moveLast()); - assertEquals("Wrong endSkipLenght", 2, + embeddedSeq.moveEnd(); + assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.movePrevious()); + assertEquals("Wrong endSkipLength", 2, (seq.offset() + seq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length())); } } Index: lexer/src/org/netbeans/api/lexer/Token.java =================================================================== RCS file: /cvs/lexer/src/org/netbeans/api/lexer/Token.java,v retrieving revision 1.5 diff -u -r1.5 Token.java --- lexer/src/org/netbeans/api/lexer/Token.java 4 Dec 2006 15:43:58 -0000 1.5 +++ lexer/src/org/netbeans/api/lexer/Token.java 26 Jan 2007 20:19:43 -0000 @@ -149,6 +149,9 @@ * Get the offset at which this token is present in the input * or -1 if this token is flyweight (and therefore does not store offset). *
+ * Note: Use of {@link TokenSequence#offset()} is usually preferred over + * this method. + *
* For flyweight tokens the real offset of the token may only be determined * by doing {@link TokenSequence#offset()} when positioned on the particular * flyweight token. Index: lexer/src/org/netbeans/api/lexer/TokenSequence.java =================================================================== RCS file: /cvs/lexer/src/org/netbeans/api/lexer/TokenSequence.java,v retrieving revision 1.7 diff -u -r1.7 TokenSequence.java --- lexer/src/org/netbeans/api/lexer/TokenSequence.java 4 Dec 2006 15:43:59 -0000 1.7 +++ lexer/src/org/netbeans/api/lexer/TokenSequence.java 26 Jan 2007 20:19:43 -0000 @@ -29,16 +29,55 @@ import org.netbeans.lib.lexer.token.AbstractToken; /** - * Token sequence allows to move between tokens - * of a token hierarchy in forward/backward direction - * and by index/offset positioning. + * Token sequence allows to iterate between tokens + * of a token hierarchy. *
- * It may be obtained by {@link TokenHierarchy#tokenSequence()}. - *
- * A typical use is a forward iteration through the tokens: + * Token sequence for top-level language of a token hierarchy + * may be obtained by {@link TokenHierarchy#tokenSequence()}. + * + *

+ * Use of token sequence is a two-step operation: + *

    + *
  1. + * Position token sequence before token that should first be retrieved + * (or behind desired token when iterating backwards). + *
    + * One of the following ways may be used: + *
      + *
    • {@link #move(int)} positions TS before token that either starts + * at the given offset or "contains" it. + *
      • + *
    • {@link #moveIndex(int)} positions TS before n-th token in the underlying + * token list. + *
      • + *
    • {@link #moveStart()} positions TS before the first token.
      • + *
    • {@link #moveEnd()} positions TS behind the last token.
      • + *
    • Do nothing - TS is positioned before the first token automatically by default.
      • + *
    + * Token sequence will always be positioned between tokens + * when using one of the operations above + * ({@link #token()} will return null to signal between-tokens location). + *
    + *
    2. + * + *
  2. + * Start iterating through the tokens in forward/backward direction + * by using {@link #moveNext()} or {@link #movePrevious()}. + *
    + * If moveNext() or movePrevious() returned + * true then TS is positioned + * over a concrete token retrievable by {@link #token()}. + *
    + * Its offset can be retrieved by {@link #offset()}. + *
    3. + *
+ *

+ * + *

+ * An example of forward iteration through the tokens: * 

  *   TokenSequence ts = tokenHierarchy.tokenSequence();
- *   // Possible positioning by ts.move()
+ *   // Possible positioning by ts.move(offset) or ts.moveIndex(index)
  *   while (ts.moveNext()) {
  *       Token t = ts.token();
  *       if (t.id() == ...) { ... }
@@ -46,10 +85,7 @@
  *       if (ts.offset() == ...) { ... }
  *   }
  *
- *
- * Token sequence provides correct offset information - * for the token to which the sequence is positioned - * (some tokens may be flyweight and do not hold the offset by themselves). + *

* *

* This class should be used by a single thread only. @@ -65,7 +101,7 @@ private AbstractToken token; // 16 bytes - private int tokenIndex = -1; // 20 bytes + private int tokenIndex; // 20 bytes /** * Offset in the input at which the current token is located @@ -113,19 +149,30 @@ } /** - * Get instance of current token to which this token sequence points to. + * Get token to which this token sequence points to or null + * if TS is positioned between tokens + * ({@link #moveNext()} or {@link #movePrevious()} were not called yet). *
- * It is necessary to call {@link #moveNext()} before first calling this method. + * A typical iteration usage: + * 

+     *   TokenSequence ts = tokenHierarchy.tokenSequence();
+     *   // Possible positioning by ts.move(offset) or ts.moveIndex(index)
+     *   while (ts.moveNext()) {
+     *       Token t = ts.token();
+     *       if (t.id() == ...) { ... }
+     *       if (TokenUtilities.equals(t.text(), "mytext")) { ... }
+     *       if (ts.offset() == ...) { ... }
+     *   }
+     *
* - *

* The returned token instance may be flyweight - * (returns true from {@link Token#isFlyweight()}) + * ({@link Token#isFlyweight()} returns true) * which means that its {@link Token#offset(TokenHierarchy)} will return -1. *
* To find a correct offset use {@link #offset()}. *
- * Or if its necessary to have a non-flyweigt the {@link #offsetToken()} - * may be used. + * Or if its necessary to revert to a regular non-flyweigt token + * the {@link #offsetToken()} may be used. *

* *

@@ -133,13 +180,13 @@ * The token instance should not be held across the input source modifications. *

* - * @return non-null token instance. + * @return token instance to which this token sequence is currently positioned + * or null if this token sequence is not positioned to any token which may + * happen after TS creation or after use of {@link #move(int)} or {@link moveIndex(int)}. + * * @see #offsetToken() - * @throws IllegalStateException if this token sequence was not positioned - * to any token yet. */ public Token token() { - checkToken(); return token; } @@ -149,6 +196,8 @@ *
* If the current token is flyweight then this method replaces it * with the corresponding non-flyweight token which it then returns. + *
+ * Subsequent calls to {@link #token()} will also return this non-flyweight token. * *

* This method may be handy if the token instance is referenced in a standalone way @@ -156,11 +205,10 @@ * to get the appropriate offset from the token itself * later when a token sequence will not be available. *

- * @throws IllegalStateException if this token sequence was not positioned - * to any token yet. + * @throws IllegalStateException if {@link #token()} returns null. */ public Token offsetToken() { - checkToken(); + checkTokenNotNull(); if (token.isFlyweight()) { token = tokenList.replaceFlyToken(tokenIndex, token, offset()); } @@ -179,11 +227,10 @@ * best performance with a constant time complexity. * * @return >=0 absolute offset of the current token in the underlying input. - * @throws IllegalStateException if this token sequence was not positioned - * to any token yet. + * @throws IllegalStateException if {@link #token()} returns null. */ public int offset() { - checkToken(); + checkTokenNotNull(); if (tokenOffset == -1) { tokenOffset = tokenList.tokenOffset(tokenIndex); } @@ -191,10 +238,29 @@ } /** - * Get the index of the current token in the complete list of tokens. - * - * @return >=0 index of the current token or -1 - * if this token sequence is initially located in front of the first token. + * Get an index of token to which (or before which) this TS is currently positioned. + *
+ *

+ * Initially or after {@link #move(int)} or {@link #moveIndex(int)} + * token sequence is positioned between tokens: + * 

+     *          Token[0]   Token[1]   ...   Token[n]
+     *        ^          ^                ^
+     * Index: 0          1                n
+     *
+ *

+ * + *

+ * After use of {@link #moveNext()} or {@link #movePrevious()} + * the token sequence is positioned over one of the actual tokens: + * 

+     *          Token[0]   Token[1]   ...   Token[n]
+     *             ^          ^                ^
+     * Index:      0          1                n
+     *
+ *

+ * + * @return >=0 index of token to which (or before which) this TS is currently positioned. */ public int index() { return tokenIndex; @@ -212,11 +278,10 @@ * or LanguageProvider). * * @return embedded sequence or null if no embedding exists for this token. - * @throws IllegalStateException if this token sequence was not positioned - * to any token yet. + * @throws IllegalStateException if {@link #token()} returns null. */ public TokenSequence embedded() { - checkToken(); + checkTokenNotNull(); return embeddedImpl(null); } @@ -247,15 +312,18 @@ * Get embedded token sequence if the token * to which this token sequence is currently positioned * has a language embedding. + * + * @throws IllegalStateException if {@link #token()} returns null. */ public TokenSequence embedded(Language embeddedLanguage) { - checkToken(); + checkTokenNotNull(); return embeddedImpl(embeddedLanguage); } /** * Create language embedding without joining of the embedded sections. * + * @throws IllegalStateException if {@link #token()} returns null. * @see #createEmbedding(Language, int, int, boolean) */ public boolean createEmbedding(Language embeddedLanguage, @@ -291,59 +359,72 @@ * Only the embedded sections with the same language path can be joined. * @return true if the embedding was created successfully or false if an embedding * with the given language already exists for this token. + * @throws IllegalStateException if {@link #token()} returns null. */ public boolean createEmbedding(Language embeddedLanguage, int startSkipLength, int endSkipLength, boolean joinSections) { - checkToken(); + checkTokenNotNull(); return EmbeddingContainer.createEmbedding(tokenList, tokenIndex, embeddedLanguage, startSkipLength, endSkipLength, joinSections); } /** * Move to the next token in this token sequence. - *
+ * + *

* The next token may not necessarily start at the offset where - * the current token ends (there may be gaps between tokens - * caused by use of a token id filter). + * the previous token ends (there may be gaps between tokens + * caused by token filtering). {@link #offset()} should be used + * for offset retrieval. + *

* * @return true if the sequence was successfully moved to the next token - * or false if stays on the original token because there are no more tokens + * or false if it was not moved before there are no more tokens * in the forward direction. * @throws ConcurrentModificationException if this token sequence * is no longer valid because of an underlying mutable input source modification. */ public boolean moveNext() { checkModCount(); - tokenIndex++; + if (token != null) // Token already fetched + tokenIndex++; Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(tokenIndex); if (tokenOrEmbeddingContainer != null) { AbstractToken origToken = token; - assignToken(tokenOrEmbeddingContainer); + token = LexerUtilsConstants.token(tokenOrEmbeddingContainer); + // If origToken == null then the right offset might already be pre-computed from move() if (tokenOffset != -1) { - // If the token list is continuous or the fetched token - // is flyweight (there cannot be a gap before flyweight token) - // the original offset can be just increased - // by the original token's length. - if (tokenList.isContinuous() || token.isFlyweight()) { - tokenOffset += origToken.length(); // advance by previous token's length - } else // Offset must be recomputed - tokenOffset = -1; // mark the offset to be recomputed + if (origToken != null) { + // If the token list is continuous or the fetched token + // is flyweight (there cannot be a gap before flyweight token) + // the original offset can be just increased + // by the original token's length. + if (tokenList.isContinuous() || token.isFlyweight()) { + tokenOffset += origToken.length(); // advance by previous token's length + } else // Offset must be recomputed + tokenOffset = -1; // mark the offset to be recomputed + } else // Not valid token previously + tokenOffset = -1; } return true; } - tokenIndex--; + if (token != null) // Unsuccessful move from existing token + tokenIndex--; return false; } /** - * Move to the previous token in this token sequence. - *
- * The next token may not necessarily end at the offset where - * the present token starts (there may be gaps between tokens - * caused by use of a token id filter). + * Move to a previous token in this token sequence. + * + *

+ * The previous token may not necessarily end at the offset where + * the previous token started (there may be gaps between tokens + * caused by token filtering). {@link #offset()} should be used + * for offset retrieval. + *

* * @return true if the sequence was successfully moved to the previous token - * or false if stayed on the original token because there are no more tokens + * or false if it was not moved because there are no more tokens * in the backward direction. * @throws ConcurrentModificationException if this token sequence * is no longer valid because of an underlying mutable input source modification. @@ -353,7 +434,7 @@ if (tokenIndex > 0) { AbstractToken origToken = token; tokenIndex--; - assignToken(); + token = LexerUtilsConstants.token(tokenList.tokenOrEmbeddingContainer(tokenIndex)); if (tokenOffset != -1) { // If the token list is continuous or the original token // is flyweight (there cannot be a gap before flyweight token) @@ -364,8 +445,6 @@ } else { // mark the offset to be computed upon call to offset() tokenOffset = -1; } - } else { - tokenOffset = -1; // mark the offset to be computed upon call to offset() } return true; @@ -374,92 +453,102 @@ } /** - * Move the token sequence to point to the token with the given index. + * Position token sequence between index-1 + * and index tokens. + *
+ * TS will be positioned in the following way: + * 
+     *          Token[0]   ...   Token[index-1]   Token[index] ...
+     *        ^                ^                ^
+     * Index: 0             index-1           index
+     *
+ * + *

+ * Subsequent {@link #moveNext()} or {@link #movePrevious()} is needed to fetch + * a concrete token in the desired direction. + *
+ * Subsequent {@link #moveNext()} will position TS over Token[index] + * (or {@link #movePrevious()} will position TS over Token[index-1]) + * so that {@link #token()} != null. * * @param index index of the token to which this sequence * should be positioned. - * @return true if the sequence was moved to the token - * with the given index. Returns false - * if index < 0 or index < tokenCount. - * In such case the current token sequence's position stays unchanged. + *
+ * If index >= {@link #tokenCount()} + * then the TS will be positioned to {@link #tokenCount()}. + *
+ * If index < 0 then the TS will be positioned to index 0. + * + * @return difference between requested index and the index to which TS + * is really set. * @throws ConcurrentModificationException if this token sequence * is no longer valid because of an underlying mutable input source modification. */ - public boolean moveIndex(int index) { + public int moveIndex(int index) { checkModCount(); - if (index < 0) { - return false; - } - Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index); - if (tokenOrEmbeddingContainer != null) { // enough tokens - this.tokenIndex = index; - assignToken(tokenOrEmbeddingContainer); - tokenOffset = -1; - return true; - - } else // Token at the requested index does not exist - leave orig. index - return false; + if (index >= 0) { + Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index); + if (tokenOrEmbeddingContainer != null) { // enough tokens + resetTokenIndex(index); + } else // Token at the requested index does not exist - leave orig. index + resetTokenIndex(tokenCount()); + } else // index < 0 + resetTokenIndex(0); + return index - tokenIndex; } /** - * Move the token sequence to be positioned to the token - * that "contains" the requested offset (the offset is at the begining - * or inside of the token). - *
- * If the offset is too big the token sequence will be positioned - * to the last token and the return value will - * be the distance between the requested offset - * and the start offset of the token to which the token sequence - * will be positioned.. - *
- * If there are no tokens in the sequence then {@link Integer#MAX_VALUE} - * will be returned. - * - *

- * The underlying token list may contain gaps that are not covered - * by any tokens and if the offset is contained in such gap then - * the token sequence will be positioned to the token that precedes the gap. - *

- * - * Example: + * Move the token sequence to be positioned before the first token. + *
+ * This is equivalent to moveIndex(0). + */ + public void moveStart() { + moveIndex(0); + } + + /** + * Move the token sequence to be positioned behind the last token. + *
+ * This is equivalent to moveIndex(tokenCount()). + */ + public void moveEnd() { + moveIndex(tokenCount()); + } + + /** + * Move token sequence to be positioned between index-1 + * and index tokens where Token[index] either starts at offset + * or "contains" the offset. + *
* 
-     *   int diff = tokenSequence.move(targetOffset);
-     *   // diff equals to (targetOffset - tokenSequence.token().offset())
-     *   if (diff >= 0 && diff < tokenSequence.token().length()) {
-     *       // Within token bounds - tokenSequence.token() starts at or "contains" targetOffset
-     *
-     *   } else if (diff == Integer.MAX_VALUE) {
-     *       // No tokens in the token sequence at all.
-     *       // Token sequence is not positioned to any token.
-     *
-     *   } else {
-     *       // 1. diff >= tokenSequence.token().length()
-     *       //   a) targetOffset is above the end of the last token in the sequence.
-     *       //     Token sequence is positioned to the last token in the sequence.
-     *       //   b) there are text areas not covered by any tokens
-     *       //     due to skipped tokens (skipTokenIds was used
-     *       //     in TokenHierarchy.create()) and the targetOffset points to such gap.
-     *       //     Token sequence is positioned to the preceding token.
-     *       // 
-     *       // 2. diff < 0
-     *       //   a) targetOffset < 0
-     *       //   b) targetOffset >= 0 but there is a text area
-     *       //     at the begining that is not covered by any tokens
-     *       //     (skipTokenIds was used in TokenHierarchy.create())
-     *       //     Token sequence is positioned to the first token in the sequence.
-     *   }
+     *        +----------+-----+----------------+--------------+------
+     *        | Token[0] | ... | Token[index-1] | Token[index] | ...
+     *        | "public" | ... | "static"       | "int"        | ...
+     *        +----------+-----+----------------+--------------+------
+     *        ^                ^                ^
+     * Index: 0             index-1           index
+     * Offset:                                  ---^ (if offset points to 'i','n' or 't')
      *
* + *

+ * Subsequent {@link #moveNext()} or {@link #movePrevious()} is needed to fetch + * a concrete token. + *
+ * If the offset is too big then the token sequence will be positioned + * behind the last token. + *

+ * + *

+ * If token filtering is used there may be gaps that are not covered + * by any tokens and if the offset is contained in such gap then + * the token sequence will be positioned before the token that follows the gap. + *

* - * @param offset absolute offset in the input to which - * the token sequence should be moved. * + * @param offset absolute offset to which the token sequence should be moved. * @return difference between the reqeuested offset - * and the absolute starting offset of the token - * to which the the token sequence gets moved. - *
- * Returns {@link Integer#MAX_VALUE} if there are no tokens in the sequence. - * In such case there is no active token. + * and the start offset of the token + * before which the the token sequence gets positioned. * * @throws ConcurrentModificationException if this token sequence * is no longer valid because of an underlying mutable input source modification. @@ -474,7 +563,8 @@ if (tokenList.tokenOrEmbeddingContainer(0) == null) { // really no tokens at all // In this case the token sequence could not be positioned yet // so no need to reset "index" or other vars - return Integer.MAX_VALUE; + resetTokenIndex(0); + return offset; } // Re-get the present token count (could be created a chunk of tokens at once) tokenCount = tokenList.tokenCountCurrent(); @@ -499,14 +589,14 @@ tokenLength = t.length(); tokenCount++; - } else { // no more tokens => break - break; + } else { // no more tokens => position behind last token + resetTokenIndex(tokenCount); + tokenOffset = prevTokenOffset + tokenLength; // May assign the token's offset in advance + return offset - tokenOffset; } } - tokenIndex = tokenCount - 1; - // Absolute token's start offset - tokenOffset = prevTokenOffset; - assignToken(); + resetTokenIndex(tokenCount - 1); + tokenOffset = prevTokenOffset; // May assign the token's offset in advance return offset - prevTokenOffset; } @@ -525,47 +615,41 @@ high = mid - 1; } else { // Token starting exactly at offset found - tokenIndex = mid; + resetTokenIndex(mid); tokenOffset = midStartOffset; - assignToken(); return 0; // right at the token begining } } // Not found exactly and high + 1 == low => high < low - // Check whether the token at "high" contains the offset + // BTW there may be gaps between tokens; if offset is in gap then position to higher token if (high >= 0) { // could be -1 AbstractToken t = LexerUtilsConstants.token(tokenList, high); prevTokenOffset = tokenList.tokenOffset(high); + // If gaps allowed check whether the token at "high" contains the offset + if (!tokenList.isContinuous() && offset > prevTokenOffset + t.length()) { + // Offset in the gap above the "high" token + high++; + prevTokenOffset += t.length(); + } } else { // at least one token exists => use token at index 0 high = 0; prevTokenOffset = tokenList.tokenOffset(0); // result may differ from 0 } - - tokenIndex = high; + resetTokenIndex(high); tokenOffset = prevTokenOffset; - assignToken(); return offset - prevTokenOffset; } - - /** - * Move to the first token in this token sequence. - * - * @return true if the sequence was positioned on the first token - * or false if there are no tokens in the sequence. - */ - public boolean moveFirst() { - return moveIndex(0); - } /** - * Move to the last token in this token sequence. - * - * @return true if the sequence was positioned on the last token - * or false if there are no tokens in the sequence. + * Check whether this TS contains zero tokens. + *
+ * This check is strongly preferred over tokenCount() == 0. + * + * @see #tokenCount() */ - public boolean moveLast() { - return moveIndex(tokenCount() - 1); // Can be -1 but handled in move(index) + public boolean isEmpty() { + return (tokenIndex == 0 && tokenList.tokenOrEmbeddingContainer(0) == null); } /** @@ -628,15 +712,14 @@ return parentTokenIndexes; } - private void assignToken(Object tokenOrEmbeddingContainer) { - token = LexerUtilsConstants.token(tokenOrEmbeddingContainer); - } - - private void assignToken() { - assignToken(tokenList.tokenOrEmbeddingContainer(tokenIndex)); + private void resetTokenIndex(int index) { + // Position to the given index e.g. by move() and moveIndex() + tokenIndex = index; + token = null; + tokenOffset = -1; } - private void checkToken() { + private void checkTokenNotNull() { if (token == null) { throw new IllegalStateException( "No token fetched by moveNext() from token sequence yet: index=" + tokenIndex Index: lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java =================================================================== RCS file: /cvs/lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java,v retrieving revision 1.5 diff -u -r1.5 LexerUtilsConstants.java --- lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java 4 Dec 2006 15:44:00 -0000 1.5 +++ lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java 26 Jan 2007 20:19:43 -0000 @@ -237,6 +237,16 @@ sb.append(": "); AbstractToken token = token(tokenOrEmbeddingContainer); sb.append(token.dumpInfo(tokenHierarchy)); + int la = tokenList.lookahead(i); + if (la != 0) { + sb.append(", la="); + sb.append(la); + } + Object state= tokenList.state(i); + if (state != null) { + sb.append(", s="); + sb.append(state); + } sb.append('\n'); } return sb; Index: lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java =================================================================== RCS file: /cvs/lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java,v retrieving revision 1.6 diff -u -r1.6 SubSequenceTokenList.java --- lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java 4 Dec 2006 15:44:00 -0000 1.6 +++ lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java 26 Jan 2007 20:19:43 -0000 @@ -92,7 +92,7 @@ if (limitStartOffset > 0) { int diff = move(limitStartOffset); if (diff != Integer.MAX_VALUE) { // some tokens exist - if (diff >= lastToken.length()) { + if (diff >= lastToken.length()) { // lastToken initialized in move() lastTokenIndex++; Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(lastTokenIndex); if (tokenOrEmbeddingContainer != null && Index: lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java =================================================================== RCS file: /cvs/lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java,v retrieving revision 1.5 diff -u -r1.5 TokenSequenceTest.java --- lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java 4 Dec 2006 15:44:04 -0000 1.5 +++ lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java 26 Jan 2007 20:19:44 -0000 @@ -48,13 +48,7 @@ TokenHierarchy hi = TokenHierarchy.create(text, SimpleTokenId.language()); TokenSequence ts = hi.tokenSequence(); - // Fail if no "move*" method called yet - try { - ts.token(); // should throw exception - fail("IllegalStateException expecxted"); - } catch (IllegalStateException e) { - // expected - } + assertNull(ts.token()); assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0); @@ -69,20 +63,38 @@ assertEquals(ts.tokenCount(), 3); assertEquals(0, ts.move(0)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0); assertEquals(2, ts.move(2)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0); assertEquals(-1, ts.move(-1)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0); assertEquals(0, ts.move(3)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3); assertEquals(0, ts.move(4)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4); assertEquals(1, ts.move(5)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4); - assertEquals(5, ts.move(9)); + assertEquals(1, ts.move(9)); + assertNull(ts.token()); + assertFalse(ts.moveNext()); + assertTrue(ts.movePrevious()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4); - assertEquals(96, ts.move(100)); + assertEquals(92, ts.move(100)); + assertNull(ts.token()); + assertFalse(ts.moveNext()); + assertTrue(ts.movePrevious()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4); @@ -107,6 +119,70 @@ sub = ts.subSequence(3, 3); assertFalse(sub.moveNext()); } + + public void testMoveNextPrevious() { + String text = "abc+defg-"; + TokenHierarchy hi = TokenHierarchy.create(text, SimpleTokenId.language()); + TokenSequence ts = hi.tokenSequence(); + assertEquals(0, ts.move(4)); + assertNull(ts.token()); + //assertEquals(4, ts.offset()); + assertTrue(ts.movePrevious()); + LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3); + + ts = hi.tokenSequence(); + ts.move(5); + assertTrue(ts.moveNext()); + LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4); + assertTrue(ts.movePrevious()); + LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3); + + // Move single char before token's end + ts = hi.tokenSequence(); + ts.move(7); + assertTrue(ts.moveNext()); + LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4); + assertTrue(ts.movePrevious()); + LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3); + + // Move past all tokens + ts = hi.tokenSequence(); + ts.move(text.length()); + assertTrue(ts.movePrevious()); + LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", text.length() - 1); + + // Move at the begining + ts = hi.tokenSequence(); + ts.move(0); + assertFalse(ts.movePrevious()); + assertTrue(ts.moveNext()); + LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0); + } + + public void testMoveIndex() { + String text = "abc+defg-"; + TokenHierarchy hi = TokenHierarchy.create(text, SimpleTokenId.language()); + TokenSequence ts = hi.tokenSequence(); + assertEquals(0, ts.index()); + assertNull(ts.token()); + //assertEquals(4, ts.offset()); + assertTrue(ts.moveNext()); + assertEquals(0, ts.offset()); + LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0); + assertTrue(ts.moveNext()); + LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3); + assertEquals(3, ts.offset()); + ts.move(2); + assertEquals(0, ts.index()); + assertTrue(ts.moveNext()); + assertEquals(0, ts.offset()); + assertEquals(0, ts.index()); + ts.moveIndex(1); + assertEquals(1, ts.index()); + assertTrue(ts.moveNext()); + assertEquals(1, ts.index()); + assertEquals(3, ts.offset()); + } public void testMoveSkipTokens() { String text = "-abc+defg--hi"; @@ -122,12 +198,7 @@ TokenSequence ts = hi.tokenSequence(); // Fail if no "move*" method called yet - try { - ts.token(); // should throw exception - fail("IllegalStateException expecxted"); - } catch (IllegalStateException e) { - // expected - } + assertNull(ts.token()); assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); @@ -137,10 +208,12 @@ LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11); assertFalse(ts.moveNext()); assertEquals(3, ts.tokenCount()); - assertTrue(ts.moveFirst()); + assertEquals(0, ts.moveIndex(0)); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); assertTrue(ts.moveNext()); - assertTrue(ts.moveLast()); + assertEquals(0, ts.moveIndex(ts.tokenCount())); + assertTrue(ts.movePrevious()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11); assertFalse(ts.moveNext()); @@ -151,16 +224,28 @@ LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); assertEquals(-1, ts.move(0)); // below filtered-out token + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); assertEquals(0, ts.move(1)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); assertEquals(1, ts.move(2)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); assertEquals(3, ts.move(4)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); assertEquals(0, ts.move(5)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 5); assertEquals(1, ts.move(12)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11); @@ -172,48 +257,56 @@ LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5); assertFalse(sub.moveNext()); - // Test moveFirst() and moveLast() on subsequence + // Test moves to first and last tokens on subsequence sub = ts.subSequence(1, 6); - assertTrue(sub.moveLast()); + assertEquals(0, sub.moveIndex(sub.tokenCount())); + assertEquals(2, sub.tokenCount()); // "abc" and "defg" (others filtered out + assertTrue(sub.movePrevious()); LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5); - assertTrue(sub.moveFirst()); + assertEquals(0, sub.moveIndex(0)); + assertTrue(sub.moveNext()); LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "abc", 1); sub = ts.subSequence(4); assertTrue(sub.moveNext()); LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5); assertTrue(sub.moveNext()); - LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11); + LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11); assertFalse(sub.moveNext()); sub = ts.subSequence(12, 15); assertTrue(sub.moveNext()); - LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11); + LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11); sub = ts.subSequence(12, 15); assertEquals(-2, sub.move(9)); - LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11); + assertNull(sub.token()); + assertTrue(sub.moveNext()); + LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11); sub = ts.subSequence(13, 15); assertFalse(sub.moveNext()); - assertFalse(sub.moveFirst()); - assertFalse(sub.moveLast()); + assertEquals(0, sub.moveIndex(0)); + assertNull(sub.token()); + assertFalse(sub.moveNext()); + assertEquals(0, sub.moveIndex(sub.tokenCount())); + assertNull(sub.token()); assertEquals(0, sub.tokenCount()); sub = ts.subSequence(-3, 1); assertFalse(sub.moveNext()); sub = ts.subSequence(13, 15); - assertEquals(Integer.MAX_VALUE, sub.move(5)); + assertEquals(5, sub.move(5)); sub = ts.subSequence(-3, 1); - assertEquals(Integer.MAX_VALUE, sub.move(1)); + assertEquals(1, sub.move(1)); // Reversed bounds => should be empty token sequence sub = ts.subSequence(6, 1); assertFalse(sub.moveNext()); sub = ts.subSequence(6, 1); - assertEquals(Integer.MAX_VALUE, sub.move(6)); + assertEquals(6, sub.move(6)); } public void DtestMoveSkipTokens2() throws IOException { @@ -244,32 +337,26 @@ // Expect no tokens assertFalse(ts.moveNext()); - // Exception if ts.token() called - try { - ts.token(); // should throw exception - fail("IllegalStateException expecxted"); - } catch (IllegalStateException e) { - // expected - } + assertNull(ts.token()); - assertEquals(ts.move(0), Integer.MAX_VALUE); - assertEquals(ts.move(10), Integer.MAX_VALUE); + assertEquals(0, ts.move(0)); + assertEquals(10, ts.move(10)); // Test subsequences TokenSequence sub = ts.subSequence(1, 6); assertFalse(sub.moveNext()); sub = ts.subSequence(1, 6); - assertEquals(Integer.MAX_VALUE, sub.move(1)); + assertEquals(1, sub.move(1)); sub = ts.subSequence(0); assertFalse(sub.moveNext()); sub = ts.subSequence(0); - assertEquals(Integer.MAX_VALUE, sub.move(0)); + assertEquals(0, sub.move(0)); sub = ts.subSequence(1); assertFalse(sub.moveNext()); sub = ts.subSequence(1); - assertEquals(Integer.MAX_VALUE, sub.move(0)); + assertEquals(0, sub.move(0)); } public void testTokenSize() { @@ -284,7 +371,8 @@ assertFalse(ts.moveNext()); TokenList tokenList = LexerTestUtilities.tokenList(ts); - ts.moveIndex(0); // move to "abc" + ts.moveIndex(0); // move before "abc" + assertTrue(ts.moveNext()); // Test DefaultToken size assertSame(DefaultToken.class, ts.token().getClass()); assertSize("Token instance too big", Collections.singletonList(ts.token()), 24, Index: lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java =================================================================== RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java,v retrieving revision 1.5 diff -u -r1.5 LanguageManagerTest.java --- lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java 26 Oct 2006 20:45:25 -0000 1.5 +++ lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java 26 Jan 2007 20:19:44 -0000 @@ -130,7 +130,8 @@ public void testCachingE() { TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language()); TokenSequence tokens = th.tokenSequence(); - tokens.moveFirst(); + tokens.moveStart(); + assertEquals(true, tokens.moveNext()); TokenSequence embeddedA = tokens.embedded(); assertNotNull("There should be an embedded language", embeddedA); @@ -144,7 +145,8 @@ public void testGCedE() { TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language()); TokenSequence tokens = th.tokenSequence(); - tokens.moveFirst(); + tokens.moveStart(); + assertEquals(true, tokens.moveNext()); TokenSequence embedded = tokens.embedded(); assertNotNull("There should be an embedded language", embedded); @@ -164,7 +166,8 @@ public void testCacheRefreshedE() { TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language()); TokenSequence tokens = th.tokenSequence(); - tokens.moveFirst(); + tokens.moveStart(); + assertEquals(true, tokens.moveNext()); TokenSequence embeddedA = tokens.embedded(); assertNotNull("There should be an embedded language", embeddedA); Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java =================================================================== RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java,v retrieving revision 1.5 diff -u -r1.5 LexerTestUtilities.java --- lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java 4 Dec 2006 15:44:05 -0000 1.5 +++ lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java 26 Jan 2007 20:19:44 -0000 @@ -70,6 +70,7 @@ public static void assertTokenEquals(String message, TokenSequence ts, TokenId id, String text, int offset) { message = messagePrefix(message); Token t = ts.token(); + TestCase.assertNotNull("Token is null", t); TokenId tId = t.id(); TestCase.assertEquals(message + "Invalid token.id()", id, tId); CharSequence tText = t.text(); @@ -312,17 +313,21 @@ } TokenHierarchy thBatch = TokenHierarchy.create(docText, language); boolean success = false; + TokenSequence batchTS = thBatch.tokenSequence(); try { // Compare lookaheads and states as well - assertTokenSequencesEqual(thBatch.tokenSequence(), thBatch, + assertTokenSequencesEqual(batchTS, thBatch, thInc.tokenSequence(), thInc, true); success = true; } finally { if (!success) { + // Go forward two tokens to have an extra tokens context + batchTS.moveNext(); + batchTS.moveNext(); System.err.println("BATCH token sequence dump:\n" + thBatch.tokenSequence()); TokenHierarchy lastHi = (TokenHierarchy)doc.getProperty(LAST_TOKEN_HIERARCHY); if (lastHi != null) { - System.err.println("PREVIOUS token sequence dump:\n" + lastHi.tokenSequence()); + System.err.println("PREVIOUS batch token sequence dump:\n" + lastHi.tokenSequence()); } } } Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java =================================================================== RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java,v retrieving revision 1.5 diff -u -r1.5 TokenHierarchySnapshotTest.java --- lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java 4 Dec 2006 15:44:06 -0000 1.5 +++ lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java 26 Jan 2007 20:19:44 -0000 @@ -76,7 +76,8 @@ // Check that all the non-fly tokens are mutable ts = snapshot1.tokenSequence(); - assertTrue(ts.moveIndex(0)); + assertEquals(0, ts.moveIndex(0)); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenSequencesEqual(hi1.tokenSequence(), hi1, snapshot1.tokenSequence(), snapshot1, false); @@ -84,7 +85,8 @@ doc.insertString(4, "+", null); // Check that the snapshot token sequence can be further operated. - assertTrue(ts.moveIndex(0)); + assertEquals(0, ts.moveIndex(0)); + assertTrue(ts.moveNext()); assertNotNull(ts.token()); // Check that the tokens except '+' are live @@ -101,7 +103,8 @@ // Check that all the non-fly tokens are mutable ts = snapshot2.tokenSequence(); - assertTrue(ts.moveIndex(0)); + assertEquals(0, ts.moveIndex(0)); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenSequencesEqual(hi2.tokenSequence(), hi2, snapshot2.tokenSequence(), snapshot2, false); Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java =================================================================== RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java,v retrieving revision 1.6 diff -u -r1.6 SimpleLexerBatchTest.java --- lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java 17 Jan 2007 13:15:29 -0000 1.6 +++ lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java 26 Jan 2007 20:19:44 -0000 @@ -81,7 +81,8 @@ assertFalse(ts.moveNext()); // Go back to block comment - assertTrue(ts.moveIndex(commentIndex)); + assertEquals(0, ts.moveIndex(commentIndex)); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.BLOCK_COMMENT, commentText, commentTextStartOffset); // Test embedded token sequence Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java =================================================================== RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java,v retrieving revision 1.6 diff -u -r1.6 SimpleLexerIncTest.java --- lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java 4 Dec 2006 15:44:07 -0000 1.6 +++ lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java 26 Jan 2007 20:19:44 -0000 @@ -90,19 +90,23 @@ // Check TokenSequence.move() int relOffset = ts.move(50); // past the end of all tokens - assertEquals(relOffset, 50 - offset); + assertEquals(relOffset, 50 - (offset + 3)); + assertTrue(ts.movePrevious()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset); relOffset = ts.move(6); // right at begining of "-" assertEquals(relOffset, 0); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", 6); relOffset = ts.move(-5); // to first token "abc" assertEquals(relOffset, -5); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0); relOffset = ts.move(5); // to first token "abc" assertEquals(relOffset, 1); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "uv", 4); Index: web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JSPHyperlinkProvider.java =================================================================== RCS file: /cvs/web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JSPHyperlinkProvider.java,v retrieving revision 1.12 diff -u -r1.12 JSPHyperlinkProvider.java --- web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JSPHyperlinkProvider.java 24 Jan 2007 21:55:07 -0000 1.12 +++ web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JSPHyperlinkProvider.java 26 Jan 2007 20:21:12 -0000 @@ -99,7 +99,8 @@ TokenHierarchy tokenHierarchy = TokenHierarchy.get(bdoc); TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); - if(tokenSequence.move(offset) == Integer.MAX_VALUE) { + tokenSequence.move(offset); + if (!tokenSequence.moveNext() && !tokenSequence.movePrevious()) { return false; //no token found } Token token = tokenSequence.token(); @@ -148,10 +149,12 @@ // is it a bean in EL? tokenSequence.move(offset); //reset tokenSequence + tokenSequence.moveNext(); TokenSequence elTokenSequence = tokenSequence.embedded(ELTokenId.language()); if (elTokenSequence != null){ ELExpression exp = new ELExpression((JspSyntaxSupport)bdoc.getSyntaxSupport()); elTokenSequence.move(offset); + elTokenSequence.moveNext(); if (elTokenSequence.token().id() == ELTokenId.DOT){ return false; @@ -203,7 +206,8 @@ TokenHierarchy tokenHierarchy = TokenHierarchy.get(bdoc); TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); - if(tokenSequence.move(offset) == Integer.MAX_VALUE) { + tokenSequence.move(offset); + if (!tokenSequence.moveNext() && !tokenSequence.movePrevious()) { return null; //no token found } Token token = tokenSequence.token(); @@ -218,6 +222,7 @@ if (elTokenSequence != null){ ELExpression exp = new ELExpression((JspSyntaxSupport)bdoc.getSyntaxSupport()); elTokenSequence.move(offset); + elTokenSequence.moveNext(); int elEnd = elTokenSequence.offset() + elTokenSequence.token().length(); int res = exp.parse(elEnd); @@ -261,7 +266,8 @@ TokenHierarchy tokenHierarchy = TokenHierarchy.get(bdoc); TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); - if(tokenSequence.move(offset) == Integer.MAX_VALUE) { + tokenSequence.move(offset); + if (!tokenSequence.moveNext() && !tokenSequence.movePrevious()) { return; //no token found } Token token = tokenSequence.token(); @@ -272,6 +278,7 @@ ELExpression exp = new ELExpression((JspSyntaxSupport)bdoc.getSyntaxSupport()); elTokenSequence.move(offset); + elTokenSequence.moveNext(); int elEnd = elTokenSequence.offset() + elTokenSequence.token().length(); int res = exp.parse(elEnd); if (res == ELExpression.EL_START ){ @@ -306,6 +313,7 @@ } tokenSequence.move(offset);//reset tokenSequence + tokenSequence.moveNext(); FileObject fObj = getTagFile(tokenSequence, jspSup); if ( fObj != null) openInEditor(fObj); @@ -430,7 +438,8 @@ TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); while (index > 0){ - if(tokenSequence.move(index) == Integer.MAX_VALUE) { + tokenSequence.move(index); + if (!tokenSequence.moveNext() && !tokenSequence.movePrevious()) { return; //no token found } Token token = tokenSequence.token(); Index: web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JspParserErrorAnnotation.java =================================================================== RCS file: /cvs/web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JspParserErrorAnnotation.java,v retrieving revision 1.4 diff -u -r1.4 JspParserErrorAnnotation.java --- web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JspParserErrorAnnotation.java 15 Dec 2006 11:14:49 -0000 1.4 +++ web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JspParserErrorAnnotation.java 26 Jan 2007 20:21:12 -0000 @@ -108,7 +108,8 @@ } TokenHierarchy tokenHierarchy = TokenHierarchy.get(document); TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); - if(tokenSequence.move(offset - 1) == Integer.MAX_VALUE) { + tokenSequence.move(offset - 1); + if (!tokenSequence.moveNext() && !tokenSequence.movePrevious()) { //no token textOnLine(docline); return ; Index: web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JspSyntaxSupport.java =================================================================== RCS file: /cvs/web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JspSyntaxSupport.java,v retrieving revision 1.89 diff -u -r1.89 JspSyntaxSupport.java --- web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JspSyntaxSupport.java 21 Dec 2006 14:01:43 -0000 1.89 +++ web/jspsyntax/src/org/netbeans/modules/web/core/syntax/JspSyntaxSupport.java 26 Jan 2007 20:21:13 -0000 @@ -2024,7 +2024,7 @@ //HTML language is not top level one ts = hi.tokenSequence(); int diff = ts.move(offset); - if(diff == Integer.MAX_VALUE) { + if(!ts.moveNext() && !ts.movePrevious()) { return null; //no token found } else { ts = ts.embedded(language); Index: web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java =================================================================== RCS file: /cvs/web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java,v retrieving revision 1.2 diff -u -r1.2 SimplifiedJSPServlet.java --- web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java 22 Jan 2007 16:04:40 -0000 1.2 +++ web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java 26 Jan 2007 20:21:13 -0000 @@ -98,7 +98,7 @@ TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc); TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); //get top level token sequence - if(!tokenSequence.moveFirst()) { + if(!tokenSequence.moveNext()) { return ; //no tokens in token sequence } Index: web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/ELExpression.java =================================================================== RCS file: /cvs/web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/ELExpression.java,v retrieving revision 1.13 diff -u -r1.13 ELExpression.java --- web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/ELExpression.java 24 Jan 2007 21:55:07 -0000 1.13 +++ web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/ELExpression.java 26 Jan 2007 20:21:13 -0000 @@ -110,6 +110,7 @@ TokenSequence jspTokenSequence = hi.tokenSequence(); jspTokenSequence.move(offset); + jspTokenSequence.moveNext(); Token jspToken = jspTokenSequence.token(); isDefferedExecution = jspToken.text().toString().startsWith("#{"); //NOI18N @@ -119,7 +120,8 @@ } } - if(ts.move(offset) == Integer.MAX_VALUE) { + ts.move(offset); + if (!ts.moveNext() && !ts.movePrevious()) { return NOT_EL; } Index: web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/JavaJSPCompletionProvider.java =================================================================== RCS file: /cvs/web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/JavaJSPCompletionProvider.java,v retrieving revision 1.17 diff -u -r1.17 JavaJSPCompletionProvider.java --- web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/JavaJSPCompletionProvider.java 9 Jan 2007 09:37:45 -0000 1.17 +++ web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/JavaJSPCompletionProvider.java 26 Jan 2007 20:21:13 -0000 @@ -70,7 +70,8 @@ TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc); TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); - if(tokenSequence.move(offset) != Integer.MAX_VALUE) { + tokenSequence.move(offset); + if (tokenSequence.moveNext() || tokenSequence.movePrevious()) { Object tokenID = tokenSequence.token().id(); if (tokenID == JspTokenId.SCRIPTLET){ return true; Index: web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/JspCompletionProvider.java =================================================================== RCS file: /cvs/web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/JspCompletionProvider.java,v retrieving revision 1.10 diff -u -r1.10 JspCompletionProvider.java --- web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/JspCompletionProvider.java 15 Dec 2006 11:14:51 -0000 1.10 +++ web/jspsyntax/src/org/netbeans/modules/web/core/syntax/completion/JspCompletionProvider.java 26 Jan 2007 20:21:13 -0000 @@ -203,8 +203,8 @@ TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc); TokenSequence tokenSequence = JspSyntaxSupport.tokenSequence(tokenHierarchy, HTMLTokenId.language(), adjustedOffset); if(tokenSequence != null) { - int diff = tokenSequence.move(adjustedOffset); - if(diff >= tokenSequence.token().length() || diff == Integer.MAX_VALUE) { + tokenSequence.move(adjustedOffset); + if (!tokenSequence.moveNext() && !tokenSequence.movePrevious()) { return; //no token found } Index: xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java =================================================================== RCS file: /cvs/xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java,v retrieving revision 1.3 diff -u -r1.3 TagBasedLexerFormatter.java --- xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java 13 Jan 2007 14:55:58 -0000 1.3 +++ xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java 26 Jan 2007 20:21:42 -0000 @@ -86,17 +86,18 @@ * e.g.