Index: editor/lib2/src/org/netbeans/modules/editor/lib2/highlighting/SyntaxHighlighting.java =================================================================== RCS file: /cvs/editor/lib2/src/org/netbeans/modules/editor/lib2/highlighting/SyntaxHighlighting.java,v retrieving revision 1.4 diff -u -r1.4 SyntaxHighlighting.java --- editor/lib2/src/org/netbeans/modules/editor/lib2/highlighting/SyntaxHighlighting.java 23 Jan 2007 06:08:06 -0000 1.4 +++ editor/lib2/src/org/netbeans/modules/editor/lib2/highlighting/SyntaxHighlighting.java 24 Jan 2007 10:58:02 -0000 @@ -35,7 +35,6 @@ import org.netbeans.api.editor.settings.FontColorSettings; import org.netbeans.api.lexer.Language; import org.netbeans.api.lexer.LanguagePath; -import org.netbeans.api.lexer.TokenChange; import org.netbeans.api.lexer.TokenHierarchy; import org.netbeans.api.lexer.TokenHierarchyEvent; import org.netbeans.api.lexer.TokenHierarchyListener; @@ -98,57 +97,14 @@ synchronized (this) { version++; } - - TokenChange tc = evt.tokenChange(); - int addedLenght = 0; - int removedLength = 0; - - if (tc.addedTokenCount() > 0) { - addedLenght = getTokensLength(tc.currentTokenSequence(), tc.index(), tc.addedTokenCount()); - } - - if (tc.removedTokenCount() > 0) { - removedLength = getTokensLength(tc.removedTokenSequence(), 0, tc.removedTokenCount()); - } - int changeStart = tc.offset(); - int changeEnd; - - if (addedLenght == 0 && removedLength == 0) { - // The real length couldn't be computed for some reason - changeEnd = Integer.MAX_VALUE; - } else { - changeEnd = changeStart + Math.max(addedLenght, removedLength); - } - - fireHighlightsChange(changeStart, changeEnd); + fireHighlightsChange(evt.affectedStartOffset(), evt.affectedEndOffset()); } // ---------------------------------------------------------------------- // Private implementation // ---------------------------------------------------------------------- - private int getTokensLength(TokenSequence seq, int startIdx, int tokenCount) { - assert startIdx >= 0 && startIdx < seq.tokenCount() : - "Invalid startIdx: " + startIdx + ", sequence lenght: " + seq.tokenCount(); - assert tokenCount > 0 && startIdx + tokenCount <= seq.tokenCount() : - "Invalid tokenCount: " + tokenCount + ", startIdx: " + startIdx + - ", sequence lenght: " + seq.tokenCount(); - - int startOffset = -1; - int endOffset = -1; - - if (seq.moveIndex(startIdx)) { - startOffset = seq.offset(); - } - - if (seq.moveIndex(startIdx + tokenCount - 1)) { - endOffset = seq.offset() + seq.token().length(); - } - - return startOffset == -1 || endOffset == -1 ? 0 : endOffset - startOffset; - } - private final class HSImpl implements HighlightsSequence { private static final int S_NORMAL = 1; @@ -192,7 +148,8 @@ case S_EMBEDDED_HEAD: // The current token contains embedded language and we have processed it's head TokenSequence seq = sequences.get(sequences.size() - 1); - if (seq.moveFirst()) { + seq.moveStart(); + if (seq.moveNext()) { state = S_NORMAL; } else { throw new IllegalStateException("Invalid state"); @@ -217,7 +174,7 @@ // We have moved to the next normal token, so look what it is TokenSequence seq = sequences.get(sequences.size() - 1); TokenSequence embeddedSeq = seq.embedded(); - while (embeddedSeq != null && embeddedSeq.moveFirst()) { + while (embeddedSeq != null && embeddedSeq.moveNext()) { sequences.add(sequences.size(), embeddedSeq); if (embeddedSeq.offset() > seq.offset()) { state = S_EMBEDDED_HEAD; @@ -254,7 +211,8 @@ } case S_EMBEDDED_TAIL: { TokenSequence seq = sequences.get(sequences.size() - 1); - if (seq.moveLast()) { + seq.moveEnd(); + if (seq.movePrevious()) { return seq.offset() + seq.token().length(); } else { throw new IllegalStateException("Invalid state"); @@ -284,7 +242,8 @@ } case S_EMBEDDED_HEAD: { TokenSequence seq = sequences.get(sequences.size() - 1); - if (seq.moveFirst()) { + seq.moveStart(); + if (seq.moveNext()) { return seq.offset(); } else { TokenSequence embeddingSeq = sequences.get(sequences.size() - 2); @@ -443,7 +402,8 @@ } else { if (sequences.size() > 1) { TokenSequence embeddingSeq = sequences.get(sequences.size() - 2); - if (seq.moveLast()) { + seq.moveEnd(); + if (seq.movePrevious()) { if ((seq.offset() + seq.token().length()) < (embeddingSeq.offset() + embeddingSeq.token().length())) { return S_EMBEDDED_TAIL; } else { Index: html/editor/lib/src/org/netbeans/editor/ext/html/HTMLSyntaxSupport.java =================================================================== RCS file: /cvs/html/editor/lib/src/org/netbeans/editor/ext/html/HTMLSyntaxSupport.java,v retrieving revision 1.32 diff -u -r1.32 HTMLSyntaxSupport.java --- html/editor/lib/src/org/netbeans/editor/ext/html/HTMLSyntaxSupport.java 5 Dec 2006 14:49:22 -0000 1.32 +++ html/editor/lib/src/org/netbeans/editor/ext/html/HTMLSyntaxSupport.java 24 Jan 2007 10:58:11 -0000 @@ -133,7 +133,8 @@ // if the carret is after HTML tag ( after char '>' ), ship inside the tag if (token.id() == HTMLTokenId.TAG_CLOSE_SYMBOL) { - if(ts.moveIndex(ts.index() - 2)) { + ts.moveIndex(ts.index() - 2); + if(ts.moveNext()) { token = ts.token(); } } Index: java/editor/src/org/netbeans/modules/editor/java/JavaCompletionProvider.java =================================================================== RCS file: /cvs/java/editor/src/org/netbeans/modules/editor/java/JavaCompletionProvider.java,v retrieving revision 1.73 diff -u -r1.73 JavaCompletionProvider.java --- java/editor/src/org/netbeans/modules/editor/java/JavaCompletionProvider.java 19 Jan 2007 15:05:40 -0000 1.73 +++ java/editor/src/org/netbeans/modules/editor/java/JavaCompletionProvider.java 24 Jan 2007 10:58:47 -0000 @@ -1101,9 +1101,8 @@ int openLtNum = 0; JavaTokenId lastNonWhitespaceTokenId = null; TokenSequence ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language()); - if (ts.move(expEndPos) > 0) - ts.moveNext(); - do { + ts.move(expEndPos); + while (ts.moveNext()) { if (ts.offset() >= offset) { break; } @@ -1136,7 +1135,7 @@ default: lastNonWhitespaceTokenId = ts.token().id(); } - } while (ts.moveNext()); + } if (!afterDot) { if (expEndPos <= offset) insideExpression(env, new TreePath(path, fa.getExpression())); @@ -3403,8 +3402,10 @@ private TokenSequence findLastNonWhitespaceToken(Env env, int startPos, int endPos) { TokenSequence ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language()); - if (ts.move(endPos) == 0) - ts.movePrevious(); + if (ts.move(endPos) == 0) // When right at the token end + ts.movePrevious(); // move to previous token + else + ts.moveNext(); // otherwise move to the token that "contains" the offset do { int offset = ts.offset(); if (offset < startPos) @@ -3541,8 +3542,10 @@ if (upToOffset) { TokenSequence ts = controller.getTokenHierarchy().tokenSequence(JavaTokenId.language()); if (ts.moveNext()) { //check for empty file - if (ts.move(offset) == 0) - ts.movePrevious(); + if (ts.move(offset) == 0) // When right at the token end + ts.movePrevious(); // Move to previous token + else + ts.moveNext(); // otherwise move to the token that "contains" the offset if (ts.offset() < offset && (ts.token().id() == JavaTokenId.IDENTIFIER || "keyword".equals(ts.token().id().primaryCategory()))) { //TODO: Use isKeyword(...) when available prefix = ts.token().toString().substring(0, offset - ts.offset()); offset = ts.offset(); Index: java/editor/src/org/netbeans/modules/editor/java/Utilities.java =================================================================== RCS file: /cvs/java/editor/src/org/netbeans/modules/editor/java/Utilities.java,v retrieving revision 1.13 diff -u -r1.13 Utilities.java --- java/editor/src/org/netbeans/modules/editor/java/Utilities.java 11 Jan 2007 10:55:50 -0000 1.13 +++ java/editor/src/org/netbeans/modules/editor/java/Utilities.java 24 Jan 2007 10:58:47 -0000 @@ -98,6 +98,7 @@ return false; if (!ts.moveNext() || ts.move(offset) == 0) return true; + ts.moveNext(); // Move to the next token after move(offset) switch(ts.token().id()) { case DOUBLE_LITERAL: if (ts.token().text().charAt(0) == '.') @@ -127,6 +128,7 @@ TokenSequence ts = hierarchy.tokenSequence(); while(ts != null && ts.moveNext()) { ts.move(offset); + ts.moveNext(); if (ts.language() == JavaTokenId.language()) return (TokenSequence)ts; ts = ts.embedded(); Index: java/hints/src/org/netbeans/modules/java/hints/JavaHintsProvider.java =================================================================== RCS file: /cvs/java/hints/src/org/netbeans/modules/java/hints/JavaHintsProvider.java,v retrieving revision 1.65 diff -u -r1.65 JavaHintsProvider.java --- java/hints/src/org/netbeans/modules/java/hints/JavaHintsProvider.java 16 Jan 2007 09:39:22 -0000 1.65 +++ java/hints/src/org/netbeans/modules/java/hints/JavaHintsProvider.java 24 Jan 2007 10:58:49 -0000 @@ -175,23 +175,23 @@ TokenSequence ts = info.getTokenHierarchy().tokenSequence(); ts.move(offset); - - Token t = ts.token(); - - if (t.id() == JavaTokenId.DOT) { - ts.moveNext(); - t = ts.token(); - } else { - if (t.id() == JavaTokenId.LT) { + if (ts.moveNext()) { + Token t = ts.token(); + + if (t.id() == JavaTokenId.DOT) { ts.moveNext(); t = ts.token(); + } else { + if (t.id() == JavaTokenId.LT) { + ts.moveNext(); + t = ts.token(); + } + } + + if (t.id() == JavaTokenId.IDENTIFIER) { + return ts.offsetToken(); } } - - if (t.id() == JavaTokenId.IDENTIFIER) { - return ts.offsetToken(); - } - return null; } @@ -256,7 +256,7 @@ int diff = ts.move((int) getPrefferedPosition(info, d)); - if (diff >= 0 && diff < ts.token().length()) { + if (ts.moveNext() && diff >= 0 && diff < ts.token().length()) { Token t = ts.token(); if (t.id() == JavaTokenId.IDENTIFIER) { Index: java/source/src/org/netbeans/api/java/source/TreeUtilities.java =================================================================== RCS file: /cvs/java/source/src/org/netbeans/api/java/source/TreeUtilities.java,v retrieving revision 1.13 diff -u -r1.13 TreeUtilities.java --- java/source/src/org/netbeans/api/java/source/TreeUtilities.java 11 Jan 2007 12:24:03 -0000 1.13 +++ java/source/src/org/netbeans/api/java/source/TreeUtilities.java 24 Jan 2007 10:58:59 -0000 @@ -158,7 +158,8 @@ return path; TokenSequence tokenList = tokensFor(path.getLeaf(), sourcePositions); - if (tokenList.moveLast() && tokenList.offset() < pos) { + tokenList.moveEnd(); + if (tokenList.movePrevious() && tokenList.offset() < pos) { switch (tokenList.token().id()) { case GTGTGT: case GTGT: Index: java/source/src/org/netbeans/modules/java/source/save/CasualDiff.java =================================================================== RCS file: /cvs/java/source/src/org/netbeans/modules/java/source/save/CasualDiff.java,v retrieving revision 1.51 diff -u -r1.51 CasualDiff.java --- java/source/src/org/netbeans/modules/java/source/save/CasualDiff.java 19 Jan 2007 15:12:32 -0000 1.51 +++ java/source/src/org/netbeans/modules/java/source/save/CasualDiff.java 24 Jan 2007 10:58:59 -0000 @@ -316,6 +316,7 @@ // skip the section when printing anonymous class if (anonClass == false) { tokenSequence.move(oldT.pos); + tokenSequence.moveNext(); // First skip as move() does not position to token directly tokenSequence.moveNext(); insertHint = TokenUtilities.moveNext(tokenSequence, tokenSequence.offset()); localPointer = diffModifiers(oldT.mods, newT.mods, oldT, localPointer); @@ -508,7 +509,6 @@ // there was not any parameter in original tree. int startOffset = oldT.restype != null ? oldT.restype.getStartPosition() : oldT.getStartPosition(); - tokenSequence.move(startOffset); TokenUtilities.moveFwdToToken(tokenSequence, startOffset, JavaTokenId.RPAREN); posHint = tokenSequence.offset(); } else { @@ -533,6 +533,7 @@ // now check, that there is a whitespace. It is not mandatory, we // have to ensure. If whitespace is not present, take body beginning tokenSequence.move(posHint); + tokenSequence.moveNext(); if (tokenSequence.token().id() != JavaTokenId.WHITESPACE) { ++posHint; } @@ -1441,8 +1442,9 @@ ResultItem item = result[j]; switch (item.operation) { case MODIFY: { - // perhaps I shouldn't support this! - if (tokenSequence.moveIndex(matrix[i][4])) { + // perhaps I shouldn't support this! + tokenSequence.moveIndex(matrix[i][4]); + if (tokenSequence.moveNext()) { testPos = tokenSequence.offset(); if (JavaTokenId.COMMA == tokenSequence.token().id()) testPos += JavaTokenId.COMMA.fixedText().length(); @@ -1488,7 +1490,8 @@ assert startOffset != -1 && endOffset != -1 : "Invalid offset!"; //printer.print(origText.substring(lastOldPos, startOffset)); //append(Diff.delete(startOffset, endOffset)); - if (tokenSequence.moveIndex(matrix[i][4])) { + tokenSequence.moveIndex(matrix[i][4]); + if (tokenSequence.moveNext()) { testPos = tokenSequence.offset(); if (JavaTokenId.COMMA == tokenSequence.token().id()) testPos += JavaTokenId.COMMA.fixedText().length(); @@ -1502,7 +1505,8 @@ break; } case NOCHANGE: { - if (tokenSequence.moveIndex(matrix[i][4])) { + tokenSequence.moveIndex(matrix[i][4]); + if (tokenSequence.moveNext()) { testPos = tokenSequence.offset(); if (JavaTokenId.COMMA == tokenSequence.token().id()) testPos += JavaTokenId.COMMA.fixedText().length(); @@ -1521,6 +1525,7 @@ return -1; } tokenSequence.moveIndex(tokenIndex); + tokenSequence.moveNext(); return tokenSequence.offset(); } Index: java/source/src/org/netbeans/modules/java/source/save/PositionEstimator.java =================================================================== RCS file: /cvs/java/source/src/org/netbeans/modules/java/source/save/PositionEstimator.java,v retrieving revision 1.2 diff -u -r1.2 PositionEstimator.java --- java/source/src/org/netbeans/modules/java/source/save/PositionEstimator.java 17 Dec 2006 16:37:47 -0000 1.2 +++ java/source/src/org/netbeans/modules/java/source/save/PositionEstimator.java 24 Jan 2007 10:59:00 -0000 @@ -139,6 +139,7 @@ int treeEnd = (int) positions.getEndPosition(compilationUnit, item); seq.move(treeStart); + seq.moveNext(); int startIndex = seq.index(); // go back to opening/closing curly, semicolon or other @@ -147,9 +148,11 @@ seq.moveNext(); int start = seq.index(); seq.move(treeEnd); + // seq.index() set (no seq.moveNext() necessary) matrix[i++] = new int[] { start, startIndex, seq.index() }; if (i == size) { seq.move(treeEnd); + // seq.index() set (no seq.moveNext() necessary) matrix[i][0] = seq.index(); } } @@ -163,6 +166,7 @@ // to decide. if (tokenIndex == -1) return -1; seq.moveIndex(tokenIndex); + seq.moveNext(); return index == 0 ? goAfterLastNewLine(seq) : goAfterFirstNewLine(seq); } @@ -209,9 +213,15 @@ @Override() public int[] getPositions(int index) { int tokenIndex = matrix[index][0]; - seq.moveIndex(tokenIndex); + if (tokenIndex != -1) { + seq.moveIndex(tokenIndex); + seq.moveNext(); + } int begin = goAfterLastNewLine(seq); - seq.moveIndex(matrix[index][2]); + if (matrix[index][2] != -1) { + seq.moveIndex(matrix[index][2]); + seq.moveNext(); + } int end = goAfterFirstNewLine(seq); return new int [] { begin, end }; } @@ -262,6 +272,7 @@ if (treeEnd < 0) continue; seq.move(treeStart); + seq.moveNext(); int startIndex = seq.index(); // go back to opening/closing curly, semicolon or other // token java-compiler important token. @@ -285,6 +296,7 @@ // to decide. if (tokenIndex == -1) return -1; seq.moveIndex(tokenIndex); + seq.moveNext(); return goAfterFirstNewLine(seq); } @@ -296,7 +308,10 @@ public int[] getPositions(int index) { int begin = getInsertPos(index); - seq.moveIndex(matrix[index][4]); + if (matrix[index][4] != -1) { + seq.moveIndex(matrix[index][4]); + seq.moveNext(); + } int end = goAfterFirstNewLine(seq); return new int [] { begin, end }; } @@ -360,6 +375,7 @@ // to decide. if (tokenIndex == -1) return -1; seq.moveIndex(tokenIndex); + seq.moveNext(); int off = goAfterFirstNewLine(seq); return off; } @@ -372,7 +388,10 @@ public int[] getPositions(int index) { int begin = getInsertPos(index); - seq.moveIndex(matrix[index][4]); + if (matrix[index][4] != -1) { + seq.moveIndex(matrix[index][4]); + seq.moveNext(); + } int end = goAfterFirstNewLine(seq); return new int [] { begin, end }; } @@ -497,16 +516,19 @@ // todo (#pf): remove - used for debugging reasons, doesn't do good job public void tablePrint(int[][] matrix, TokenSequence seq) { for (int i = 0; i < matrix.length; i++) { - for (int j = 0; j < 5; j++) { + for (int j = 0; j < matrix[i].length; j++) { int item = matrix[i][j]; String s = "(nothing)"; if (item > -1) { seq.moveIndex(item); + seq.moveNext(); s = "'" + seq.token().text(); } s += "' "; - System.err.print(s.substring(0, 25)); + // System.err.print(s.substring(0, 25)); + System.err.print(item + "\t"); } + System.err.println(""); } } Index: java/source/src/org/netbeans/modules/java/source/save/TokenUtilities.java =================================================================== RCS file: /cvs/java/source/src/org/netbeans/modules/java/source/save/TokenUtilities.java,v retrieving revision 1.1 diff -u -r1.1 TokenUtilities.java --- java/source/src/org/netbeans/modules/java/source/save/TokenUtilities.java 29 Nov 2006 21:54:48 -0000 1.1 +++ java/source/src/org/netbeans/modules/java/source/save/TokenUtilities.java 24 Jan 2007 10:59:00 -0000 @@ -65,6 +65,7 @@ */ static int moveNext(TokenSequence tokenSequence, final int pos) { tokenSequence.move(pos); + tokenSequence.moveNext(); // Assumes the pos is located within input bounds JavaTokenId type = tokenSequence.token().id(); while (JavaTokenId.WHITESPACE.equals(type) || JavaTokenId.BLOCK_COMMENT.equals(type) || @@ -82,6 +83,7 @@ JavaTokenId id) { tokenSequence.move(pos); + tokenSequence.moveNext(); // Assumes the pos is located within input bounds while (!id.equals(tokenSequence.token().id())) { if (!tokenSequence.moveNext()) return -1; @@ -94,6 +96,7 @@ JavaTokenId id) { tokenSequence.move(pos); + tokenSequence.moveNext(); // Assumes the pos is located within input bounds while (!id.equals(tokenSequence.token().id())) { if (!tokenSequence.movePrevious()) return -1; @@ -106,6 +109,7 @@ String text) { tokenSequence.move(pos); + tokenSequence.moveNext(); // Assumes the pos is located within input bounds while (!text.equals(tokenSequence.token().text())) { if (!tokenSequence.moveNext()) return -1; @@ -118,6 +122,7 @@ String text) { tokenSequence.move(pos); + tokenSequence.moveNext(); // Assumes the pos is located within input bounds while (!text.equals(tokenSequence.token().text())) { if (!tokenSequence.movePrevious()) return -1; Index: java/source/src/org/netbeans/modules/java/source/save/TreeDiff.java =================================================================== RCS file: /cvs/java/source/src/org/netbeans/modules/java/source/save/TreeDiff.java,v retrieving revision 1.6 diff -u -r1.6 TreeDiff.java --- java/source/src/org/netbeans/modules/java/source/save/TreeDiff.java 11 Jan 2007 12:24:36 -0000 1.6 +++ java/source/src/org/netbeans/modules/java/source/save/TreeDiff.java 24 Jan 2007 10:59:00 -0000 @@ -1247,7 +1247,8 @@ ResultItem item = result[j]; switch (item.operation) { case MODIFY: { - if (tokenSequence.moveIndex(matrix[i][4])) { + tokenSequence.moveIndex(matrix[i][4]); + if (tokenSequence.moveNext()) { testPos = tokenSequence.offset(); if (JavaTokenId.COMMA == tokenSequence.token().id()) testPos += JavaTokenId.COMMA.fixedText().length(); @@ -1283,7 +1284,8 @@ int endOffset = toOff(s.tail(j) || s.next(j) ? matrix[i+1][2] : matrix[i][4]); assert startOffset != -1 && endOffset != -1 : "Invalid offset!"; append(Diff.delete(startOffset, endOffset)); - if (tokenSequence.moveIndex(matrix[i][4])) { + tokenSequence.moveIndex(matrix[i][4]); + if (tokenSequence.moveNext()) { testPos = tokenSequence.offset(); if (JavaTokenId.COMMA == tokenSequence.token().id()) testPos += JavaTokenId.COMMA.fixedText().length(); @@ -1293,7 +1295,8 @@ break; } case NOCHANGE: { - if (tokenSequence.moveIndex(matrix[i][4])) { + tokenSequence.moveIndex(matrix[i][4]); + if (tokenSequence.moveNext()) { testPos = tokenSequence.offset(); if (JavaTokenId.COMMA == tokenSequence.token().id()) testPos += JavaTokenId.COMMA.fixedText().length(); Index: lexer/arch.xml =================================================================== RCS file: /cvs/lexer/arch.xml,v retrieving revision 1.8 diff -u -r1.8 arch.xml --- lexer/arch.xml 6 Dec 2006 15:26:46 -0000 1.8 +++ lexer/arch.xml 24 Jan 2007 10:59:14 -0000 @@ -300,20 +300,18 @@ TokenHierarchy hi = TokenHierarchy.get(document); TokenSequence ts = hi.tokenSequence(); // If necessary move ts to the requested offset - int diff = ts.move(offset); - if (diff != Integer.MAX_VALUE)) { // Some tokens exist - do { - Token t = ts.token(); - if (t.id() == ...) { ... } - if (TokenUtilities.equals(t.text(), "mytext")) { ... } - if (ts.offset() == ...) { ... } + ts.move(offset); + while (ts.moveNext()) { + Token t = ts.token(); + if (t.id() == ...) { ... } + if (TokenUtilities.equals(t.text(), "mytext")) { ... } + if (ts.offset() == ...) { ... } - // Possibly retrieve embedded token sequence - TokenSequence embedded = ts.embedded(); - if (embedded != null) { // Token has a valid language embedding - ... - } - } while (ts.moveNext()); + // Possibly retrieve embedded token sequence + TokenSequence embedded = ts.embedded(); + if (embedded != null) { // Token has a valid language embedding + ... + } } } finally { document.readUnlock(); Index: lexer/manifest.mf =================================================================== RCS file: /cvs/lexer/manifest.mf,v retrieving revision 1.13 diff -u -r1.13 manifest.mf --- lexer/manifest.mf 14 Dec 2006 13:03:01 -0000 1.13 +++ lexer/manifest.mf 24 Jan 2007 10:59:14 -0000 @@ -1,4 +1,4 @@ OpenIDE-Module: org.netbeans.modules.lexer/2 OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/lexer/Bundle.properties -OpenIDE-Module-Specification-Version: 1.12.0 +OpenIDE-Module-Specification-Version: 1.13.0 OpenIDE-Module-Recommends: org.netbeans.spi.lexer.LanguageProvider Index: lexer/api/apichanges.xml =================================================================== RCS file: /cvs/lexer/api/apichanges.xml,v retrieving revision 1.10 diff -u -r1.10 apichanges.xml --- lexer/api/apichanges.xml 14 Dec 2006 13:03:02 -0000 1.10 +++ lexer/api/apichanges.xml 24 Jan 2007 10:59:15 -0000 @@ -91,6 +91,32 @@ + + + Changed TokenSequence.move() and moveIndex() use + + + + + +

+ Changed the TokenSequence.move() to position before the particular token + that "contains" the token (or after the last token if the given offset + is too high). Additional moveNext() is necessary + to actually move to the next token. +
+ TokenSequence.moveIndex() was modified in a similar way + (see javadocs). +
+ TokenSequence.moveFirst() and moveLast() + were replaced by moveStart() that positionins before + the first token and by moveEnd() that positions + after the last token. + +

+ + + Added Lexer.release() Index: lexer/editorbridge/src/org/netbeans/modules/lexer/editorbridge/LexerLayer.java =================================================================== RCS file: /cvs/lexer/editorbridge/src/org/netbeans/modules/lexer/editorbridge/LexerLayer.java,v retrieving revision 1.9 diff -u -r1.9 LexerLayer.java --- lexer/editorbridge/src/org/netbeans/modules/lexer/editorbridge/LexerLayer.java 5 Jan 2007 03:30:36 -0000 1.9 +++ lexer/editorbridge/src/org/netbeans/modules/lexer/editorbridge/LexerLayer.java 24 Jan 2007 10:59:15 -0000 @@ -108,8 +108,8 @@ if (active) { pastSequences = new Stack(); tokenSequence = hi.tokenSequence(); - int relOffset = tokenSequence.move(startOffset); - if (relOffset != Integer.MAX_VALUE) { + tokenSequence.move(startOffset); + if (tokenSequence.moveNext()) { updateTokenEndOffsetAndColoring(startOffset); } else { // no tokens active = false; @@ -178,7 +178,8 @@ break; } } else { - if (embed.move(offset) != Integer.MAX_VALUE) { + embed.move(offset); + if (embed.moveNext()) { pastSequences.push(tokenSequence); tokenSequence = embed; } else { Index: lexer/nbbridge/test/unit/src/org/netbeans/modules/lexer/nbbridge/test/MimeLookupLanguageProviderTest.java =================================================================== RCS file: /cvs/lexer/nbbridge/test/unit/src/org/netbeans/modules/lexer/nbbridge/test/MimeLookupLanguageProviderTest.java,v retrieving revision 1.4 diff -u -r1.4 MimeLookupLanguageProviderTest.java --- lexer/nbbridge/test/unit/src/org/netbeans/modules/lexer/nbbridge/test/MimeLookupLanguageProviderTest.java 26 Oct 2006 20:45:21 -0000 1.4 +++ lexer/nbbridge/test/unit/src/org/netbeans/modules/lexer/nbbridge/test/MimeLookupLanguageProviderTest.java 24 Jan 2007 10:59:15 -0000 @@ -75,7 +75,8 @@ assertEquals("Wrong language", "text/x-simple-plain", lang.mimeType()); for(int i = 0; i < seq.tokenCount(); i++) { - seq.move(i); + seq.moveIndex(i); + assertTrue(seq.moveNext()); Token token = seq.token(); if (token.id() == SimplePlainTokenId.WORD) { @@ -86,11 +87,13 @@ assertNotNull("Can't find language of the embedded sequence", embeddedLang); assertEquals("Wrong language of the embedded sequence", "text/x-simple-char", embeddedLang.mimeType()); - assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveFirst()); - assertEquals("Wrong startSkipLenght", 1, embeddedSeq.offset() - seq.offset()); + embeddedSeq.moveStart(); + assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveNext()); + assertEquals("Wrong startSkipLength", 1, embeddedSeq.offset() - seq.offset()); - assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.moveLast()); - assertEquals("Wrong endSkipLenght", 2, + embeddedSeq.moveEnd(); + assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.movePrevious()); + assertEquals("Wrong endSkipLength", 2, (seq.offset() + seq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length())); } } Index: lexer/src/org/netbeans/api/lexer/TokenSequence.java =================================================================== RCS file: /cvs/lexer/src/org/netbeans/api/lexer/TokenSequence.java,v retrieving revision 1.7 diff -u -r1.7 TokenSequence.java --- lexer/src/org/netbeans/api/lexer/TokenSequence.java 4 Dec 2006 15:43:59 -0000 1.7 +++ lexer/src/org/netbeans/api/lexer/TokenSequence.java 24 Jan 2007 10:59:16 -0000 @@ -38,7 +38,7 @@ * A typical use is a forward iteration through the tokens: * 
  *   TokenSequence ts = tokenHierarchy.tokenSequence();
- *   // Possible positioning by ts.move()
+ *   // Possible positioning by ts.move(offset);
  *   while (ts.moveNext()) {
  *       Token t = ts.token();
  *       if (t.id() == ...) { ... }
@@ -65,7 +65,7 @@
     
     private AbstractToken token; // 16 bytes
     
-    private int tokenIndex = -1; // 20 bytes
+    private int tokenIndex; // 20 bytes
     
     /**
      * Offset in the input at which the current token is located
@@ -115,9 +115,18 @@
     /**
      * Get instance of current token to which this token sequence points to.
      * 

-     * It is necessary to call {@link #moveNext()} before first calling this method.
+     * A typical iteration usage:
+     * 
+     *   TokenSequence ts = tokenHierarchy.tokenSequence();
+     *   // Possible positioning by ts.move(offset);
+     *   while (ts.moveNext()) {
+     *       Token t = ts.token();
+     *       if (t.id() == ...) { ... }
+     *       if (TokenUtilities.equals(t.text(), "mytext")) { ... }
+     *       if (ts.offset() == ...) { ... }
+     *   }
+     * 

      *
-     * 

      * The returned token instance may be flyweight
      * (returns true from {@link Token#isFlyweight()})
      * which means that its {@link Token#offset(TokenHierarchy)} will return -1.
@@ -133,13 +142,13 @@
      * The token instance should not be held across the input source modifications.
      * 

      *
-     * @return non-null token instance.
+     * @return token instance to which this token sequence is currently positioned
+     *  or null if this token sequence is not positioned to any token which may
+     *  happen after TS creation or after use of {@link #move(int)} or {@link moveIndex(int)}.
+     * 
      * @see #offsetToken()
-     * @throws IllegalStateException if this token sequence was not positioned
-     *  to any token yet.
      */
     public Token token() {
-        checkToken();
         return token;
     }
     
@@ -160,7 +169,7 @@
      *  to any token yet.
      */
     public Token offsetToken() {
-        checkToken();
+        checkTokenNotNull();
         if (token.isFlyweight()) {
             token = tokenList.replaceFlyToken(tokenIndex, token, offset());
         }
@@ -183,7 +192,7 @@
      *  to any token yet.
      */
     public int offset() {
-        checkToken();
+        checkTokenNotNull();
         if (tokenOffset == -1) {
             tokenOffset = tokenList.tokenOffset(tokenIndex);
         }
@@ -191,10 +200,11 @@
     }
     
     /**
-     * Get the index of the current token in the complete list of tokens.
+     * Get an index of token to which (or before which) this TS is currently positioned.
+     * 

+     * Zero is returned after TS creation.
      *
-     * @return >=0 index of the current token or -1
-     *  if this token sequence is initially located in front of the first token.
+     * @return >=0 index of the current token.
      */
     public int index() {
         return tokenIndex;
@@ -216,7 +226,7 @@
      *  to any token yet.
      */
     public TokenSequence embedded() {
-        checkToken();
+        checkTokenNotNull();
         return embeddedImpl(null);
     }
     
@@ -249,7 +259,7 @@
      * has a language embedding.
      */
     public  TokenSequence embedded(Language embeddedLanguage) {
-        checkToken();
+        checkTokenNotNull();
         return embeddedImpl(embeddedLanguage);
     }
 
@@ -294,7 +304,7 @@
      */
     public boolean createEmbedding(Language embeddedLanguage,
     int startSkipLength, int endSkipLength, boolean joinSections) {
-        checkToken();
+        checkTokenNotNull();
         return EmbeddingContainer.createEmbedding(tokenList, tokenIndex,
                 embeddedLanguage, startSkipLength, endSkipLength, joinSections);
     }
@@ -302,48 +312,62 @@
     /**
      * Move to the next token in this token sequence.
      * 

+     * This method can be used in case when TS is positioned before a token
+     * (after its creation or after use of {@link #move(int)} or {@link #moveIndex(int)})
+     * or in case the TS already points to a concrete token {@link #token()} != null
+     * - in such case it moves to next token and {@link #index()} gets increased by one.
+     * 

      * The next token may not necessarily start at the offset where
-     * the current token ends (there may be gaps between tokens
-     * caused by use of a token id filter).
+     * the previous token ends (there may be gaps between tokens
+     * caused by token filtering).
      *
      * @return true if the sequence was successfully moved to the next token
-     *  or false if stays on the original token because there are no more tokens
+     *  or false if it was not moved before there are no more tokens
      *  in the forward direction.
      * @throws ConcurrentModificationException if this token sequence
      *  is no longer valid because of an underlying mutable input source modification.
      */
     public boolean moveNext() {
         checkModCount();
-        tokenIndex++;
+        if (token != null) // Token already fetched
+            tokenIndex++;
         Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(tokenIndex);
         if (tokenOrEmbeddingContainer != null) {
             AbstractToken origToken = token;
-            assignToken(tokenOrEmbeddingContainer);
+            token = LexerUtilsConstants.token(tokenOrEmbeddingContainer);
+            // If origToken == null then the right offset might already be pre-computed from move()
             if (tokenOffset != -1) {
-                // If the token list is continuous or the fetched token
-                // is flyweight (there cannot be a gap before flyweight token)
-                // the original offset can be just increased
-                // by the original token's length.
-                if (tokenList.isContinuous() || token.isFlyweight()) {
-                    tokenOffset += origToken.length(); // advance by previous token's length
-                } else // Offset must be recomputed
-                    tokenOffset = -1; // mark the offset to be recomputed
+                if (origToken != null) {
+                    // If the token list is continuous or the fetched token
+                    // is flyweight (there cannot be a gap before flyweight token)
+                    // the original offset can be just increased
+                    // by the original token's length.
+                    if (tokenList.isContinuous() || token.isFlyweight()) {
+                        tokenOffset += origToken.length(); // advance by previous token's length
+                    } else // Offset must be recomputed
+                        tokenOffset = -1; // mark the offset to be recomputed
+                } else // Not valid token previously
+                    tokenOffset = -1;
             }
             return true;
         }
-        tokenIndex--;
+        if (token != null) // Unsuccessful move from existing token
+            tokenIndex--;
         return false;
     }
 
     /**
-     * Move to the previous token in this token sequence.
+     * Move to a previous token in this token sequence.
+     * 

+     * This method can be used in case when TS is positioned after a token
+     * (after its creation or after use of {@link #move(int)} or {@link #moveIndex(int)})
+     * or in case the TS already points to a concrete token {@link #token()} != null.
+     * 

+     * In both cases {@link #index()} gets decreased by one.
      * 

-     * The next token may not necessarily end at the offset where
-     * the present token starts (there may be gaps between tokens
-     * caused by use of a token id filter).
      *
      * @return true if the sequence was successfully moved to the previous token
-     *  or false if stayed on the original token because there are no more tokens
+     *  or false if it was not moved because there are no more tokens
      *  in the backward direction.
      * @throws ConcurrentModificationException if this token sequence
      *  is no longer valid because of an underlying mutable input source modification.
@@ -353,7 +377,7 @@
         if (tokenIndex > 0) {
             AbstractToken origToken = token;
             tokenIndex--;
-            assignToken();
+            token = LexerUtilsConstants.token(tokenList.tokenOrEmbeddingContainer(tokenIndex));
             if (tokenOffset != -1) {
                 // If the token list is continuous or the original token
                 // is flyweight (there cannot be a gap before flyweight token)
@@ -364,8 +388,6 @@
                 } else { // mark the offset to be computed upon call to offset()
                     tokenOffset = -1;
                 }
-            } else {
-                tokenOffset = -1; // mark the offset to be computed upon call to offset()
             }
             return true;
 
@@ -374,92 +396,95 @@
     }
 
     /**
-     * Move the token sequence to point to the token with the given index.
+     * Move TS to be positioned before a token with the given index.
+     * 

+     * Example: Fetching tokens starting at the given index:
+     * 
+     * TokenSequence ts = tokenHierarchy.tokenSequence();
+     * ts.moveIndex(index);
+     * while (ts.moveNext()) {
+     *     Token t = ts.token();
+     *     ...
+     * }
+     * 

      *
      * @param index index of the token to which this sequence
      *   should be positioned.
-     * @return true if the sequence was moved to the token
-     *   with the given index. Returns false
-     *   if index < 0 or index < tokenCount.
-     *   In such case the current token sequence's position stays unchanged.
+     *   

+     *   If index >= {@link #tokenCount()}
+     *   then the TS will be positioned after the last token.
+     *   

+     *   If index < 0
+     *   then the TS will be positioned before first token (at index 0).
+     * @return difference between requested index and the index to which TS
+     *   is really set.
      * @throws ConcurrentModificationException if this token sequence
      *  is no longer valid because of an underlying mutable input source modification.
      */
-    public boolean moveIndex(int index) {
+    public int moveIndex(int index) {
         checkModCount();
-        if (index < 0) {
-            return false;
-        }
-        Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index);
-        if (tokenOrEmbeddingContainer != null) { // enough tokens
-            this.tokenIndex = index;
-            assignToken(tokenOrEmbeddingContainer);
-            tokenOffset = -1;
-            return true;
-
-        } else // Token at the requested index does not exist - leave orig. index
-            return false;
+        if (index >= 0) {
+            Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index);
+            if (tokenOrEmbeddingContainer != null) { // enough tokens
+                resetTokenIndex(index);
+            } else // Token at the requested index does not exist - leave orig. index
+                resetTokenIndex(tokenCount());
+        } else // index < 0
+            resetTokenIndex(0);
+        return index - tokenIndex;
+    }
+    
+    /**
+     * Move the token sequence to be positioned before the first token.
+     * 

+     * This is equivalent to moveIndex(0).
+     */
+    public void moveStart() {
+        moveIndex(0);
     }
     
     /**
-     * Move the token sequence to be positioned to the token
+     * Move the token sequence to be positioned after the last token.
+     * 

+     * This is equivalent to moveIndex(tokenCount()).
+     */
+    public void moveEnd() {
+        moveIndex(tokenCount());
+    }
+    
+    /**
+     * Move the token sequence to be positioned before the token
      * that "contains" the requested offset (the offset is at the begining
      * or inside of the token).
-     * 

-     * If the offset is too big the token sequence will be positioned
-     * to the last token and the return value will
-     * be the distance between the requested offset
-     * and the start offset of the token to which the token sequence
-     * will be positioned..
-     * 

-     * If there are no tokens in the sequence then {@link Integer#MAX_VALUE}
-     * will be returned.
-     *
+     * 

+     * Typical usage pattern:
+     * 
+     * TokenSequence ts = ...
+     * ts.move(offset);
+     * while (ts.moveNext()) {
+     *     Token t = ts.token();
+     *     ...
+     * }
+     * 

+     * 
+     * If the offset is too big then the token sequence will be positioned
+     * behind the last token.
      * 

-     * The underlying token list may contain gaps that are not covered
+     * If token filtering is used there may be gaps that are not covered
      * by any tokens and if the offset is contained in such gap then
-     * the token sequence will be positioned to the token that precedes the gap.
+     * the token sequence will be positioned before the token that follows the gap.
      * 

      *
-     * Example:
-     * 
-     *   int diff = tokenSequence.move(targetOffset);
-     *   // diff equals to (targetOffset - tokenSequence.token().offset())
-     *   if (diff >= 0 && diff < tokenSequence.token().length()) {
-     *       // Within token bounds - tokenSequence.token() starts at or "contains" targetOffset
-     *
-     *   } else if (diff == Integer.MAX_VALUE) {
-     *       // No tokens in the token sequence at all.
-     *       // Token sequence is not positioned to any token.
-     *
-     *   } else {
-     *       // 1. diff >= tokenSequence.token().length()
-     *       //   a) targetOffset is above the end of the last token in the sequence.
-     *       //     Token sequence is positioned to the last token in the sequence.
-     *       //   b) there are text areas not covered by any tokens
-     *       //     due to skipped tokens (skipTokenIds was used
-     *       //     in TokenHierarchy.create()) and the targetOffset points to such gap.
-     *       //     Token sequence is positioned to the preceding token.
-     *       // 
-     *       // 2. diff < 0
-     *       //   a) targetOffset < 0
-     *       //   b) targetOffset >= 0 but there is a text area
-     *       //     at the begining that is not covered by any tokens
-     *       //     (skipTokenIds was used in TokenHierarchy.create())
-     *       //     Token sequence is positioned to the first token in the sequence.
-     *   }
-     * 

-     * 
      *
      * @param offset absolute offset in the input to which
      *  the token sequence should be moved.
      *
      * @return difference between the reqeuested offset
-     *  and the absolute starting offset of the token
-     *  to which the the token sequence gets moved.
-     *  

-     *  Returns {@link Integer#MAX_VALUE} if there are no tokens in the sequence.
-     *  In such case there is no active token.
+     *  and the start offset of the token
+     *  before which the the token sequence gets positioned.
+     *  

+     *  If the TS is positioned after the last token the difference is computed
+     *  against the end offset of the last token.
      * 
      * @throws ConcurrentModificationException if this token sequence
      *  is no longer valid because of an underlying mutable input source modification.
@@ -474,7 +499,8 @@
             if (tokenList.tokenOrEmbeddingContainer(0) == null) { // really no tokens at all
                 // In this case the token sequence could not be positioned yet
                 // so no need to reset "index" or other vars
-                return Integer.MAX_VALUE;
+                resetTokenIndex(0);
+                return offset;
             }
             // Re-get the present token count (could be created a chunk of tokens at once)
             tokenCount = tokenList.tokenCountCurrent();
@@ -499,14 +525,14 @@
                     tokenLength = t.length();
                     tokenCount++;
 
-                } else { // no more tokens => break
-                    break;
+                } else { // no more tokens => position after last token
+                    resetTokenIndex(tokenCount);
+                    tokenOffset = prevTokenOffset + tokenLength; // May assign the token's offset in advance
+                    return offset - tokenOffset;
                 }
             }
-            tokenIndex = tokenCount - 1;
-            // Absolute token's start offset 
-            tokenOffset = prevTokenOffset;
-            assignToken();
+            resetTokenIndex(tokenCount - 1);
+            tokenOffset = prevTokenOffset; // May assign the token's offset in advance
             return offset - prevTokenOffset;
         }
         
@@ -525,50 +551,33 @@
                 high = mid - 1;
             } else {
                 // Token starting exactly at offset found
-                tokenIndex = mid;
+                resetTokenIndex(mid);
                 tokenOffset = midStartOffset;
-                assignToken();
                 return 0; // right at the token begining
             }
         }
         
         // Not found exactly and high + 1 == low => high < low
-        // Check whether the token at "high" contains the offset
+        // BTW there may be gaps between tokens; if offset is in gap then position to higher token
         if (high >= 0) { // could be -1
             AbstractToken t = LexerUtilsConstants.token(tokenList, high);
             prevTokenOffset = tokenList.tokenOffset(high);
+            // If gaps allowed check whether the token at "high" contains the offset
+            if (!tokenList.isContinuous() && offset > prevTokenOffset + t.length()) {
+                // Offset in the gap above the "high" token
+                high++;
+                prevTokenOffset += t.length();
+            }
         } else { // at least one token exists => use token at index 0
             high = 0;
             prevTokenOffset = tokenList.tokenOffset(0); // result may differ from 0
         }
-
-        tokenIndex = high;
+        resetTokenIndex(high);
         tokenOffset = prevTokenOffset;
-        assignToken();
         return offset - prevTokenOffset;
     }
 
     /**
-     * Move to the first token in this token sequence.
-     *
-     * @return true if the sequence was positioned on the first token
-     *  or false if there are no tokens in the sequence.
-     */
-    public boolean moveFirst() {
-        return moveIndex(0);
-    }
-    
-    /**
-     * Move to the last token in this token sequence.
-     *
-     * @return true if the sequence was positioned on the last token
-     *  or false if there are no tokens in the sequence.
-     */
-    public boolean moveLast() {
-        return moveIndex(tokenCount() - 1); // Can be -1 but handled in move(index)
-    }
-
-    /**
      * Return total count of tokens in this sequence.
      * 

      * Note: Calling this method will lead
@@ -628,15 +637,14 @@
         return parentTokenIndexes;
     }
 
-    private void assignToken(Object tokenOrEmbeddingContainer) {
-        token = LexerUtilsConstants.token(tokenOrEmbeddingContainer);
-    }
-    
-    private void assignToken() {
-        assignToken(tokenList.tokenOrEmbeddingContainer(tokenIndex));
+    private void resetTokenIndex(int index) {
+        // Position to the given index e.g. by move() and moveIndex()
+        tokenIndex = index;
+        token = null;
+        tokenOffset = -1;
     }
 
-    private void checkToken() {
+    private void checkTokenNotNull() {
         if (token == null) {
             throw new IllegalStateException(
                 "No token fetched by moveNext() from token sequence yet: index=" + tokenIndex
Index: lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java
===================================================================
RCS file: /cvs/lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java,v
retrieving revision 1.5
diff -u -r1.5 LexerUtilsConstants.java
--- lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java	4 Dec 2006 15:44:00 -0000	1.5
+++ lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java	24 Jan 2007 10:59:16 -0000
@@ -237,6 +237,16 @@
             sb.append(": ");
             AbstractToken token = token(tokenOrEmbeddingContainer);
             sb.append(token.dumpInfo(tokenHierarchy));
+            int la = tokenList.lookahead(i);
+            if (la != 0) {
+                sb.append(", la=");
+                sb.append(la);
+            }
+            Object state= tokenList.state(i);
+            if (state != null) {
+                sb.append(", s=");
+                sb.append(state);
+            }
             sb.append('\n');
         }
         return sb;
Index: lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java
===================================================================
RCS file: /cvs/lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java,v
retrieving revision 1.6
diff -u -r1.6 SubSequenceTokenList.java
--- lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java	4 Dec 2006 15:44:00 -0000	1.6
+++ lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java	24 Jan 2007 10:59:16 -0000
@@ -92,7 +92,7 @@
         if (limitStartOffset > 0) {
             int diff = move(limitStartOffset);
             if (diff != Integer.MAX_VALUE) { // some tokens exist
-                if (diff >= lastToken.length()) {
+                if (diff >= lastToken.length()) { // lastToken initialized in move()
                     lastTokenIndex++;
                     Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(lastTokenIndex);
                     if (tokenOrEmbeddingContainer != null &&
Index: lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java
===================================================================
RCS file: /cvs/lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java,v
retrieving revision 1.5
diff -u -r1.5 TokenSequenceTest.java
--- lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java	4 Dec 2006 15:44:04 -0000	1.5
+++ lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java	24 Jan 2007 10:59:16 -0000
@@ -48,13 +48,7 @@
         TokenHierarchy hi = TokenHierarchy.create(text, SimpleTokenId.language());
         TokenSequence ts = hi.tokenSequence();
         
-        // Fail if no "move*" method called yet
-        try {
-            ts.token(); // should throw exception
-            fail("IllegalStateException expecxted");
-        } catch (IllegalStateException e) {
-            // expected
-        }
+        assertNull(ts.token());
 
         assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
@@ -69,20 +63,38 @@
         assertEquals(ts.tokenCount(), 3);
         
         assertEquals(0, ts.move(0));
+        assertNull(ts.token());
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
         assertEquals(2, ts.move(2));
+        assertNull(ts.token());
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
         assertEquals(-1, ts.move(-1));
+        assertNull(ts.token());
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
         assertEquals(0, ts.move(3));
+        assertNull(ts.token());
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
         assertEquals(0, ts.move(4));
+        assertNull(ts.token());
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
         assertEquals(1, ts.move(5));
+        assertNull(ts.token());
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
-        assertEquals(5, ts.move(9));
+        assertEquals(1, ts.move(9));
+        assertNull(ts.token());
+        assertFalse(ts.moveNext());
+        assertTrue(ts.movePrevious());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
-        assertEquals(96, ts.move(100));
+        assertEquals(92, ts.move(100));
+        assertNull(ts.token());
+        assertFalse(ts.moveNext());
+        assertTrue(ts.movePrevious());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
 
         
@@ -107,6 +119,70 @@
         sub = ts.subSequence(3, 3);
         assertFalse(sub.moveNext());
     }
+    
+    public void testMoveNextPrevious() {
+        String text = "abc+defg-";
+        TokenHierarchy hi = TokenHierarchy.create(text, SimpleTokenId.language());
+        TokenSequence ts = hi.tokenSequence();
+        assertEquals(0, ts.move(4));
+        assertNull(ts.token());
+        //assertEquals(4, ts.offset());
+        assertTrue(ts.movePrevious());
+        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
+        
+        ts = hi.tokenSequence();
+        ts.move(5);
+        assertTrue(ts.moveNext());
+        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
+        assertTrue(ts.movePrevious());
+        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
+
+        // Move single char before token's end
+        ts = hi.tokenSequence();
+        ts.move(7);
+        assertTrue(ts.moveNext());
+        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
+        assertTrue(ts.movePrevious());
+        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
+    
+        // Move past all tokens
+        ts = hi.tokenSequence();
+        ts.move(text.length());
+        assertTrue(ts.movePrevious());
+        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", text.length() - 1);
+
+        // Move at the begining
+        ts = hi.tokenSequence();
+        ts.move(0);
+        assertFalse(ts.movePrevious());
+        assertTrue(ts.moveNext());
+        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
+    }
+    
+    public void testMoveIndex() {
+        String text = "abc+defg-";
+        TokenHierarchy hi = TokenHierarchy.create(text, SimpleTokenId.language());
+        TokenSequence ts = hi.tokenSequence();
+        assertEquals(0, ts.index());
+        assertNull(ts.token());
+        //assertEquals(4, ts.offset());
+        assertTrue(ts.moveNext());
+        assertEquals(0, ts.offset());
+        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
+        assertTrue(ts.moveNext());
+        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
+        assertEquals(3, ts.offset());
+        ts.move(2);
+        assertEquals(0, ts.index());
+        assertTrue(ts.moveNext());
+        assertEquals(0, ts.offset());
+        assertEquals(0, ts.index());
+        ts.moveIndex(1);
+        assertEquals(1, ts.index());
+        assertTrue(ts.moveNext());
+        assertEquals(1, ts.index());
+        assertEquals(3, ts.offset());
+    }
 
     public void testMoveSkipTokens() {
         String text = "-abc+defg--hi";
@@ -122,12 +198,7 @@
         TokenSequence ts = hi.tokenSequence();
         
         // Fail if no "move*" method called yet
-        try {
-            ts.token(); // should throw exception
-            fail("IllegalStateException expecxted");
-        } catch (IllegalStateException e) {
-            // expected
-        }
+        assertNull(ts.token());
 
         assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
@@ -137,10 +208,12 @@
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
         assertFalse(ts.moveNext());
         assertEquals(3, ts.tokenCount());
-        assertTrue(ts.moveFirst());
+        assertEquals(0, ts.moveIndex(0));
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
         assertTrue(ts.moveNext());
-        assertTrue(ts.moveLast());
+        assertEquals(0, ts.moveIndex(ts.tokenCount()));
+        assertTrue(ts.movePrevious());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
         assertFalse(ts.moveNext());
 
@@ -151,16 +224,28 @@
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
         
         assertEquals(-1, ts.move(0)); // below filtered-out token
+        assertNull(ts.token());
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
         assertEquals(0, ts.move(1));
+        assertNull(ts.token());
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
         assertEquals(1, ts.move(2));
+        assertNull(ts.token());
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
         assertEquals(3, ts.move(4));
+        assertNull(ts.token());
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
         assertEquals(0, ts.move(5));
+        assertNull(ts.token());
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 5);
         assertEquals(1, ts.move(12));
+        assertNull(ts.token());
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
         
         
@@ -172,48 +257,56 @@
         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
         assertFalse(sub.moveNext());
         
-        // Test moveFirst() and moveLast() on subsequence
+        // Test moves to first and last tokens on subsequence
         sub = ts.subSequence(1, 6);
-        assertTrue(sub.moveLast());
+        assertEquals(0, sub.moveIndex(sub.tokenCount()));
+        assertEquals(2, sub.tokenCount()); // "abc" and "defg" (others filtered out
+        assertTrue(sub.movePrevious());
         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
-        assertTrue(sub.moveFirst());
+        assertEquals(0, sub.moveIndex(0));
+        assertTrue(sub.moveNext());
         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "abc", 1);
         
         sub = ts.subSequence(4);
         assertTrue(sub.moveNext());
         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
         assertTrue(sub.moveNext());
-        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
+        LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
         assertFalse(sub.moveNext());
 
         sub = ts.subSequence(12, 15);
         assertTrue(sub.moveNext());
-        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
+        LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
 
         sub = ts.subSequence(12, 15);
         assertEquals(-2, sub.move(9));
-        LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
+        assertNull(sub.token());
+        assertTrue(sub.moveNext());
+        LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
 
         sub = ts.subSequence(13, 15);
         assertFalse(sub.moveNext());
-        assertFalse(sub.moveFirst());
-        assertFalse(sub.moveLast());
+        assertEquals(0, sub.moveIndex(0));
+        assertNull(sub.token());
+        assertFalse(sub.moveNext());
+        assertEquals(0, sub.moveIndex(sub.tokenCount()));
+        assertNull(sub.token());
         assertEquals(0, sub.tokenCount());
 
         sub = ts.subSequence(-3, 1);
         assertFalse(sub.moveNext());
 
         sub = ts.subSequence(13, 15);
-        assertEquals(Integer.MAX_VALUE, sub.move(5));
+        assertEquals(5, sub.move(5));
 
         sub = ts.subSequence(-3, 1);
-        assertEquals(Integer.MAX_VALUE, sub.move(1));
+        assertEquals(1, sub.move(1));
 
         // Reversed bounds => should be empty token sequence
         sub = ts.subSequence(6, 1);
         assertFalse(sub.moveNext());
         sub = ts.subSequence(6, 1);
-        assertEquals(Integer.MAX_VALUE, sub.move(6));
+        assertEquals(6, sub.move(6));
     }
 
     public void DtestMoveSkipTokens2() throws IOException {
@@ -244,32 +337,26 @@
 
         // Expect no tokens
         assertFalse(ts.moveNext());
-        // Exception if ts.token() called
-        try {
-            ts.token(); // should throw exception
-            fail("IllegalStateException expecxted");
-        } catch (IllegalStateException e) {
-            // expected
-        }
+        assertNull(ts.token());
         
-        assertEquals(ts.move(0), Integer.MAX_VALUE);
-        assertEquals(ts.move(10), Integer.MAX_VALUE);
+        assertEquals(0, ts.move(0));
+        assertEquals(10, ts.move(10));
         
         // Test subsequences
         TokenSequence sub = ts.subSequence(1, 6);
         assertFalse(sub.moveNext());
         sub = ts.subSequence(1, 6);
-        assertEquals(Integer.MAX_VALUE, sub.move(1));
+        assertEquals(1, sub.move(1));
         
         sub = ts.subSequence(0);
         assertFalse(sub.moveNext());
         sub = ts.subSequence(0);
-        assertEquals(Integer.MAX_VALUE, sub.move(0));
+        assertEquals(0, sub.move(0));
 
         sub = ts.subSequence(1);
         assertFalse(sub.moveNext());
         sub = ts.subSequence(1);
-        assertEquals(Integer.MAX_VALUE, sub.move(0));
+        assertEquals(0, sub.move(0));
     }
     
     public void testTokenSize() {
@@ -284,7 +371,8 @@
         assertFalse(ts.moveNext());
         
         TokenList tokenList = LexerTestUtilities.tokenList(ts);
-        ts.moveIndex(0); // move to "abc"
+        ts.moveIndex(0); // move before "abc"
+        assertTrue(ts.moveNext());
         // Test DefaultToken size
         assertSame(DefaultToken.class, ts.token().getClass());
         assertSize("Token instance too big", Collections.singletonList(ts.token()), 24,
Index: lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java
===================================================================
RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java,v
retrieving revision 1.5
diff -u -r1.5 LanguageManagerTest.java
--- lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java	26 Oct 2006 20:45:25 -0000	1.5
+++ lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java	24 Jan 2007 10:59:16 -0000
@@ -130,7 +130,8 @@
     public void testCachingE() {
         TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
         TokenSequence tokens = th.tokenSequence();
-        tokens.moveFirst();
+        tokens.moveStart();
+        assertEquals(true, tokens.moveNext());
         
         TokenSequence embeddedA = tokens.embedded();
         assertNotNull("There should be an embedded language", embeddedA);
@@ -144,7 +145,8 @@
     public void testGCedE() {
         TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
         TokenSequence tokens = th.tokenSequence();
-        tokens.moveFirst();
+        tokens.moveStart();
+        assertEquals(true, tokens.moveNext());
         
         TokenSequence embedded = tokens.embedded();
         assertNotNull("There should be an embedded language", embedded);
@@ -164,7 +166,8 @@
     public void testCacheRefreshedE() {
         TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language());
         TokenSequence tokens = th.tokenSequence();
-        tokens.moveFirst();
+        tokens.moveStart();
+        assertEquals(true, tokens.moveNext());
         
         TokenSequence embeddedA = tokens.embedded();
         assertNotNull("There should be an embedded language", embeddedA);
Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java
===================================================================
RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java,v
retrieving revision 1.5
diff -u -r1.5 LexerTestUtilities.java
--- lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java	4 Dec 2006 15:44:05 -0000	1.5
+++ lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java	24 Jan 2007 10:59:16 -0000
@@ -70,6 +70,7 @@
     public static void assertTokenEquals(String message, TokenSequence ts, TokenId id, String text, int offset) {
         message = messagePrefix(message);
         Token t = ts.token();
+        TestCase.assertNotNull("Token is null", t);
         TokenId tId = t.id();
         TestCase.assertEquals(message + "Invalid token.id()", id, tId);
         CharSequence tText = t.text();
@@ -312,17 +313,21 @@
         }
         TokenHierarchy thBatch = TokenHierarchy.create(docText, language);
         boolean success = false;
+        TokenSequence batchTS = thBatch.tokenSequence();
         try {
             // Compare lookaheads and states as well
-            assertTokenSequencesEqual(thBatch.tokenSequence(), thBatch,
+            assertTokenSequencesEqual(batchTS, thBatch,
                     thInc.tokenSequence(), thInc, true);
             success = true;
         } finally {
             if (!success) {
+                // Go forward two tokens to have an extra tokens context
+                batchTS.moveNext();
+                batchTS.moveNext();
                 System.err.println("BATCH token sequence dump:\n" + thBatch.tokenSequence());
                 TokenHierarchy lastHi = (TokenHierarchy)doc.getProperty(LAST_TOKEN_HIERARCHY);
                 if (lastHi != null) {
-                    System.err.println("PREVIOUS token sequence dump:\n" + lastHi.tokenSequence());
+                    System.err.println("PREVIOUS batch token sequence dump:\n" + lastHi.tokenSequence());
                 }
             }
         }
Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java
===================================================================
RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java,v
retrieving revision 1.5
diff -u -r1.5 TokenHierarchySnapshotTest.java
--- lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java	4 Dec 2006 15:44:06 -0000	1.5
+++ lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java	24 Jan 2007 10:59:16 -0000
@@ -76,7 +76,8 @@
 
         // Check that all the non-fly tokens are mutable
         ts = snapshot1.tokenSequence();
-        assertTrue(ts.moveIndex(0));
+        assertEquals(0, ts.moveIndex(0));
+        assertTrue(ts.moveNext());
         
         LexerTestUtilities.assertTokenSequencesEqual(hi1.tokenSequence(), hi1,
                 snapshot1.tokenSequence(), snapshot1, false);
@@ -84,7 +85,8 @@
         doc.insertString(4, "+", null);
 
         // Check that the snapshot token sequence can be further operated.
-        assertTrue(ts.moveIndex(0));
+        assertEquals(0, ts.moveIndex(0));
+        assertTrue(ts.moveNext());
         assertNotNull(ts.token());
 
         // Check that the tokens except '+' are live
@@ -101,7 +103,8 @@
 
         // Check that all the non-fly tokens are mutable
         ts = snapshot2.tokenSequence();
-        assertTrue(ts.moveIndex(0));
+        assertEquals(0, ts.moveIndex(0));
+        assertTrue(ts.moveNext());
 
         LexerTestUtilities.assertTokenSequencesEqual(hi2.tokenSequence(), hi2,
                 snapshot2.tokenSequence(), snapshot2, false);
Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java
===================================================================
RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java,v
retrieving revision 1.6
diff -u -r1.6 SimpleLexerBatchTest.java
--- lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java	17 Jan 2007 13:15:29 -0000	1.6
+++ lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java	24 Jan 2007 10:59:16 -0000
@@ -81,7 +81,8 @@
         assertFalse(ts.moveNext());
 
         // Go back to block comment
-        assertTrue(ts.moveIndex(commentIndex));
+        assertEquals(0, ts.moveIndex(commentIndex));
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.BLOCK_COMMENT, commentText, commentTextStartOffset);
 
         // Test embedded token sequence
Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java
===================================================================
RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java,v
retrieving revision 1.6
diff -u -r1.6 SimpleLexerIncTest.java
--- lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java	4 Dec 2006 15:44:07 -0000	1.6
+++ lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java	24 Jan 2007 10:59:17 -0000
@@ -90,19 +90,23 @@
         
         // Check TokenSequence.move()
         int relOffset = ts.move(50); // past the end of all tokens
-        assertEquals(relOffset, 50 - offset);
+        assertEquals(relOffset, 50 - (offset + 3));
+        assertTrue(ts.movePrevious());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset);
 
         relOffset = ts.move(6); // right at begining of "-"
         assertEquals(relOffset, 0);
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", 6);
 
         relOffset = ts.move(-5); // to first token "abc"
         assertEquals(relOffset, -5);
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
 
         relOffset = ts.move(5); // to first token "abc"
         assertEquals(relOffset, 1);
+        assertTrue(ts.moveNext());
         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "uv", 4);
 
 
Index: openide/text/src/org/openide/text/CloneableEditorSupport.java
===================================================================
RCS file: /cvs/openide/text/src/org/openide/text/CloneableEditorSupport.java,v
retrieving revision 1.28
diff -u -r1.28 CloneableEditorSupport.java
--- openide/text/src/org/openide/text/CloneableEditorSupport.java	12 Jan 2007 10:16:57 -0000	1.28
+++ openide/text/src/org/openide/text/CloneableEditorSupport.java	24 Jan 2007 10:59:59 -0000
@@ -1,3 +1,4 @@
+
 /*
  * The contents of this file are subject to the terms of the Common Development
  * and Distribution License (the License). You may not use this file except in
Index: web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java
===================================================================
RCS file: /cvs/web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java,v
retrieving revision 1.2
diff -u -r1.2 SimplifiedJSPServlet.java
--- web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java	22 Jan 2007 16:04:40 -0000	1.2
+++ web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java	24 Jan 2007 11:00:55 -0000
@@ -98,7 +98,7 @@
         TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);
         TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); //get top level token sequence
         
-        if(!tokenSequence.moveFirst()) {
+        if(!tokenSequence.moveNext()) {
             return ; //no tokens in token sequence
         }
         
Index: xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java
===================================================================
RCS file: /cvs/xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java,v
retrieving revision 1.3
diff -u -r1.3 TagBasedLexerFormatter.java
--- xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java	13 Jan 2007 14:55:58 -0000	1.3
+++ xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java	24 Jan 2007 11:01:22 -0000
@@ -86,17 +86,18 @@
          * e.g.