+ Changed the TokenSequence.move()
to position before the particular token
+ that "contains" the token (or after the last token if the given offset
+ is too high). Additional moveNext()
is necessary
+ to actually move to the next token.
+
+ TokenSequence.moveIndex()
was modified in a similar way
+ (see javadocs).
+
+ TokenSequence.moveFirst()
and moveLast()
+ were replaced by moveStart()
that positionins before
+ the first token and by moveEnd()
that positions
+ after the last token.
+
+
* TokenSequence ts = tokenHierarchy.tokenSequence(); - * // Possible positioning by ts.move() + * // Possible positioning by ts.move(offset); * while (ts.moveNext()) { * Token t = ts.token(); * if (t.id() == ...) { ... } @@ -65,7 +65,7 @@ private AbstractTokentoken; // 16 bytes - private int tokenIndex = -1; // 20 bytes + private int tokenIndex; // 20 bytes /** * Offset in the input at which the current token is located @@ -115,9 +115,18 @@ /** * Get instance of current token to which this token sequence points to. *
- * It is necessary to call {@link #moveNext()} before first calling this method. + * A typical iteration usage: + *+ * TokenSequence ts = tokenHierarchy.tokenSequence(); + * // Possible positioning by ts.move(offset); + * while (ts.moveNext()) { + * Token t = ts.token(); + * if (t.id() == ...) { ... } + * if (TokenUtilities.equals(t.text(), "mytext")) { ... } + * if (ts.offset() == ...) { ... } + * } + ** - ** The returned token instance may be flyweight * (returns true from {@link Token#isFlyweight()}) * which means that its {@link Token#offset(TokenHierarchy)} will return -1. @@ -133,13 +142,13 @@ * The token instance should not be held across the input source modifications. *
* - * @return non-null token instance. + * @return token instance to which this token sequence is currently positioned + * or null if this token sequence is not positioned to any token which may + * happen after TS creation or after use of {@link #move(int)} or {@link moveIndex(int)}. + * * @see #offsetToken() - * @throws IllegalStateException if this token sequence was not positioned - * to any token yet. */ public Tokentoken() { - checkToken(); return token; } @@ -160,7 +169,7 @@ * to any token yet. */ public Token offsetToken() { - checkToken(); + checkTokenNotNull(); if (token.isFlyweight()) { token = tokenList.replaceFlyToken(tokenIndex, token, offset()); } @@ -183,7 +192,7 @@ * to any token yet. */ public int offset() { - checkToken(); + checkTokenNotNull(); if (tokenOffset == -1) { tokenOffset = tokenList.tokenOffset(tokenIndex); } @@ -191,10 +200,11 @@ } /** - * Get the index of the current token in the complete list of tokens. + * Get an index of token to which (or before which) this TS is currently positioned. + *
+ * Zero is returned after TS creation. * - * @return >=0 index of the current token or-1
- * if this token sequence is initially located in front of the first token. + * @return >=0 index of the current token. */ public int index() { return tokenIndex; @@ -216,7 +226,7 @@ * to any token yet. */ public TokenSequence extends TokenId> embedded() { - checkToken(); + checkTokenNotNull(); return embeddedImpl(null); } @@ -249,7 +259,7 @@ * has a language embedding. */ publicTokenSequence embedded(Language embeddedLanguage) { - checkToken(); + checkTokenNotNull(); return embeddedImpl(embeddedLanguage); } @@ -294,7 +304,7 @@ */ public boolean createEmbedding(Language extends TokenId> embeddedLanguage, int startSkipLength, int endSkipLength, boolean joinSections) { - checkToken(); + checkTokenNotNull(); return EmbeddingContainer.createEmbedding(tokenList, tokenIndex, embeddedLanguage, startSkipLength, endSkipLength, joinSections); } @@ -302,48 +312,62 @@ /** * Move to the next token in this token sequence. *
+ * This method can be used in case when TS is positioned before a token + * (after its creation or after use of {@link #move(int)} or {@link #moveIndex(int)}) + * or in case the TS already points to a concrete token{@link #token()} != null
+ * - in such case it moves to next token and {@link #index()} gets increased by one. + *
* The next token may not necessarily start at the offset where - * the current token ends (there may be gaps between tokens - * caused by use of a token id filter). + * the previous token ends (there may be gaps between tokens + * caused by token filtering). * * @return true if the sequence was successfully moved to the next token - * or false if stays on the original token because there are no more tokens + * or false if it was not moved before there are no more tokens * in the forward direction. * @throws ConcurrentModificationException if this token sequence * is no longer valid because of an underlying mutable input source modification. */ public boolean moveNext() { checkModCount(); - tokenIndex++; + if (token != null) // Token already fetched + tokenIndex++; Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(tokenIndex); if (tokenOrEmbeddingContainer != null) { AbstractToken origToken = token; - assignToken(tokenOrEmbeddingContainer); + token = LexerUtilsConstants.token(tokenOrEmbeddingContainer); + // If origToken == null then the right offset might already be pre-computed from move() if (tokenOffset != -1) { - // If the token list is continuous or the fetched token - // is flyweight (there cannot be a gap before flyweight token) - // the original offset can be just increased - // by the original token's length. - if (tokenList.isContinuous() || token.isFlyweight()) { - tokenOffset += origToken.length(); // advance by previous token's length - } else // Offset must be recomputed - tokenOffset = -1; // mark the offset to be recomputed + if (origToken != null) { + // If the token list is continuous or the fetched token + // is flyweight (there cannot be a gap before flyweight token) + // the original offset can be just increased + // by the original token's length. + if (tokenList.isContinuous() || token.isFlyweight()) { + tokenOffset += origToken.length(); // advance by previous token's length + } else // Offset must be recomputed + tokenOffset = -1; // mark the offset to be recomputed + } else // Not valid token previously + tokenOffset = -1; } return true; } - tokenIndex--; + if (token != null) // Unsuccessful move from existing token + tokenIndex--; return false; } /** - * Move to the previous token in this token sequence. + * Move to a previous token in this token sequence. + *
+ * This method can be used in case when TS is positioned after a token + * (after its creation or after use of {@link #move(int)} or {@link #moveIndex(int)}) + * or in case the TS already points to a concrete token{@link #token()} != null
. + *
+ * In both cases {@link #index()} gets decreased by one. *
- * The next token may not necessarily end at the offset where - * the present token starts (there may be gaps between tokens - * caused by use of a token id filter). * * @return true if the sequence was successfully moved to the previous token - * or false if stayed on the original token because there are no more tokens + * or false if it was not moved because there are no more tokens * in the backward direction. * @throws ConcurrentModificationException if this token sequence * is no longer valid because of an underlying mutable input source modification. @@ -353,7 +377,7 @@ if (tokenIndex > 0) { AbstractToken origToken = token; tokenIndex--; - assignToken(); + token = LexerUtilsConstants.token(tokenList.tokenOrEmbeddingContainer(tokenIndex)); if (tokenOffset != -1) { // If the token list is continuous or the original token // is flyweight (there cannot be a gap before flyweight token) @@ -364,8 +388,6 @@ } else { // mark the offset to be computed upon call to offset() tokenOffset = -1; } - } else { - tokenOffset = -1; // mark the offset to be computed upon call to offset() } return true; @@ -374,92 +396,95 @@ } /** - * Move the token sequence to point to the token with the given index. + * Move TS to be positioned before a token with the given index. + *
+ * Example: Fetching tokens starting at the given index: + *+ * TokenSequence ts = tokenHierarchy.tokenSequence(); + * ts.moveIndex(index); + * while (ts.moveNext()) { + * Token t = ts.token(); + * ... + * } + ** * @param index index of the token to which this sequence * should be positioned. - * @returntrue
if the sequence was moved to the token - * with the given index. Returnsfalse
- * ifindex < 0
orindex < tokenCount
. - * In such case the current token sequence's position stays unchanged. + *
+ * Ifindex >= {@link #tokenCount()}
+ * then the TS will be positioned after the last token. + *
+ * Ifindex < 0
+ * then the TS will be positioned before first token (at index 0). + * @return difference between requested index and the index to which TS + * is really set. * @throws ConcurrentModificationException if this token sequence * is no longer valid because of an underlying mutable input source modification. */ - public boolean moveIndex(int index) { + public int moveIndex(int index) { checkModCount(); - if (index < 0) { - return false; - } - Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index); - if (tokenOrEmbeddingContainer != null) { // enough tokens - this.tokenIndex = index; - assignToken(tokenOrEmbeddingContainer); - tokenOffset = -1; - return true; - - } else // Token at the requested index does not exist - leave orig. index - return false; + if (index >= 0) { + Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index); + if (tokenOrEmbeddingContainer != null) { // enough tokens + resetTokenIndex(index); + } else // Token at the requested index does not exist - leave orig. index + resetTokenIndex(tokenCount()); + } else // index < 0 + resetTokenIndex(0); + return index - tokenIndex; + } + + /** + * Move the token sequence to be positioned before the first token. + *
+ * This is equivalent tomoveIndex(0)
. + */ + public void moveStart() { + moveIndex(0); } /** - * Move the token sequence to be positioned to the token + * Move the token sequence to be positioned after the last token. + *
+ * This is equivalent tomoveIndex(tokenCount())
. + */ + public void moveEnd() { + moveIndex(tokenCount()); + } + + /** + * Move the token sequence to be positioned before the token * that "contains" the requested offset (the offset is at the begining * or inside of the token). - *
- * If the offset is too big the token sequence will be positioned - * to the last token and the return value will - * be the distance between the requested offset - * and the start offset of the token to which the token sequence - * will be positioned.. - *
- * If there are no tokens in the sequence then {@link Integer#MAX_VALUE} - * will be returned. - * + *
+ * Typical usage pattern: + *+ * TokenSequence ts = ... + * ts.move(offset); + * while (ts.moveNext()) { + * Token t = ts.token(); + * ... + * } + *+ * + * If the offset is too big then the token sequence will be positioned + * behind the last token. *- * The underlying token list may contain gaps that are not covered + * If token filtering is used there may be gaps that are not covered * by any tokens and if the offset is contained in such gap then - * the token sequence will be positioned to the token that precedes the gap. + * the token sequence will be positioned before the token that follows the gap. *
* - * Example: - *- * int diff = tokenSequence.move(targetOffset); - * // diff equals to (targetOffset - tokenSequence.token().offset()) - * if (diff >= 0 && diff < tokenSequence.token().length()) { - * // Within token bounds - tokenSequence.token() starts at or "contains" targetOffset - * - * } else if (diff == Integer.MAX_VALUE) { - * // No tokens in the token sequence at all. - * // Token sequence is not positioned to any token. - * - * } else { - * // 1. diff >= tokenSequence.token().length() - * // a) targetOffset is above the end of the last token in the sequence. - * // Token sequence is positioned to the last token in the sequence. - * // b) there are text areas not covered by any tokens - * // due to skipped tokens (skipTokenIds was used - * // in TokenHierarchy.create()) and the targetOffset points to such gap. - * // Token sequence is positioned to the preceding token. - * // - * // 2. diff < 0 - * // a) targetOffset < 0 - * // b) targetOffset >= 0 but there is a text area - * // at the begining that is not covered by any tokens - * // (skipTokenIds was used in TokenHierarchy.create()) - * // Token sequence is positioned to the first token in the sequence. - * } - *- * * * @param offset absolute offset in the input to which * the token sequence should be moved. * * @return difference between the reqeuested offset - * and the absolute starting offset of the token - * to which the the token sequence gets moved. - *
- * Returns {@link Integer#MAX_VALUE} if there are no tokens in the sequence. - * In such case there is no active token. + * and the start offset of the token + * before which the the token sequence gets positioned. + *
+ * If the TS is positioned after the last token the difference is computed + * against the end offset of the last token. * * @throws ConcurrentModificationException if this token sequence * is no longer valid because of an underlying mutable input source modification. @@ -474,7 +499,8 @@ if (tokenList.tokenOrEmbeddingContainer(0) == null) { // really no tokens at all // In this case the token sequence could not be positioned yet // so no need to reset "index" or other vars - return Integer.MAX_VALUE; + resetTokenIndex(0); + return offset; } // Re-get the present token count (could be created a chunk of tokens at once) tokenCount = tokenList.tokenCountCurrent(); @@ -499,14 +525,14 @@ tokenLength = t.length(); tokenCount++; - } else { // no more tokens => break - break; + } else { // no more tokens => position after last token + resetTokenIndex(tokenCount); + tokenOffset = prevTokenOffset + tokenLength; // May assign the token's offset in advance + return offset - tokenOffset; } } - tokenIndex = tokenCount - 1; - // Absolute token's start offset - tokenOffset = prevTokenOffset; - assignToken(); + resetTokenIndex(tokenCount - 1); + tokenOffset = prevTokenOffset; // May assign the token's offset in advance return offset - prevTokenOffset; } @@ -525,50 +551,33 @@ high = mid - 1; } else { // Token starting exactly at offset found - tokenIndex = mid; + resetTokenIndex(mid); tokenOffset = midStartOffset; - assignToken(); return 0; // right at the token begining } } // Not found exactly and high + 1 == low => high < low - // Check whether the token at "high" contains the offset + // BTW there may be gaps between tokens; if offset is in gap then position to higher token if (high >= 0) { // could be -1 AbstractToken t = LexerUtilsConstants.token(tokenList, high); prevTokenOffset = tokenList.tokenOffset(high); + // If gaps allowed check whether the token at "high" contains the offset + if (!tokenList.isContinuous() && offset > prevTokenOffset + t.length()) { + // Offset in the gap above the "high" token + high++; + prevTokenOffset += t.length(); + } } else { // at least one token exists => use token at index 0 high = 0; prevTokenOffset = tokenList.tokenOffset(0); // result may differ from 0 } - - tokenIndex = high; + resetTokenIndex(high); tokenOffset = prevTokenOffset; - assignToken(); return offset - prevTokenOffset; } /** - * Move to the first token in this token sequence. - * - * @return true if the sequence was positioned on the first token - * or false if there are no tokens in the sequence. - */ - public boolean moveFirst() { - return moveIndex(0); - } - - /** - * Move to the last token in this token sequence. - * - * @return true if the sequence was positioned on the last token - * or false if there are no tokens in the sequence. - */ - public boolean moveLast() { - return moveIndex(tokenCount() - 1); // Can be -1 but handled in move(index) - } - - /** * Return total count of tokens in this sequence. *
* Note: Calling this method will lead @@ -628,15 +637,14 @@ return parentTokenIndexes; } - private void assignToken(Object tokenOrEmbeddingContainer) { - token = LexerUtilsConstants.token(tokenOrEmbeddingContainer); - } - - private void assignToken() { - assignToken(tokenList.tokenOrEmbeddingContainer(tokenIndex)); + private void resetTokenIndex(int index) { + // Position to the given index e.g. by move() and moveIndex() + tokenIndex = index; + token = null; + tokenOffset = -1; } - private void checkToken() { + private void checkTokenNotNull() { if (token == null) { throw new IllegalStateException( "No token fetched by moveNext() from token sequence yet: index=" + tokenIndex Index: lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java =================================================================== RCS file: /cvs/lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java,v retrieving revision 1.5 diff -u -r1.5 LexerUtilsConstants.java --- lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java 4 Dec 2006 15:44:00 -0000 1.5 +++ lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java 17 Jan 2007 16:49:12 -0000 @@ -237,6 +237,16 @@ sb.append(": "); AbstractToken token = token(tokenOrEmbeddingContainer); sb.append(token.dumpInfo(tokenHierarchy)); + int la = tokenList.lookahead(i); + if (la != 0) { + sb.append(", la="); + sb.append(la); + } + Object state= tokenList.state(i); + if (state != null) { + sb.append(", s="); + sb.append(state); + } sb.append('\n'); } return sb; Index: lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java =================================================================== RCS file: /cvs/lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java,v retrieving revision 1.6 diff -u -r1.6 SubSequenceTokenList.java --- lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java 4 Dec 2006 15:44:00 -0000 1.6 +++ lexer/src/org/netbeans/lib/lexer/SubSequenceTokenList.java 17 Jan 2007 16:49:13 -0000 @@ -92,7 +92,7 @@ if (limitStartOffset > 0) { int diff = move(limitStartOffset); if (diff != Integer.MAX_VALUE) { // some tokens exist - if (diff >= lastToken.length()) { + if (diff >= lastToken.length()) { // lastToken initialized in move() lastTokenIndex++; Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(lastTokenIndex); if (tokenOrEmbeddingContainer != null && Index: lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java =================================================================== RCS file: /cvs/lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java,v retrieving revision 1.5 diff -u -r1.5 TokenSequenceTest.java --- lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java 4 Dec 2006 15:44:04 -0000 1.5 +++ lexer/test/unit/src/org/netbeans/api/lexer/TokenSequenceTest.java 17 Jan 2007 16:49:13 -0000 @@ -48,13 +48,7 @@ TokenHierarchy> hi = TokenHierarchy.create(text, SimpleTokenId.language()); TokenSequence extends TokenId> ts = hi.tokenSequence(); - // Fail if no "move*" method called yet - try { - ts.token(); // should throw exception - fail("IllegalStateException expecxted"); - } catch (IllegalStateException e) { - // expected - } + assertNull(ts.token()); assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0); @@ -69,20 +63,38 @@ assertEquals(ts.tokenCount(), 3); assertEquals(0, ts.move(0)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0); assertEquals(2, ts.move(2)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0); assertEquals(-1, ts.move(-1)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0); assertEquals(0, ts.move(3)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3); assertEquals(0, ts.move(4)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4); assertEquals(1, ts.move(5)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4); - assertEquals(5, ts.move(9)); + assertEquals(1, ts.move(9)); + assertNull(ts.token()); + assertFalse(ts.moveNext()); + assertTrue(ts.movePrevious()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4); - assertEquals(96, ts.move(100)); + assertEquals(92, ts.move(100)); + assertNull(ts.token()); + assertFalse(ts.moveNext()); + assertTrue(ts.movePrevious()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4); @@ -122,12 +134,7 @@ TokenSequence extends TokenId> ts = hi.tokenSequence(); // Fail if no "move*" method called yet - try { - ts.token(); // should throw exception - fail("IllegalStateException expecxted"); - } catch (IllegalStateException e) { - // expected - } + assertNull(ts.token()); assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); @@ -137,10 +144,12 @@ LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11); assertFalse(ts.moveNext()); assertEquals(3, ts.tokenCount()); - assertTrue(ts.moveFirst()); + assertEquals(0, ts.moveIndex(0)); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); assertTrue(ts.moveNext()); - assertTrue(ts.moveLast()); + assertEquals(0, ts.moveIndex(ts.tokenCount())); + assertTrue(ts.movePrevious()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11); assertFalse(ts.moveNext()); @@ -151,16 +160,28 @@ LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); assertEquals(-1, ts.move(0)); // below filtered-out token + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); assertEquals(0, ts.move(1)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); assertEquals(1, ts.move(2)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); assertEquals(3, ts.move(4)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1); assertEquals(0, ts.move(5)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 5); assertEquals(1, ts.move(12)); + assertNull(ts.token()); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11); @@ -172,48 +193,56 @@ LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5); assertFalse(sub.moveNext()); - // Test moveFirst() and moveLast() on subsequence + // Test moves to first and last tokens on subsequence sub = ts.subSequence(1, 6); - assertTrue(sub.moveLast()); + assertEquals(0, sub.moveIndex(sub.tokenCount())); + assertEquals(2, sub.tokenCount()); // "abc" and "defg" (others filtered out + assertTrue(sub.movePrevious()); LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5); - assertTrue(sub.moveFirst()); + assertEquals(0, sub.moveIndex(0)); + assertTrue(sub.moveNext()); LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "abc", 1); sub = ts.subSequence(4); assertTrue(sub.moveNext()); LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5); assertTrue(sub.moveNext()); - LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11); + LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11); assertFalse(sub.moveNext()); sub = ts.subSequence(12, 15); assertTrue(sub.moveNext()); - LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11); + LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11); sub = ts.subSequence(12, 15); assertEquals(-2, sub.move(9)); - LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11); + assertNull(sub.token()); + assertTrue(sub.moveNext()); + LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11); sub = ts.subSequence(13, 15); assertFalse(sub.moveNext()); - assertFalse(sub.moveFirst()); - assertFalse(sub.moveLast()); + assertEquals(0, sub.moveIndex(0)); + assertNull(sub.token()); + assertFalse(sub.moveNext()); + assertEquals(0, sub.moveIndex(sub.tokenCount())); + assertNull(sub.token()); assertEquals(0, sub.tokenCount()); sub = ts.subSequence(-3, 1); assertFalse(sub.moveNext()); sub = ts.subSequence(13, 15); - assertEquals(Integer.MAX_VALUE, sub.move(5)); + assertEquals(5, sub.move(5)); sub = ts.subSequence(-3, 1); - assertEquals(Integer.MAX_VALUE, sub.move(1)); + assertEquals(1, sub.move(1)); // Reversed bounds => should be empty token sequence sub = ts.subSequence(6, 1); assertFalse(sub.moveNext()); sub = ts.subSequence(6, 1); - assertEquals(Integer.MAX_VALUE, sub.move(6)); + assertEquals(6, sub.move(6)); } public void DtestMoveSkipTokens2() throws IOException { @@ -244,32 +273,26 @@ // Expect no tokens assertFalse(ts.moveNext()); - // Exception if ts.token() called - try { - ts.token(); // should throw exception - fail("IllegalStateException expecxted"); - } catch (IllegalStateException e) { - // expected - } + assertNull(ts.token()); - assertEquals(ts.move(0), Integer.MAX_VALUE); - assertEquals(ts.move(10), Integer.MAX_VALUE); + assertEquals(0, ts.move(0)); + assertEquals(10, ts.move(10)); // Test subsequences TokenSequence extends TokenId> sub = ts.subSequence(1, 6); assertFalse(sub.moveNext()); sub = ts.subSequence(1, 6); - assertEquals(Integer.MAX_VALUE, sub.move(1)); + assertEquals(1, sub.move(1)); sub = ts.subSequence(0); assertFalse(sub.moveNext()); sub = ts.subSequence(0); - assertEquals(Integer.MAX_VALUE, sub.move(0)); + assertEquals(0, sub.move(0)); sub = ts.subSequence(1); assertFalse(sub.moveNext()); sub = ts.subSequence(1); - assertEquals(Integer.MAX_VALUE, sub.move(0)); + assertEquals(0, sub.move(0)); } public void testTokenSize() { @@ -284,7 +307,8 @@ assertFalse(ts.moveNext()); TokenList tokenList = LexerTestUtilities.tokenList(ts); - ts.moveIndex(0); // move to "abc" + ts.moveIndex(0); // move before "abc" + assertTrue(ts.moveNext()); // Test DefaultToken size assertSame(DefaultToken.class, ts.token().getClass()); assertSize("Token instance too big", Collections.singletonList(ts.token()), 24, Index: lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java =================================================================== RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java,v retrieving revision 1.5 diff -u -r1.5 LanguageManagerTest.java --- lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java 26 Oct 2006 20:45:25 -0000 1.5 +++ lexer/test/unit/src/org/netbeans/lib/lexer/LanguageManagerTest.java 17 Jan 2007 16:49:13 -0000 @@ -130,7 +130,8 @@ public void testCachingE() { TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language()); TokenSequence tokens = th.tokenSequence(); - tokens.moveFirst(); + tokens.moveStart(); + assertEquals(true, tokens.moveNext()); TokenSequence embeddedA = tokens.embedded(); assertNotNull("There should be an embedded language", embeddedA); @@ -144,7 +145,8 @@ public void testGCedE() { TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language()); TokenSequence tokens = th.tokenSequence(); - tokens.moveFirst(); + tokens.moveStart(); + assertEquals(true, tokens.moveNext()); TokenSequence embedded = tokens.embedded(); assertNotNull("There should be an embedded language", embedded); @@ -164,7 +166,8 @@ public void testCacheRefreshedE() { TokenHierarchy th = TokenHierarchy.create("abc", SimplePlainTokenId.language()); TokenSequence tokens = th.tokenSequence(); - tokens.moveFirst(); + tokens.moveStart(); + assertEquals(true, tokens.moveNext()); TokenSequence embeddedA = tokens.embedded(); assertNotNull("There should be an embedded language", embeddedA); Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java =================================================================== RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java,v retrieving revision 1.5 diff -u -r1.5 LexerTestUtilities.java --- lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java 4 Dec 2006 15:44:05 -0000 1.5 +++ lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java 17 Jan 2007 16:49:14 -0000 @@ -70,6 +70,7 @@ public static void assertTokenEquals(String message, TokenSequence extends TokenId> ts, TokenId id, String text, int offset) { message = messagePrefix(message); Token extends TokenId> t = ts.token(); + TestCase.assertNotNull("Token is null", t); TokenId tId = t.id(); TestCase.assertEquals(message + "Invalid token.id()", id, tId); CharSequence tText = t.text(); @@ -312,17 +313,21 @@ } TokenHierarchy> thBatch = TokenHierarchy.create(docText, language); boolean success = false; + TokenSequence> batchTS = thBatch.tokenSequence(); try { // Compare lookaheads and states as well - assertTokenSequencesEqual(thBatch.tokenSequence(), thBatch, + assertTokenSequencesEqual(batchTS, thBatch, thInc.tokenSequence(), thInc, true); success = true; } finally { if (!success) { + // Go forward two tokens to have an extra tokens context + batchTS.moveNext(); + batchTS.moveNext(); System.err.println("BATCH token sequence dump:\n" + thBatch.tokenSequence()); TokenHierarchy> lastHi = (TokenHierarchy>)doc.getProperty(LAST_TOKEN_HIERARCHY); if (lastHi != null) { - System.err.println("PREVIOUS token sequence dump:\n" + lastHi.tokenSequence()); + System.err.println("PREVIOUS batch token sequence dump:\n" + lastHi.tokenSequence()); } } } Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java =================================================================== RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java,v retrieving revision 1.5 diff -u -r1.5 TokenHierarchySnapshotTest.java --- lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java 4 Dec 2006 15:44:06 -0000 1.5 +++ lexer/test/unit/src/org/netbeans/lib/lexer/test/inc/TokenHierarchySnapshotTest.java 17 Jan 2007 16:49:14 -0000 @@ -76,7 +76,8 @@ // Check that all the non-fly tokens are mutable ts = snapshot1.tokenSequence(); - assertTrue(ts.moveIndex(0)); + assertEquals(0, ts.moveIndex(0)); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenSequencesEqual(hi1.tokenSequence(), hi1, snapshot1.tokenSequence(), snapshot1, false); @@ -84,7 +85,8 @@ doc.insertString(4, "+", null); // Check that the snapshot token sequence can be further operated. - assertTrue(ts.moveIndex(0)); + assertEquals(0, ts.moveIndex(0)); + assertTrue(ts.moveNext()); assertNotNull(ts.token()); // Check that the tokens except '+' are live @@ -101,7 +103,8 @@ // Check that all the non-fly tokens are mutable ts = snapshot2.tokenSequence(); - assertTrue(ts.moveIndex(0)); + assertEquals(0, ts.moveIndex(0)); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenSequencesEqual(hi2.tokenSequence(), hi2, snapshot2.tokenSequence(), snapshot2, false); Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java =================================================================== RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java,v retrieving revision 1.6 diff -u -r1.6 SimpleLexerBatchTest.java --- lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java 17 Jan 2007 13:15:29 -0000 1.6 +++ lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerBatchTest.java 17 Jan 2007 16:49:14 -0000 @@ -81,7 +81,8 @@ assertFalse(ts.moveNext()); // Go back to block comment - assertTrue(ts.moveIndex(commentIndex)); + assertEquals(0, ts.moveIndex(commentIndex)); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.BLOCK_COMMENT, commentText, commentTextStartOffset); // Test embedded token sequence Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java =================================================================== RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java,v retrieving revision 1.6 diff -u -r1.6 SimpleLexerIncTest.java --- lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java 4 Dec 2006 15:44:07 -0000 1.6 +++ lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexerIncTest.java 17 Jan 2007 16:49:16 -0000 @@ -90,19 +90,23 @@ // Check TokenSequence.move() int relOffset = ts.move(50); // past the end of all tokens - assertEquals(relOffset, 50 - offset); + assertEquals(relOffset, 50 - (offset + 3)); + assertTrue(ts.movePrevious()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset); relOffset = ts.move(6); // right at begining of "-" assertEquals(relOffset, 0); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", 6); relOffset = ts.move(-5); // to first token "abc" assertEquals(relOffset, -5); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0); relOffset = ts.move(5); // to first token "abc" assertEquals(relOffset, 1); + assertTrue(ts.moveNext()); LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "uv", 4); Index: openide/text/src/org/openide/text/CloneableEditorSupport.java =================================================================== RCS file: /cvs/openide/text/src/org/openide/text/CloneableEditorSupport.java,v retrieving revision 1.28 diff -u -r1.28 CloneableEditorSupport.java --- openide/text/src/org/openide/text/CloneableEditorSupport.java 12 Jan 2007 10:16:57 -0000 1.28 +++ openide/text/src/org/openide/text/CloneableEditorSupport.java 17 Jan 2007 16:50:39 -0000 @@ -1,3 +1,4 @@ + /* * The contents of this file are subject to the terms of the Common Development * and Distribution License (the License). You may not use this file except in Index: web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java =================================================================== RCS file: /cvs/web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java,v retrieving revision 1.1 diff -u -r1.1 SimplifiedJSPServlet.java --- web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java 4 Jan 2007 15:13:17 -0000 1.1 +++ web/jspsyntax/src/org/netbeans/modules/web/core/syntax/SimplifiedJSPServlet.java 17 Jan 2007 16:53:01 -0000 @@ -97,7 +97,7 @@ TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc); TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); //get top level token sequence - if(!tokenSequence.moveFirst()) { + if(!tokenSequence.moveNext()) { return ; //no tokens in token sequence } Index: xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java =================================================================== RCS file: /cvs/xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java,v retrieving revision 1.3 diff -u -r1.3 TagBasedLexerFormatter.java --- xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java 13 Jan 2007 14:55:58 -0000 1.3 +++ xml/tageditorsupport/src/org/netbeans/modules/editor/structure/formatting/TagBasedLexerFormatter.java 17 Jan 2007 16:53:58 -0000 @@ -86,17 +86,18 @@ * e.g.