Attachment #39153 for bug #91184

This Bugzilla instance is a read-only archive of historic NetBeans bug reports. To report a bug in NetBeans please follow the project's instructions for reporting issues.

View | Details | Raw Unified | Return to bug 91184
Collapse All | Expand All

Lines 129-135 Link Here

(-)java/editor/nbproject/project.xml (-2 / +2 lines)
129	<compile-dependency/>	129	<compile-dependency/>
130	<run-dependency>	130	<run-dependency>
131	<release-version>1</release-version>	131	<release-version>1</release-version>
132	<specification-version>1.0</specification-version>	132	<specification-version>1.2</specification-version>
133	</run-dependency>	133	</run-dependency>
134	</dependency>	134	</dependency>
135	<dependency>	135	<dependency>
Lines 146-152 Link Here
146	<compile-dependency/>	146	<compile-dependency/>
147	<run-dependency>	147	<run-dependency>
148	<release-version>2</release-version>	148	<release-version>2</release-version>
149	<specification-version>1.4</specification-version>	149	<specification-version>1.14</specification-version>
150	</run-dependency>	150	</run-dependency>
151	</dependency>	151	</dependency>
152	<dependency>	152	<dependency>




                if (ts.token().text().charAt(0) == '.')
                    break;
            case CHAR_LITERAL:
            case CHAR_LITERAL_INCOMPLETE:
            case FLOAT_LITERAL:
            case FLOAT_LITERAL_INVALID:
            case INT_LITERAL:
            case INVALID_COMMENT_END:
            case JAVADOC_COMMENT:
            case JAVADOC_COMMENT_INCOMPLETE:
            case LONG_LITERAL:
            case STRING_LITERAL:
            case STRING_LITERAL_INCOMPLETE:
            case LINE_COMMENT:
            case BLOCK_COMMENT:
            case BLOCK_COMMENT_INCOMPLETE:
                return false;
        }
        return true;

Lines 1-5 Link Here

(-)java/lexer/manifest.mf (-1 / +1 lines)
1	OpenIDE-Module: org.netbeans.modules.java.lexer/1	1	OpenIDE-Module: org.netbeans.modules.java.lexer/1
2	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/java/lexer/Bundle.properties	2	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/java/lexer/Bundle.properties
3	OpenIDE-Module-Specification-Version: 1.1	3	OpenIDE-Module-Specification-Version: 1.2
4	OpenIDE-Module-Layer: org/netbeans/lib/java/lexer/layer.xml	4	OpenIDE-Module-Layer: org/netbeans/lib/java/lexer/layer.xml
5		5

Lines 29-35 Link Here

(-)java/lexer/nbproject/project.xml (-1 / +1 lines)
29	<compile-dependency/>	29	<compile-dependency/>
30	<run-dependency>	30	<run-dependency>
31	<release-version>2</release-version>	31	<release-version>2</release-version>
32	<specification-version>1.12</specification-version>	32	<specification-version>1.14</specification-version>
33	</run-dependency>	33	</run-dependency>
34	</dependency>	34	</dependency>
35	</module-dependencies>	35	</module-dependencies>




    BLOCK_COMMENT(null, "comment"),
    JAVADOC_COMMENT(null, "comment"),
    
    // Errors
    CHAR_LITERAL_INCOMPLETE(null, "character"),
    STRING_LITERAL_INCOMPLETE(null, "string"),
    BLOCK_COMMENT_INCOMPLETE(null, "comment"),
    JAVADOC_COMMENT_INCOMPLETE(null, "comment"),
    INVALID_COMMENT_END("*/", "error"),
    FLOAT_LITERAL_INVALID(null, "number");


        @Override
        protected Map<String,Collection<JavaTokenId>> createTokenCategories() {
            Map<String,Collection<JavaTokenId>> cats = new HashMap<String,Collection<JavaTokenId>>();
            // Incomplete tokens
            cats.put("incomplete", EnumSet.of(
                JavaTokenId.CHAR_LITERAL_INCOMPLETE,
                JavaTokenId.STRING_LITERAL_INCOMPLETE,
                JavaTokenId.BLOCK_COMMENT_INCOMPLETE,
                JavaTokenId.JAVADOC_COMMENT_INCOMPLETE
            ));
            // Additional literals being a lexical error
            cats.put("error", EnumSet.of(
                JavaTokenId.CHAR_LITERAL_INCOMPLETE,
                JavaTokenId.STRING_LITERAL_INCOMPLETE,
                JavaTokenId.BLOCK_COMMENT_INCOMPLETE,
                JavaTokenId.JAVADOC_COMMENT_INCOMPLETE,
                JavaTokenId.FLOAT_LITERAL_INVALID
            ));
            // Literals category
            EnumSet<JavaTokenId> l = EnumSet.of(
                JavaTokenId.INT_LITERAL,
                JavaTokenId.LONG_LITERAL,
                JavaTokenId.FLOAT_LITERAL,
                JavaTokenId.DOUBLE_LITERAL,
                JavaTokenId.CHAR_LITERAL

            );
            l.add(JavaTokenId.STRING_LITERAL);
                JavaTokenId.CHAR_LITERAL_INCOMPLETE,
                JavaTokenId.STRING_LITERAL,
                JavaTokenId.STRING_LITERAL_INCOMPLETE
            ));
            cats.put("literal", l);

            return cats;

                case JAVADOC_COMMENT:
                    return LanguageEmbedding.create(JavadocTokenId.language(), 3, 2);
                case STRING_LITERAL:
                case STRING_LITERAL_INCOMPLETE:
                    return LanguageEmbedding.create(JavaStringTokenId.language(), 1, 1);
            }
            return null; // No embedding




package org.netbeans.lib.java.lexer;

import org.netbeans.api.java.lexer.JavaTokenId;
import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.spi.lexer.Lexer;
import org.netbeans.spi.lexer.LexerInput;

                            case '\r': input.consumeNewline();
                            case '\n':
                            case EOF:
                                return tokenFactory.createToken(JavaTokenId.STRING_LITERAL,
                                        input.readLength(), PartType.START);
                        }

                case '\'': // char literal

                            case '\r': input.consumeNewline();
                            case '\n':
                            case EOF:
                                return tokenFactory.createToken(JavaTokenId.CHAR_LITERAL,
                                        input.readLength(), PartType.START);
                        }

                case '/':

                                            if (c == '/')
                                                return token(JavaTokenId.JAVADOC_COMMENT);
                                            else if (c == EOF)
                                                return tokenFactory.createToken(JavaTokenId.JAVADOC_COMMENT,
                                                        input.readLength(), PartType.START);
                                        }
                                        if (c == EOF)
                                            return tokenFactory.createToken(JavaTokenId.JAVADOC_COMMENT,
                                                        input.readLength(), PartType.START);
                                        c = input.read();
                                    }


                                        if (c == '/')
                                            return token(JavaTokenId.BLOCK_COMMENT);
                                        else if (c == EOF)
                                            return tokenFactory.createToken(JavaTokenId.BLOCK_COMMENT,
                                                    input.readLength(), PartType.START);
                                    }
                                    if (c == EOF)
                                        return tokenFactory.createToken(JavaTokenId.BLOCK_COMMENT,
                                                input.readLength(), PartType.START);
                                }
                            }
                    } // end of switch()




BLOCK_COMMENT   "/**/"
JAVADOC_COMMENT  "/***/"
JAVADOC_COMMENT  "/**\n*javadoc-comment*/"
BLOCK_COMMENT   "/* a", la=1
----- EOF -----

.t.e.s.t. Identifiers

WHITESPACE      " ", la=1
CHAR_LITERAL    "'\\n'"
WHITESPACE      " ", la=1
CHAR_LITERAL    "'a", la=1
----- EOF -----

.t.e.s.t. String Literals

WHITESPACE      "\n", la=1
STRING_LITERAL  ""\\\\\\\\\\\\\\"\\"""
WHITESPACE      "\n", la=1
STRING_LITERAL  ""\\n\\" \\"a", la=1
----- EOF -----

.t.e.s.t. Number Literals

Lines 56-62 Link Here

(-)java/lexer/test/unit/src/org/netbeans/lib/java/lexer/JavaLanguageTest.java (-1 / +1 lines)
56	Set testCats = language.tokenCategories();	56	Set testCats = language.tokenCategories();
57	Collection cats = Arrays.asList(new String[] {	57	Collection cats = Arrays.asList(new String[] {
58	"error", "identifier", "operator", "separator", "whitespace", "error", "comment",	58	"error", "identifier", "operator", "separator", "whitespace", "error", "comment",
59	"keyword", "string", "character", "number", "literal", "incomplete",	59	"keyword", "string", "character", "number", "literal",
60	});	60	});
61	LexerTestUtilities.assertCollectionsEqual("Invalid categories", cats, testCats);	61	LexerTestUtilities.assertCollectionsEqual("Invalid categories", cats, testCats);
62		62




import org.netbeans.api.java.lexer.JavaStringTokenId;
import org.netbeans.api.java.lexer.JavaTokenId;
import org.netbeans.api.java.lexer.JavadocTokenId;
import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.api.lexer.TokenSequence;

        LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.BLOCK_COMMENT, "/**/");
        LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.JAVADOC_COMMENT, "/***/");
        LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.JAVADOC_COMMENT, "/**\n*javadoc-comment*/");
        LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.BLOCK_COMMENT, "/* a");
        assertEquals(PartType.START, ts.token().partType());
    }
    
    public void testIdentifiers() {

        LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
        LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL, "'\\n'");
        LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
        LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL, "'a");
        assertEquals(PartType.START, ts.token().partType());
    }
    
    public void testStringLiterals() {

        LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
        LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"\\n\"");
        LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
        LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"a");
        assertEquals(PartType.START, ts.token().partType());
    }
    
    public void testNumberLiterals() {

Lines 42-48 Link Here

(-)java/source/test/unit/src/org/netbeans/api/java/source/gen/WhitespaceIgnoringDiff.java (-1 / +1 lines)
42	public WhitespaceIgnoringDiff() {	42	public WhitespaceIgnoringDiff() {
43	}	43	}
44		44
45	private Set<JavaTokenId> IGNORED_TOKENS = EnumSet.of(JavaTokenId.WHITESPACE, JavaTokenId.LINE_COMMENT, JavaTokenId.BLOCK_COMMENT, JavaTokenId.BLOCK_COMMENT_INCOMPLETE, JavaTokenId.JAVADOC_COMMENT, JavaTokenId.JAVADOC_COMMENT_INCOMPLETE);	45	private Set<JavaTokenId> IGNORED_TOKENS = EnumSet.of(JavaTokenId.WHITESPACE, JavaTokenId.LINE_COMMENT, JavaTokenId.BLOCK_COMMENT, JavaTokenId.JAVADOC_COMMENT);
46		46
47	public boolean diff(File first, File second, File diff) throws IOException {	47	public boolean diff(File first, File second, File diff) throws IOException {
48	boolean result = diffImpl(first, second);	48	boolean result = diffImpl(first, second);

Lines 54-60 Link Here

(-)javadoc/nbproject/project.xml (-2 / +2 lines)
54	<compile-dependency/>	54	<compile-dependency/>
55	<run-dependency>	55	<run-dependency>
56	<release-version>1</release-version>	56	<release-version>1</release-version>
57	<specification-version>1.1</specification-version>	57	<specification-version>1.2</specification-version>
58	</run-dependency>	58	</run-dependency>
59	</dependency>	59	</dependency>
60	<dependency>	60	<dependency>
Lines 80-86 Link Here
80	<compile-dependency/>	80	<compile-dependency/>
81	<run-dependency>	81	<run-dependency>
82	<release-version>2</release-version>	82	<release-version>2</release-version>
83	<specification-version>1.13</specification-version>	83	<specification-version>1.14</specification-version>
84	</run-dependency>	84	</run-dependency>
85	</dependency>	85	</dependency>
86	<dependency>	86	<dependency>

Lines 578-584 Link Here

(-)javadoc/src/org/netbeans/modules/javadoc/hints/JavadocUtilities.java (-1 lines)
578	IGNORE_TOKES = new HashSet<TokenId>();	578	IGNORE_TOKES = new HashSet<TokenId>();
579	IGNORE_TOKES.add(JavaTokenId.WHITESPACE);	579	IGNORE_TOKES.add(JavaTokenId.WHITESPACE);
580	IGNORE_TOKES.add(JavaTokenId.BLOCK_COMMENT);	580	IGNORE_TOKES.add(JavaTokenId.BLOCK_COMMENT);
581	IGNORE_TOKES.add(JavaTokenId.BLOCK_COMMENT_INCOMPLETE);
582	}	581	}
583		582
584	}	583	}




import java.util.Map;
import org.netbeans.api.languages.CharInput;
import org.netbeans.api.languages.ASTToken;
import org.netbeans.api.lexer.PartType;
import org.netbeans.api.languages.LanguagesManager;
import org.netbeans.api.languages.ParseException;
import org.netbeans.api.lexer.Token;

                tokenId,
                v.endOffset - v.startOffset,
                (TokenProperties) v.property,
                PartType.COMPLETE
            );
        else
            return tokenFactory.createPropertyToken (
                tokenId,
                v.endOffset - v.startOffset,
                new TokenPropProvider(v.property),
                PartType.COMPLETE
            );
    }
        

    
    // innerclasses ............................................................
    
    private static final class TokenPropProvider implements TokenPropertyProvider {
        
        private final Object value;
        
        TokenPropProvider(Object value) {
            this.value = value;
        }
        
        public Object getValue (Token token, Object key) {
            if ("type".equals(key))
                return value;
            return null;
        }

    }
            return "type";
        }
    };
    
    static class TokenProperties implements TokenPropertyProvider {
        

            return null;
        }

        public Object getValue (Token token, Object tokenStoreKey, Object tokenStoreValue) {
            return null;
        }

        public Object tokenStoreKey() {
            return null;
        }
    };
    
    static class Vojta {

Lines 1-4 Link Here

(-)lexer/manifest.mf (-1 / +1 lines)
1	OpenIDE-Module: org.netbeans.modules.lexer/2	1	OpenIDE-Module: org.netbeans.modules.lexer/2
2	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/lexer/Bundle.properties	2	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/lexer/Bundle.properties
3	OpenIDE-Module-Specification-Version: 1.13.0	3	OpenIDE-Module-Specification-Version: 1.14.0
4	OpenIDE-Module-Recommends: org.netbeans.spi.lexer.LanguageProvider	4	OpenIDE-Module-Recommends: org.netbeans.spi.lexer.LanguageProvider




  <changes>
  <change id="api.tokensequence.move.semantics">
      <api name="api"/>
      <summary>Added PartInfo for token parts support</summary>
      <version major="1" minor="14"/>
      <date day="1" month="3" year="2007"/>
      <author login="mmetelka"/>
      <compatibility source="incompatible" deletion="yes" addition="yes" modification="yes"/>
      <description>
        <p>
            Added <code>PartInfo</code> enum and <code>Token.partInfo()</code>
            that identifies whether the token is COMPLETE or which part
            of a complete token this part represents (START, INNER or END).
        </p>
      </description>
  </change>

  <change id="api.tokensequence.move.semantics">
      <api name="api"/>
      <summary>Changed TokenSequence.move() and moveIndex() use</summary>
      <version major="1" minor="13"/>
      <date day="16" month="1" year="2007"/>




/*
 * The contents of this file are subject to the terms of the Common Development
 * and Distribution License (the License). You may not use this file except in
 * compliance with the License.
 *
 * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
 * or http://www.netbeans.org/cddl.txt.
 *
 * When distributing Covered Code, include this CDDL Header Notice in each file
 * and include the License file at http://www.netbeans.org/cddl.txt.
 * If applicable, add the following below the CDDL Header, with the fields
 * enclosed by brackets [] replaced by your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 */

package org.netbeans.api.lexer;

/**
 * Whether {@link Token} represents a complete token
 * or just a part of a complete token.
 * <br/>
 * A complete token may consist of one start token, zero or more middle tokens
 * and zero or more end tokens (there may be incomplete token
 * at the end of input e.g. an incomplete block comment so there is just a start
 * part of a token).
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public enum PartType {

    /**
     * A given token represents a complete token.
     */
    COMPLETE,

    /**
     * A given token represents initial part of a complete token.
     */
    START,

    /**
     * A given token represents middle part of a complete token.
     */
    MIDDLE,

    /**
     * A given token represents end part of a complete token.
     */
    END;

}

Lines 194-199 Link Here

(-)lexer/src/org/netbeans/api/lexer/Token.java (+6 lines)
194	* @return true if the token is flyweight or false otherwise.	194	* @return true if the token is flyweight or false otherwise.
195	*/	195	*/
196	public abstract boolean isFlyweight();	196	public abstract boolean isFlyweight();
		197
		198	/**
		199	* Check whether this token represents a complete token
		200	* or whether it's a part of a complete token.
		201	*/
		202	public abstract PartType partType();
197		203
198	/**	204	/**
199	* Check whether this token has preprocessed text	205	* Check whether this token has preprocessed text




import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.GapList;
import org.netbeans.lib.lexer.token.ComplexToken;
import org.netbeans.lib.lexer.token.PreprocessedTextToken;
import org.netbeans.spi.lexer.CharPreprocessor;
import org.netbeans.spi.lexer.Lexer;
import org.netbeans.spi.lexer.LexerInput;
import org.netbeans.lib.lexer.token.AbstractToken;
import org.netbeans.lib.lexer.token.ComplexToken;
import org.netbeans.lib.lexer.token.PropertyCustomTextToken;
import org.netbeans.spi.lexer.LexerRestartInfo;
import org.netbeans.spi.lexer.TokenFactory;


        PreprocessedTextStorage storage = preprocessorOperation.createPreprocessedTextStorage(
                token.text(), extraPreprocessedChars);
        
        if (token.getClass() == ComplexToken.class) {
            ((ComplexToken)token).initPrep(storage, error);
        } else {
            ((PreprocessedTextToken)token).initPrep(storage, error);
        }




            || (state1 != null && state1.equals(state2));
    }
    
    public static <T extends TokenId> Object getTokenProperty(Token<T> token,
    TokenPropertyProvider propertyProvider, Object key, Object tokenStoreValue) {
        Object tokenStoreKey = propertyProvider.tokenStoreKey();
        if (tokenStoreKey != null && tokenStoreKey.equals(key)) { // token store value
            return propertyProvider.getValue(token, tokenStoreKey, tokenStoreValue);
        } else {
            return propertyProvider.getValue(token, key);
        }
    }
    
    public static String idToString(TokenId id) {
        return id.name() + '[' + id.ordinal() + ']'; // NOI18N;
    }





package org.netbeans.lib.lexer.token;

import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenId;

     *
     * @return non-null identification of this token.
     */
    @Override
    public final T id() {
        return id;
    }

    /**
     * Get text represented by this token.
     */
    @Override
    public CharSequence text() {
        if (tokenList != null) {
            if (tokenList.getClass() == EmbeddedTokenList.class) {

        this.rawOffset = rawOffset;
    }

    @Override
    public final boolean isFlyweight() {
        return (rawOffset == -1);
    }

        setRawOffset(-1);
    }
    
    @Override
    public PartType partType() {
        return PartType.COMPLETE;
    }

    @Override
    public boolean isCustomText() {
        return false;
    }

    @Override
    public final int offset(TokenHierarchy<?> tokenHierarchy) {
        if (rawOffset == -1) { // flyweight token
            return -1;

        }
    }
    
    @Override
    public boolean isPreprocessedText() {
        return false;
    }
    
    @Override
    public CharSequence preprocessedText() {
        return null;
    }
    
    @Override
    public String preprocessError() {
        return null;
    }

    @Override
    public int preprocessErrorIndex() {
        return -1;
    }
    
    @Override
    public boolean hasProperties() {
        return false;
    }
    
    @Override
    public Object getProperty(Object key) {
        return null;
    }




/*
 * The contents of this file are subject to the terms of the Common Development
 * and Distribution License (the License). You may not use this file except in
 * compliance with the License.
 *
 * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
 * or http://www.netbeans.org/cddl.txt.
 *
 * When distributing Covered Code, include this CDDL Header Notice in each file
 * and include the License file at http://www.netbeans.org/cddl.txt.
 * If applicable, add the following below the CDDL Header, with the fields
 * enclosed by brackets [] replaced by your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 */

package org.netbeans.lib.lexer.token;

import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.lib.lexer.PreprocessedTextStorage;
import org.netbeans.spi.lexer.CharPreprocessor;
import org.netbeans.spi.lexer.TokenPropertyProvider;

/**
 * Token that holds information about preprocessed characters
 * and also carries properties.
 *
 * <p>
 * Instances of this token are more costly than other token types
 * because in addition to regular information they store preprocessed
 * text of the token.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class ComplexToken<T extends TokenId> extends PreprocessedTextToken<T> {

    private final TokenPropertyProvider propertyProvider; // 36 bytes (32-super + 4)

    private final CharSequence customText; // 40 bytes
    
    private final PartType partType; // 44 bytes

    public ComplexToken(T id, int length,
    TokenPropertyProvider propertyProvider, CharSequence customText, PartType partType) {
        super(id, length);
        this.propertyProvider = propertyProvider;
        this.customText = customText;
        this.partType = partType;
    }

    @Override
    public boolean hasProperties() {
        return (propertyProvider != null);
    }

    @Override
    public Object getProperty(Object key) {
        return (propertyProvider != null) ? propertyProvider.getValue(this, key) : null;
    }
    
    @Override
    public CharSequence text() {
        return (customText != null) ? customText : super.text();
    }
    
    @Override
    public PartType partType() {
        return partType;
    }

    @Override
    protected String dumpInfoTokenType() {
        return "PPrT"; // NOI18N "PrepToken"
    }
    
}





package org.netbeans.lib.lexer.token;

import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.TokenId;

/**

 */

public class CustomTextToken<T extends TokenId> extends DefaultToken<T> {
    
    private final CharSequence text; // 28 bytes (24-super + 4)
    
    private final PartType partType; // 32 bytes
    
    /**
     * @param id non-null identification of the token.
     * @param length length of the token.
     * @param text non-null text of the token.
     */
    public CustomTextToken(T id, int length, CharSequence text, PartType partType) {
        super(id, length);
        assert (text != null);
        this.text = text;
        this.partType = partType;
    }
    
    @Override
    public final CharSequence text() {
        return text;
    }
    
    @Override
    protected String dumpInfoTokenType() {
        return "CusT"; // NOI18N "TextToken" or "FlyToken"
    }




        this.length = 0;
    }

    @Override
    public final int length() {
        return length;
    }

    @Override
    protected String dumpInfoTokenType() {
        return "DefT"; // NOI18N "TextToken" or "FlyToken"
    }




        this.prepError = prepError;
    }
    
    @Override
    public synchronized CharSequence preprocessedText() {
        return prepTextStorage;
    }
    
    @Override
    public synchronized boolean isPreprocessedText() {
        return (prepTextStorage != null);
    }
    
    @Override
    public String preprocessError() {
        return (prepError != null) ? prepError.message() : null;
    }

    @Override
    public int preprocessErrorIndex() {
        return (prepError != null) ? prepError.index() : -1;
    }

    @Override
    protected String dumpInfoTokenType() {
        return "PreT"; // NOI18N "PrepToken"
    }




/*
 * The contents of this file are subject to the terms of the Common Development
 * and Distribution License (the License). You may not use this file except in
 * compliance with the License.
 *
 * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
 * or http://www.netbeans.org/cddl.txt.
 *
 * When distributing Covered Code, include this CDDL Header Notice in each file
 * and include the License file at http://www.netbeans.org/cddl.txt.
 * If applicable, add the following below the CDDL Header, with the fields
 * enclosed by brackets [] replaced by your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 */

package org.netbeans.lib.lexer.token;

import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.CharPreprocessorError;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.lib.lexer.PreprocessedTextStorage;
import org.netbeans.spi.lexer.TokenPropertyProvider;

/**
 * Token that holds information about preprocessed characters.
 *
 * <p>
 * Instances of this token are more costly than other token types
 * because in addition to regular information they store preprocessed
 * text of the token.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class PropertyCustomTextToken<T extends TokenId> extends CustomTextToken<T> {
    
    private TokenPropertyProvider propertyProvider; // 36 bytes (32-super + 4)
    
    private Object tokenStoreValue; // 40 bytes
    
    private PreprocessedTextStorage prepTextStorage; // 44 bytes
    
    private CharPreprocessorError prepError; // 48 bytes
    
    public PropertyCustomTextToken(T id, int length, CharSequence text,
    TokenPropertyProvider propertyProvider, Object tokenStoreValue) {
        super(id, length, text);
        this.propertyProvider = propertyProvider;
        this.tokenStoreValue = tokenStoreValue;
    }
    
    public void initPrep(PreprocessedTextStorage prepTextStorage, CharPreprocessorError prepError) {
        this.prepTextStorage = prepTextStorage;
        this.prepError = prepError;
    }
    
    public synchronized CharSequence preprocessedText() {
        return prepTextStorage;
    }
    
    public synchronized boolean isPreprocessedText() {
        return true;
    }
    
    public String preprocessError() {
        return (prepError != null) ? prepError.message() : null;
    }

    public int preprocessErrorIndex() {
        return (prepError != null) ? prepError.index() : -1;
    }

    public boolean hasProperties() {
        return true;
    }

    public Object getProperty(Object key) {
        return LexerUtilsConstants.getTokenProperty(this, propertyProvider, key, tokenStoreValue);
    }

    protected String dumpInfoTokenType() {
        return "PPCT"; // NOI18N
    }
    
}




/*
 * The contents of this file are subject to the terms of the Common Development
 * and Distribution License (the License). You may not use this file except in
 * compliance with the License.
 *
 * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
 * or http://www.netbeans.org/cddl.txt.
 *
 * When distributing Covered Code, include this CDDL Header Notice in each file
 * and include the License file at http://www.netbeans.org/cddl.txt.
 * If applicable, add the following below the CDDL Header, with the fields
 * enclosed by brackets [] replaced by your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 */

package org.netbeans.lib.lexer.token;

import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.lib.lexer.PreprocessedTextStorage;
import org.netbeans.spi.lexer.CharPreprocessor;
import org.netbeans.spi.lexer.TokenPropertyProvider;

/**
 * Token that holds information about preprocessed characters
 * and also carries properties.
 *
 * <p>
 * Instances of this token are more costly than other token types
 * because in addition to regular information they store preprocessed
 * text of the token.
 *
 * @author Miloslav Metelka
 * @version 1.00
 */

public final class PropertyPreprocessedTextToken<T extends TokenId> extends PreprocessedTextToken<T> {

    private TokenPropertyProvider propertyProvider; // 28 bytes (24-super + 4)

    private Object tokenStoreValue; // 32 bytes

    public PropertyPreprocessedTextToken(T id, int length,
    TokenPropertyProvider propertyProvider, Object tokenStoreValue) {
        super(id, length);
        this.propertyProvider = propertyProvider;
        this.tokenStoreValue = tokenStoreValue;
    }

    public boolean hasProperties() {
        return true;
    }

    public Object getProperty(Object key) {
        return LexerUtilsConstants.getTokenProperty(this, propertyProvider, key, tokenStoreValue);
    }

    protected String dumpInfoTokenType() {
        return "PPrT"; // NOI18N "PrepToken"
    }
    
}






package org.netbeans.lib.lexer.token;

import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.spi.lexer.TokenPropertyProvider;


public final class PropertyToken<T extends TokenId> extends DefaultToken<T> {
    
    private final TokenPropertyProvider propertyProvider; // 28 bytes (24-super + 4)
    
    private final PartType partType; // 32 bytes
    
    public PropertyToken(T id, int length,
    TokenPropertyProvider propertyProvider, PartType partType) {
        super(id, length);
        this.propertyProvider = propertyProvider;
        this.partType = partType;
    }
    
    @Override
    public boolean hasProperties() {
        return (propertyProvider != null);
    }

    @Override
    public Object getProperty(Object key) {
        return (propertyProvider != null) ? propertyProvider.getValue(this, key) : null;
    }
    
    @Override
    public PartType partType() {
        return partType;
    }

    @Override
    protected String dumpInfoTokenType() {
        return "ProT"; // NOI18N "PrepToken"
    }
    
}




        this.text = text;
    }

    @Override
    public final int length() {
        return text.length();
    }

    @Override
    public final CharSequence text() {
        return text;
    }

        return new TextToken<T>(id(), tokenList, rawOffset, text());
    }
    
    @Override
    protected String dumpInfoTokenType() {
        return isFlyweight() ? "FlyT" : "TexT"; // NOI18N "TextToken" or "FlyToken"
    }




package org.netbeans.spi.lexer;

import java.util.Set;
import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.CharSequenceUtilities;

import org.netbeans.lib.lexer.TokenIdImpl;
import org.netbeans.lib.lexer.token.CustomTextToken;
import org.netbeans.lib.lexer.token.DefaultToken;
import org.netbeans.lib.lexer.token.ComplexToken;
import org.netbeans.lib.lexer.token.PreprocessedTextToken;
import org.netbeans.lib.lexer.token.ComplexToken;
import org.netbeans.lib.lexer.token.PropertyPreprocessedTextToken;
import org.netbeans.lib.lexer.token.PropertyToken;
import org.netbeans.lib.lexer.token.TextToken;


    }

    /**
     * Create regular token instance with an explicit length and part type.
     *
     * @param id non-null token id recognized by the lexer.
     * @param length >=0 length of the token to be created. The length must not
     *  exceed the number of characters read from the lexer input.
     * @param partType whether this token is complete token or a part of a complete token.
     * @return non-null regular token instance.
     *  <br/>
     *  If there were any characters preprocessed by {@link CharPreprocessor}
     *  then a special token instance will be created for it.
     *  <br/>
     *  {@link #SKIP_TOKEN} will be returned
     *  if tokens for the given token id should be skipped
     *  because of token id filter.
     */
    public Token<T> createToken(T id, int length, PartType partType) {
        checkPartTypeNonNull(partType);
        if (partType == PartType.COMPLETE)
            return createToken(id, length);

        if (isSkipToken(id)) {
            operation.tokenRecognized(length, true);
            return skipToken();
        } else { // Do not skip the token
            if (operation.tokenRecognized(length, false)) { // Create preprocessed token
                return new ComplexToken<T>(id, operation.tokenLength(), null, null, partType);
            } else {
                return new PropertyToken<T>(id, operation.tokenLength(), null, partType);
            }
        }
    }

    /**
     * Get flyweight token for the given arguments.
     * <br/>
     * <b>Note:</b> The returned token will not be flyweight under certain

     * @param length >=0 length of the token to be created. The length must not
     *  exceed the number of characters read from the lexer input.
     * @param propertyProvider non-null token property provider.
     * @param partType whether this token is complete or just a part of complete token.
     *  or null if there is no explicit value (value will be computed lazily when asked).
     *  <br/>
     *  See {@link TokenPropertyProvider} for examples how this parameter may be used.
     * @return non-null property token instance.
     *  <br/>

     *  because of token id filter.
     */
    public Token<T> createPropertyToken(T id, int length,
    TokenPropertyProvider propertyProvider, PartType partType) {
        checkPartTypeNonNull(partType);
        if (isSkipToken(id)) {
            operation.tokenRecognized(length, true);
            return skipToken();
        } else { // Do not skip the token
            if (operation.tokenRecognized(length, false)) { // Create preprocessed token
                return new ComplexToken<T>(id, operation.tokenLength(),
                    propertyProvider, null, partType);
            } else {
                return new PropertyToken<T>(id, operation.tokenLength(),
                    propertyProvider, partType);
            }
        }
    }

     * Create token with a custom text that possibly differs from the text
     * represented by the token in the input text.
     */
    public Token<T> createCustomTextToken(T id, CharSequence text, int length, PartType partType) {
        checkPartTypeNonNull(partType);
        if (isSkipToken(id)) {
            operation.tokenRecognized(length, true);
            return skipToken();
        } else { // Do not skip the token
            if (operation.tokenRecognized(length, false)) { // Create preprocessed token
                return new ComplexToken<T>(id, operation.tokenLength(), null, text, partType);
                        operation.tokenLength(), text, null, null);
            } else {
                return new CustomTextToken<T>(id, operation.tokenLength(), text, partType);
            }
        }
    }

    @SuppressWarnings("unchecked") // NOI18N
    private Token<T> skipToken() {
        return SKIP_TOKEN;
    }
    
    private void checkPartTypeNonNull(PartType partType) {
        if (partType == null)
            throw new IllegalArgumentException("partType must be non-null");
    }
    
}




/**
 * Provides extra properties of a token.
 * <br/>
 * Normally each token has an extra instance of the property provider:
 * <pre>
 * final class MyTokenPropertyProvider implements TokenPropertyProvider {
 *
 *     private final Object value;
 *
 *     TokenPropProvider(Object value) {
 *         this.value = value;
 *     }
 *      
 *     public Object getValue (Token token, Object key) {
 *         if ("type".equals(key))
 *             return value;
 *         return null;
 *     }
 *
 * }
 * </pre>
 * <br/>
 * However multiple flyweight instances of the provider may be used to save memory
 * if there are just several values for a property.
 * <br/>
 * Example of two instances of a provider for boolean property "key":
 * <pre>
 * final class MyTokenPropertyProvider implements TokenPropertyProvider {
 * Generally this interface can be used in multiple ways:
 * <ul>
 *  <li>
 *    A new instance of the provider per each token.
 *    <br/>
 *    Example of storing a value for "key" key in the provider:
 *    <pre>
 *    final class MyTokenPropertyProvider implements TokenPropertyProvider {
 * 
 *       private final Object value;
 * 
 *       MyTokenPropertyProvider(Object value) {
 *           this.value = value;
 *       }
 *
 *       public Object getValue(Token token, Object key) {
 *           if ("key".equals(key)) {
 *               return value;
 *           }
 *           return null;
 *       }
 * 
 *       public Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue) {
 *           return null; // not using token's storage
 *       }
 * 
 *       public Object tokenStoreKey() {
 *           return null; // not using token's storage
 *       }
 * 
 *    }
 *    </pre>
 *  </li>
 * 
 *  <li>
 *    A single instance of the provider for multiple tokens.
 *    Each token may have a specific value of the given property.
 *    <br/>
 *    This might be achieved by returning the particular key
 *    from {@link #tokenStoreKey()} and using the token store value
 *    for the storage of the property value.
 *    <br/>
 *    Example of storing a value for "key" by the token's storage:
 *    <pre>
 *    final class MyTokenPropertyProvider implements TokenPropertyProvider {
 * 
 *       static final MyTokenPropertyProvider INSTANCE = new MyTokenPropertyProvider();
 * 
 *       private MyTokenPropertyProvider() {
 *       }
 *
 *     static final MyTokenPropertyProvider TRUE = new MyTokenPropertyProvider(Boolean.TRUE);
 *
 *     static final MyTokenPropertyProvider FALSE = new MyTokenPropertyProvider(Boolean.FALSE);
 * 
 *       public Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue) {
 *           // Assuming that the appropriate tokenStoreValue was passed to
 *           // TokenFactory.createPropertyToken(id, length, propertyProvider, tokenStoreValue);
 *           return tokenStoreValue;
 *       }
 * 
 *       public Object tokenStoreKey() {
 *           return "key";
 *       }
 * 
 *    }
 *    </pre>
 *  </li>
 * 
 *  <li>
 *    Multiple flyweight instances of the provider.
 *    This might be useful if there is just several values for the property.
 *    <br/>
 *    Example of two instances of a provider for "key":
 *    <pre>
 *    final class MyTokenPropertyProvider implements TokenPropertyProvider {
 * 
 *       static final MyTokenPropertyProvider TRUE = new MyTokenPropertyProvider(Boolean.TRUE);
 * 
 *       static final MyTokenPropertyProvider FALSE = new MyTokenPropertyProvider(Boolean.FALSE);
 * 
 *       private final Boolean value;
 * 
 *       private MyTokenPropertyProvider(Boolean value) {
 *           this.value = value;
 *       }
 *
 *       public Object getValue(Token token, Object key) {
 *           if ("key".equals(key)) {
 *               return value;
 *           }
 *           return null;
 *       }
 * 
 *       public Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue) {
 *           return null; // not using token's storage
 *       }
 * 
 *       public Object tokenStoreKey() {
 *           return null; // not using token's storage
 *       }
 * 
 *     private final Boolean value;
 *
 *     private MyTokenPropertyProvider(Boolean value) {
 *         this.value = value;
 *     }
 *
 *     public Object getValue(Token token, Object key) {
 *         if ("key".equals(key)) {
 *             return value;
 *         }
 *         return null;
 *     }
 *
 * }
 * </pre>
 * <br/>
 * A special kind of token <code>PropertyToken</code> allows to carry token properties.
 *
 * @author Miloslav Metelka
 * @version 1.00

public interface TokenPropertyProvider<T extends TokenId> {
    
    /**
     * Get value of a token property.
     *
     * @param token non-null token for which the property is being retrieved.
     *  It might be useful if the property would be computed dynamically.
     * @param key non-null key for which the value should be retrieved.
     * @return value of the property or null if there is no value for the given key.
     */
    Object getValue(Token token, Object key);

    /**
     * Get value of a token-store property.
     * <br/>
     * This method is only invoked if {@link #tokenStoreKey()} returned non-null value.
     * <br/>
     * When called for the first time the <code>tokenStoreValue</code>
     * will have the value given to
     * {@link TokenFactory#createPropertyToken(TokenId,int,TokenPropertyProvider,Object)}.
     * <br/>
     * For subsequent invocations of this method the value returned from
     * a last call to it (for the given token) will be used.
     *
     * @param token non-null token for which the property is being retrieved.
     * @param tokenStoreKey non-null key for which the value should be retrieved.
     * @param tokenStoreValue value that was is currently stored in the token.
     * @return value for the tokenStoreKey. The value will be both returned
     *  and stored in the token.
     */
    Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue);
    
    /**
     * Get a key of the property that is stored in the token.
     *
     * @return key of the property which is stored in the token itself
     *  or null if the property provider does not want any property
     *  to be stored in the token.
     */
    Object tokenStoreKey();

}

Lines 90-95 Link Here

(-)lexer/test/unit/src/org/netbeans/api/lexer/CustomTokenClassTest.java (+4 lines)
90	return false;	90	return false;
91	}	91	}
92		92
		93	public PartType partType() {
		94	return null;
		95	}
		96
93	}	97	}
94		98
95	}	99	}

Lines 217-222 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java (+1 lines)
217	TestCase.assertEquals(message + "Invalid token.state()", state(ts), state(ts2));	217	TestCase.assertEquals(message + "Invalid token.state()", state(ts), state(ts2));
218	}	218	}
219	TestCase.assertEquals(message + "Invalid token length", t.length(), t2.length());	219	TestCase.assertEquals(message + "Invalid token length", t.length(), t2.length());
		220	TestCase.assertEquals(message + "Invalid token part", t.partType(), t2.partType());
220	}	221	}
221		222
222	/**	223	/**

Lines 27-33 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/dump/TokenDumpCheck.java (-1 lines)
27	import java.util.ArrayList;	27	import java.util.ArrayList;
28	import java.util.Arrays;	28	import java.util.Arrays;
29	import java.util.List;	29	import java.util.List;
30	import java.util.regex.Pattern;
31	import org.netbeans.api.lexer.Language;	30	import org.netbeans.api.lexer.Language;
32	import org.netbeans.api.lexer.Token;	31	import org.netbeans.api.lexer.Token;
33	import org.netbeans.api.lexer.TokenHierarchy;	32	import org.netbeans.api.lexer.TokenHierarchy;





package org.netbeans.lib.lexer.test.dump;

import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.spi.lexer.Lexer;
import org.netbeans.spi.lexer.LexerInput;

                case EOF:
                    input.backup(1);
                    return tokenFactory.createPropertyToken(id, input.readLength(),
                        new UnicodeCharValueProvider(new Character(ch)), PartType.COMPLETE);
            }
        }
        input.backup(1);


    private static final class UnicodeCharValueProvider implements TokenPropertyProvider {
        
        private Character ch;
        
        UnicodeCharValueProvider(Character ch) {
            this.ch = ch;
        }
        
        public Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue) {
            return tokenStoreValue;
        }
        
        public Object getValue(Token token, Object key) {
            if (TokenDumpTokenId.UNICODE_CHAR_TOKEN_PROPERTY.equals(key))
                return ch;
            return null; // no non-tokenStore value
        }
        
    }

Lines 46-52 Link Here

(-)web/jspsyntax/lexer/nbproject/project.xml (-2 / +2 lines)
46	<compile-dependency/>	46	<compile-dependency/>
47	<run-dependency>	47	<run-dependency>
48	<release-version>1</release-version>	48	<release-version>1</release-version>
49	<specification-version>1.1</specification-version>	49	<specification-version>1.2</specification-version>
50	</run-dependency>	50	</run-dependency>
51	</dependency>	51	</dependency>
52	<dependency>	52	<dependency>
Lines 55-61 Link Here
55	<compile-dependency/>	55	<compile-dependency/>
56	<run-dependency>	56	<run-dependency>
57	<release-version>2</release-version>	57	<release-version>2</release-version>
58	<specification-version>1.12</specification-version>	58	<specification-version>1.14</specification-version>
59	</run-dependency>	59	</run-dependency>
60	</dependency>	60	</dependency>
61	</module-dependencies>	61	</module-dependencies>

Lines 24-29 Link Here

(-)web/jspsyntax/lexer/src/org/netbeans/lib/jsp/lexer/JspLexer.java (-10 / +12 lines)
24	import org.netbeans.api.jsp.lexer.JspTokenId;	24	import org.netbeans.api.jsp.lexer.JspTokenId;
25	import org.netbeans.api.lexer.InputAttributes;	25	import org.netbeans.api.lexer.InputAttributes;
26	import org.netbeans.api.lexer.LanguagePath;	26	import org.netbeans.api.lexer.LanguagePath;
		27	import org.netbeans.api.lexer.PartType;
27	import org.netbeans.api.lexer.Token;	28	import org.netbeans.api.lexer.Token;
28	import org.netbeans.spi.jsp.lexer.JspParseData;	29	import org.netbeans.spi.jsp.lexer.JspParseData;
29	import org.netbeans.spi.lexer.Lexer;	30	import org.netbeans.spi.lexer.Lexer;
Lines 1112-1118 Link Here
1112	throw new IllegalStateException("Unsupported scriptlet type " + lexerStateJspScriptlet);	1113	throw new IllegalStateException("Unsupported scriptlet type " + lexerStateJspScriptlet);
1113	}	1114	}
1114		1115
1115	return tokenFactory.createPropertyToken(tokenId, input.readLength(), new JspTokenPropertyProvider(), scriptletType);	1116	return tokenFactory.createPropertyToken(tokenId, input.readLength(),
		1117	new JspTokenPropertyProvider(scriptletType), PartType.COMPLETE);
1116	}	1118	}
1117		1119
1118	private void checkToken(JspTokenId tokenId) {	1120	private void checkToken(JspTokenId tokenId) {
Lines 1127-1145 Link Here
1127		1129
1128	private static class JspTokenPropertyProvider implements TokenPropertyProvider {	1130	private static class JspTokenPropertyProvider implements TokenPropertyProvider {
1129		1131
1130	public Object getValue(Token token, Object key) {	1132	private final JspTokenId.JavaCodeType scriptletType;
1131	return null;	1133
		1134	JspTokenPropertyProvider(JspTokenId.JavaCodeType scriptletType) {
		1135	this.scriptletType = scriptletType;
1132	}	1136	}
1133		1137
1134	public Object getValue(Token token, Object tokenStoreKey,	1138	public Object getValue(Token token, Object key) {
1135	Object tokenStoreValue) {	1139	if (JspTokenId.SCRIPTLET_TOKEN_TYPE_PROPERTY.equals(key))
1136	return tokenStoreValue;	1140	return scriptletType;
		1141	return null;
1137	}	1142	}
1138		1143
1139	public Object tokenStoreKey() {	1144	}
1140	return JspTokenId.SCRIPTLET_TOKEN_TYPE_PROPERTY;
1141	}
1142	}
1143		1145
1144	}	1146	}
1145		1147

Return to bug 91184

Lines 25-30 Link Here

(-)languages/engine/src/org/netbeans/modules/languages/lexer/SLexer.java (-21 / +14 lines)
25	import java.util.Map;	25	import java.util.Map;
26	import org.netbeans.api.languages.CharInput;	26	import org.netbeans.api.languages.CharInput;
27	import org.netbeans.api.languages.ASTToken;	27	import org.netbeans.api.languages.ASTToken;
		28	import org.netbeans.api.lexer.PartType;
28	import org.netbeans.api.languages.LanguagesManager;	29	import org.netbeans.api.languages.LanguagesManager;
29	import org.netbeans.api.languages.ParseException;	30	import org.netbeans.api.languages.ParseException;
30	import org.netbeans.api.lexer.Token;	31	import org.netbeans.api.lexer.Token;
Lines 232-245 Link Here
232	tokenId,	233	tokenId,
233	v.endOffset - v.startOffset,	234	v.endOffset - v.startOffset,
234	(TokenProperties) v.property,	235	(TokenProperties) v.property,
235	null	236	PartType.COMPLETE
236	);	237	);
237	else	238	else
238	return tokenFactory.createPropertyToken (	239	return tokenFactory.createPropertyToken (
239	tokenId,	240	tokenId,
240	v.endOffset - v.startOffset,	241	v.endOffset - v.startOffset,
241	tokenPropertyProvider,	242	new TokenPropProvider(v.property),
242	v.property	243	PartType.COMPLETE
243	);	244	);
244	}	245	}
245		246
Lines 264-285 Link Here
264		265
265	// innerclasses ............................................................	266	// innerclasses ............................................................
266		267
267	private static TokenPropertyProvider tokenPropertyProvider = new TokenPropertyProvider () {	268	private static final class TokenPropProvider implements TokenPropertyProvider {
268		269
269	public Object getValue (Token token, Object key) {	270	private final Object value;
270	return null;	271
		272	TokenPropProvider(Object value) {
		273	this.value = value;
271	}	274	}
272		275
273	public Object getValue (Token token, Object tokenStoreKey, Object tokenStoreValue) {	276	public Object getValue (Token token, Object key) {
274	if (tokenStoreKey.equals ("type"))	277	if ("type".equals(key))
275	return tokenStoreValue;	278	return value;
276	return null;	279	return null;
277	}	280	}
278		281
279	public Object tokenStoreKey() {	282	}
280	return "type";
281	}
282	};
283		283
284	static class TokenProperties implements TokenPropertyProvider {	284	static class TokenProperties implements TokenPropertyProvider {
285		285
Lines 304-316 Link Here
304	return null;	304	return null;
305	}	305	}
306		306
307	public Object getValue (Token token, Object tokenStoreKey, Object tokenStoreValue) {
308	return null;
309	}
310
311	public Object tokenStoreKey() {
312	return null;
313	}
314	};	307	};
315		308
316	static class Vojta {	309	static class Vojta {

Lines 257-272 Link Here

(-)lexer/src/org/netbeans/lib/lexer/LexerUtilsConstants.java (-10 lines)
257	\|\| (state1 != null && state1.equals(state2));	257	\|\| (state1 != null && state1.equals(state2));
258	}	258	}
259		259
260	public static <T extends TokenId> Object getTokenProperty(Token<T> token,
261	TokenPropertyProvider propertyProvider, Object key, Object tokenStoreValue) {
262	Object tokenStoreKey = propertyProvider.tokenStoreKey();
263	if (tokenStoreKey != null && tokenStoreKey.equals(key)) { // token store value
264	return propertyProvider.getValue(token, tokenStoreKey, tokenStoreValue);
265	} else {
266	return propertyProvider.getValue(token, key);
267	}
268	}
269
270	public static String idToString(TokenId id) {	260	public static String idToString(TokenId id) {
271	return id.name() + '[' + id.ordinal() + ']'; // NOI18N;	261	return id.name() + '[' + id.ordinal() + ']'; // NOI18N;
272	}	262	}

Lines 101-119 Link Here

(-)java/editor/src/org/netbeans/modules/editor/java/Utilities.java (-4 lines)
101	if (ts.token().text().charAt(0) == '.')	101	if (ts.token().text().charAt(0) == '.')
102	break;	102	break;
103	case CHAR_LITERAL:	103	case CHAR_LITERAL:
104	case CHAR_LITERAL_INCOMPLETE:
105	case FLOAT_LITERAL:	104	case FLOAT_LITERAL:
106	case FLOAT_LITERAL_INVALID:	105	case FLOAT_LITERAL_INVALID:
107	case INT_LITERAL:	106	case INT_LITERAL:
108	case INVALID_COMMENT_END:	107	case INVALID_COMMENT_END:
109	case JAVADOC_COMMENT:	108	case JAVADOC_COMMENT:
110	case JAVADOC_COMMENT_INCOMPLETE:
111	case LONG_LITERAL:	109	case LONG_LITERAL:
112	case STRING_LITERAL:	110	case STRING_LITERAL:
113	case STRING_LITERAL_INCOMPLETE:
114	case LINE_COMMENT:	111	case LINE_COMMENT:
115	case BLOCK_COMMENT:	112	case BLOCK_COMMENT:
116	case BLOCK_COMMENT_INCOMPLETE:
117	return false;	113	return false;
118	}	114	}
119	return true;	115	return true;

Lines 162-172 Link Here

(-)java/lexer/src/org/netbeans/api/java/lexer/JavaTokenId.java (-24 / +3 lines)
162	BLOCK_COMMENT(null, "comment"),	162	BLOCK_COMMENT(null, "comment"),
163	JAVADOC_COMMENT(null, "comment"),	163	JAVADOC_COMMENT(null, "comment"),
164		164
165	// Errors and incomplete tokens	165	// Errors
166	CHAR_LITERAL_INCOMPLETE(null, "character"),
167	STRING_LITERAL_INCOMPLETE(null, "string"),
168	BLOCK_COMMENT_INCOMPLETE(null, "comment"),
169	JAVADOC_COMMENT_INCOMPLETE(null, "comment"),
170	INVALID_COMMENT_END("*/", "error"),	166	INVALID_COMMENT_END("*/", "error"),
171	FLOAT_LITERAL_INVALID(null, "number");	167	FLOAT_LITERAL_INVALID(null, "number");
172		168
Lines 203-237 Link Here
203	@Override	199	@Override
204	protected Map<String,Collection<JavaTokenId>> createTokenCategories() {	200	protected Map<String,Collection<JavaTokenId>> createTokenCategories() {
205	Map<String,Collection<JavaTokenId>> cats = new HashMap<String,Collection<JavaTokenId>>();	201	Map<String,Collection<JavaTokenId>> cats = new HashMap<String,Collection<JavaTokenId>>();
206	// Incomplete tokens
207	cats.put("incomplete", EnumSet.of(
208	JavaTokenId.CHAR_LITERAL_INCOMPLETE,
209	JavaTokenId.STRING_LITERAL_INCOMPLETE,
210	JavaTokenId.BLOCK_COMMENT_INCOMPLETE,
211	JavaTokenId.JAVADOC_COMMENT_INCOMPLETE
212	));
213	// Additional literals being a lexical error	202	// Additional literals being a lexical error
214	cats.put("error", EnumSet.of(	203	cats.put("error", EnumSet.of(
215	JavaTokenId.CHAR_LITERAL_INCOMPLETE,
216	JavaTokenId.STRING_LITERAL_INCOMPLETE,
217	JavaTokenId.BLOCK_COMMENT_INCOMPLETE,
218	JavaTokenId.JAVADOC_COMMENT_INCOMPLETE,
219	JavaTokenId.FLOAT_LITERAL_INVALID	204	JavaTokenId.FLOAT_LITERAL_INVALID
220	));	205	));
221	// Complete and incomplete literals	206	// Literals category
222	EnumSet<JavaTokenId> l = EnumSet.of(	207	EnumSet<JavaTokenId> l = EnumSet.of(
223	JavaTokenId.INT_LITERAL,	208	JavaTokenId.INT_LITERAL,
224	JavaTokenId.LONG_LITERAL,	209	JavaTokenId.LONG_LITERAL,
225	JavaTokenId.FLOAT_LITERAL,	210	JavaTokenId.FLOAT_LITERAL,
226	JavaTokenId.DOUBLE_LITERAL,	211	JavaTokenId.DOUBLE_LITERAL,
227	JavaTokenId.CHAR_LITERAL	212	JavaTokenId.CHAR_LITERAL
228
229	);	213	);
230	l.addAll(EnumSet.of(	214	l.add(JavaTokenId.STRING_LITERAL);
231	JavaTokenId.CHAR_LITERAL_INCOMPLETE,
232	JavaTokenId.STRING_LITERAL,
233	JavaTokenId.STRING_LITERAL_INCOMPLETE
234	));
235	cats.put("literal", l);	215	cats.put("literal", l);
236		216
237	return cats;	217	return cats;
Lines 250-256 Link Here
250	case JAVADOC_COMMENT:	230	case JAVADOC_COMMENT:
251	return LanguageEmbedding.create(JavadocTokenId.language(), 3, 2);	231	return LanguageEmbedding.create(JavadocTokenId.language(), 3, 2);
252	case STRING_LITERAL:	232	case STRING_LITERAL:
253	case STRING_LITERAL_INCOMPLETE:
254	return LanguageEmbedding.create(JavaStringTokenId.language(), 1, 1);	233	return LanguageEmbedding.create(JavaStringTokenId.language(), 1, 1);
255	}	234	}
256	return null; // No embedding	235	return null; // No embedding

Lines 20-25 Link Here

(-)java/lexer/src/org/netbeans/lib/java/lexer/JavaLexer.java (-6 / +13 lines)
20	package org.netbeans.lib.java.lexer;	20	package org.netbeans.lib.java.lexer;
21		21
22	import org.netbeans.api.java.lexer.JavaTokenId;	22	import org.netbeans.api.java.lexer.JavaTokenId;
		23	import org.netbeans.api.lexer.PartType;
23	import org.netbeans.api.lexer.Token;	24	import org.netbeans.api.lexer.Token;
24	import org.netbeans.spi.lexer.Lexer;	25	import org.netbeans.spi.lexer.Lexer;
25	import org.netbeans.spi.lexer.LexerInput;	26	import org.netbeans.spi.lexer.LexerInput;
Lines 79-85 Link Here
79	case '\r': input.consumeNewline();	80	case '\r': input.consumeNewline();
80	case '\n':	81	case '\n':
81	case EOF:	82	case EOF:
82	return token(JavaTokenId.STRING_LITERAL_INCOMPLETE);	83	return tokenFactory.createToken(JavaTokenId.STRING_LITERAL,
		84	input.readLength(), PartType.START);
83	}	85	}
84		86
85	case '\'': // char literal	87	case '\'': // char literal
Lines 93-99 Link Here
93	case '\r': input.consumeNewline();	95	case '\r': input.consumeNewline();
94	case '\n':	96	case '\n':
95	case EOF:	97	case EOF:
96	return token(JavaTokenId.CHAR_LITERAL_INCOMPLETE);	98	return tokenFactory.createToken(JavaTokenId.CHAR_LITERAL,
		99	input.readLength(), PartType.START);
97	}	100	}
98		101
99	case '/':	102	case '/':
Lines 120-129 Link Here
120	if (c == '/')	123	if (c == '/')
121	return token(JavaTokenId.JAVADOC_COMMENT);	124	return token(JavaTokenId.JAVADOC_COMMENT);
122	else if (c == EOF)	125	else if (c == EOF)
123	return token(JavaTokenId.JAVADOC_COMMENT_INCOMPLETE);	126	return tokenFactory.createToken(JavaTokenId.JAVADOC_COMMENT,
		127	input.readLength(), PartType.START);
124	}	128	}
125	if (c == EOF)	129	if (c == EOF)
126	return token(JavaTokenId.JAVADOC_COMMENT_INCOMPLETE);	130	return tokenFactory.createToken(JavaTokenId.JAVADOC_COMMENT,
		131	input.readLength(), PartType.START);
127	c = input.read();	132	c = input.read();
128	}	133	}
129		134
Lines 135-144 Link Here
135	if (c == '/')	140	if (c == '/')
136	return token(JavaTokenId.BLOCK_COMMENT);	141	return token(JavaTokenId.BLOCK_COMMENT);
137	else if (c == EOF)	142	else if (c == EOF)
138	return token(JavaTokenId.BLOCK_COMMENT_INCOMPLETE);	143	return tokenFactory.createToken(JavaTokenId.BLOCK_COMMENT,
		144	input.readLength(), PartType.START);
139	}	145	}
140	if (c == EOF)	146	if (c == EOF)
141	return token(JavaTokenId.BLOCK_COMMENT_INCOMPLETE);	147	return tokenFactory.createToken(JavaTokenId.BLOCK_COMMENT,
		148	input.readLength(), PartType.START);
142	}	149	}
143	}	150	}
144	} // end of switch()	151	} // end of switch()

Lines 3-9 Link Here

(-)java/lexer/test/unit/data/testfiles/testInput.java.txt.tokens.txt (-3 / +3 lines)
3	BLOCK_COMMENT "/**/"	3	BLOCK_COMMENT "/**/"
4	JAVADOC_COMMENT "/***/"	4	JAVADOC_COMMENT "/***/"
5	JAVADOC_COMMENT "/*\njavadoc-comment*/"	5	JAVADOC_COMMENT "/*\njavadoc-comment*/"
6	BLOCK_COMMENT_INCOMPLETE "/* a", la=1	6	BLOCK_COMMENT "/* a", la=1
7	----- EOF -----	7	----- EOF -----
8		8
9	.t.e.s.t. Identifiers	9	.t.e.s.t. Identifiers
Lines 37-43 Link Here
37	WHITESPACE " ", la=1	37	WHITESPACE " ", la=1
38	CHAR_LITERAL "'\\n'"	38	CHAR_LITERAL "'\\n'"
39	WHITESPACE " ", la=1	39	WHITESPACE " ", la=1
40	CHAR_LITERAL_INCOMPLETE "'a", la=1	40	CHAR_LITERAL "'a", la=1
41	----- EOF -----	41	----- EOF -----
42		42
43	.t.e.s.t. String Literals	43	.t.e.s.t. String Literals
Lines 51-57 Link Here
51	WHITESPACE "\n", la=1	51	WHITESPACE "\n", la=1
52	STRING_LITERAL ""\\\\\\\\\\\\\\"\\"""	52	STRING_LITERAL ""\\\\\\\\\\\\\\"\\"""
53	WHITESPACE "\n", la=1	53	WHITESPACE "\n", la=1
54	STRING_LITERAL_INCOMPLETE ""\\n\\" \\"a", la=1	54	STRING_LITERAL ""\\n\\" \\"a", la=1
55	----- EOF -----	55	----- EOF -----
56		56
57	.t.e.s.t. Number Literals	57	.t.e.s.t. Number Literals

Lines 23-28 Link Here

(-)java/lexer/test/unit/src/org/netbeans/lib/java/lexer/JavaLexerBatchTest.java (-3 / +7 lines)
23	import org.netbeans.api.java.lexer.JavaStringTokenId;	23	import org.netbeans.api.java.lexer.JavaStringTokenId;
24	import org.netbeans.api.java.lexer.JavaTokenId;	24	import org.netbeans.api.java.lexer.JavaTokenId;
25	import org.netbeans.api.java.lexer.JavadocTokenId;	25	import org.netbeans.api.java.lexer.JavadocTokenId;
		26	import org.netbeans.api.lexer.PartType;
26	import org.netbeans.api.lexer.TokenHierarchy;	27	import org.netbeans.api.lexer.TokenHierarchy;
27	import org.netbeans.api.lexer.TokenId;	28	import org.netbeans.api.lexer.TokenId;
28	import org.netbeans.api.lexer.TokenSequence;	29	import org.netbeans.api.lexer.TokenSequence;
Lines 55-61 Link Here
55	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.BLOCK_COMMENT, "/**/");	56	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.BLOCK_COMMENT, "/**/");
56	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.JAVADOC_COMMENT, "/***/");	57	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.JAVADOC_COMMENT, "/***/");
57	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.JAVADOC_COMMENT, "/*\njavadoc-comment*/");	58	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.JAVADOC_COMMENT, "/*\njavadoc-comment*/");
58	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.BLOCK_COMMENT_INCOMPLETE, "/* a");	59	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.BLOCK_COMMENT, "/* a");
		60	assertEquals(PartType.START, ts.token().partType());
59	}	61	}
60		62
61	public void testIdentifiers() {	63	public void testIdentifiers() {
Lines 95-101 Link Here
95	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");	97	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
96	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL, "'\\n'");	98	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL, "'\\n'");
97	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");	99	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
98	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL_INCOMPLETE, "'a");	100	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL, "'a");
		101	assertEquals(PartType.START, ts.token().partType());
99	}	102	}
100		103
101	public void testStringLiterals() {	104	public void testStringLiterals() {
Lines 115-121 Link Here
115	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");	118	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
116	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"\\n\"");	119	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"\\n\"");
117	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");	120	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
118	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL_INCOMPLETE, "\"a");	121	LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"a");
		122	assertEquals(PartType.START, ts.token().partType());
119	}	123	}
120		124
121	public void testNumberLiterals() {	125	public void testNumberLiterals() {

Lines 93-98 Link Here

(-)lexer/api/apichanges.xml (+16 lines)
93	<changes>	93	<changes>
94	<change id="api.tokensequence.move.semantics">	94	<change id="api.tokensequence.move.semantics">
95	<api name="api"/>	95	<api name="api"/>
		96	<summary>Added PartInfo for token parts support</summary>
		97	<version major="1" minor="14"/>
		98	<date day="1" month="3" year="2007"/>
		99	<author login="mmetelka"/>
		100	<compatibility source="incompatible" deletion="yes" addition="yes" modification="yes"/>
		101	<description>
		102	<p>
		103	Added <code>PartInfo</code> enum and <code>Token.partInfo()</code>
		104	that identifies whether the token is COMPLETE or which part
		105	of a complete token this part represents (START, INNER or END).
		106	</p>
		107	</description>
		108	</change>
		109
		110	<change id="api.tokensequence.move.semantics">
		111	<api name="api"/>
96	<summary>Changed TokenSequence.move() and moveIndex() use</summary>	112	<summary>Changed TokenSequence.move() and moveIndex() use</summary>
97	<version major="1" minor="13"/>	113	<version major="1" minor="13"/>
98	<date day="16" month="1" year="2007"/>	114	<date day="16" month="1" year="2007"/>

Added Link Here

(-)lexer/src/org/netbeans/api/lexer/PartType.java (+57 lines)
1	/*
2	* The contents of this file are subject to the terms of the Common Development
3	* and Distribution License (the License). You may not use this file except in
4	* compliance with the License.
5	*
6	* You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7	* or http://www.netbeans.org/cddl.txt.
8	*
9	* When distributing Covered Code, include this CDDL Header Notice in each file
10	* and include the License file at http://www.netbeans.org/cddl.txt.
11	* If applicable, add the following below the CDDL Header, with the fields
12	* enclosed by brackets [] replaced by your own identifying information:
13	* "Portions Copyrighted [year] [name of copyright owner]"
14	*
15	* The Original Software is NetBeans. The Initial Developer of the Original
16	* Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17	* Microsystems, Inc. All Rights Reserved.
18	*/
19
20	package org.netbeans.api.lexer;
21
22	/**
23	* Whether {@link Token} represents a complete token
24	* or just a part of a complete token.
25	* <br/>
26	* A complete token may consist of one start token, zero or more middle tokens
27	* and zero or more end tokens (there may be incomplete token
28	* at the end of input e.g. an incomplete block comment so there is just a start
29	* part of a token).
30	*
31	* @author Miloslav Metelka
32	* @version 1.00
33	*/
34
35	public enum PartType {
36
37	/**
38	* A given token represents a complete token.
39	*/
40	COMPLETE,
41
42	/**
43	* A given token represents initial part of a complete token.
44	*/
45	START,
46
47	/**
48	* A given token represents middle part of a complete token.
49	*/
50	MIDDLE,
51
52	/**
53	* A given token represents end part of a complete token.
54	*/
55	END;
56
57	}

Lines 19-24 Link Here

(-)lexer/src/org/netbeans/lib/lexer/token/AbstractToken.java (+17 lines)
19		19
20	package org.netbeans.lib.lexer.token;	20	package org.netbeans.lib.lexer.token;
21		21
		22	import org.netbeans.api.lexer.PartType;
22	import org.netbeans.api.lexer.Token;	23	import org.netbeans.api.lexer.Token;
23	import org.netbeans.api.lexer.TokenHierarchy;	24	import org.netbeans.api.lexer.TokenHierarchy;
24	import org.netbeans.api.lexer.TokenId;	25	import org.netbeans.api.lexer.TokenId;
Lines 63-68 Link Here
63	*	64	*
64	* @return non-null identification of this token.	65	* @return non-null identification of this token.
65	*/	66	*/
		67	@Override
66	public final T id() {	68	public final T id() {
67	return id;	69	return id;
68	}	70	}
Lines 70-75 Link Here
70	/**	72	/**
71	* Get text represented by this token.	73	* Get text represented by this token.
72	*/	74	*/
		75	@Override
73	public CharSequence text() {	76	public CharSequence text() {
74	if (tokenList != null) {	77	if (tokenList != null) {
75	if (tokenList.getClass() == EmbeddedTokenList.class) {	78	if (tokenList.getClass() == EmbeddedTokenList.class) {
Lines 113-118 Link Here
113	this.rawOffset = rawOffset;	116	this.rawOffset = rawOffset;
114	}	117	}
115		118
		119	@Override
116	public final boolean isFlyweight() {	120	public final boolean isFlyweight() {
117	return (rawOffset == -1);	121	return (rawOffset == -1);
118	}	122	}
Lines 121-130 Link Here
121	setRawOffset(-1);	125	setRawOffset(-1);
122	}	126	}
123		127
		128	@Override
		129	public PartType partType() {
		130	return PartType.COMPLETE;
		131	}
		132
		133	@Override
124	public boolean isCustomText() {	134	public boolean isCustomText() {
125	return false;	135	return false;
126	}	136	}
127		137
		138	@Override
128	public final int offset(TokenHierarchy<?> tokenHierarchy) {	139	public final int offset(TokenHierarchy<?> tokenHierarchy) {
129	if (rawOffset == -1) { // flyweight token	140	if (rawOffset == -1) { // flyweight token
130	return -1;	141	return -1;
Lines 140-165 Link Here
140	}	151	}
141	}	152	}
142		153
		154	@Override
143	public boolean isPreprocessedText() {	155	public boolean isPreprocessedText() {
144	return false;	156	return false;
145	}	157	}
146		158
		159	@Override
147	public CharSequence preprocessedText() {	160	public CharSequence preprocessedText() {
148	return null;	161	return null;
149	}	162	}
150		163
		164	@Override
151	public String preprocessError() {	165	public String preprocessError() {
152	return null;	166	return null;
153	}	167	}
154		168
		169	@Override
155	public int preprocessErrorIndex() {	170	public int preprocessErrorIndex() {
156	return -1;	171	return -1;
157	}	172	}
158		173
		174	@Override
159	public boolean hasProperties() {	175	public boolean hasProperties() {
160	return false;	176	return false;
161	}	177	}
162		178
		179	@Override
163	public Object getProperty(Object key) {	180	public Object getProperty(Object key) {
164	return null;	181	return null;
165	}	182	}

Lines 58-67 Link Here

(-)lexer/src/org/netbeans/lib/lexer/token/DefaultToken.java (+2 lines)
58	this.length = 0;	58	this.length = 0;
59	}	59	}
60		60
		61	@Override
61	public final int length() {	62	public final int length() {
62	return length;	63	return length;
63	}	64	}
64		65
		66	@Override
65	protected String dumpInfoTokenType() {	67	protected String dumpInfoTokenType() {
66	return "DefT"; // NOI18N "TextToken" or "FlyToken"	68	return "DefT"; // NOI18N "TextToken" or "FlyToken"
67	}	69	}

Lines 50-71 Link Here

(-)lexer/src/org/netbeans/lib/lexer/token/PreprocessedTextToken.java (-1 / +6 lines)
50	this.prepError = prepError;	50	this.prepError = prepError;
51	}	51	}
52		52
		53	@Override
53	public synchronized CharSequence preprocessedText() {	54	public synchronized CharSequence preprocessedText() {
54	return prepTextStorage;	55	return prepTextStorage;
55	}	56	}
56		57
		58	@Override
57	public synchronized boolean isPreprocessedText() {	59	public synchronized boolean isPreprocessedText() {
58	return true;	60	return (prepTextStorage != null);
59	}	61	}
60		62
		63	@Override
61	public String preprocessError() {	64	public String preprocessError() {
62	return (prepError != null) ? prepError.message() : null;	65	return (prepError != null) ? prepError.message() : null;
63	}	66	}
64		67
		68	@Override
65	public int preprocessErrorIndex() {	69	public int preprocessErrorIndex() {
66	return (prepError != null) ? prepError.index() : -1;	70	return (prepError != null) ? prepError.index() : -1;
67	}	71	}
68		72
		73	@Override
69	protected String dumpInfoTokenType() {	74	protected String dumpInfoTokenType() {
70	return "PreT"; // NOI18N "PrepToken"	75	return "PreT"; // NOI18N "PrepToken"
71	}	76	}

Lines 69-78 Link Here

(-)lexer/src/org/netbeans/lib/lexer/token/TextToken.java (+3 lines)
69	this.text = text;	69	this.text = text;
70	}	70	}
71		71
		72	@Override
72	public final int length() {	73	public final int length() {
73	return text.length();	74	return text.length();
74	}	75	}
75		76
		77	@Override
76	public final CharSequence text() {	78	public final CharSequence text() {
77	return text;	79	return text;
78	}	80	}
Lines 81-86 Link Here
81	return new TextToken<T>(id(), tokenList, rawOffset, text());	83	return new TextToken<T>(id(), tokenList, rawOffset, text());
82	}	84	}
83		85
		86	@Override
84	protected String dumpInfoTokenType() {	87	protected String dumpInfoTokenType() {
85	return isFlyweight() ? "FlyT" : "TexT"; // NOI18N "TextToken" or "FlyToken"	88	return isFlyweight() ? "FlyT" : "TexT"; // NOI18N "TextToken" or "FlyToken"
86	}	89	}

Lines 20-25 Link Here

(-)lexer/src/org/netbeans/spi/lexer/TokenFactory.java (-13 / +51 lines)
20	package org.netbeans.spi.lexer;	20	package org.netbeans.spi.lexer;
21		21
22	import java.util.Set;	22	import java.util.Set;
		23	import org.netbeans.api.lexer.PartType;
23	import org.netbeans.api.lexer.Token;	24	import org.netbeans.api.lexer.Token;
24	import org.netbeans.api.lexer.TokenId;	25	import org.netbeans.api.lexer.TokenId;
25	import org.netbeans.lib.editor.util.CharSequenceUtilities;	26	import org.netbeans.lib.editor.util.CharSequenceUtilities;
Lines 28-36 Link Here
28	import org.netbeans.lib.lexer.TokenIdImpl;	29	import org.netbeans.lib.lexer.TokenIdImpl;
29	import org.netbeans.lib.lexer.token.CustomTextToken;	30	import org.netbeans.lib.lexer.token.CustomTextToken;
30	import org.netbeans.lib.lexer.token.DefaultToken;	31	import org.netbeans.lib.lexer.token.DefaultToken;
		32	import org.netbeans.lib.lexer.token.ComplexToken;
31	import org.netbeans.lib.lexer.token.PreprocessedTextToken;	33	import org.netbeans.lib.lexer.token.PreprocessedTextToken;
32	import org.netbeans.lib.lexer.token.PropertyCustomTextToken;	34	import org.netbeans.lib.lexer.token.ComplexToken;
33	import org.netbeans.lib.lexer.token.PropertyPreprocessedTextToken;
34	import org.netbeans.lib.lexer.token.PropertyToken;	35	import org.netbeans.lib.lexer.token.PropertyToken;
35	import org.netbeans.lib.lexer.token.TextToken;	36	import org.netbeans.lib.lexer.token.TextToken;
36		37
Lines 104-109 Link Here
104	}	105	}
105		106
106	/**	107	/**
		108	* Create regular token instance with an explicit length and part type.
		109	*
		110	* @param id non-null token id recognized by the lexer.
		111	* @param length >=0 length of the token to be created. The length must not
		112	* exceed the number of characters read from the lexer input.
		113	* @param partType whether this token is complete token or a part of a complete token.
		114	* @return non-null regular token instance.
		115	* <br/>
		116	* If there were any characters preprocessed by {@link CharPreprocessor}
		117	* then a special token instance will be created for it.
		118	* <br/>
		119	* {@link #SKIP_TOKEN} will be returned
		120	* if tokens for the given token id should be skipped
		121	* because of token id filter.
		122	*/
		123	public Token<T> createToken(T id, int length, PartType partType) {
		124	checkPartTypeNonNull(partType);
		125	if (partType == PartType.COMPLETE)
		126	return createToken(id, length);
		127
		128	if (isSkipToken(id)) {
		129	operation.tokenRecognized(length, true);
		130	return skipToken();
		131	} else { // Do not skip the token
		132	if (operation.tokenRecognized(length, false)) { // Create preprocessed token
		133	return new ComplexToken<T>(id, operation.tokenLength(), null, null, partType);
		134	} else {
		135	return new PropertyToken<T>(id, operation.tokenLength(), null, partType);
		136	}
		137	}
		138	}
		139
		140	/**
107	* Get flyweight token for the given arguments.	141	* Get flyweight token for the given arguments.
108	* <br/>	142	* <br/>
109	* <b>Note:</b> The returned token will not be flyweight under certain	143	* <b>Note:</b> The returned token will not be flyweight under certain
Lines 165-173 Link Here
165	* @param length >=0 length of the token to be created. The length must not	199	* @param length >=0 length of the token to be created. The length must not
166	* exceed the number of characters read from the lexer input.	200	* exceed the number of characters read from the lexer input.
167	* @param propertyProvider non-null token property provider.	201	* @param propertyProvider non-null token property provider.
168	* @param tokenStoreValue explicit value for property stored in the token itself	202	* @param partType whether this token is complete or just a part of complete token.
169	* or null if there is no explicit value (value will be computed lazily when asked).
170	* <br/>
171	* See {@link TokenPropertyProvider} for examples how this parameter may be used.	203	* See {@link TokenPropertyProvider} for examples how this parameter may be used.
172	* @return non-null property token instance.	204	* @return non-null property token instance.
173	* <br/>	205	* <br/>
Lines 179-195 Link Here
179	* because of token id filter.	211	* because of token id filter.
180	*/	212	*/
181	public Token<T> createPropertyToken(T id, int length,	213	public Token<T> createPropertyToken(T id, int length,
182	TokenPropertyProvider propertyProvider, Object tokenStoreValue) {	214	TokenPropertyProvider propertyProvider, PartType partType) {
		215	checkPartTypeNonNull(partType);
183	if (isSkipToken(id)) {	216	if (isSkipToken(id)) {
184	operation.tokenRecognized(length, true);	217	operation.tokenRecognized(length, true);
185	return skipToken();	218	return skipToken();
186	} else { // Do not skip the token	219	} else { // Do not skip the token
187	if (operation.tokenRecognized(length, false)) { // Create preprocessed token	220	if (operation.tokenRecognized(length, false)) { // Create preprocessed token
188	return new PropertyPreprocessedTextToken<T>(id, operation.tokenLength(),	221	return new ComplexToken<T>(id, operation.tokenLength(),
189	propertyProvider, tokenStoreValue);	222	propertyProvider, null, partType);
190	} else {	223	} else {
191	return new PropertyToken<T>(id, operation.tokenLength(),	224	return new PropertyToken<T>(id, operation.tokenLength(),
192	propertyProvider, tokenStoreValue);	225	propertyProvider, partType);
193	}	226	}
194	}	227	}
195	}	228	}
Lines 198-213 Link Here
198	* Create token with a custom text that possibly differs from the text	231	* Create token with a custom text that possibly differs from the text
199	* represented by the token in the input text.	232	* represented by the token in the input text.
200	*/	233	*/
201	public Token<T> createCustomTextToken(T id, CharSequence text, int length) {	234	public Token<T> createCustomTextToken(T id, CharSequence text, int length, PartType partType) {
		235	checkPartTypeNonNull(partType);
202	if (isSkipToken(id)) {	236	if (isSkipToken(id)) {
203	operation.tokenRecognized(length, true);	237	operation.tokenRecognized(length, true);
204	return skipToken();	238	return skipToken();
205	} else { // Do not skip the token	239	} else { // Do not skip the token
206	if (operation.tokenRecognized(length, false)) { // Create preprocessed token	240	if (operation.tokenRecognized(length, false)) { // Create preprocessed token
207	return new PropertyCustomTextToken<T>(id,	241	return new ComplexToken<T>(id, operation.tokenLength(), null, text, partType);
208	operation.tokenLength(), text, null, null);
209	} else {	242	} else {
210	return new CustomTextToken<T>(id, operation.tokenLength(), text);	243	return new CustomTextToken<T>(id, operation.tokenLength(), text, partType);
211	}	244	}
212	}	245	}
213	}	246	}
Lines 220-225 Link Here
220	@SuppressWarnings("unchecked") // NOI18N	253	@SuppressWarnings("unchecked") // NOI18N
221	private Token<T> skipToken() {	254	private Token<T> skipToken() {
222	return SKIP_TOKEN;	255	return SKIP_TOKEN;
		256	}
		257
		258	private void checkPartTypeNonNull(PartType partType) {
		259	if (partType == null)
		260	throw new IllegalArgumentException("partType must be non-null");
223	}	261	}
224		262
225	}	263	}

Lines 25-145 Link Here

(-)lexer/src/org/netbeans/spi/lexer/TokenPropertyProvider.java (-141 / +45 lines)
25	/**	25	/**
26	* Provides extra properties of a token.	26	* Provides extra properties of a token.
27	* <br/>	27	* <br/>
28	* A special kind of token <code>PropertyToken</code> allows to carry token properties.	28	* Normally each token has an extra instance of the property provider:
		29	* <pre>
		30	* final class MyTokenPropertyProvider implements TokenPropertyProvider {
		31	*
		32	* private final Object value;
		33	*
		34	* TokenPropProvider(Object value) {
		35	* this.value = value;
		36	* }
		37	*
		38	* public Object getValue (Token token, Object key) {
		39	* if ("type".equals(key))
		40	* return value;
		41	* return null;
		42	* }
		43	*
		44	* }
		45	* </pre>
29	* <br/>	46	* <br/>
30	* That token may store a value of one property in its instance	47	* However multiple flyweight instances of the provider may be used to save memory
31	* (see <code>PropertyToken.tokenStoreValue</code>. If the provider	48	* if there are just several values for a property.
32	* wants to use that field for storing of the value it needs to return	49	* <br/>
33	* the corresponding key for that value from {@link #tokenStoreKey()}.	50	* Example of two instances of a provider for boolean property "key":
34	*	51	* <pre>
35	* <p/>	52	* final class MyTokenPropertyProvider implements TokenPropertyProvider {
36	* Generally this interface can be used in multiple ways:
37	* <ul>
38	* <li>
39	* A new instance of the provider per each token.
40	* <br/>
41	* Example of storing a value for "key" key in the provider:
42	* <pre>
43	* final class MyTokenPropertyProvider implements TokenPropertyProvider {
44	*
45	* private final Object value;
46	*
47	* MyTokenPropertyProvider(Object value) {
48	* this.value = value;
49	* }
50	*
51	* public Object getValue(Token token, Object key) {
52	* if ("key".equals(key)) {
53	* return value;
54	* }
55	* return null;
56	* }
57	*
58	* public Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue) {
59	* return null; // not using token's storage
60	* }
61	*
62	* public Object tokenStoreKey() {
63	* return null; // not using token's storage
64	* }
65	*
66	* }
67	* </pre>
68	* </li>
69	*
70	* <li>
71	* A single instance of the provider for multiple tokens.
72	* Each token may have a specific value of the given property.
73	* <br/>
74	* This might be achieved by returning the particular key
75	* from {@link #tokenStoreKey()} and using the token store value
76	* for the storage of the property value.
77	* <br/>
78	* Example of storing a value for "key" by the token's storage:
79	* <pre>
80	* final class MyTokenPropertyProvider implements TokenPropertyProvider {
81	*
82	* static final MyTokenPropertyProvider INSTANCE = new MyTokenPropertyProvider();
83	*
84	* private MyTokenPropertyProvider() {
85	* }
86	*	53	*
87	* public Object getValue(Token token, Object key) {	54	* static final MyTokenPropertyProvider TRUE = new MyTokenPropertyProvider(Boolean.TRUE);
88	* return null; // no extra properties	55	*
89	* }	56	* static final MyTokenPropertyProvider FALSE = new MyTokenPropertyProvider(Boolean.FALSE);
90	*
91	* public Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue) {
92	* // Assuming that the appropriate tokenStoreValue was passed to
93	* // TokenFactory.createPropertyToken(id, length, propertyProvider, tokenStoreValue);
94	* return tokenStoreValue;
95	* }
96	*
97	* public Object tokenStoreKey() {
98	* return "key";
99	* }
100	*
101	* }
102	* </pre>
103	* </li>
104	*
105	* <li>
106	* Multiple flyweight instances of the provider.
107	* This might be useful if there is just several values for the property.
108	* <br/>
109	* Example of two instances of a provider for "key":
110	* <pre>
111	* final class MyTokenPropertyProvider implements TokenPropertyProvider {
112	*
113	* static final MyTokenPropertyProvider TRUE = new MyTokenPropertyProvider(Boolean.TRUE);
114	*
115	* static final MyTokenPropertyProvider FALSE = new MyTokenPropertyProvider(Boolean.FALSE);
116	*
117	* private final Boolean value;
118	*
119	* private MyTokenPropertyProvider(Boolean value) {
120	* this.value = value;
121	* }
122	*
123	* public Object getValue(Token token, Object key) {
124	* if ("key".equals(key)) {
125	* return value;
126	* }
127	* return null;
128	* }
129	*
130	* public Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue) {
131	* return null; // not using token's storage
132	* }
133	*
134	* public Object tokenStoreKey() {
135	* return null; // not using token's storage
136	* }
137	*	57	*
138	* }	58	* private final Boolean value;
139	* </pre>	59	*
140	* </li>	60	* private MyTokenPropertyProvider(Boolean value) {
141	* </ul>	61	* this.value = value;
		62	* }
		63	*
		64	* public Object getValue(Token token, Object key) {
		65	* if ("key".equals(key)) {
		66	* return value;
		67	* }
		68	* return null;
		69	* }
142	*	70	*
		71	* }
		72	* </pre>
		73	* <br/>
		74	* A special kind of token <code>PropertyToken</code> allows to carry token properties.
143	*	75	*
144	* @author Miloslav Metelka	76	* @author Miloslav Metelka
145	* @version 1.00	77	* @version 1.00
Lines 148-188 Link Here
148	public interface TokenPropertyProvider<T extends TokenId> {	80	public interface TokenPropertyProvider<T extends TokenId> {
149		81
150	/**	82	/**
151	* Get value of a property which is not a token-store property.	83	* Get value of a token property.
152	*	84	*
153	* @param token non-null token for which the property is being retrieved.	85	* @param token non-null token for which the property is being retrieved.
		86	* It might be useful if the property would be computed dynamically.
154	* @param key non-null key for which the value should be retrieved.	87	* @param key non-null key for which the value should be retrieved.
155	* @return value of the property or null if there is no value for the given key.	88	* @return value of the property or null if there is no value for the given key.
156	*/	89	*/
157	Object getValue(Token token, Object key);	90	Object getValue(Token token, Object key);
158
159	/**
160	* Get value of a token-store property.
161	* <br/>
162	* This method is only invoked if {@link #tokenStoreKey()} returned non-null value.
163	* <br/>
164	* When called for the first time the <code>tokenStoreValue</code>
165	* will have the value given to
166	* {@link TokenFactory#createPropertyToken(TokenId,int,TokenPropertyProvider,Object)}.
167	* <br/>
168	* For subsequent invocations of this method the value returned from
169	* a last call to it (for the given token) will be used.
170	*
171	* @param token non-null token for which the property is being retrieved.
172	* @param tokenStoreKey non-null key for which the value should be retrieved.
173	* @param tokenStoreValue value that was is currently stored in the token.
174	* @return value for the tokenStoreKey. The value will be both returned
175	* and stored in the token.
176	*/
177	Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue);
178
179	/**
180	* Get a key of the property that is stored in the token.
181	*
182	* @return key of the property which is stored in the token itself
183	* or null if the property provider does not want any property
184	* to be stored in the token.
185	*/
186	Object tokenStoreKey();
187		91
188	}	92	}

Lines 19-24 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/dump/TokenDumpLexer.java (-10 / +9 lines)
19		19
20	package org.netbeans.lib.lexer.test.dump;	20	package org.netbeans.lib.lexer.test.dump;
21		21
		22	import org.netbeans.api.lexer.PartType;
22	import org.netbeans.api.lexer.Token;	23	import org.netbeans.api.lexer.Token;
23	import org.netbeans.spi.lexer.Lexer;	24	import org.netbeans.spi.lexer.Lexer;
24	import org.netbeans.spi.lexer.LexerInput;	25	import org.netbeans.spi.lexer.LexerInput;
Lines 182-188 Link Here
182	case EOF:	183	case EOF:
183	input.backup(1);	184	input.backup(1);
184	return tokenFactory.createPropertyToken(id, input.readLength(),	185	return tokenFactory.createPropertyToken(id, input.readLength(),
185	UnicodeCharValueProvider.INSTANCE, new Character(ch));	186	new UnicodeCharValueProvider(new Character(ch)), PartType.COMPLETE);
186	}	187	}
187	}	188	}
188	input.backup(1);	189	input.backup(1);
Lines 198-215 Link Here
198		199
199	private static final class UnicodeCharValueProvider implements TokenPropertyProvider {	200	private static final class UnicodeCharValueProvider implements TokenPropertyProvider {
200		201
201	static final UnicodeCharValueProvider INSTANCE = new UnicodeCharValueProvider();	202	private Character ch;
202		203
203	public Object getValue(Token token, Object key) {	204	UnicodeCharValueProvider(Character ch) {
204	return null; // no non-tokenStore value	205	this.ch = ch;
205	}
206
207	public Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue) {
208	return tokenStoreValue;
209	}	206	}
210		207
211	public Object tokenStoreKey() {	208	public Object getValue(Token token, Object key) {
212	return TokenDumpTokenId.UNICODE_CHAR_TOKEN_PROPERTY;	209	if (TokenDumpTokenId.UNICODE_CHAR_TOKEN_PROPERTY.equals(key))
		210	return ch;
		211	return null; // no non-tokenStore value
213	}	212	}
214		213
215	}	214	}