# This patch file was generated by NetBeans IDE
# Following Index: paths are relative to: /s/nb/t/nb_all/lexer
# This patch can be applied using context Tools: Patch action on respective folder.
# It uses platform neutral UTF-8 encoding and \n newlines.
# Above lines and this line are ignored by the patching process.
Index: test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleTokenId.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleTokenId.java Base (1.3)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleTokenId.java Locally Modified (Based On 1.3)
***************
*** 25,32 ****
  import org.netbeans.spi.lexer.LanguageEmbedding;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerInput;
! import org.netbeans.spi.lexer.TokenFactory;
  
  /**
   * Simple implementation of enumerated token id.
--- 25,31 ----
  import org.netbeans.spi.lexer.LanguageEmbedding;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerRestartInfo;
  
  /**
   * Simple implementation of enumerated token id.
***************
*** 107,116 ****
              return cats;
          }
  
!         protected Lexer<SimpleTokenId> createLexer(
!         LexerInput input, TokenFactory<SimpleTokenId> tokenFactory, Object state,
!         LanguagePath languagePath, InputAttributes inputAttributes) {
!             return new SimpleLexer(input, tokenFactory, state, languagePath, inputAttributes);
          }
          
          public LanguageEmbedding embedding(
--- 106,113 ----
              return cats;
          }
  
!         protected Lexer<SimpleTokenId> createLexer(LexerRestartInfo<SimpleTokenId> info) {
!             return new SimpleLexer(info);
          }
          
          public LanguageEmbedding embedding(
Index: src/org/netbeans/lib/lexer/LanguageManager.java
*** /s/nb/t/nb_all/lexer/src/org/netbeans/lib/lexer/LanguageManager.java Base (1.4)
--- /s/nb/t/nb_all/lexer/src/org/netbeans/lib/lexer/LanguageManager.java Locally Modified (Based On 1.4)
***************
*** 37,44 ****
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.LanguageProvider;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerInput;
! import org.netbeans.spi.lexer.TokenFactory;
  import org.openide.util.Lookup;
  import org.openide.util.LookupEvent;
  import org.openide.util.LookupListener;
--- 37,43 ----
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.LanguageProvider;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerRestartInfo;
  import org.openide.util.Lookup;
  import org.openide.util.LookupEvent;
  import org.openide.util.LookupListener;
***************
*** 50,56 ****
  public final class LanguageManager extends LanguageProvider implements LookupListener, PropertyChangeListener {
      
      private static final LanguageDescription<TokenId> NO_LANG = new LanguageHierarchy<TokenId>() {
!         protected Lexer<TokenId> createLexer(LexerInput input, TokenFactory<TokenId> tokenFactory, Object state, LanguagePath languagePath, InputAttributes inputAttributes) {
              return null;
          }
          protected Collection<TokenId> createTokenIds() {
--- 49,55 ----
  public final class LanguageManager extends LanguageProvider implements LookupListener, PropertyChangeListener {
      
      private static final LanguageDescription<TokenId> NO_LANG = new LanguageHierarchy<TokenId>() {
!         protected Lexer<TokenId> createLexer(LexerRestartInfo<TokenId> info) {
              return null;
          }
          protected Collection<TokenId> createTokenIds() {
Index: test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleCharTokenId.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleCharTokenId.java Base (1.3)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleCharTokenId.java Locally Modified (Based On 1.3)
***************
*** 23,30 ****
  import org.netbeans.spi.lexer.LanguageEmbedding;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerInput;
! import org.netbeans.spi.lexer.TokenFactory;
  
  /**
   * Token identifications of the simple plain language.
--- 23,29 ----
  import org.netbeans.spi.lexer.LanguageEmbedding;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerRestartInfo;
  
  /**
   * Token identifications of the simple plain language.
***************
*** 52,61 ****
              return EnumSet.allOf(SimpleCharTokenId.class);
          }
          
!         public Lexer<SimpleCharTokenId> createLexer(
!         LexerInput input, TokenFactory<SimpleCharTokenId> tokenFactory, Object state,
!         LanguagePath languagePath, InputAttributes inputAttributes) {
!             return new SimpleCharLexer(input, tokenFactory, state, languagePath, inputAttributes);
          }
  
          public LanguageEmbedding embedding(
--- 51,58 ----
              return EnumSet.allOf(SimpleCharTokenId.class);
          }
          
!         public Lexer<SimpleCharTokenId> createLexer(LexerRestartInfo<SimpleCharTokenId> info) {
!             return new SimpleCharLexer(info);
          }
  
          public LanguageEmbedding embedding(
Index: test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleStringTokenId.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleStringTokenId.java Base (1.3)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleStringTokenId.java Locally Modified (Based On 1.3)
***************
*** 24,31 ****
  import org.netbeans.spi.lexer.LanguageEmbedding;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerInput;
! import org.netbeans.spi.lexer.TokenFactory;
  
  /**
   * Token ids for simple string embedding - copied from JavaStringTokenId
--- 24,30 ----
  import org.netbeans.spi.lexer.LanguageEmbedding;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerRestartInfo;
  
  /**
   * Token ids for simple string embedding - copied from JavaStringTokenId
***************
*** 73,82 ****
              return null; // no extra categories
          }
  
!         public Lexer<SimpleStringTokenId> createLexer(
!         LexerInput input, TokenFactory<SimpleStringTokenId> tokenFactory, Object state,
!         LanguagePath languagePath, InputAttributes inputAttributes) {
!             return new SimpleStringLexer(input, tokenFactory, state, languagePath, inputAttributes);
          }
  
          public LanguageEmbedding embedding(
--- 72,79 ----
              return null; // no extra categories
          }
  
!         public Lexer<SimpleStringTokenId> createLexer(LexerRestartInfo<SimpleStringTokenId> info) {
!             return new SimpleStringLexer(info);
          }
  
          public LanguageEmbedding embedding(
Index: src/org/netbeans/spi/lexer/LexerRestartInfo.java
*** /s/nb/t/nb_all/lexer/src/org/netbeans/spi/lexer/LexerRestartInfo.java No Base Revision
--- /s/nb/t/nb_all/lexer/src/org/netbeans/spi/lexer/LexerRestartInfo.java Locally New
***************
*** 1,0 ****
--- 1,82 ----
+ /*
+  * The contents of this file are subject to the terms of the Common Development
+  * and Distribution License (the License). You may not use this file except in
+  * compliance with the License.
+  *
+  * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
+  * or http://www.netbeans.org/cddl.txt.
+  *
+  * When distributing Covered Code, include this CDDL Header Notice in each file
+  * and include the License file at http://www.netbeans.org/cddl.txt.
+  * If applicable, add the following below the CDDL Header, with the fields
+  * enclosed by brackets [] replaced by your own identifying information:
+  * "Portions Copyrighted [year] [name of copyright owner]"
+  *
+  * The Original Software is NetBeans. The Initial Developer of the Original
+  * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
+  * Microsystems, Inc. All Rights Reserved.
+  */
+ 
+ package org.netbeans.spi.lexer;
+ 
+ import org.netbeans.api.lexer.InputAttributes;
+ import org.netbeans.api.lexer.LanguagePath;
+ import org.netbeans.api.lexer.TokenId;
+ 
+ /**
+  * Lexer restart info contains all the necessary information for restarting
+  * of a lexer mainly the lexer input, state and token factory.
+  *
+  * @author Miloslav Metelka
+  * @since
+  */
+ 
+ public final class LexerRestartInfo<T extends TokenId> {
+ 
+     private final LexerInput input;
+     
+     private final TokenFactory<T> tokenFactory;
+     
+     private final Object state;
+     
+     private final LanguagePath languagePath;
+     
+     private final InputAttributes inputAttributes;
+     
+     LexerRestartInfo(LexerInput input,
+     TokenFactory<T> tokenFactory, Object state,
+     LanguagePath languagePath, InputAttributes inputAttributes) {
+         this.input = input;
+         this.tokenFactory = tokenFactory;
+         this.state = state;
+         this.languagePath = languagePath;
+         this.inputAttributes = inputAttributes;
+     }
+     
+     public LexerInput input() {
+         return input;
+     }
+ 
+     public TokenFactory<T> tokenFactory() {
+         return tokenFactory;
+     }
+     
+     public Object state() {
+         return state;
+     }
+     
+     public LanguagePath languagePath() {
+         return languagePath;
+     }
+     
+     public InputAttributes inputAttributes() {
+         return inputAttributes;
+     }
+     
+     public Object getAttributeValue(Object key) {
+         return (inputAttributes != null)
+                 ? inputAttributes.getValue(languagePath, key)
+                 : null;
+     }
+ 
+ }
Index: test/unit/src/org/netbeans/lib/lexer/test/state/StateLexer.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/state/StateLexer.java Base (1.2)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/state/StateLexer.java Locally Modified (Based On 1.2)
***************
*** 13,28 ****
  
  package org.netbeans.lib.lexer.test.state;
  
- import java.util.HashMap;
- import java.util.Map;
  import org.netbeans.api.lexer.InputAttributes;
- import org.netbeans.api.lexer.LanguagePath;
  import org.netbeans.api.lexer.Token;
  import org.netbeans.spi.lexer.Lexer;
  import org.netbeans.spi.lexer.LexerInput;
  import org.netbeans.spi.lexer.TokenFactory;
  
--- 13,23 ----
  
  package org.netbeans.lib.lexer.test.state;
  
  import org.netbeans.api.lexer.InputAttributes;
  import org.netbeans.api.lexer.Token;
  import org.netbeans.spi.lexer.Lexer;
  import org.netbeans.spi.lexer.LexerInput;
+ import org.netbeans.spi.lexer.LexerRestartInfo;
  import org.netbeans.spi.lexer.TokenFactory;
  
  /**
***************
*** 46,70 ****
  
      private TokenFactory<StateTokenId> tokenFactory;
      
!     private LanguagePath languagePath;
      
      private InputAttributes inputAttributes;
      
!     StateLexer(LexerInput input, TokenFactory<StateTokenId> tokenFactory, Object state,
!     LanguagePath languagePath, InputAttributes inputAttributes) {
!         this.input = input;
!         this.tokenFactory = tokenFactory;
!         this.state = state;
!         this.languagePath = languagePath;
!         this.inputAttributes = inputAttributes;
  
!         this.useIntStates = (inputAttributes != null)
!                 ? Boolean.TRUE.equals((Boolean)inputAttributes.getValue(languagePath, "states"))
!                 : false;
!         
!         Object expectedRestartState = (inputAttributes != null)
!                 ? inputAttributes.getValue(languagePath, "restartState")
!                 : null;
          if (expectedRestartState != null && !expectedRestartState.equals(state)) {
              throw new IllegalStateException("Expected restart state " + expectedRestartState + ", but real is " + state);
          }
--- 44,61 ----
  
      private TokenFactory<StateTokenId> tokenFactory;
      
!     private LexerRestartInfo<StateTokenId> info;
      
      private InputAttributes inputAttributes;
      
!     StateLexer(LexerRestartInfo<StateTokenId> info) {
!         this.input = info.input();
!         this.tokenFactory = info.tokenFactory();
!         this.state = info.state();
!         this.info = info;
  
!         this.useIntStates = Boolean.TRUE.equals(info.getAttributeValue("states"));
!         Object expectedRestartState = info.getAttributeValue("restartState");
          if (expectedRestartState != null && !expectedRestartState.equals(state)) {
              throw new IllegalStateException("Expected restart state " + expectedRestartState + ", but real is " + state);
          }
***************
*** 75,81 ****
      }
  
      public Token<StateTokenId> nextToken() {
!         boolean returnNullToken = (inputAttributes != null && Boolean.TRUE.equals(inputAttributes.getValue(languagePath, "returnNullToken")));
          while (true) {
              int c = input.read();
              if (returnNullToken) // Test early return of null token
--- 66,72 ----
      }
  
      public Token<StateTokenId> nextToken() {
!         boolean returnNullToken = Boolean.TRUE.equals(info.getAttributeValue("returnNullToken"));
          while (true) {
              int c = input.read();
              if (returnNullToken) // Test early return of null token
Index: test/unit/src/org/netbeans/lib/lexer/test/simple/SimplePlainTokenId.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimplePlainTokenId.java Base (1.3)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimplePlainTokenId.java Locally Modified (Based On 1.3)
***************
*** 23,30 ****
  import org.netbeans.spi.lexer.LanguageEmbedding;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerInput;
! import org.netbeans.spi.lexer.TokenFactory;
  
  /**
   * Token identifications of the simple plain language.
--- 23,29 ----
  import org.netbeans.spi.lexer.LanguageEmbedding;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerRestartInfo;
  
  /**
   * Token identifications of the simple plain language.
***************
*** 57,66 ****
              return EnumSet.allOf(SimplePlainTokenId.class);
          }
          
!         public Lexer<SimplePlainTokenId> createLexer(
!         LexerInput input, TokenFactory<SimplePlainTokenId> tokenFactory, Object state,
!         LanguagePath languagePath, InputAttributes inputAttributes) {
!             return new SimplePlainLexer(input, tokenFactory, state, languagePath, inputAttributes);
          }
  
          public LanguageEmbedding embedding(
--- 56,63 ----
              return EnumSet.allOf(SimplePlainTokenId.class);
          }
          
!         public Lexer<SimplePlainTokenId> createLexer(LexerRestartInfo<SimplePlainTokenId> info) {
!             return new SimplePlainLexer(info);
          }
  
          public LanguageEmbedding embedding(
Index: test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleJavadocLexer.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleJavadocLexer.java Base (1.2)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleJavadocLexer.java Locally Modified (Based On 1.2)
***************
*** 13,26 ****
  
  package org.netbeans.lib.lexer.test.simple;
  
- import org.netbeans.api.lexer.InputAttributes;
- import org.netbeans.api.lexer.LanguagePath;
  import org.netbeans.api.lexer.Token;
  import org.netbeans.spi.lexer.Lexer;
  import org.netbeans.spi.lexer.LexerInput;
  import org.netbeans.spi.lexer.TokenFactory;
  
--- 13,22 ----
  
  package org.netbeans.lib.lexer.test.simple;
  
  import org.netbeans.api.lexer.Token;
  import org.netbeans.spi.lexer.Lexer;
  import org.netbeans.spi.lexer.LexerInput;
+ import org.netbeans.spi.lexer.LexerRestartInfo;
  import org.netbeans.spi.lexer.TokenFactory;
  
  /**
***************
*** 35,45 ****
      
      private TokenFactory<SimpleJavadocTokenId> tokenFactory;
      
!     public SimpleJavadocLexer(LexerInput input, TokenFactory<SimpleJavadocTokenId> tokenFactory, Object state,
!     LanguagePath languagePath, InputAttributes inputAttributes) {
!         this.input = input;
!         this.tokenFactory = tokenFactory;
!         assert (state == null); // passed argument always null
      }
      
      public Object state() {
--- 34,43 ----
      
      private TokenFactory<SimpleJavadocTokenId> tokenFactory;
      
!     public SimpleJavadocLexer(LexerRestartInfo<SimpleJavadocTokenId> info) {
!         this.input = info.input();
!         this.tokenFactory = info.tokenFactory();
!         assert (info.state() == null); // passed argument always null
      }
      
      public Object state() {
Index: test/unit/src/org/netbeans/lib/lexer/test/simple/GenLanguage.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/GenLanguage.java Base (1.2)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/GenLanguage.java Locally Modified (Based On 1.2)
***************
*** 25,32 ****
  import org.netbeans.spi.lexer.LanguageEmbedding;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerInput;
! import org.netbeans.spi.lexer.TokenFactory;
  
  /**
   * Example of language that could be generated by a lexer generator.
--- 25,31 ----
  import org.netbeans.spi.lexer.LanguageEmbedding;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerRestartInfo;
  
  /**
   * Example of language that could be generated by a lexer generator.
***************
*** 88,95 ****
              return cats;
          }
  
!         public Lexer createLexer(LexerInput input, TokenFactory tokenFactory, Object state,
!         LanguagePath languagePath, InputAttributes inputAttributes) {
              return null;
          }
  
--- 87,93 ----
              return cats;
          }
  
!         public Lexer createLexer(LexerRestartInfo info) {
              return null;
          }
  
Index: test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleCharLexer.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleCharLexer.java Base (1.2)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleCharLexer.java Locally Modified (Based On 1.2)
***************
*** 13,26 ****
  
  package org.netbeans.lib.lexer.test.simple;
  
- import org.netbeans.api.lexer.InputAttributes;
- import org.netbeans.api.lexer.LanguagePath;
  import org.netbeans.api.lexer.Token;
  import org.netbeans.spi.lexer.Lexer;
  import org.netbeans.spi.lexer.LexerInput;
  import org.netbeans.spi.lexer.TokenFactory;
  
--- 13,22 ----
  
  package org.netbeans.lib.lexer.test.simple;
  
  import org.netbeans.api.lexer.Token;
  import org.netbeans.spi.lexer.Lexer;
  import org.netbeans.spi.lexer.LexerInput;
+ import org.netbeans.spi.lexer.LexerRestartInfo;
  import org.netbeans.spi.lexer.TokenFactory;
  
  /**
***************
*** 31,40 ****
      
      private TokenFactory<SimpleCharTokenId> tokenFactory;
      
!     SimpleCharLexer(LexerInput input, TokenFactory<SimpleCharTokenId> tokenFactory, Object state,
!     LanguagePath languagePath, InputAttributes inputAttributes) {
!         this.input = input;
!         this.tokenFactory = tokenFactory;
      }
      
      public Token<SimpleCharTokenId> nextToken() {
--- 30,38 ----
      
      private TokenFactory<SimpleCharTokenId> tokenFactory;
      
!     SimpleCharLexer(LexerRestartInfo<SimpleCharTokenId> info) {
!         this.input = info.input();
!         this.tokenFactory = info.tokenFactory();
      }
      
      public Token<SimpleCharTokenId> nextToken() {
Index: test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLanguageProvider.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLanguageProvider.java Base (1.5)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLanguageProvider.java Locally Modified (Based On 1.5)
***************
*** 31,38 ****
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.LanguageProvider;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerInput;
! import org.netbeans.spi.lexer.TokenFactory;
  
  /**
   *
--- 31,37 ----
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.LanguageProvider;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerRestartInfo;
  
  /**
   *
***************
*** 89,95 ****
              return Collections.emptyList();
          }
  
!         protected Lexer<TokenId> createLexer(LexerInput input, TokenFactory<TokenId> tokenFactory, Object state, LanguagePath languagePath, InputAttributes inputAttributes) {
              return null;
          }
  
--- 88,94 ----
              return Collections.emptyList();
          }
  
!         protected Lexer<TokenId> createLexer(LexerRestartInfo<TokenId> info) {
              return null;
          }
  
Index: src/org/netbeans/lib/lexer/LexerSpiPackageAccessor.java
*** /s/nb/t/nb_all/lexer/src/org/netbeans/lib/lexer/LexerSpiPackageAccessor.java Base (1.3)
--- /s/nb/t/nb_all/lexer/src/org/netbeans/lib/lexer/LexerSpiPackageAccessor.java Locally Modified (Based On 1.3)
***************
*** 31,36 ****
--- 31,37 ----
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
  import org.netbeans.spi.lexer.LexerInput;
+ import org.netbeans.spi.lexer.LexerRestartInfo;
  import org.netbeans.spi.lexer.MutableTextInput;
  import org.netbeans.spi.lexer.TokenFactory;
  import org.netbeans.spi.lexer.TokenValidator;
***************
*** 67,74 ****
  
      public abstract <T extends TokenId> Map<String,Collection<T>> createTokenCategories(LanguageHierarchy<T> languageHierarchy);
      
!     public abstract Lexer createLexer(LanguageHierarchy languageHierarchy,
!     LexerInput input, TokenFactory tokenFactory, Object state,
      LanguagePath languagePath, InputAttributes inputAttributes);
      
      public abstract String mimeType(LanguageHierarchy languageHierarchy);
--- 68,77 ----
  
      public abstract <T extends TokenId> Map<String,Collection<T>> createTokenCategories(LanguageHierarchy<T> languageHierarchy);
      
!     public abstract Lexer createLexer(LanguageHierarchy languageHierarchy, LexerRestartInfo info);
!     
!     public abstract <T extends TokenId> LexerRestartInfo<T> createLexerRestartInfo(
!     LexerInput input, TokenFactory<T> tokenFactory, Object state,
      LanguagePath languagePath, InputAttributes inputAttributes);
      
      public abstract String mimeType(LanguageHierarchy languageHierarchy);
Index: src/org/netbeans/spi/lexer/LanguageHierarchy.java
*** /s/nb/t/nb_all/lexer/src/org/netbeans/spi/lexer/LanguageHierarchy.java Base (1.4)
--- /s/nb/t/nb_all/lexer/src/org/netbeans/spi/lexer/LanguageHierarchy.java Locally Modified (Based On 1.4)
***************
*** 197,205 ****
       *  (e.g. there might be a version of the language that the lexer should check).
       *  It may be null if there are no extra attributes.
       */
!     protected abstract Lexer<T> createLexer(LexerInput input,
!     TokenFactory<T> tokenFactory, Object state,
!     LanguagePath languagePath, InputAttributes inputAttributes);
      
      /**
       * Gets the mime type of the language constructed from this language hierarchy.
--- 197,203 ----
       *  (e.g. there might be a version of the language that the lexer should check).
       *  It may be null if there are no extra attributes.
       */
!     protected abstract Lexer<T> createLexer(LexerRestartInfo<T> info);
      
      /**
       * Gets the mime type of the language constructed from this language hierarchy.
***************
*** 347,356 ****
          }
  
          @SuppressWarnings("unchecked")
!         public Lexer createLexer(LanguageHierarchy languageHierarchy,
!         LexerInput input, TokenFactory tokenFactory, Object state,
          LanguagePath languagePath, InputAttributes inputAttributes) {
!             return languageHierarchy.createLexer(input, tokenFactory, state, languagePath, inputAttributes);
          }
  
          @SuppressWarnings("unchecked")
--- 345,358 ----
          }
  
          @SuppressWarnings("unchecked")
!         public Lexer createLexer(LanguageHierarchy languageHierarchy, LexerRestartInfo info) {
!             return languageHierarchy.createLexer(info);
!         }
! 
!         public <T extends TokenId> LexerRestartInfo<T> createLexerRestartInfo(
!         LexerInput input, TokenFactory<T> tokenFactory, Object state,
          LanguagePath languagePath, InputAttributes inputAttributes) {
!             return new LexerRestartInfo<T>(input, tokenFactory, state, languagePath, inputAttributes);
          }
  
          @SuppressWarnings("unchecked")
Index: src/org/netbeans/lib/lexer/LexerInputOperation.java
*** /s/nb/t/nb_all/lexer/src/org/netbeans/lib/lexer/LexerInputOperation.java Base (1.2)
--- /s/nb/t/nb_all/lexer/src/org/netbeans/lib/lexer/LexerInputOperation.java Locally Modified (Based On 1.2)
***************
*** 31,36 ****
--- 31,37 ----
  import org.netbeans.lib.lexer.token.AbstractToken;
  import org.netbeans.lib.lexer.token.PreprocessedTextToken;
  import org.netbeans.lib.lexer.token.PropertyCustomTextToken;
+ import org.netbeans.spi.lexer.LexerRestartInfo;
  import org.netbeans.spi.lexer.TokenFactory;
  
  /**
***************
*** 129,141 ****
          LexerInput lexerInput = LexerSpiPackageAccessor.get().createLexerInput(
                  (preprocessorOperation != null) ? preprocessorOperation : this);
  
!         lexer = LexerSpiPackageAccessor.get().createLexer(
!                 languageOperation.languageHierarchy(),
                  lexerInput, tokenFactory, lexerRestartState,
                  tokenList.languagePath(), inputAttributes());
      }
  
--- 130,141 ----
          LexerInput lexerInput = LexerSpiPackageAccessor.get().createLexerInput(
                  (preprocessorOperation != null) ? preprocessorOperation : this);
  
!         @SuppressWarnings("unchecked")
!         LexerRestartInfo info = LexerSpiPackageAccessor.get().createLexerRestartInfo(
                  lexerInput, tokenFactory, lexerRestartState,
                  tokenList.languagePath(), inputAttributes());
+         lexer = LexerSpiPackageAccessor.get().createLexer(
+                 languageOperation.languageHierarchy(), info);
      }
  
      public abstract int read(int index);
Index: test/unit/src/org/netbeans/lib/lexer/test/simple/SimplePlainLexer.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimplePlainLexer.java Base (1.2)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimplePlainLexer.java Locally Modified (Based On 1.2)
***************
*** 13,26 ****
  
  package org.netbeans.lib.lexer.test.simple;
  
- import org.netbeans.api.lexer.InputAttributes;
- import org.netbeans.api.lexer.LanguagePath;
  import org.netbeans.api.lexer.Token;
  import org.netbeans.spi.lexer.Lexer;
  import org.netbeans.spi.lexer.LexerInput;
  import org.netbeans.spi.lexer.TokenFactory;
  
--- 13,22 ----
  
  package org.netbeans.lib.lexer.test.simple;
  
  import org.netbeans.api.lexer.Token;
  import org.netbeans.spi.lexer.Lexer;
  import org.netbeans.spi.lexer.LexerInput;
+ import org.netbeans.spi.lexer.LexerRestartInfo;
  import org.netbeans.spi.lexer.TokenFactory;
  
  /**
***************
*** 41,51 ****
      
      private int state = INIT;
      
!     SimplePlainLexer(LexerInput input, TokenFactory<SimplePlainTokenId> tokenFactory, Object state,
!     LanguagePath languagePath, InputAttributes inputAttributes) {
!         this.input = input;
!         this.tokenFactory = tokenFactory;
!         assert (state == null);
      }
      
      public Token<SimplePlainTokenId> nextToken() {
--- 40,49 ----
      
      private int state = INIT;
      
!     SimplePlainLexer(LexerRestartInfo<SimplePlainTokenId> info) {
!         this.input = info.input();
!         this.tokenFactory = info.tokenFactory();
!         assert (info.state() == null);
      }
      
      public Token<SimplePlainTokenId> nextToken() {
Index: test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleJavadocTokenId.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleJavadocTokenId.java Base (1.3)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleJavadocTokenId.java Locally Modified (Based On 1.3)
***************
*** 24,31 ****
  import org.netbeans.spi.lexer.LanguageEmbedding;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerInput;
! import org.netbeans.spi.lexer.TokenFactory;
  
  /**
   * Token ids for simple javadoc language 
--- 24,30 ----
  import org.netbeans.spi.lexer.LanguageEmbedding;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerRestartInfo;
  
  /**
   * Token ids for simple javadoc language 
***************
*** 67,76 ****
              return null; // no extra categories
          }
  
!         public Lexer<SimpleJavadocTokenId> createLexer(
!         LexerInput input, TokenFactory<SimpleJavadocTokenId> tokenFactory, Object state,
!         LanguagePath languagePath, InputAttributes inputAttributes) {
!             return new SimpleJavadocLexer(input, tokenFactory, state, languagePath, inputAttributes);
          }
  
          public LanguageEmbedding embedding(
--- 66,73 ----
              return null; // no extra categories
          }
  
!         public Lexer<SimpleJavadocTokenId> createLexer(LexerRestartInfo<SimpleJavadocTokenId> info) {
!             return new SimpleJavadocLexer(info);
          }
  
          public LanguageEmbedding embedding(
Index: test/unit/src/org/netbeans/api/lexer/LanguagePathTest.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/api/lexer/LanguagePathTest.java Base (1.4)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/api/lexer/LanguagePathTest.java Locally Modified (Based On 1.4)
***************
*** 16,26 ****
  import java.util.Collection;
  import java.util.EnumSet;
  import org.netbeans.junit.NbTestCase;
- import org.netbeans.spi.lexer.LanguageEmbedding;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerInput;
! import org.netbeans.spi.lexer.TokenFactory;
  
  /**
   *
--- 16,24 ----
  import java.util.Collection;
  import java.util.EnumSet;
  import org.netbeans.junit.NbTestCase;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerRestartInfo;
  
  /**
   *
***************
*** 98,106 ****
              this.mimeType = mimeType;
          }
          
!         protected Lexer<TestTokenId> createLexer(
!         LexerInput input, TokenFactory<TestTokenId> tokenFactory, Object state,
!         LanguagePath languagePath, InputAttributes inputAttributes) {
              return null;
          }
  
--- 96,102 ----
              this.mimeType = mimeType;
          }
          
!         protected Lexer<TestTokenId> createLexer(LexerRestartInfo<TestTokenId> info) {
              return null;
          }
  
Index: test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexer.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexer.java Base (1.2)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexer.java Locally Modified (Based On 1.2)
***************
*** 15,28 ****
  
  import java.util.HashMap;
  import java.util.Map;
- import org.netbeans.api.lexer.InputAttributes;
- import org.netbeans.api.lexer.LanguagePath;
  import org.netbeans.api.lexer.Token;
  import org.netbeans.spi.lexer.Lexer;
  import org.netbeans.spi.lexer.LexerInput;
  import org.netbeans.spi.lexer.TokenFactory;
  
--- 15,24 ----
  
  import java.util.HashMap;
  import java.util.Map;
  import org.netbeans.api.lexer.Token;
  import org.netbeans.spi.lexer.Lexer;
  import org.netbeans.spi.lexer.LexerInput;
+ import org.netbeans.spi.lexer.LexerRestartInfo;
  import org.netbeans.spi.lexer.TokenFactory;
  
  /**
***************
*** 46,60 ****
      
      private final int version;
  
!     SimpleLexer(LexerInput input, TokenFactory<SimpleTokenId> tokenFactory, Object state,
!     LanguagePath languagePath, InputAttributes inputAttributes) {
!         this.input = input;
!         this.tokenFactory = tokenFactory;
!         assert (state == null); // never set to non-null value in state()
  
!         Integer ver = (inputAttributes != null)
!                 ? (Integer)inputAttributes.getValue(languagePath, "version")
!                 : null;
          this.version = (ver != null) ? ver.intValue() : 2;
      }
  
--- 45,56 ----
      
      private final int version;
  
!     SimpleLexer(LexerRestartInfo<SimpleTokenId> info) {
!         this.input = info.input();
!         this.tokenFactory = info.tokenFactory();
!         assert (info.state() == null); // never set to non-null value in state()
  
!         Integer ver = (Integer)info.getAttributeValue("version");
          this.version = (ver != null) ? ver.intValue() : 2;
      }
  
Index: test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleStringLexer.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleStringLexer.java Base (1.2)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleStringLexer.java Locally Modified (Based On 1.2)
***************
*** 13,26 ****
  
  package org.netbeans.lib.lexer.test.simple;
  
- import org.netbeans.api.lexer.InputAttributes;
- import org.netbeans.api.lexer.LanguagePath;
  import org.netbeans.api.lexer.Token;
  import org.netbeans.spi.lexer.Lexer;
  import org.netbeans.spi.lexer.LexerInput;
  import org.netbeans.spi.lexer.TokenFactory;
  
--- 13,22 ----
  
  package org.netbeans.lib.lexer.test.simple;
  
  import org.netbeans.api.lexer.Token;
  import org.netbeans.spi.lexer.Lexer;
  import org.netbeans.spi.lexer.LexerInput;
+ import org.netbeans.spi.lexer.LexerRestartInfo;
  import org.netbeans.spi.lexer.TokenFactory;
  
  /**
***************
*** 35,45 ****
      
      private TokenFactory<SimpleStringTokenId> tokenFactory;
      
!     public SimpleStringLexer(LexerInput input, TokenFactory<SimpleStringTokenId> tokenFactory, Object state,
!     LanguagePath languagePath, InputAttributes inputAttributes) {
!         this.input = input;
!         this.tokenFactory = tokenFactory;
!         assert (state == null); // passed argument always null
      }
      
      public Object state() {
--- 34,43 ----
      
      private TokenFactory<SimpleStringTokenId> tokenFactory;
      
!     public SimpleStringLexer(LexerRestartInfo<SimpleStringTokenId> info) {
!         this.input = info.input();
!         this.tokenFactory = info.tokenFactory();
!         assert (info.state() == null); // passed argument always null
      }
      
      public Object state() {
Index: test/unit/src/org/netbeans/lib/lexer/test/state/StateTokenId.java
*** /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/state/StateTokenId.java Base (1.1)
--- /s/nb/t/nb_all/lexer/test/unit/src/org/netbeans/lib/lexer/test/state/StateTokenId.java Locally Modified (Based On 1.1)
***************
*** 15,28 ****
  
  import java.util.Collection;
  import java.util.EnumSet;
- import org.netbeans.api.lexer.InputAttributes;
  import org.netbeans.api.lexer.LanguageDescription;
- import org.netbeans.api.lexer.LanguagePath;
  import org.netbeans.api.lexer.TokenId;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerInput;
! import org.netbeans.spi.lexer.TokenFactory;
  
  /**
   * @author mmetelka
--- 15,25 ----
  
  import java.util.Collection;
  import java.util.EnumSet;
  import org.netbeans.api.lexer.LanguageDescription;
  import org.netbeans.api.lexer.TokenId;
  import org.netbeans.spi.lexer.LanguageHierarchy;
  import org.netbeans.spi.lexer.Lexer;
! import org.netbeans.spi.lexer.LexerRestartInfo;
  
  /**
   * @author mmetelka
***************
*** 51,60 ****
              return EnumSet.allOf(StateTokenId.class);
          }
  
!         protected Lexer<StateTokenId> createLexer(
!         LexerInput input, TokenFactory<StateTokenId> tokenFactory, Object state,
!         LanguagePath languagePath, InputAttributes inputAttributes) {
!             return new StateLexer(input, tokenFactory, state, languagePath, inputAttributes);
          }
          
      }.language();
--- 48,55 ----
              return EnumSet.allOf(StateTokenId.class);
          }
  
!         protected Lexer<StateTokenId> createLexer(LexerRestartInfo<StateTokenId> info) {
!             return new StateLexer(info);
          }
          
      }.language();