<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<!--NewPage-->
<HTML>
<HEAD>
<!-- Generated by javadoc (build 1.6.0_18) on Fri Dec 09 20:48:13 GMT 2011 -->
<TITLE>
Tokenizer
</TITLE>
<META NAME="date" CONTENT="2011-12-09">
<LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../../stylesheet.css" TITLE="Style">
<SCRIPT type="text/javascript">
function windowTitle()
{
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="Tokenizer";
}
}
</SCRIPT>
<NOSCRIPT>
</NOSCRIPT>
</HEAD>
<BODY BGCOLOR="white" onload="windowTitle();">
<HR>
<!-- ========= START OF TOP NAVBAR ======= -->
<A NAME="navbar_top"><!-- --></A>
<A HREF="#skip-navbar_top" title="Skip navigation links"></A>
<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0" SUMMARY="">
<TR>
<TD COLSPAN=2 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
<A NAME="navbar_top_firstrow"><!-- --></A>
<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3" SUMMARY="">
<TR ALIGN="center" VALIGN="top">
<TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../overview-summary.html"><FONT CLASS="NavBarFont1"><B>Overview</B></FONT></A> </TD>
<TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="package-summary.html"><FONT CLASS="NavBarFont1"><B>Package</B></FONT></A> </TD>
<TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> <FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT> </TD>
<TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="package-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A> </TD>
<TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A> </TD>
<TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A> </TD>
<TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A> </TD>
</TR>
</TABLE>
</TD>
<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
</EM>
</TD>
</TR>
<TR>
<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
<A HREF="../../../../../net/sf/saxon/expr/parser/Token.html" title="class in net.sf.saxon.expr.parser"><B>PREV CLASS</B></A>
<A HREF="../../../../../net/sf/saxon/expr/parser/TypeChecker.html" title="class in net.sf.saxon.expr.parser"><B>NEXT CLASS</B></A></FONT></TD>
<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
<A HREF="../../../../../index.html?net/sf/saxon/expr/parser/Tokenizer.html" target="_top"><B>FRAMES</B></A>
<A HREF="Tokenizer.html" target="_top"><B>NO FRAMES</B></A>
<SCRIPT type="text/javascript">
<!--
if(window==top) {
document.writeln('<A HREF="../../../../../allclasses-noframe.html"><B>All Classes</B></A>');
}
//-->
</SCRIPT>
<NOSCRIPT>
<A HREF="../../../../../allclasses-noframe.html"><B>All Classes</B></A>
</NOSCRIPT>
</FONT></TD>
</TR>
<TR>
<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
SUMMARY: NESTED | <A HREF="#field_summary">FIELD</A> | <A HREF="#constructor_summary">CONSTR</A> | <A HREF="#method_summary">METHOD</A></FONT></TD>
<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
DETAIL: <A HREF="#field_detail">FIELD</A> | <A HREF="#constructor_detail">CONSTR</A> | <A HREF="#method_detail">METHOD</A></FONT></TD>
</TR>
</TABLE>
<A NAME="skip-navbar_top"></A>
<!-- ========= END OF TOP NAVBAR ========= -->
<HR>
<!-- ======== START OF CLASS DATA ======== -->
<H2>
<FONT SIZE="-1">
net.sf.saxon.expr.parser</FONT>
<BR>
Class Tokenizer</H2>
<PRE>
<A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">java.lang.Object</A>
<IMG SRC="../../../../../resources/inherit.gif" ALT="extended by "><B>net.sf.saxon.expr.parser.Tokenizer</B>
</PRE>
<HR>
<DL>
<DT><PRE>public final class <B>Tokenizer</B><DT>extends <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</A></DL>
</PRE>
<P>
Tokenizer for expressions and inputs.
This code was originally derived from James Clark's xt, though it has been greatly modified since.
See copyright notice at end of file.
<P>
<P>
<HR>
<P>
<!-- =========== FIELD SUMMARY =========== -->
<A NAME="field_summary"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="2"><FONT SIZE="+2">
<B>Field Summary</B></FONT></TH>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>static int</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#BARE_NAME_STATE">BARE_NAME_STATE</A></B></CODE>
<BR>
State in which a name is NOT to be merged with what comes next, for example "("</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> int</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#currentToken">currentToken</A></B></CODE>
<BR>
The number identifying the most recently read token</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> int</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#currentTokenStartOffset">currentTokenStartOffset</A></B></CODE>
<BR>
The position in the input expression where the current token starts</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</A></CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#currentTokenValue">currentTokenValue</A></B></CODE>
<BR>
The string value of the most recently read token</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>static int</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#DEFAULT_STATE">DEFAULT_STATE</A></B></CODE>
<BR>
Initial default state of the Tokenizer</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> boolean</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#disallowUnionKeyword">disallowUnionKeyword</A></B></CODE>
<BR>
Flag to disallow "union" as a synonym for "|" when parsing XSLT 2.0 patterns</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</A></CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#input">input</A></B></CODE>
<BR>
The string being parsed</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> int</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#inputOffset">inputOffset</A></B></CODE>
<BR>
The current position within the input string</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> boolean</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#isXQuery">isXQuery</A></B></CODE>
<BR>
Flag to indicate that this is XQuery as distinct from XPath</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>static int</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#OPERATOR_STATE">OPERATOR_STATE</A></B></CODE>
<BR>
State in which the next thing to be read is an operator</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>static int</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#SEQUENCE_TYPE_STATE">SEQUENCE_TYPE_STATE</A></B></CODE>
<BR>
State in which the next thing to be read is a SequenceType</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> int</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#startLineNumber">startLineNumber</A></B></CODE>
<BR>
The starting line number (for XPath in XSLT, the line number in the stylesheet)</TD>
</TR>
</TABLE>
<!-- ======== CONSTRUCTOR SUMMARY ======== -->
<A NAME="constructor_summary"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="2"><FONT SIZE="+2">
<B>Constructor Summary</B></FONT></TH>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#Tokenizer()">Tokenizer</A></B>()</CODE>
<BR>
</TD>
</TR>
</TABLE>
<!-- ========== METHOD SUMMARY =========== -->
<A NAME="method_summary"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="2"><FONT SIZE="+2">
<B>Method Summary</B></FONT></TH>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> int</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#getColumnNumber()">getColumnNumber</A></B>()</CODE>
<BR>
Get the column number of the current token</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> int</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#getColumnNumber(int)">getColumnNumber</A></B>(int offset)</CODE>
<BR>
Return the column number corresponding to a given offset in the expression</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> long</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#getLineAndColumn(int)">getLineAndColumn</A></B>(int offset)</CODE>
<BR>
Get the line and column number corresponding to a given offset in the input expression,
as a long value with the line number in the top half
and the column number in the lower half</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> int</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#getLineNumber()">getLineNumber</A></B>()</CODE>
<BR>
Get the line number of the current token</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> int</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#getLineNumber(int)">getLineNumber</A></B>(int offset)</CODE>
<BR>
Return the line number corresponding to a given offset in the expression</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> int</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#getState()">getState</A></B>()</CODE>
<BR>
Get the current tokenizer state</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#incrementLineNumber(int)">incrementLineNumber</A></B>(int offset)</CODE>
<BR>
Increment the line number, making a record of where in the input string the newline character occurred.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#lookAhead()">lookAhead</A></B>()</CODE>
<BR>
Look ahead by one token.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#next()">next</A></B>()</CODE>
<BR>
Get the next token from the input expression.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> char</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#nextChar()">nextChar</A></B>()</CODE>
<BR>
Read next character directly.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</A></CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#recentText(int)">recentText</A></B>(int offset)</CODE>
<BR>
Get the most recently read text (for use in an error message)</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#setState(int)">setState</A></B>(int state)</CODE>
<BR>
Set the tokenizer into a special state</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#tokenize(java.lang.String, int, int, int)">tokenize</A></B>(<A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</A> input,
int start,
int end,
int lineNumber)</CODE>
<BR>
Prepare a string for tokenization.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#treatCurrentAsOperator()">treatCurrentAsOperator</A></B>()</CODE>
<BR>
Force the current token to be treated as an operator if possible</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE> void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../../net/sf/saxon/expr/parser/Tokenizer.html#unreadChar()">unreadChar</A></B>()</CODE>
<BR>
Step back one character.</TD>
</TR>
</TABLE>
<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
<TH ALIGN="left"><B>Methods inherited from class java.lang.<A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</A></B></TH>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD><CODE><A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true#clone()" title="class or interface in java.lang">clone</A>, <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true#equals(java.lang.Object)" title="class or interface in java.lang">equals</A>, <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true#finalize()" title="class or interface in java.lang">finalize</A>, <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true#getClass()" title="class or interface in java.lang">getClass</A>, <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true#hashCode()" title="class or interface in java.lang">hashCode</A>, <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true#notify()" title="class or interface in java.lang">notify</A>, <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true#notifyAll()" title="class or interface in java.lang">notifyAll</A>, <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true#toString()" title="class or interface in java.lang">toString</A>, <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true#wait()" title="class or interface in java.lang">wait</A>, <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true#wait(long)" title="class or interface in java.lang">wait</A>, <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/Object.html?is-external=true#wait(long, int)" title="class or interface in java.lang">wait</A></CODE></TD>
</TR>
</TABLE>
<P>
<!-- ============ FIELD DETAIL =========== -->
<A NAME="field_detail"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="1"><FONT SIZE="+2">
<B>Field Detail</B></FONT></TH>
</TR>
</TABLE>
<A NAME="DEFAULT_STATE"><!-- --></A><H3>
DEFAULT_STATE</H3>
<PRE>
public static final int <B>DEFAULT_STATE</B></PRE>
<DL>
<DD>Initial default state of the Tokenizer
<P>
<DL>
<DT><B>See Also:</B><DD><A HREF="../../../../../constant-values.html#net.sf.saxon.expr.parser.Tokenizer.DEFAULT_STATE">Constant Field Values</A></DL>
</DL>
<HR>
<A NAME="BARE_NAME_STATE"><!-- --></A><H3>
BARE_NAME_STATE</H3>
<PRE>
public static final int <B>BARE_NAME_STATE</B></PRE>
<DL>
<DD>State in which a name is NOT to be merged with what comes next, for example "("
<P>
<DL>
<DT><B>See Also:</B><DD><A HREF="../../../../../constant-values.html#net.sf.saxon.expr.parser.Tokenizer.BARE_NAME_STATE">Constant Field Values</A></DL>
</DL>
<HR>
<A NAME="SEQUENCE_TYPE_STATE"><!-- --></A><H3>
SEQUENCE_TYPE_STATE</H3>
<PRE>
public static final int <B>SEQUENCE_TYPE_STATE</B></PRE>
<DL>
<DD>State in which the next thing to be read is a SequenceType
<P>
<DL>
<DT><B>See Also:</B><DD><A HREF="../../../../../constant-values.html#net.sf.saxon.expr.parser.Tokenizer.SEQUENCE_TYPE_STATE">Constant Field Values</A></DL>
</DL>
<HR>
<A NAME="OPERATOR_STATE"><!-- --></A><H3>
OPERATOR_STATE</H3>
<PRE>
public static final int <B>OPERATOR_STATE</B></PRE>
<DL>
<DD>State in which the next thing to be read is an operator
<P>
<DL>
<DT><B>See Also:</B><DD><A HREF="../../../../../constant-values.html#net.sf.saxon.expr.parser.Tokenizer.OPERATOR_STATE">Constant Field Values</A></DL>
</DL>
<HR>
<A NAME="startLineNumber"><!-- --></A><H3>
startLineNumber</H3>
<PRE>
public int <B>startLineNumber</B></PRE>
<DL>
<DD>The starting line number (for XPath in XSLT, the line number in the stylesheet)
<P>
<DL>
</DL>
</DL>
<HR>
<A NAME="currentToken"><!-- --></A><H3>
currentToken</H3>
<PRE>
public int <B>currentToken</B></PRE>
<DL>
<DD>The number identifying the most recently read token
<P>
<DL>
</DL>
</DL>
<HR>
<A NAME="currentTokenValue"><!-- --></A><H3>
currentTokenValue</H3>
<PRE>
public <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</A> <B>currentTokenValue</B></PRE>
<DL>
<DD>The string value of the most recently read token
<P>
<DL>
</DL>
</DL>
<HR>
<A NAME="currentTokenStartOffset"><!-- --></A><H3>
currentTokenStartOffset</H3>
<PRE>
public int <B>currentTokenStartOffset</B></PRE>
<DL>
<DD>The position in the input expression where the current token starts
<P>
<DL>
</DL>
</DL>
<HR>
<A NAME="input"><!-- --></A><H3>
input</H3>
<PRE>
public <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</A> <B>input</B></PRE>
<DL>
<DD>The string being parsed
<P>
<DL>
</DL>
</DL>
<HR>
<A NAME="inputOffset"><!-- --></A><H3>
inputOffset</H3>
<PRE>
public int <B>inputOffset</B></PRE>
<DL>
<DD>The current position within the input string
<P>
<DL>
</DL>
</DL>
<HR>
<A NAME="disallowUnionKeyword"><!-- --></A><H3>
disallowUnionKeyword</H3>
<PRE>
public boolean <B>disallowUnionKeyword</B></PRE>
<DL>
<DD>Flag to disallow "union" as a synonym for "|" when parsing XSLT 2.0 patterns
<P>
<DL>
</DL>
</DL>
<HR>
<A NAME="isXQuery"><!-- --></A><H3>
isXQuery</H3>
<PRE>
public boolean <B>isXQuery</B></PRE>
<DL>
<DD>Flag to indicate that this is XQuery as distinct from XPath
<P>
<DL>
</DL>
</DL>
<!-- ========= CONSTRUCTOR DETAIL ======== -->
<A NAME="constructor_detail"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="1"><FONT SIZE="+2">
<B>Constructor Detail</B></FONT></TH>
</TR>
</TABLE>
<A NAME="Tokenizer()"><!-- --></A><H3>
Tokenizer</H3>
<PRE>
public <B>Tokenizer</B>()</PRE>
<DL>
</DL>
<!-- ============ METHOD DETAIL ========== -->
<A NAME="method_detail"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="1"><FONT SIZE="+2">
<B>Method Detail</B></FONT></TH>
</TR>
</TABLE>
<A NAME="getState()"><!-- --></A><H3>
getState</H3>
<PRE>
public int <B>getState</B>()</PRE>
<DL>
<DD>Get the current tokenizer state
<P>
<DD><DL>
<DT><B>Returns:</B><DD>the current state</DL>
</DD>
</DL>
<HR>
<A NAME="setState(int)"><!-- --></A><H3>
setState</H3>
<PRE>
public void <B>setState</B>(int state)</PRE>
<DL>
<DD>Set the tokenizer into a special state
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>state</CODE> - the new state</DL>
</DD>
</DL>
<HR>
<A NAME="tokenize(java.lang.String, int, int, int)"><!-- --></A><H3>
tokenize</H3>
<PRE>
public void <B>tokenize</B>(<A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</A> input,
int start,
int end,
int lineNumber)
throws <A HREF="../../../../../net/sf/saxon/trans/XPathException.html" title="class in net.sf.saxon.trans">XPathException</A></PRE>
<DL>
<DD>Prepare a string for tokenization.
The actual tokens are obtained by calls on next()
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>input</CODE> - the string to be tokenized<DD><CODE>start</CODE> - start point within the string<DD><CODE>end</CODE> - end point within the string (last character not read):
-1 means end of string<DD><CODE>lineNumber</CODE> - the linenumber in the source where the expression appears
<DT><B>Throws:</B>
<DD><CODE><A HREF="../../../../../net/sf/saxon/trans/XPathException.html" title="class in net.sf.saxon.trans">XPathException</A></CODE> - if a lexical error occurs, e.g. unmatched
string quotes</DL>
</DD>
</DL>
<HR>
<A NAME="next()"><!-- --></A><H3>
next</H3>
<PRE>
public void <B>next</B>()
throws <A HREF="../../../../../net/sf/saxon/trans/XPathException.html" title="class in net.sf.saxon.trans">XPathException</A></PRE>
<DL>
<DD>Get the next token from the input expression. The type of token is returned in the
currentToken variable, the string value of the token in currentTokenValue.
<P>
<DD><DL>
<DT><B>Throws:</B>
<DD><CODE><A HREF="../../../../../net/sf/saxon/trans/XPathException.html" title="class in net.sf.saxon.trans">XPathException</A></CODE> - if a lexical error is detected</DL>
</DD>
</DL>
<HR>
<A NAME="treatCurrentAsOperator()"><!-- --></A><H3>
treatCurrentAsOperator</H3>
<PRE>
public void <B>treatCurrentAsOperator</B>()</PRE>
<DL>
<DD>Force the current token to be treated as an operator if possible
<P>
<DD><DL>
</DL>
</DD>
</DL>
<HR>
<A NAME="lookAhead()"><!-- --></A><H3>
lookAhead</H3>
<PRE>
public void <B>lookAhead</B>()
throws <A HREF="../../../../../net/sf/saxon/trans/XPathException.html" title="class in net.sf.saxon.trans">XPathException</A></PRE>
<DL>
<DD>Look ahead by one token. This method does the real tokenization work.
The method is normally called internally, but the XQuery parser also
calls it to resume normal tokenization after dealing with pseudo-XML
syntax.
<P>
<DD><DL>
<DT><B>Throws:</B>
<DD><CODE><A HREF="../../../../../net/sf/saxon/trans/XPathException.html" title="class in net.sf.saxon.trans">XPathException</A></CODE> - if a lexical error occurs</DL>
</DD>
</DL>
<HR>
<A NAME="nextChar()"><!-- --></A><H3>
nextChar</H3>
<PRE>
public char <B>nextChar</B>()
throws <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/StringIndexOutOfBoundsException.html?is-external=true" title="class or interface in java.lang">StringIndexOutOfBoundsException</A></PRE>
<DL>
<DD>Read next character directly. Used by the XQuery parser when parsing pseudo-XML syntax
<P>
<DD><DL>
<DT><B>Returns:</B><DD>the next character from the input
<DT><B>Throws:</B>
<DD><CODE><A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/StringIndexOutOfBoundsException.html?is-external=true" title="class or interface in java.lang">StringIndexOutOfBoundsException</A></CODE> - if an attempt is made to read beyond
the end of the string. This will only occur in the event of a syntax error in the
input.</DL>
</DD>
</DL>
<HR>
<A NAME="incrementLineNumber(int)"><!-- --></A><H3>
incrementLineNumber</H3>
<PRE>
public void <B>incrementLineNumber</B>(int offset)</PRE>
<DL>
<DD>Increment the line number, making a record of where in the input string the newline character occurred.
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>offset</CODE> - the place in the input string where the newline occurred</DL>
</DD>
</DL>
<HR>
<A NAME="unreadChar()"><!-- --></A><H3>
unreadChar</H3>
<PRE>
public void <B>unreadChar</B>()</PRE>
<DL>
<DD>Step back one character. If this steps back to a previous line, adjust the line number.
<P>
<DD><DL>
</DL>
</DD>
</DL>
<HR>
<A NAME="recentText(int)"><!-- --></A><H3>
recentText</H3>
<PRE>
public <A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</A> <B>recentText</B>(int offset)</PRE>
<DL>
<DD>Get the most recently read text (for use in an error message)
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>offset</CODE> - the offset of the offending token, if known, or -1 to use the current offset
<DT><B>Returns:</B><DD>a chunk of text leading up to the error</DL>
</DD>
</DL>
<HR>
<A NAME="getLineNumber()"><!-- --></A><H3>
getLineNumber</H3>
<PRE>
public int <B>getLineNumber</B>()</PRE>
<DL>
<DD>Get the line number of the current token
<P>
<DD><DL>
<DT><B>Returns:</B><DD>the line number</DL>
</DD>
</DL>
<HR>
<A NAME="getColumnNumber()"><!-- --></A><H3>
getColumnNumber</H3>
<PRE>
public int <B>getColumnNumber</B>()</PRE>
<DL>
<DD>Get the column number of the current token
<P>
<DD><DL>
<DT><B>Returns:</B><DD>the column number</DL>
</DD>
</DL>
<HR>
<A NAME="getLineAndColumn(int)"><!-- --></A><H3>
getLineAndColumn</H3>
<PRE>
public long <B>getLineAndColumn</B>(int offset)</PRE>
<DL>
<DD>Get the line and column number corresponding to a given offset in the input expression,
as a long value with the line number in the top half
and the column number in the lower half
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>offset</CODE> - the byte offset in the expression
<DT><B>Returns:</B><DD>the line and column number, packed together</DL>
</DD>
</DL>
<HR>
<A NAME="getLineNumber(int)"><!-- --></A><H3>
getLineNumber</H3>
<PRE>
public int <B>getLineNumber</B>(int offset)</PRE>
<DL>
<DD>Return the line number corresponding to a given offset in the expression
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>offset</CODE> - the byte offset in the expression
<DT><B>Returns:</B><DD>the line number</DL>
</DD>
</DL>
<HR>
<A NAME="getColumnNumber(int)"><!-- --></A><H3>
getColumnNumber</H3>
<PRE>
public int <B>getColumnNumber</B>(int offset)</PRE>
<DL>
<DD>Return the column number corresponding to a given offset in the expression
<P>
<DD><DL>
<DT><B>Parameters:</B><DD><CODE>offset</CODE> - the byte offset in the expression
<DT><B>Returns:</B><DD>the column number</DL>
</DD>
</DL>
<!-- ========= END OF CLASS DATA ========= -->
<HR>
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<A NAME="navbar_bottom"><!-- --></A>
<A HREF="#skip-navbar_bottom" title="Skip navigation links"></A>
<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0" SUMMARY="">
<TR>
<TD COLSPAN=2 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
<A NAME="navbar_bottom_firstrow"><!-- --></A>
<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3" SUMMARY="">
<TR ALIGN="center" VALIGN="top">
<TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../overview-summary.html"><FONT CLASS="NavBarFont1"><B>Overview</B></FONT></A> </TD>
<TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="package-summary.html"><FONT CLASS="NavBarFont1"><B>Package</B></FONT></A> </TD>
<TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> <FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT> </TD>
<TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="package-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A> </TD>
<TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A> </TD>
<TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A> </TD>
<TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A> </TD>
</TR>
</TABLE>
</TD>
<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
</EM>
</TD>
</TR>
<TR>
<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
<A HREF="../../../../../net/sf/saxon/expr/parser/Token.html" title="class in net.sf.saxon.expr.parser"><B>PREV CLASS</B></A>
<A HREF="../../../../../net/sf/saxon/expr/parser/TypeChecker.html" title="class in net.sf.saxon.expr.parser"><B>NEXT CLASS</B></A></FONT></TD>
<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
<A HREF="../../../../../index.html?net/sf/saxon/expr/parser/Tokenizer.html" target="_top"><B>FRAMES</B></A>
<A HREF="Tokenizer.html" target="_top"><B>NO FRAMES</B></A>
<SCRIPT type="text/javascript">
<!--
if(window==top) {
document.writeln('<A HREF="../../../../../allclasses-noframe.html"><B>All Classes</B></A>');
}
//-->
</SCRIPT>
<NOSCRIPT>
<A HREF="../../../../../allclasses-noframe.html"><B>All Classes</B></A>
</NOSCRIPT>
</FONT></TD>
</TR>
<TR>
<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
SUMMARY: NESTED | <A HREF="#field_summary">FIELD</A> | <A HREF="#constructor_summary">CONSTR</A> | <A HREF="#method_summary">METHOD</A></FONT></TD>
<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
DETAIL: <A HREF="#field_detail">FIELD</A> | <A HREF="#constructor_detail">CONSTR</A> | <A HREF="#method_detail">METHOD</A></FONT></TD>
</TR>
</TABLE>
<A NAME="skip-navbar_bottom"></A>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
<HR>
Copyright (c) 2004-2011 Saxonica Limited. All rights reserved.
</BODY>
</HTML>
