<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> <html> <head> <meta http-equiv="content-type" content="text/html; charset=UTF-8"> <title>Lexer examples</title> </head> <body><div class="manualnavbar" style="text-align: center;"> <div class="prev" style="text-align: left; float: left;"><a href="parle.examples.html">Examples</a></div> <div class="next" style="text-align: right; float: right;"><a href="parle.examples.parser.html">Parser examples</a></div> <div class="up"><a href="parle.examples.html">Examples</a></div> <div class="home"><a href="index.html">PHP Manual</a></div> </div><hr /><div id="parle.examples.lexer" class="section"> <h2 class="title">Lexer examples</h2> <div class="example" id="example-5977"> <p><strong>Example #1 Tokenize comma separated integer list</strong></p> <div class="example-contents"> <div class="phpcode"><code><span style="color: #000000"> <span style="color: #0000BB"><?php<br /><br /></span><span style="color: #007700">use </span><span style="color: #0000BB">Parle</span><span style="color: #007700">\</span><span style="color: #0000BB">Token</span><span style="color: #007700">;<br />use </span><span style="color: #0000BB">Parle</span><span style="color: #007700">\</span><span style="color: #0000BB">Lexer</span><span style="color: #007700">;<br />use </span><span style="color: #0000BB">Parle</span><span style="color: #007700">\</span><span style="color: #0000BB">LexerException</span><span style="color: #007700">;<br /><br /></span><span style="color: #FF8000">/* name => id */<br /></span><span style="color: #0000BB">$token </span><span style="color: #007700">= array(<br /> </span><span style="color: #DD0000">"COMMA" </span><span style="color: #007700">=> </span><span style="color: #0000BB">1</span><span style="color: #007700">,<br /> </span><span style="color: #DD0000">"CRLF" </span><span style="color: #007700">=> </span><span style="color: #0000BB">2</span><span style="color: #007700">,<br /> </span><span style="color: #DD0000">"DECIMAL" </span><span style="color: #007700">=> </span><span style="color: #0000BB">3</span><span style="color: #007700">,<br />);<br /></span><span style="color: #FF8000">/* id => name */<br /></span><span style="color: #0000BB">$tokenIdToName </span><span style="color: #007700">= </span><span style="color: #0000BB">array_flip</span><span style="color: #007700">(</span><span style="color: #0000BB">$token</span><span style="color: #007700">);<br /><br /></span><span style="color: #0000BB">$lex </span><span style="color: #007700">= new </span><span style="color: #0000BB">Lexer</span><span style="color: #007700">;<br /></span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">push</span><span style="color: #007700">(</span><span style="color: #DD0000">"[\x2c]"</span><span style="color: #007700">, </span><span style="color: #0000BB">$token</span><span style="color: #007700">[</span><span style="color: #DD0000">"COMMA"</span><span style="color: #007700">]);<br /></span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">push</span><span style="color: #007700">(</span><span style="color: #DD0000">"[\r][\n]"</span><span style="color: #007700">, </span><span style="color: #0000BB">$token</span><span style="color: #007700">[</span><span style="color: #DD0000">"CRLF"</span><span style="color: #007700">]);<br /></span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">push</span><span style="color: #007700">(</span><span style="color: #DD0000">"[\d]+"</span><span style="color: #007700">, </span><span style="color: #0000BB">$token</span><span style="color: #007700">[</span><span style="color: #DD0000">"DECIMAL"</span><span style="color: #007700">]);<br /></span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">build</span><span style="color: #007700">();<br /><br /></span><span style="color: #0000BB">$in </span><span style="color: #007700">= </span><span style="color: #DD0000">"0,1,2\r\n3,42,5\r\n6,77,8\r\n"</span><span style="color: #007700">;<br /><br /></span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">consume</span><span style="color: #007700">(</span><span style="color: #0000BB">$in</span><span style="color: #007700">);<br /><br />do {<br /> </span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">advance</span><span style="color: #007700">();<br /> </span><span style="color: #0000BB">$tok </span><span style="color: #007700">= </span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">getToken</span><span style="color: #007700">();<br /><br /> if (</span><span style="color: #0000BB">Token</span><span style="color: #007700">::</span><span style="color: #0000BB">UNKNOWN </span><span style="color: #007700">== </span><span style="color: #0000BB">$tok</span><span style="color: #007700">-></span><span style="color: #0000BB">id</span><span style="color: #007700">) {<br /> throw new </span><span style="color: #0000BB">LexerException</span><span style="color: #007700">(</span><span style="color: #DD0000">"Unknown token '</span><span style="color: #007700">{</span><span style="color: #0000BB">$tok</span><span style="color: #007700">-></span><span style="color: #0000BB">value</span><span style="color: #007700">}</span><span style="color: #DD0000">' at offset </span><span style="color: #007700">{</span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">marker</span><span style="color: #007700">}</span><span style="color: #DD0000">."</span><span style="color: #007700">);<br /> }<br /><br /> echo </span><span style="color: #DD0000">"TOKEN: "</span><span style="color: #007700">, </span><span style="color: #0000BB">$tokenIdToName</span><span style="color: #007700">[</span><span style="color: #0000BB">$tok</span><span style="color: #007700">-></span><span style="color: #0000BB">id</span><span style="color: #007700">], </span><span style="color: #0000BB">PHP_EOL</span><span style="color: #007700">;<br />} while (</span><span style="color: #0000BB">Token</span><span style="color: #007700">::</span><span style="color: #0000BB">EOI </span><span style="color: #007700">!= </span><span style="color: #0000BB">$tok</span><span style="color: #007700">-></span><span style="color: #0000BB">id</span><span style="color: #007700">);</span> </span> </code></div> </div> </div> <div class="example" id="example-5978"> <p><strong>Example #2 Tokenize assign statement</strong></p> <div class="example-contents"> <div class="phpcode"><code><span style="color: #000000"> <span style="color: #0000BB"><?php<br /><br /></span><span style="color: #007700">use </span><span style="color: #0000BB">Parle</span><span style="color: #007700">\{</span><span style="color: #0000BB">Token</span><span style="color: #007700">, </span><span style="color: #0000BB">Lexer</span><span style="color: #007700">};<br /><br /></span><span style="color: #0000BB">$lex </span><span style="color: #007700">= new </span><span style="color: #0000BB">Lexer</span><span style="color: #007700">;<br /><br /></span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">push</span><span style="color: #007700">(</span><span style="color: #DD0000">"\$[a-zA-Z_][a-zA-Z0-9_]*"</span><span style="color: #007700">, </span><span style="color: #0000BB">1</span><span style="color: #007700">);<br /></span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">push</span><span style="color: #007700">(</span><span style="color: #DD0000">"="</span><span style="color: #007700">, </span><span style="color: #0000BB">2</span><span style="color: #007700">);<br /></span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">push</span><span style="color: #007700">(</span><span style="color: #DD0000">"\d+"</span><span style="color: #007700">, </span><span style="color: #0000BB">3</span><span style="color: #007700">);<br /></span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">push</span><span style="color: #007700">(</span><span style="color: #DD0000">";"</span><span style="color: #007700">, </span><span style="color: #0000BB">4</span><span style="color: #007700">);<br /><br /></span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">build</span><span style="color: #007700">();<br /><br /></span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">consume</span><span style="color: #007700">(</span><span style="color: #DD0000">'$x = 42; $y = 24;'</span><span style="color: #007700">);<br /><br /><br />do {<br /> </span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">advance</span><span style="color: #007700">();<br /> </span><span style="color: #0000BB">$tok </span><span style="color: #007700">= </span><span style="color: #0000BB">$lex</span><span style="color: #007700">-></span><span style="color: #0000BB">getToken</span><span style="color: #007700">();<br /> </span><span style="color: #0000BB">var_dump</span><span style="color: #007700">(</span><span style="color: #0000BB">$tok</span><span style="color: #007700">);<br />} while (</span><span style="color: #0000BB">Token</span><span style="color: #007700">::</span><span style="color: #0000BB">EOI </span><span style="color: #007700">!= </span><span style="color: #0000BB">$tok</span><span style="color: #007700">-></span><span style="color: #0000BB">id</span><span style="color: #007700">);</span> </span> </code></div> </div> </div> </div><hr /><div class="manualnavbar" style="text-align: center;"> <div class="prev" style="text-align: left; float: left;"><a href="parle.examples.html">Examples</a></div> <div class="next" style="text-align: right; float: right;"><a href="parle.examples.parser.html">Parser examples</a></div> <div class="up"><a href="parle.examples.html">Examples</a></div> <div class="home"><a href="index.html">PHP Manual</a></div> </div></body></html>