<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<title>SphinxBase: src/libsphinxbase/lm/ngram_model_arpa.c Source File</title>
<link href="tabs.css" rel="stylesheet" type="text/css"/>
<link href="navtree.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="navtree.js"></script>
<script type="text/javascript" src="resize.js"></script>
<script type="text/javascript">
$(document).ready(initResizable);
</script>
<link href="doxygen.css" rel="stylesheet" type="text/css"/>
</head>
<body>
<!-- Generated by Doxygen 1.7.3 -->
<div id="top">
<div id="titlearea">
<table cellspacing="0" cellpadding="0">
<tbody>
<tr style="height: 56px;">
<td style="padding-left: 0.5em;">
<div id="projectname">SphinxBase <span id="projectnumber">0.6</span></div>
</td>
</tr>
</tbody>
</table>
</div>
<div id="navrow1" class="tabs">
<ul class="tablist">
<li><a href="index.html"><span>Main Page</span></a></li>
<li><a href="pages.html"><span>Related Pages</span></a></li>
<li><a href="annotated.html"><span>Data Structures</span></a></li>
<li class="current"><a href="files.html"><span>Files</span></a></li>
</ul>
</div>
<div id="navrow2" class="tabs2">
<ul class="tablist">
<li><a href="files.html"><span>File List</span></a></li>
<li><a href="globals.html"><span>Globals</span></a></li>
</ul>
</div>
</div>
<div id="side-nav" class="ui-resizable side-nav-resizable">
<div id="nav-tree">
<div id="nav-tree-contents">
</div>
</div>
<div id="splitbar" style="-moz-user-select:none;"
class="ui-resizable-handle">
</div>
</div>
<script type="text/javascript">
initNavTree('ngram__model__arpa_8c.html','');
</script>
<div id="doc-content">
<div class="header">
<div class="headertitle">
<h1>src/libsphinxbase/lm/ngram_model_arpa.c</h1> </div>
</div>
<div class="contents">
<div class="fragment"><pre class="fragment"><a name="l00001"></a>00001 <span class="comment">/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */</span>
<a name="l00002"></a>00002 <span class="comment">/* ====================================================================</span>
<a name="l00003"></a>00003 <span class="comment"> * Copyright (c) 1999-2007 Carnegie Mellon University. All rights</span>
<a name="l00004"></a>00004 <span class="comment"> * reserved.</span>
<a name="l00005"></a>00005 <span class="comment"> *</span>
<a name="l00006"></a>00006 <span class="comment"> * Redistribution and use in source and binary forms, with or without</span>
<a name="l00007"></a>00007 <span class="comment"> * modification, are permitted provided that the following conditions</span>
<a name="l00008"></a>00008 <span class="comment"> * are met:</span>
<a name="l00009"></a>00009 <span class="comment"> *</span>
<a name="l00010"></a>00010 <span class="comment"> * 1. Redistributions of source code must retain the above copyright</span>
<a name="l00011"></a>00011 <span class="comment"> * notice, this list of conditions and the following disclaimer. </span>
<a name="l00012"></a>00012 <span class="comment"> *</span>
<a name="l00013"></a>00013 <span class="comment"> * 2. Redistributions in binary form must reproduce the above copyright</span>
<a name="l00014"></a>00014 <span class="comment"> * notice, this list of conditions and the following disclaimer in</span>
<a name="l00015"></a>00015 <span class="comment"> * the documentation and/or other materials provided with the</span>
<a name="l00016"></a>00016 <span class="comment"> * distribution.</span>
<a name="l00017"></a>00017 <span class="comment"> *</span>
<a name="l00018"></a>00018 <span class="comment"> * This work was supported in part by funding from the Defense Advanced </span>
<a name="l00019"></a>00019 <span class="comment"> * Research Projects Agency and the National Science Foundation of the </span>
<a name="l00020"></a>00020 <span class="comment"> * United States of America, and the CMU Sphinx Speech Consortium.</span>
<a name="l00021"></a>00021 <span class="comment"> *</span>
<a name="l00022"></a>00022 <span class="comment"> * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND </span>
<a name="l00023"></a>00023 <span class="comment"> * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, </span>
<a name="l00024"></a>00024 <span class="comment"> * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR</span>
<a name="l00025"></a>00025 <span class="comment"> * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY</span>
<a name="l00026"></a>00026 <span class="comment"> * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</span>
<a name="l00027"></a>00027 <span class="comment"> * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT </span>
<a name="l00028"></a>00028 <span class="comment"> * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, </span>
<a name="l00029"></a>00029 <span class="comment"> * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY </span>
<a name="l00030"></a>00030 <span class="comment"> * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT </span>
<a name="l00031"></a>00031 <span class="comment"> * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE </span>
<a name="l00032"></a>00032 <span class="comment"> * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</span>
<a name="l00033"></a>00033 <span class="comment"> *</span>
<a name="l00034"></a>00034 <span class="comment"> * ====================================================================</span>
<a name="l00035"></a>00035 <span class="comment"> *</span>
<a name="l00036"></a>00036 <span class="comment"> */</span>
<a name="l00037"></a>00037 <span class="comment">/*</span>
<a name="l00038"></a>00038 <span class="comment"> * \file ngram_model_arpa.c ARPA format language models</span>
<a name="l00039"></a>00039 <span class="comment"> *</span>
<a name="l00040"></a>00040 <span class="comment"> * Author: David Huggins-Daines <dhuggins@cs.cmu.edu></span>
<a name="l00041"></a>00041 <span class="comment"> */</span>
<a name="l00042"></a>00042
<a name="l00043"></a>00043 <span class="preprocessor">#include "sphinxbase/ckd_alloc.h"</span>
<a name="l00044"></a>00044 <span class="preprocessor">#include <string.h></span>
<a name="l00045"></a>00045 <span class="preprocessor">#include <limits.h></span>
<a name="l00046"></a>00046 <span class="preprocessor">#include <assert.h></span>
<a name="l00047"></a>00047
<a name="l00048"></a>00048 <span class="preprocessor">#include "sphinxbase/err.h"</span>
<a name="l00049"></a>00049 <span class="preprocessor">#include "sphinxbase/pio.h"</span>
<a name="l00050"></a>00050 <span class="preprocessor">#include "sphinxbase/listelem_alloc.h"</span>
<a name="l00051"></a>00051 <span class="preprocessor">#include "sphinxbase/strfuncs.h"</span>
<a name="l00052"></a>00052
<a name="l00053"></a>00053 <span class="preprocessor">#include "ngram_model_arpa.h"</span>
<a name="l00054"></a>00054
<a name="l00055"></a>00055 <span class="keyword">static</span> <a class="code" href="structngram__funcs__s.html" title="Implementation-specific functions for operating on ngram_model_t objects.">ngram_funcs_t</a> ngram_model_arpa_funcs;
<a name="l00056"></a>00056
<a name="l00057"></a>00057 <span class="preprocessor">#define TSEG_BASE(m,b) ((m)->lm3g.tseg_base[(b)>>LOG_BG_SEG_SZ])</span>
<a name="l00058"></a>00058 <span class="preprocessor"></span><span class="preprocessor">#define FIRST_BG(m,u) ((m)->lm3g.unigrams[u].bigrams)</span>
<a name="l00059"></a>00059 <span class="preprocessor"></span><span class="preprocessor">#define FIRST_TG(m,b) (TSEG_BASE((m),(b))+((m)->lm3g.bigrams[b].trigrams))</span>
<a name="l00060"></a>00060 <span class="preprocessor"></span>
<a name="l00061"></a>00061 <span class="comment">/*</span>
<a name="l00062"></a>00062 <span class="comment"> * Read and return #unigrams, #bigrams, #trigrams as stated in input file.</span>
<a name="l00063"></a>00063 <span class="comment"> */</span>
<a name="l00064"></a>00064 <span class="keyword">static</span> <span class="keywordtype">int</span>
<a name="l00065"></a>00065 ReadNgramCounts(<a class="code" href="structlineiter__t.html" title="Line iterator for files.">lineiter_t</a> **li, int32 * n_ug, int32 * n_bg, int32 * n_tg)
<a name="l00066"></a>00066 {
<a name="l00067"></a>00067 int32 ngram, ngram_cnt;
<a name="l00068"></a>00068
<a name="l00069"></a>00069 <span class="comment">/* skip file until past the '\data\' marker */</span>
<a name="l00070"></a>00070 <span class="keywordflow">while</span> (*li) {
<a name="l00071"></a>00071 <a class="code" href="strfuncs_8h.html#ac36a095632a4f16cf4e0fbcdb01de5ad" title="Remove whitespace from a string, modifying it in-place.">string_trim</a>((*li)->buf, <a class="code" href="strfuncs_8h.html#ab5c9ca15770a4bd3047705762b815df9a4fcbb0fe16fa4aa48723ba3ba10c26dd" title="Both ends of string.">STRING_BOTH</a>);
<a name="l00072"></a>00072 <span class="keywordflow">if</span> (strcmp((*li)->buf, <span class="stringliteral">"\\data\\"</span>) == 0)
<a name="l00073"></a>00073 <span class="keywordflow">break</span>;
<a name="l00074"></a>00074 *li = <a class="code" href="pio_8h.html#aff8df0b6928746d61b3520555263f71e" title="Move to the next line in the file.">lineiter_next</a>(*li);
<a name="l00075"></a>00075 }
<a name="l00076"></a>00076 <span class="keywordflow">if</span> (*li == NULL || strcmp((*li)->buf, <span class="stringliteral">"\\data\\"</span>) != 0) {
<a name="l00077"></a>00077 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"No \\data\\ mark in LM file\n"</span>);
<a name="l00078"></a>00078 <span class="keywordflow">return</span> -1;
<a name="l00079"></a>00079 }
<a name="l00080"></a>00080
<a name="l00081"></a>00081 *n_ug = *n_bg = *n_tg = 0;
<a name="l00082"></a>00082 <span class="keywordflow">while</span> ((*li = <a class="code" href="pio_8h.html#aff8df0b6928746d61b3520555263f71e" title="Move to the next line in the file.">lineiter_next</a>(*li))) {
<a name="l00083"></a>00083 <span class="keywordflow">if</span> (sscanf((*li)->buf, <span class="stringliteral">"ngram %d=%d"</span>, &ngram, &ngram_cnt) != 2)
<a name="l00084"></a>00084 <span class="keywordflow">break</span>;
<a name="l00085"></a>00085 <span class="keywordflow">switch</span> (ngram) {
<a name="l00086"></a>00086 <span class="keywordflow">case</span> 1:
<a name="l00087"></a>00087 *n_ug = ngram_cnt;
<a name="l00088"></a>00088 <span class="keywordflow">break</span>;
<a name="l00089"></a>00089 <span class="keywordflow">case</span> 2:
<a name="l00090"></a>00090 *n_bg = ngram_cnt;
<a name="l00091"></a>00091 <span class="keywordflow">break</span>;
<a name="l00092"></a>00092 <span class="keywordflow">case</span> 3:
<a name="l00093"></a>00093 *n_tg = ngram_cnt;
<a name="l00094"></a>00094 <span class="keywordflow">break</span>;
<a name="l00095"></a>00095 <span class="keywordflow">default</span>:
<a name="l00096"></a>00096 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Unknown ngram (%d)\n"</span>, ngram);
<a name="l00097"></a>00097 <span class="keywordflow">return</span> -1;
<a name="l00098"></a>00098 }
<a name="l00099"></a>00099 }
<a name="l00100"></a>00100 <span class="keywordflow">if</span> (*li == NULL) {
<a name="l00101"></a>00101 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"EOF while reading ngram counts\n"</span>);
<a name="l00102"></a>00102 <span class="keywordflow">return</span> -1;
<a name="l00103"></a>00103 }
<a name="l00104"></a>00104
<a name="l00105"></a>00105 <span class="comment">/* Position iterator to the unigrams header '\1-grams:\' */</span>
<a name="l00106"></a>00106 <span class="keywordflow">while</span> ((*li = <a class="code" href="pio_8h.html#aff8df0b6928746d61b3520555263f71e" title="Move to the next line in the file.">lineiter_next</a>(*li))) {
<a name="l00107"></a>00107 <a class="code" href="strfuncs_8h.html#ac36a095632a4f16cf4e0fbcdb01de5ad" title="Remove whitespace from a string, modifying it in-place.">string_trim</a>((*li)->buf, <a class="code" href="strfuncs_8h.html#ab5c9ca15770a4bd3047705762b815df9a4fcbb0fe16fa4aa48723ba3ba10c26dd" title="Both ends of string.">STRING_BOTH</a>);
<a name="l00108"></a>00108 <span class="keywordflow">if</span> (strcmp((*li)->buf, <span class="stringliteral">"\\1-grams:"</span>) == 0)
<a name="l00109"></a>00109 <span class="keywordflow">break</span>;
<a name="l00110"></a>00110 }
<a name="l00111"></a>00111 <span class="keywordflow">if</span> (*li == NULL) {
<a name="l00112"></a>00112 <a class="code" href="err_8h.html#a54ffbfe898d74595c586a1f48f32ef03" title="Print error text; Call perror(&quot;&quot;);.">E_ERROR_SYSTEM</a>(<span class="stringliteral">"Failed to read \\1-grams: mark"</span>);
<a name="l00113"></a>00113 <span class="keywordflow">return</span> -1;
<a name="l00114"></a>00114 }
<a name="l00115"></a>00115
<a name="l00116"></a>00116 <span class="keywordflow">if</span> ((*n_ug <= 0) || (*n_bg < 0) || (*n_tg < 0)) {
<a name="l00117"></a>00117 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Bad or missing ngram count\n"</span>);
<a name="l00118"></a>00118 <span class="keywordflow">return</span> -1;
<a name="l00119"></a>00119 }
<a name="l00120"></a>00120 <span class="keywordflow">return</span> 0;
<a name="l00121"></a>00121 }
<a name="l00122"></a>00122
<a name="l00123"></a>00123 <span class="comment">/*</span>
<a name="l00124"></a>00124 <span class="comment"> * Read in the unigrams from given file into the LM structure model.</span>
<a name="l00125"></a>00125 <span class="comment"> * On entry to this procedure, the iterator is positioned to the</span>
<a name="l00126"></a>00126 <span class="comment"> * header line '\1-grams:'.</span>
<a name="l00127"></a>00127 <span class="comment"> */</span>
<a name="l00128"></a>00128 <span class="keyword">static</span> <span class="keywordtype">int</span>
<a name="l00129"></a>00129 ReadUnigrams(<a class="code" href="structlineiter__t.html" title="Line iterator for files.">lineiter_t</a> **li, <a class="code" href="structngram__model__arpa__s.html" title="Subclass of ngram_model for ARPA file reading.">ngram_model_arpa_t</a> * model)
<a name="l00130"></a>00130 {
<a name="l00131"></a>00131 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base = &model-><a class="code" href="structngram__model__arpa__s.html#ab09caa709057492ca333e7349e006c5d" title="Base ngram_model_t structure.">base</a>;
<a name="l00132"></a>00132 int32 wcnt;
<a name="l00133"></a>00133 <span class="keywordtype">float</span> p1;
<a name="l00134"></a>00134
<a name="l00135"></a>00135 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"Reading unigrams\n"</span>);
<a name="l00136"></a>00136
<a name="l00137"></a>00137 wcnt = 0;
<a name="l00138"></a>00138 <span class="keywordflow">while</span> ((*li = <a class="code" href="pio_8h.html#aff8df0b6928746d61b3520555263f71e" title="Move to the next line in the file.">lineiter_next</a>(*li))) {
<a name="l00139"></a>00139 <span class="keywordtype">char</span> *wptr[3], *name;
<a name="l00140"></a>00140 float32 bo_wt = 0.0f;
<a name="l00141"></a>00141 <span class="keywordtype">int</span> n;
<a name="l00142"></a>00142
<a name="l00143"></a>00143 <a class="code" href="strfuncs_8h.html#ac36a095632a4f16cf4e0fbcdb01de5ad" title="Remove whitespace from a string, modifying it in-place.">string_trim</a>((*li)->buf, <a class="code" href="strfuncs_8h.html#ab5c9ca15770a4bd3047705762b815df9a4fcbb0fe16fa4aa48723ba3ba10c26dd" title="Both ends of string.">STRING_BOTH</a>);
<a name="l00144"></a>00144 <span class="keywordflow">if</span> (strcmp((*li)->buf, <span class="stringliteral">"\\2-grams:"</span>) == 0
<a name="l00145"></a>00145 || strcmp((*li)->buf, <span class="stringliteral">"\\end\\"</span>) == 0)
<a name="l00146"></a>00146 <span class="keywordflow">break</span>;
<a name="l00147"></a>00147
<a name="l00148"></a>00148 <span class="keywordflow">if</span> ((n = <a class="code" href="strfuncs_8h.html#a5b520fdebcca599db86faaf75a82173f" title="Convert a line to an array of &quot;words&quot;, based on whitespace separators.">str2words</a>((*li)->buf, wptr, 3)) < 2) {
<a name="l00149"></a>00149 <span class="keywordflow">if</span> ((*li)->buf[0] != <span class="charliteral">'\0'</span>)
<a name="l00150"></a>00150 <a class="code" href="err_8h.html#a6a794bec721b555ac1f2167f9e12f662" title="Print warning information to standard error stream.">E_WARN</a>(<span class="stringliteral">"Format error; unigram ignored: %s\n"</span>, (*li)->buf);
<a name="l00151"></a>00151 <span class="keywordflow">continue</span>;
<a name="l00152"></a>00152 }
<a name="l00153"></a>00153 <span class="keywordflow">else</span> {
<a name="l00154"></a>00154 p1 = (float)<a class="code" href="strfuncs_8h.html#ab708351fe7308551632a782bfad75a1e" title="Locale independent version of atof().">atof_c</a>(wptr[0]);
<a name="l00155"></a>00155 name = wptr[1];
<a name="l00156"></a>00156 <span class="keywordflow">if</span> (n == 3)
<a name="l00157"></a>00157 bo_wt = (float)<a class="code" href="strfuncs_8h.html#ab708351fe7308551632a782bfad75a1e" title="Locale independent version of atof().">atof_c</a>(wptr[2]);
<a name="l00158"></a>00158 }
<a name="l00159"></a>00159
<a name="l00160"></a>00160 <span class="keywordflow">if</span> (wcnt >= base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0]) {
<a name="l00161"></a>00161 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Too many unigrams\n"</span>);
<a name="l00162"></a>00162 <span class="keywordflow">return</span> -1;
<a name="l00163"></a>00163 }
<a name="l00164"></a>00164
<a name="l00165"></a>00165 <span class="comment">/* Associate name with word id */</span>
<a name="l00166"></a>00166 base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wcnt] = <a class="code" href="ckd__alloc_8h.html#ad313f92478859f9e4ea99d0f6e78c393" title="Macro for __ckd_salloc__.">ckd_salloc</a>(name);
<a name="l00167"></a>00167 <span class="keywordflow">if</span> ((<a class="code" href="hash__table_8h.html#aebfe63c3869c271b125a8413ee384412" title="Try to add a new entry with given key and associated value to hash table h.">hash_table_enter</a>(base-><a class="code" href="structngram__model__s.html#a75567419a8002ef6e916c81f5d9ee9ed" title="Mapping of unigram names to word IDs.">wid</a>, base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wcnt], (<span class="keywordtype">void</span> *)(<span class="keywordtype">long</span>)wcnt))
<a name="l00168"></a>00168 != (<span class="keywordtype">void</span> *)(<span class="keywordtype">long</span>)wcnt) {
<a name="l00169"></a>00169 <a class="code" href="err_8h.html#a6a794bec721b555ac1f2167f9e12f662" title="Print warning information to standard error stream.">E_WARN</a>(<span class="stringliteral">"Duplicate word in dictionary: %s\n"</span>, base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wcnt]);
<a name="l00170"></a>00170 }
<a name="l00171"></a>00171 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.unigrams[wcnt].<a class="code" href="structunigram__s.html#a488db9623272838a933cd4b768409fea" title="Unigram probability.">prob1</a>.l = <a class="code" href="logmath_8h.html#aacb4dddeed63a61fb927915f7e3a642e" title="Convert base 10 log (in floating point) to integer log in base B.">logmath_log10_to_log</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, p1);
<a name="l00172"></a>00172 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.unigrams[wcnt].<a class="code" href="structunigram__s.html#ad33b4af5b40a8d13ffae932bab003df6" title="Unigram backoff weight.">bo_wt1</a>.l = <a class="code" href="logmath_8h.html#aacb4dddeed63a61fb927915f7e3a642e" title="Convert base 10 log (in floating point) to integer log in base B.">logmath_log10_to_log</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, bo_wt);
<a name="l00173"></a>00173 wcnt++;
<a name="l00174"></a>00174 }
<a name="l00175"></a>00175
<a name="l00176"></a>00176 <span class="keywordflow">if</span> (base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0] != wcnt) {
<a name="l00177"></a>00177 <a class="code" href="err_8h.html#a6a794bec721b555ac1f2167f9e12f662" title="Print warning information to standard error stream.">E_WARN</a>(<span class="stringliteral">"lm_t.ucount(%d) != #unigrams read(%d)\n"</span>,
<a name="l00178"></a>00178 base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0], wcnt);
<a name="l00179"></a>00179 base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0] = wcnt;
<a name="l00180"></a>00180 base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a> = wcnt;
<a name="l00181"></a>00181 }
<a name="l00182"></a>00182 <span class="keywordflow">return</span> 0;
<a name="l00183"></a>00183 }
<a name="l00184"></a>00184
<a name="l00185"></a>00185 <span class="comment">/*</span>
<a name="l00186"></a>00186 <span class="comment"> * Read bigrams from given file into given model structure.</span>
<a name="l00187"></a>00187 <span class="comment"> */</span>
<a name="l00188"></a>00188 <span class="keyword">static</span> <span class="keywordtype">int</span>
<a name="l00189"></a>00189 ReadBigrams(<a class="code" href="structlineiter__t.html" title="Line iterator for files.">lineiter_t</a> **li, <a class="code" href="structngram__model__arpa__s.html" title="Subclass of ngram_model for ARPA file reading.">ngram_model_arpa_t</a> * model)
<a name="l00190"></a>00190 {
<a name="l00191"></a>00191 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base = &model-><a class="code" href="structngram__model__arpa__s.html#ab09caa709057492ca333e7349e006c5d" title="Base ngram_model_t structure.">base</a>;
<a name="l00192"></a>00192 int32 w1, w2, prev_w1, bgcount;
<a name="l00193"></a>00193 <a class="code" href="structbigram__s.html" title="Bigram structure.">bigram_t</a> *bgptr;
<a name="l00194"></a>00194
<a name="l00195"></a>00195 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"Reading bigrams\n"</span>);
<a name="l00196"></a>00196
<a name="l00197"></a>00197 bgcount = 0;
<a name="l00198"></a>00198 bgptr = model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.bigrams;
<a name="l00199"></a>00199 prev_w1 = -1;
<a name="l00200"></a>00200
<a name="l00201"></a>00201 <span class="keywordflow">while</span> ((*li = <a class="code" href="pio_8h.html#aff8df0b6928746d61b3520555263f71e" title="Move to the next line in the file.">lineiter_next</a>(*li))) {
<a name="l00202"></a>00202 float32 p, bo_wt = 0.0f;
<a name="l00203"></a>00203 int32 p2, bo_wt2;
<a name="l00204"></a>00204 <span class="keywordtype">char</span> *wptr[4], *word1, *word2;
<a name="l00205"></a>00205 <span class="keywordtype">int</span> n;
<a name="l00206"></a>00206
<a name="l00207"></a>00207 <a class="code" href="strfuncs_8h.html#ac36a095632a4f16cf4e0fbcdb01de5ad" title="Remove whitespace from a string, modifying it in-place.">string_trim</a>((*li)->buf, <a class="code" href="strfuncs_8h.html#ab5c9ca15770a4bd3047705762b815df9a4fcbb0fe16fa4aa48723ba3ba10c26dd" title="Both ends of string.">STRING_BOTH</a>);
<a name="l00208"></a>00208 wptr[3] = NULL;
<a name="l00209"></a>00209 <span class="keywordflow">if</span> ((n = <a class="code" href="strfuncs_8h.html#a5b520fdebcca599db86faaf75a82173f" title="Convert a line to an array of &quot;words&quot;, based on whitespace separators.">str2words</a>((*li)->buf, wptr, 4)) < 3) {
<a name="l00210"></a>00210 <span class="keywordflow">if</span> ((*li)->buf[0] != <span class="charliteral">'\0'</span>)
<a name="l00211"></a>00211 <span class="keywordflow">break</span>;
<a name="l00212"></a>00212 <span class="keywordflow">continue</span>;
<a name="l00213"></a>00213 }
<a name="l00214"></a>00214 <span class="keywordflow">else</span> {
<a name="l00215"></a>00215 p = (float32)<a class="code" href="strfuncs_8h.html#ab708351fe7308551632a782bfad75a1e" title="Locale independent version of atof().">atof_c</a>(wptr[0]);
<a name="l00216"></a>00216 word1 = wptr[1];
<a name="l00217"></a>00217 word2 = wptr[2];
<a name="l00218"></a>00218 <span class="keywordflow">if</span> (wptr[3])
<a name="l00219"></a>00219 bo_wt = (float32)<a class="code" href="strfuncs_8h.html#ab708351fe7308551632a782bfad75a1e" title="Locale independent version of atof().">atof_c</a>(wptr[3]);
<a name="l00220"></a>00220 }
<a name="l00221"></a>00221
<a name="l00222"></a>00222 <span class="keywordflow">if</span> ((w1 = <a class="code" href="ngram__model_8h.html#ad03d4355d4ea659815dc25bce8d83880" title="Look up numerical word ID.">ngram_wid</a>(base, word1)) == <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>) {
<a name="l00223"></a>00223 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Unknown word: %s, skipping bigram (%s %s)\n"</span>,
<a name="l00224"></a>00224 word1, word1, word2);
<a name="l00225"></a>00225 <span class="keywordflow">continue</span>;
<a name="l00226"></a>00226 }
<a name="l00227"></a>00227 <span class="keywordflow">if</span> ((w2 = <a class="code" href="ngram__model_8h.html#ad03d4355d4ea659815dc25bce8d83880" title="Look up numerical word ID.">ngram_wid</a>(base, word2)) == <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>) {
<a name="l00228"></a>00228 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Unknown word: %s, skipping bigram (%s %s)\n"</span>,
<a name="l00229"></a>00229 word2, word1, word2);
<a name="l00230"></a>00230 <span class="keywordflow">continue</span>;
<a name="l00231"></a>00231 }
<a name="l00232"></a>00232
<a name="l00233"></a>00233 <span class="comment">/* FIXME: Should use logmath_t quantization here. */</span>
<a name="l00234"></a>00234 <span class="comment">/* HACK!! to quantize probs to 4 decimal digits */</span>
<a name="l00235"></a>00235 p = (float32)((int32)(p * 10000)) / 10000;
<a name="l00236"></a>00236 bo_wt = (float32)((int32)(bo_wt * 10000)) / 10000;
<a name="l00237"></a>00237
<a name="l00238"></a>00238 p2 = <a class="code" href="logmath_8h.html#aacb4dddeed63a61fb927915f7e3a642e" title="Convert base 10 log (in floating point) to integer log in base B.">logmath_log10_to_log</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, p);
<a name="l00239"></a>00239 bo_wt2 = <a class="code" href="logmath_8h.html#aacb4dddeed63a61fb927915f7e3a642e" title="Convert base 10 log (in floating point) to integer log in base B.">logmath_log10_to_log</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, bo_wt);
<a name="l00240"></a>00240
<a name="l00241"></a>00241 <span class="keywordflow">if</span> (bgcount >= base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1]) {
<a name="l00242"></a>00242 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Too many bigrams\n"</span>);
<a name="l00243"></a>00243 <span class="keywordflow">return</span> -1;
<a name="l00244"></a>00244 }
<a name="l00245"></a>00245
<a name="l00246"></a>00246 bgptr-><a class="code" href="structbigram__s.html#af66faad954d56f5ceaf6ba47cdcd9b1f" title="Index of unigram entry for this.">wid</a> = w2;
<a name="l00247"></a>00247 bgptr-><a class="code" href="structbigram__s.html#a769b8a54f864ac4844e37cfa17ed1af0" title="Index into array of actual bigram probs.">prob2</a> = sorted_id(&model->sorted_prob2, &p2);
<a name="l00248"></a>00248 <span class="keywordflow">if</span> (base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[2] > 0)
<a name="l00249"></a>00249 bgptr-><a class="code" href="structbigram__s.html#a410991054e81251285cbc209964cdaf4" title="Index into array of actual bigram backoff wts.">bo_wt2</a> = sorted_id(&model->sorted_bo_wt2, &bo_wt2);
<a name="l00250"></a>00250
<a name="l00251"></a>00251 <span class="keywordflow">if</span> (w1 != prev_w1) {
<a name="l00252"></a>00252 <span class="keywordflow">if</span> (w1 < prev_w1) {
<a name="l00253"></a>00253 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Bigrams not in unigram order\n"</span>);
<a name="l00254"></a>00254 <span class="keywordflow">return</span> -1;
<a name="l00255"></a>00255 }
<a name="l00256"></a>00256
<a name="l00257"></a>00257 <span class="keywordflow">for</span> (prev_w1++; prev_w1 <= w1; prev_w1++)
<a name="l00258"></a>00258 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.unigrams[prev_w1].<a class="code" href="structunigram__s.html#ae148f631c0d9851b14bb9cb31c0c061d" title="Index of 1st entry in lm_t.bigrams[].">bigrams</a> = bgcount;
<a name="l00259"></a>00259 prev_w1 = w1;
<a name="l00260"></a>00260 }
<a name="l00261"></a>00261 bgcount++;
<a name="l00262"></a>00262 bgptr++;
<a name="l00263"></a>00263
<a name="l00264"></a>00264 <span class="keywordflow">if</span> ((bgcount & 0x0000ffff) == 0) {
<a name="l00265"></a>00265 <a class="code" href="err_8h.html#ae5a17ac5be8c738d3d977b1bea2d4538" title="Print logging information without header, to standard error stream.">E_INFOCONT</a>(<span class="stringliteral">"."</span>);
<a name="l00266"></a>00266 }
<a name="l00267"></a>00267 }
<a name="l00268"></a>00268 <span class="keywordflow">if</span> (*li == NULL || ((strcmp((*li)->buf, <span class="stringliteral">"\\end\\"</span>) != 0)
<a name="l00269"></a>00269 && (strcmp((*li)->buf, <span class="stringliteral">"\\3-grams:"</span>) != 0))) {
<a name="l00270"></a>00270 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Bad bigram: %s\n"</span>, (*li)->buf);
<a name="l00271"></a>00271 <span class="keywordflow">return</span> -1;
<a name="l00272"></a>00272 }
<a name="l00273"></a>00273
<a name="l00274"></a>00274 <span class="keywordflow">for</span> (prev_w1++; prev_w1 <= base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0]; prev_w1++)
<a name="l00275"></a>00275 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.unigrams[prev_w1].<a class="code" href="structunigram__s.html#ae148f631c0d9851b14bb9cb31c0c061d" title="Index of 1st entry in lm_t.bigrams[].">bigrams</a> = bgcount;
<a name="l00276"></a>00276
<a name="l00277"></a>00277 <span class="keywordflow">return</span> 0;
<a name="l00278"></a>00278 }
<a name="l00279"></a>00279
<a name="l00280"></a>00280 <span class="comment">/*</span>
<a name="l00281"></a>00281 <span class="comment"> * Very similar to ReadBigrams.</span>
<a name="l00282"></a>00282 <span class="comment"> */</span>
<a name="l00283"></a>00283 <span class="keyword">static</span> <span class="keywordtype">int</span>
<a name="l00284"></a>00284 ReadTrigrams(<a class="code" href="structlineiter__t.html" title="Line iterator for files.">lineiter_t</a> **li, <a class="code" href="structngram__model__arpa__s.html" title="Subclass of ngram_model for ARPA file reading.">ngram_model_arpa_t</a> * model)
<a name="l00285"></a>00285 {
<a name="l00286"></a>00286 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base = &model-><a class="code" href="structngram__model__arpa__s.html#ab09caa709057492ca333e7349e006c5d" title="Base ngram_model_t structure.">base</a>;
<a name="l00287"></a>00287 int32 i, w1, w2, w3, prev_w1, prev_w2, tgcount, prev_bg, bg, endbg;
<a name="l00288"></a>00288 int32 seg, prev_seg, prev_seg_lastbg;
<a name="l00289"></a>00289 <a class="code" href="structtrigram__s.html" title="Trigram structure.">trigram_t</a> *tgptr;
<a name="l00290"></a>00290 <a class="code" href="structbigram__s.html" title="Bigram structure.">bigram_t</a> *bgptr;
<a name="l00291"></a>00291
<a name="l00292"></a>00292 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"Reading trigrams\n"</span>);
<a name="l00293"></a>00293
<a name="l00294"></a>00294 tgcount = 0;
<a name="l00295"></a>00295 tgptr = model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.trigrams;
<a name="l00296"></a>00296 prev_w1 = -1;
<a name="l00297"></a>00297 prev_w2 = -1;
<a name="l00298"></a>00298 prev_bg = -1;
<a name="l00299"></a>00299 prev_seg = -1;
<a name="l00300"></a>00300
<a name="l00301"></a>00301 <span class="keywordflow">while</span> ((*li = <a class="code" href="pio_8h.html#aff8df0b6928746d61b3520555263f71e" title="Move to the next line in the file.">lineiter_next</a>(*li))) {
<a name="l00302"></a>00302 float32 p;
<a name="l00303"></a>00303 int32 p3;
<a name="l00304"></a>00304 <span class="keywordtype">char</span> *wptr[4], *word1, *word2, *word3;
<a name="l00305"></a>00305
<a name="l00306"></a>00306 <a class="code" href="strfuncs_8h.html#ac36a095632a4f16cf4e0fbcdb01de5ad" title="Remove whitespace from a string, modifying it in-place.">string_trim</a>((*li)->buf, <a class="code" href="strfuncs_8h.html#ab5c9ca15770a4bd3047705762b815df9a4fcbb0fe16fa4aa48723ba3ba10c26dd" title="Both ends of string.">STRING_BOTH</a>);
<a name="l00307"></a>00307 <span class="keywordflow">if</span> (<a class="code" href="strfuncs_8h.html#a5b520fdebcca599db86faaf75a82173f" title="Convert a line to an array of &quot;words&quot;, based on whitespace separators.">str2words</a>((*li)->buf, wptr, 4) != 4) {
<a name="l00308"></a>00308 <span class="keywordflow">if</span> ((*li)->buf[0] != <span class="charliteral">'\0'</span>)
<a name="l00309"></a>00309 <span class="keywordflow">break</span>;
<a name="l00310"></a>00310 <span class="keywordflow">continue</span>;
<a name="l00311"></a>00311 }
<a name="l00312"></a>00312 <span class="keywordflow">else</span> {
<a name="l00313"></a>00313 p = (float32)<a class="code" href="strfuncs_8h.html#ab708351fe7308551632a782bfad75a1e" title="Locale independent version of atof().">atof_c</a>(wptr[0]);
<a name="l00314"></a>00314 word1 = wptr[1];
<a name="l00315"></a>00315 word2 = wptr[2];
<a name="l00316"></a>00316 word3 = wptr[3];
<a name="l00317"></a>00317 }
<a name="l00318"></a>00318
<a name="l00319"></a>00319 <span class="keywordflow">if</span> ((w1 = <a class="code" href="ngram__model_8h.html#ad03d4355d4ea659815dc25bce8d83880" title="Look up numerical word ID.">ngram_wid</a>(base, word1)) == <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>) {
<a name="l00320"></a>00320 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Unknown word: %s, skipping trigram (%s %s %s)\n"</span>,
<a name="l00321"></a>00321 word1, word1, word2, word3);
<a name="l00322"></a>00322 <span class="keywordflow">continue</span>;
<a name="l00323"></a>00323 }
<a name="l00324"></a>00324 <span class="keywordflow">if</span> ((w2 = <a class="code" href="ngram__model_8h.html#ad03d4355d4ea659815dc25bce8d83880" title="Look up numerical word ID.">ngram_wid</a>(base, word2)) == <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>) {
<a name="l00325"></a>00325 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Unknown word: %s, skipping trigram (%s %s %s)\n"</span>,
<a name="l00326"></a>00326 word2, word1, word2, word3);
<a name="l00327"></a>00327 <span class="keywordflow">continue</span>;
<a name="l00328"></a>00328 }
<a name="l00329"></a>00329 <span class="keywordflow">if</span> ((w3 = <a class="code" href="ngram__model_8h.html#ad03d4355d4ea659815dc25bce8d83880" title="Look up numerical word ID.">ngram_wid</a>(base, word3)) == <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>) {
<a name="l00330"></a>00330 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Unknown word: %s, skipping trigram (%s %s %s)\n"</span>,
<a name="l00331"></a>00331 word3, word1, word2, word3);
<a name="l00332"></a>00332 <span class="keywordflow">continue</span>;
<a name="l00333"></a>00333 }
<a name="l00334"></a>00334
<a name="l00335"></a>00335 <span class="comment">/* FIXME: Should use logmath_t quantization here. */</span>
<a name="l00336"></a>00336 <span class="comment">/* HACK!! to quantize probs to 4 decimal digits */</span>
<a name="l00337"></a>00337 p = (float32)((int32)(p * 10000)) / 10000;
<a name="l00338"></a>00338 p3 = <a class="code" href="logmath_8h.html#aacb4dddeed63a61fb927915f7e3a642e" title="Convert base 10 log (in floating point) to integer log in base B.">logmath_log10_to_log</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, p);
<a name="l00339"></a>00339
<a name="l00340"></a>00340 <span class="keywordflow">if</span> (tgcount >= base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[2]) {
<a name="l00341"></a>00341 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Too many trigrams\n"</span>);
<a name="l00342"></a>00342 <span class="keywordflow">return</span> -1;
<a name="l00343"></a>00343 }
<a name="l00344"></a>00344
<a name="l00345"></a>00345 tgptr-><a class="code" href="structtrigram__s.html#a492e1fedb16fa11d26a941177c1305be" title="Index of unigram entry for this.">wid</a> = w3;
<a name="l00346"></a>00346 tgptr-><a class="code" href="structtrigram__s.html#a52f3fad42f7c653a469af0c1b75988a9" title="Index into array of actual trigram probs.">prob3</a> = sorted_id(&model->sorted_prob3, &p3);
<a name="l00347"></a>00347
<a name="l00348"></a>00348 <span class="keywordflow">if</span> ((w1 != prev_w1) || (w2 != prev_w2)) {
<a name="l00349"></a>00349 <span class="comment">/* Trigram for a new bigram; update tg info for all previous bigrams */</span>
<a name="l00350"></a>00350 <span class="keywordflow">if</span> ((w1 < prev_w1) || ((w1 == prev_w1) && (w2 < prev_w2))) {
<a name="l00351"></a>00351 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Trigrams not in bigram order\n"</span>);
<a name="l00352"></a>00352 <span class="keywordflow">return</span> -1;
<a name="l00353"></a>00353 }
<a name="l00354"></a>00354
<a name="l00355"></a>00355 bg = (w1 !=
<a name="l00356"></a>00356 prev_w1) ? model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.unigrams[w1].<a class="code" href="structunigram__s.html#ae148f631c0d9851b14bb9cb31c0c061d" title="Index of 1st entry in lm_t.bigrams[].">bigrams</a> : prev_bg + 1;
<a name="l00357"></a>00357 endbg = model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.unigrams[w1 + 1].<a class="code" href="structunigram__s.html#ae148f631c0d9851b14bb9cb31c0c061d" title="Index of 1st entry in lm_t.bigrams[].">bigrams</a>;
<a name="l00358"></a>00358 bgptr = model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.bigrams + bg;
<a name="l00359"></a>00359 for (; (bg < endbg) && (bgptr-><a class="code" href="structbigram__s.html#af66faad954d56f5ceaf6ba47cdcd9b1f" title="Index of unigram entry for this.">wid</a> != w2); bg++, bgptr++);
<a name="l00360"></a>00360 <span class="keywordflow">if</span> (bg >= endbg) {
<a name="l00361"></a>00361 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Missing bigram for trigram: %s"</span>, (*li)->buf);
<a name="l00362"></a>00362 <span class="keywordflow">return</span> -1;
<a name="l00363"></a>00363 }
<a name="l00364"></a>00364
<a name="l00365"></a>00365 <span class="comment">/* bg = bigram entry index for <w1,w2>. Update tseg_base */</span>
<a name="l00366"></a>00366 seg = bg >> LOG_BG_SEG_SZ;
<a name="l00367"></a>00367 <span class="keywordflow">for</span> (i = prev_seg + 1; i <= seg; i++)
<a name="l00368"></a>00368 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>[i] = tgcount;
<a name="l00369"></a>00369
<a name="l00370"></a>00370 <span class="comment">/* Update trigrams pointers for all bigrams until bg */</span>
<a name="l00371"></a>00371 if (prev_seg < seg) {
<a name="l00372"></a>00372 int32 tgoff = 0;
<a name="l00373"></a>00373
<a name="l00374"></a>00374 <span class="keywordflow">if</span> (prev_seg >= 0) {
<a name="l00375"></a>00375 tgoff = tgcount - model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>[prev_seg];
<a name="l00376"></a>00376 <span class="keywordflow">if</span> (tgoff > 65535) {
<a name="l00377"></a>00377 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Offset from tseg_base > 65535\n"</span>);
<a name="l00378"></a>00378 <span class="keywordflow">return</span> -1;
<a name="l00379"></a>00379 }
<a name="l00380"></a>00380 }
<a name="l00381"></a>00381
<a name="l00382"></a>00382 prev_seg_lastbg = ((prev_seg + 1) << LOG_BG_SEG_SZ) - 1;
<a name="l00383"></a>00383 bgptr = model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.bigrams + prev_bg;
<a name="l00384"></a>00384 <span class="keywordflow">for</span> (++prev_bg, ++bgptr; prev_bg <= prev_seg_lastbg;
<a name="l00385"></a>00385 prev_bg++, bgptr++)
<a name="l00386"></a>00386 bgptr-><a class="code" href="structbigram__s.html#a11c1cb1d7c40de624f09c372b36a6857" title="Index of 1st entry in lm_t.trigrams[], RELATIVE TO its segment base (see above)">trigrams</a> = tgoff;
<a name="l00387"></a>00387
<a name="l00388"></a>00388 for (; prev_bg <= bg; prev_bg++, bgptr++)
<a name="l00389"></a>00389 bgptr-><a class="code" href="structbigram__s.html#a11c1cb1d7c40de624f09c372b36a6857" title="Index of 1st entry in lm_t.trigrams[], RELATIVE TO its segment base (see above)">trigrams</a> = 0;
<a name="l00390"></a>00390 }
<a name="l00391"></a>00391 <span class="keywordflow">else</span> {
<a name="l00392"></a>00392 int32 tgoff;
<a name="l00393"></a>00393
<a name="l00394"></a>00394 tgoff = tgcount - model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>[prev_seg];
<a name="l00395"></a>00395 <span class="keywordflow">if</span> (tgoff > 65535) {
<a name="l00396"></a>00396 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Offset from tseg_base > 65535\n"</span>);
<a name="l00397"></a>00397 <span class="keywordflow">return</span> -1;
<a name="l00398"></a>00398 }
<a name="l00399"></a>00399
<a name="l00400"></a>00400 bgptr = model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.bigrams + prev_bg;
<a name="l00401"></a>00401 <span class="keywordflow">for</span> (++prev_bg, ++bgptr; prev_bg <= bg; prev_bg++, bgptr++)
<a name="l00402"></a>00402 bgptr-><a class="code" href="structbigram__s.html#a11c1cb1d7c40de624f09c372b36a6857" title="Index of 1st entry in lm_t.trigrams[], RELATIVE TO its segment base (see above)">trigrams</a> = tgoff;
<a name="l00403"></a>00403 }
<a name="l00404"></a>00404
<a name="l00405"></a>00405 prev_w1 = w1;
<a name="l00406"></a>00406 prev_w2 = w2;
<a name="l00407"></a>00407 prev_bg = bg;
<a name="l00408"></a>00408 prev_seg = seg;
<a name="l00409"></a>00409 }
<a name="l00410"></a>00410
<a name="l00411"></a>00411 tgcount++;
<a name="l00412"></a>00412 tgptr++;
<a name="l00413"></a>00413
<a name="l00414"></a>00414 <span class="keywordflow">if</span> ((tgcount & 0x0000ffff) == 0) {
<a name="l00415"></a>00415 <a class="code" href="err_8h.html#ae5a17ac5be8c738d3d977b1bea2d4538" title="Print logging information without header, to standard error stream.">E_INFOCONT</a>(<span class="stringliteral">"."</span>);
<a name="l00416"></a>00416 }
<a name="l00417"></a>00417 }
<a name="l00418"></a>00418 <span class="keywordflow">if</span> (*li == NULL || strcmp((*li)->buf, <span class="stringliteral">"\\end\\"</span>) != 0) {
<a name="l00419"></a>00419 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Bad trigram: %s\n"</span>, (*li)->buf);
<a name="l00420"></a>00420 <span class="keywordflow">return</span> -1;
<a name="l00421"></a>00421 }
<a name="l00422"></a>00422
<a name="l00423"></a>00423 <span class="keywordflow">for</span> (prev_bg++; prev_bg <= base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1]; prev_bg++) {
<a name="l00424"></a>00424 <span class="keywordflow">if</span> ((prev_bg & (BG_SEG_SZ - 1)) == 0)
<a name="l00425"></a>00425 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>[prev_bg >> LOG_BG_SEG_SZ] = tgcount;
<a name="l00426"></a>00426 <span class="keywordflow">if</span> ((tgcount - model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>[prev_bg >> LOG_BG_SEG_SZ]) > 65535) {
<a name="l00427"></a>00427 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Offset from tseg_base > 65535\n"</span>);
<a name="l00428"></a>00428 <span class="keywordflow">return</span> -1;
<a name="l00429"></a>00429 }
<a name="l00430"></a>00430 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.bigrams[prev_bg].<a class="code" href="structbigram__s.html#a11c1cb1d7c40de624f09c372b36a6857" title="Index of 1st entry in lm_t.trigrams[], RELATIVE TO its segment base (see above)">trigrams</a> =
<a name="l00431"></a>00431 tgcount - model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>[prev_bg >> LOG_BG_SEG_SZ];
<a name="l00432"></a>00432 }
<a name="l00433"></a>00433 <span class="keywordflow">return</span> 0;
<a name="l00434"></a>00434 }
<a name="l00435"></a>00435
<a name="l00436"></a>00436 <span class="keyword">static</span> <a class="code" href="structunigram__s.html" title="Unigram structure (common among all lm3g implementations)">unigram_t</a> *
<a name="l00437"></a>00437 new_unigram_table(int32 n_ug)
<a name="l00438"></a>00438 {
<a name="l00439"></a>00439 <a class="code" href="structunigram__s.html" title="Unigram structure (common among all lm3g implementations)">unigram_t</a> *table;
<a name="l00440"></a>00440 int32 i;
<a name="l00441"></a>00441
<a name="l00442"></a>00442 table = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_ug, <span class="keyword">sizeof</span>(<a class="code" href="structunigram__s.html" title="Unigram structure (common among all lm3g implementations)">unigram_t</a>));
<a name="l00443"></a>00443 <span class="keywordflow">for</span> (i = 0; i < n_ug; i++) {
<a name="l00444"></a>00444 table[i].<a class="code" href="structunigram__s.html#a488db9623272838a933cd4b768409fea" title="Unigram probability.">prob1</a>.l = INT_MIN;
<a name="l00445"></a>00445 table[i].<a class="code" href="structunigram__s.html#ad33b4af5b40a8d13ffae932bab003df6" title="Unigram backoff weight.">bo_wt1</a>.l = INT_MIN;
<a name="l00446"></a>00446 }
<a name="l00447"></a>00447 <span class="keywordflow">return</span> table;
<a name="l00448"></a>00448 }
<a name="l00449"></a>00449
<a name="l00450"></a>00450 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *
<a name="l00451"></a>00451 ngram_model_arpa_read(<a class="code" href="structcmd__ln__t.html" title="Opaque structure used to hold the results of command-line parsing.">cmd_ln_t</a> *config,
<a name="l00452"></a>00452 <span class="keyword">const</span> <span class="keywordtype">char</span> *file_name,
<a name="l00453"></a>00453 <a class="code" href="structlogmath__s.html">logmath_t</a> *lmath)
<a name="l00454"></a>00454 {
<a name="l00455"></a>00455 <a class="code" href="structlineiter__t.html" title="Line iterator for files.">lineiter_t</a> *li;
<a name="l00456"></a>00456 FILE *fp;
<a name="l00457"></a>00457 int32 is_pipe;
<a name="l00458"></a>00458 int32 n_unigram;
<a name="l00459"></a>00459 int32 n_bigram;
<a name="l00460"></a>00460 int32 n_trigram;
<a name="l00461"></a>00461 int32 n;
<a name="l00462"></a>00462 <a class="code" href="structngram__model__arpa__s.html" title="Subclass of ngram_model for ARPA file reading.">ngram_model_arpa_t</a> *model;
<a name="l00463"></a>00463 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base;
<a name="l00464"></a>00464
<a name="l00465"></a>00465 <span class="keywordflow">if</span> ((fp = <a class="code" href="pio_8h.html#aa3d71506049eb49cf03eff1b89ef281f" title="Like fopen, but use popen and zcat if it is determined that &quot;file&quot; is compressed (i...">fopen_comp</a>(file_name, <span class="stringliteral">"r"</span>, &is_pipe)) == NULL) {
<a name="l00466"></a>00466 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"File %s not found\n"</span>, file_name);
<a name="l00467"></a>00467 <span class="keywordflow">return</span> NULL;
<a name="l00468"></a>00468 }
<a name="l00469"></a>00469 li = <a class="code" href="pio_8h.html#a22d0125ab198f02f8bbe543417d99566" title="Start reading lines from a file.">lineiter_start</a>(fp);
<a name="l00470"></a>00470
<a name="l00471"></a>00471 <span class="comment">/* Read #unigrams, #bigrams, #trigrams from file */</span>
<a name="l00472"></a>00472 <span class="keywordflow">if</span> (ReadNgramCounts(&li, &n_unigram, &n_bigram, &n_trigram) == -1) {
<a name="l00473"></a>00473 <a class="code" href="pio_8h.html#a3f80e5d4d666426cef229dd41237d9cf" title="Stop reading lines from a file.">lineiter_free</a>(li);
<a name="l00474"></a>00474 <a class="code" href="pio_8h.html#a87592c3a2d0a00eed9eda014950beb65" title="Close a file opened using fopen_comp.">fclose_comp</a>(fp, is_pipe);
<a name="l00475"></a>00475 <span class="keywordflow">return</span> NULL;
<a name="l00476"></a>00476 }
<a name="l00477"></a>00477 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"ngrams 1=%d, 2=%d, 3=%d\n"</span>, n_unigram, n_bigram, n_trigram);
<a name="l00478"></a>00478
<a name="l00479"></a>00479 <span class="comment">/* Allocate space for LM, including initial OOVs and placeholders; initialize it */</span>
<a name="l00480"></a>00480 model = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(1, <span class="keyword">sizeof</span>(*model));
<a name="l00481"></a>00481 base = &model-><a class="code" href="structngram__model__arpa__s.html#ab09caa709057492ca333e7349e006c5d" title="Base ngram_model_t structure.">base</a>;
<a name="l00482"></a>00482 <span class="keywordflow">if</span> (n_trigram > 0)
<a name="l00483"></a>00483 n = 3;
<a name="l00484"></a>00484 <span class="keywordflow">else</span> <span class="keywordflow">if</span> (n_bigram > 0)
<a name="l00485"></a>00485 n = 2;
<a name="l00486"></a>00486 <span class="keywordflow">else</span>
<a name="l00487"></a>00487 n = 1;
<a name="l00488"></a>00488 <span class="comment">/* Initialize base model. */</span>
<a name="l00489"></a>00489 ngram_model_init(base, &ngram_model_arpa_funcs, lmath, n, n_unigram);
<a name="l00490"></a>00490 base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0] = n_unigram;
<a name="l00491"></a>00491 base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1] = n_bigram;
<a name="l00492"></a>00492 base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[2] = n_trigram;
<a name="l00493"></a>00493 base-><a class="code" href="structngram__model__s.html#a78a3253febced2cae4732044da466ee6" title="Are word strings writable?">writable</a> = TRUE;
<a name="l00494"></a>00494
<a name="l00495"></a>00495 <span class="comment">/*</span>
<a name="l00496"></a>00496 <span class="comment"> * Allocate one extra unigram and bigram entry: sentinels to terminate</span>
<a name="l00497"></a>00497 <span class="comment"> * followers (bigrams and trigrams, respectively) of previous entry.</span>
<a name="l00498"></a>00498 <span class="comment"> */</span>
<a name="l00499"></a>00499 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.unigrams = new_unigram_table(n_unigram + 1);
<a name="l00500"></a>00500 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.bigrams =
<a name="l00501"></a>00501 <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_bigram + 1, <span class="keyword">sizeof</span>(<a class="code" href="structbigram__s.html" title="Bigram structure.">bigram_t</a>));
<a name="l00502"></a>00502 <span class="keywordflow">if</span> (n_trigram > 0)
<a name="l00503"></a>00503 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.trigrams =
<a name="l00504"></a>00504 <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_trigram, <span class="keyword">sizeof</span>(<a class="code" href="structtrigram__s.html" title="Trigram structure.">trigram_t</a>));
<a name="l00505"></a>00505
<a name="l00506"></a>00506 <span class="keywordflow">if</span> (n_trigram > 0) {
<a name="l00507"></a>00507 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a> =
<a name="l00508"></a>00508 <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>((n_bigram + 1) / BG_SEG_SZ + 1,
<a name="l00509"></a>00509 <span class="keyword">sizeof</span>(int32));
<a name="l00510"></a>00510 }
<a name="l00511"></a>00511 <span class="keywordflow">if</span> (ReadUnigrams(&li, model) == -1) {
<a name="l00512"></a>00512 <a class="code" href="pio_8h.html#a87592c3a2d0a00eed9eda014950beb65" title="Close a file opened using fopen_comp.">fclose_comp</a>(fp, is_pipe);
<a name="l00513"></a>00513 <a class="code" href="ngram__model_8h.html#aec73d28e7285e539a0b44a7ac0cbe489" title="Release memory associated with an N-Gram model.">ngram_model_free</a>(base);
<a name="l00514"></a>00514 <span class="keywordflow">return</span> NULL;
<a name="l00515"></a>00515 }
<a name="l00516"></a>00516 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = #unigrams created\n"</span>, base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0]);
<a name="l00517"></a>00517
<a name="l00518"></a>00518 init_sorted_list(&model->sorted_prob2);
<a name="l00519"></a>00519 <span class="keywordflow">if</span> (base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[2] > 0)
<a name="l00520"></a>00520 init_sorted_list(&model->sorted_bo_wt2);
<a name="l00521"></a>00521
<a name="l00522"></a>00522 <span class="keywordflow">if</span> (base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1] > 0) {
<a name="l00523"></a>00523 <span class="keywordflow">if</span> (ReadBigrams(&li, model) == -1) {
<a name="l00524"></a>00524 <a class="code" href="pio_8h.html#a87592c3a2d0a00eed9eda014950beb65" title="Close a file opened using fopen_comp.">fclose_comp</a>(fp, is_pipe);
<a name="l00525"></a>00525 <a class="code" href="ngram__model_8h.html#aec73d28e7285e539a0b44a7ac0cbe489" title="Release memory associated with an N-Gram model.">ngram_model_free</a>(base);
<a name="l00526"></a>00526 <span class="keywordflow">return</span> NULL;
<a name="l00527"></a>00527 }
<a name="l00528"></a>00528
<a name="l00529"></a>00529 base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1] = FIRST_BG(model, base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0]);
<a name="l00530"></a>00530 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a273e6ea4c39d1a563cc59f00b4b6ee98" title="prob2 size">n_prob2</a> = model->sorted_prob2.<a class="code" href="structsorted__list__t.html#aa7468ec9a2fe7c61d2bc76ba43c575ce" title="first free element in list">free</a>;
<a name="l00531"></a>00531 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a34acf8c1ffaa4bb712ce1196eea59678" title="Table of actual bigram probs.">prob2</a> = vals_in_sorted_list(&model->sorted_prob2);
<a name="l00532"></a>00532 free_sorted_list(&model->sorted_prob2);
<a name="l00533"></a>00533 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = #bigrams created\n"</span>, base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1]);
<a name="l00534"></a>00534 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = #prob2 entries\n"</span>, model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a273e6ea4c39d1a563cc59f00b4b6ee98" title="prob2 size">n_prob2</a>);
<a name="l00535"></a>00535 }
<a name="l00536"></a>00536
<a name="l00537"></a>00537 <span class="keywordflow">if</span> (base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[2] > 0) {
<a name="l00538"></a>00538 <span class="comment">/* Create trigram bo-wts array */</span>
<a name="l00539"></a>00539 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#ad4bbdd65d13712fe653afe8b1de9b096" title="bo_wt2 size">n_bo_wt2</a> = model->sorted_bo_wt2.<a class="code" href="structsorted__list__t.html#aa7468ec9a2fe7c61d2bc76ba43c575ce" title="first free element in list">free</a>;
<a name="l00540"></a>00540 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a7ee629aa1b8e88529127cf4da470d80f" title="Table of actual bigram backoff weights.">bo_wt2</a> = vals_in_sorted_list(&model->sorted_bo_wt2);
<a name="l00541"></a>00541 free_sorted_list(&model->sorted_bo_wt2);
<a name="l00542"></a>00542 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = #bo_wt2 entries\n"</span>, model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#ad4bbdd65d13712fe653afe8b1de9b096" title="bo_wt2 size">n_bo_wt2</a>);
<a name="l00543"></a>00543
<a name="l00544"></a>00544 init_sorted_list(&model->sorted_prob3);
<a name="l00545"></a>00545
<a name="l00546"></a>00546 <span class="keywordflow">if</span> (ReadTrigrams(&li, model) == -1) {
<a name="l00547"></a>00547 <a class="code" href="pio_8h.html#a87592c3a2d0a00eed9eda014950beb65" title="Close a file opened using fopen_comp.">fclose_comp</a>(fp, is_pipe);
<a name="l00548"></a>00548 <a class="code" href="ngram__model_8h.html#aec73d28e7285e539a0b44a7ac0cbe489" title="Release memory associated with an N-Gram model.">ngram_model_free</a>(base);
<a name="l00549"></a>00549 <span class="keywordflow">return</span> NULL;
<a name="l00550"></a>00550 }
<a name="l00551"></a>00551
<a name="l00552"></a>00552 base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[2] = FIRST_TG(model, base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1]);
<a name="l00553"></a>00553 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#add28369f51e657ee54deed5291c84d09" title="prob3 size">n_prob3</a> = model->sorted_prob3.<a class="code" href="structsorted__list__t.html#aa7468ec9a2fe7c61d2bc76ba43c575ce" title="first free element in list">free</a>;
<a name="l00554"></a>00554 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#adc9ed2ad1f2daefdd40713a9dd371673" title="Table of actual trigram probs.">prob3</a> = vals_in_sorted_list(&model->sorted_prob3);
<a name="l00555"></a>00555 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = #trigrams created\n"</span>, base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[2]);
<a name="l00556"></a>00556 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = #prob3 entries\n"</span>, model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#add28369f51e657ee54deed5291c84d09" title="prob3 size">n_prob3</a>);
<a name="l00557"></a>00557
<a name="l00558"></a>00558 free_sorted_list(&model->sorted_prob3);
<a name="l00559"></a>00559
<a name="l00560"></a>00560 <span class="comment">/* Initialize tginfo */</span>
<a name="l00561"></a>00561 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a9be0c8197334e3ef632e9e3abdad6a4f" title="tginfo[lw2] is head of linked list of trigram information for some cached subset of bigrams (*...">tginfo</a> = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_unigram, <span class="keyword">sizeof</span>(<a class="code" href="structtginfo__s.html" title="Trigram information cache.">tginfo_t</a> *));
<a name="l00562"></a>00562 model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a2c96ba8aa34632e4d42908f9384d1b01" title="List element allocator for tginfo.">le</a> = <a class="code" href="listelem__alloc_8h.html#a4b08b49eaa74cbe9a3c95170cee78de7" title="Initialize and return a list element allocator.">listelem_alloc_init</a>(<span class="keyword">sizeof</span>(<a class="code" href="structtginfo__s.html" title="Trigram information cache.">tginfo_t</a>));
<a name="l00563"></a>00563 }
<a name="l00564"></a>00564
<a name="l00565"></a>00565 <a class="code" href="pio_8h.html#a3f80e5d4d666426cef229dd41237d9cf" title="Stop reading lines from a file.">lineiter_free</a>(li);
<a name="l00566"></a>00566 <a class="code" href="pio_8h.html#a87592c3a2d0a00eed9eda014950beb65" title="Close a file opened using fopen_comp.">fclose_comp</a>(fp, is_pipe);
<a name="l00567"></a>00567 <span class="keywordflow">return</span> base;
<a name="l00568"></a>00568 }
<a name="l00569"></a>00569
<a name="l00570"></a>00570 <span class="keywordtype">int</span>
<a name="l00571"></a>00571 ngram_model_arpa_write(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *model,
<a name="l00572"></a>00572 <span class="keyword">const</span> <span class="keywordtype">char</span> *file_name)
<a name="l00573"></a>00573 {
<a name="l00574"></a>00574 <a class="code" href="structngram__iter__s.html" title="Base iterator structure for N-grams.">ngram_iter_t</a> *itor;
<a name="l00575"></a>00575 FILE *fh;
<a name="l00576"></a>00576 <span class="keywordtype">int</span> i;
<a name="l00577"></a>00577
<a name="l00578"></a>00578 <span class="keywordflow">if</span> ((fh = fopen(file_name, <span class="stringliteral">"w"</span>)) == NULL) {
<a name="l00579"></a>00579 <a class="code" href="err_8h.html#a54ffbfe898d74595c586a1f48f32ef03" title="Print error text; Call perror(&quot;&quot;);.">E_ERROR_SYSTEM</a>(<span class="stringliteral">"Failed to open %s for writing"</span>, file_name);
<a name="l00580"></a>00580 <span class="keywordflow">return</span> -1;
<a name="l00581"></a>00581 }
<a name="l00582"></a>00582 fprintf(fh, <span class="stringliteral">"This is an ARPA-format language model file, generated by CMU Sphinx\n"</span>);
<a name="l00583"></a>00583
<a name="l00584"></a>00584 <span class="comment">/* The ARPA format doesn't require any extra information that</span>
<a name="l00585"></a>00585 <span class="comment"> * N-Gram iterators can't give us, so this is very</span>
<a name="l00586"></a>00586 <span class="comment"> * straightforward compared with DMP writing. */</span>
<a name="l00587"></a>00587
<a name="l00588"></a>00588 <span class="comment">/* Write N-gram counts. */</span>
<a name="l00589"></a>00589 fprintf(fh, <span class="stringliteral">"\\data\\\n"</span>);
<a name="l00590"></a>00590 <span class="keywordflow">for</span> (i = 0; i < model-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a>; ++i) {
<a name="l00591"></a>00591 fprintf(fh, <span class="stringliteral">"ngram %d=%d\n"</span>, i+1, model-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[i]);
<a name="l00592"></a>00592 }
<a name="l00593"></a>00593
<a name="l00594"></a>00594 <span class="comment">/* Write N-grams */</span>
<a name="l00595"></a>00595 <span class="keywordflow">for</span> (i = 0; i < model-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a>; ++i) {
<a name="l00596"></a>00596 fprintf(fh, <span class="stringliteral">"\n\\%d-grams:\n"</span>, i + 1);
<a name="l00597"></a>00597 <span class="keywordflow">for</span> (itor = <a class="code" href="ngram__model_8h.html#a12683dda2253dc45680102f02fbdb1e2" title="Iterate over all M-grams.">ngram_model_mgrams</a>(model, i); itor; itor = <a class="code" href="ngram__model_8h.html#a3a2b285c01393b3ebddaec1fefed11a4" title="Advance an M-gram iterator.">ngram_iter_next</a>(itor)) {
<a name="l00598"></a>00598 int32 <span class="keyword">const</span> *wids;
<a name="l00599"></a>00599 int32 score, bowt;
<a name="l00600"></a>00600 <span class="keywordtype">int</span> j;
<a name="l00601"></a>00601
<a name="l00602"></a>00602 wids = <a class="code" href="ngram__model_8h.html#a240c738781daa226a2fc13395dbdb514" title="Get information from the current M-gram in an iterator.">ngram_iter_get</a>(itor, &score, &bowt);
<a name="l00603"></a>00603 fprintf(fh, <span class="stringliteral">"%.4f "</span>, <a class="code" href="logmath_8h.html#a7c17cb624003975e84fbd141ca6e2e06" title="Convert integer log in base B to base 10 log (in floating point).">logmath_log_to_log10</a>(model-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, score));
<a name="l00604"></a>00604 <span class="keywordflow">for</span> (j = 0; j <= i; ++j) {
<a name="l00605"></a>00605 assert(wids[j] < model-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0]);
<a name="l00606"></a>00606 fprintf(fh, <span class="stringliteral">"%s "</span>, model-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wids[j]]);
<a name="l00607"></a>00607 }
<a name="l00608"></a>00608 <span class="keywordflow">if</span> (i < model->n-1)
<a name="l00609"></a>00609 fprintf(fh, <span class="stringliteral">"%.4f"</span>, <a class="code" href="logmath_8h.html#a7c17cb624003975e84fbd141ca6e2e06" title="Convert integer log in base B to base 10 log (in floating point).">logmath_log_to_log10</a>(model-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, bowt));
<a name="l00610"></a>00610 fprintf(fh, <span class="stringliteral">"\n"</span>);
<a name="l00611"></a>00611 }
<a name="l00612"></a>00612 }
<a name="l00613"></a>00613 fprintf(fh, <span class="stringliteral">"\n\\end\\\n"</span>);
<a name="l00614"></a>00614 <span class="keywordflow">return</span> fclose(fh);
<a name="l00615"></a>00615 }
<a name="l00616"></a>00616
<a name="l00617"></a>00617 <span class="keyword">static</span> <span class="keywordtype">int</span>
<a name="l00618"></a>00618 ngram_model_arpa_apply_weights(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base, float32 lw,
<a name="l00619"></a>00619 float32 wip, float32 uw)
<a name="l00620"></a>00620 {
<a name="l00621"></a>00621 <a class="code" href="structngram__model__arpa__s.html" title="Subclass of ngram_model for ARPA file reading.">ngram_model_arpa_t</a> *model = (<a class="code" href="structngram__model__arpa__s.html" title="Subclass of ngram_model for ARPA file reading.">ngram_model_arpa_t</a> *)base;
<a name="l00622"></a>00622 lm3g_apply_weights(base, &model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>, lw, wip, uw);
<a name="l00623"></a>00623 <span class="keywordflow">return</span> 0;
<a name="l00624"></a>00624 }
<a name="l00625"></a>00625
<a name="l00626"></a>00626 <span class="comment">/* Lousy "templating" for things that are largely the same in DMP and</span>
<a name="l00627"></a>00627 <span class="comment"> * ARPA models, except for the bigram and trigram types and some</span>
<a name="l00628"></a>00628 <span class="comment"> * names. */</span>
<a name="l00629"></a>00629 <span class="preprocessor">#define NGRAM_MODEL_TYPE ngram_model_arpa_t</span>
<a name="l00630"></a>00630 <span class="preprocessor"></span><span class="preprocessor">#include "lm3g_templates.c"</span>
<a name="l00631"></a>00631
<a name="l00632"></a>00632 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00633"></a>00633 ngram_model_arpa_free(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base)
<a name="l00634"></a>00634 {
<a name="l00635"></a>00635 <a class="code" href="structngram__model__arpa__s.html" title="Subclass of ngram_model for ARPA file reading.">ngram_model_arpa_t</a> *model = (<a class="code" href="structngram__model__arpa__s.html" title="Subclass of ngram_model for ARPA file reading.">ngram_model_arpa_t</a> *)base;
<a name="l00636"></a>00636 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.unigrams);
<a name="l00637"></a>00637 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.bigrams);
<a name="l00638"></a>00638 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.trigrams);
<a name="l00639"></a>00639 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a34acf8c1ffaa4bb712ce1196eea59678" title="Table of actual bigram probs.">prob2</a>);
<a name="l00640"></a>00640 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a7ee629aa1b8e88529127cf4da470d80f" title="Table of actual bigram backoff weights.">bo_wt2</a>);
<a name="l00641"></a>00641 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#adc9ed2ad1f2daefdd40713a9dd371673" title="Table of actual trigram probs.">prob3</a>);
<a name="l00642"></a>00642 lm3g_tginfo_free(base, &model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>);
<a name="l00643"></a>00643 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__arpa__s.html#abe1aa64587afdcb6134c0efaad622e78" title="Shared lm3g structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>);
<a name="l00644"></a>00644 }
<a name="l00645"></a>00645
<a name="l00646"></a>00646 <span class="keyword">static</span> <a class="code" href="structngram__funcs__s.html" title="Implementation-specific functions for operating on ngram_model_t objects.">ngram_funcs_t</a> ngram_model_arpa_funcs = {
<a name="l00647"></a>00647 ngram_model_arpa_free, <span class="comment">/* free */</span>
<a name="l00648"></a>00648 ngram_model_arpa_apply_weights, <span class="comment">/* apply_weights */</span>
<a name="l00649"></a>00649 lm3g_template_score, <span class="comment">/* score */</span>
<a name="l00650"></a>00650 lm3g_template_raw_score, <span class="comment">/* raw_score */</span>
<a name="l00651"></a>00651 lm3g_template_add_ug, <span class="comment">/* add_ug */</span>
<a name="l00652"></a>00652 lm3g_template_flush, <span class="comment">/* flush */</span>
<a name="l00653"></a>00653 lm3g_template_iter, <span class="comment">/* iter */</span>
<a name="l00654"></a>00654 lm3g_template_mgrams, <span class="comment">/* mgrams */</span>
<a name="l00655"></a>00655 lm3g_template_successors, <span class="comment">/* successors */</span>
<a name="l00656"></a>00656 lm3g_template_iter_get, <span class="comment">/* iter_get */</span>
<a name="l00657"></a>00657 lm3g_template_iter_next, <span class="comment">/* iter_next */</span>
<a name="l00658"></a>00658 lm3g_template_iter_free <span class="comment">/* iter_free */</span>
<a name="l00659"></a>00659 };
</pre></div></div>
</div>
<div id="nav-path" class="navpath">
<ul>
<li class="navelem"><b>ngram_model_arpa.c</b> </li>
<li class="footer">Generated on Tue Apr 19 2011 for SphinxBase by 
<a href="http://www.doxygen.org/index.html">
<img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.7.3 </li>
</ul>
</div>
</body>
</html>
distrib
>
Fedora
>
14
>
x86_64
>
media
>
updates
>
by-pkgid
>
0b420d0fce195cf4115dc6a3be5c2da2
>
files
>
296
