<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<title>SphinxBase: src/libsphinxbase/lm/ngram_model_dmp.c Source File</title>
<link href="tabs.css" rel="stylesheet" type="text/css"/>
<link href="navtree.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="navtree.js"></script>
<script type="text/javascript" src="resize.js"></script>
<script type="text/javascript">
$(document).ready(initResizable);
</script>
<link href="doxygen.css" rel="stylesheet" type="text/css"/>
</head>
<body>
<!-- Generated by Doxygen 1.7.3 -->
<div id="top">
<div id="titlearea">
<table cellspacing="0" cellpadding="0">
<tbody>
<tr style="height: 56px;">
<td style="padding-left: 0.5em;">
<div id="projectname">SphinxBase <span id="projectnumber">0.6</span></div>
</td>
</tr>
</tbody>
</table>
</div>
<div id="navrow1" class="tabs">
<ul class="tablist">
<li><a href="index.html"><span>Main Page</span></a></li>
<li><a href="pages.html"><span>Related Pages</span></a></li>
<li><a href="annotated.html"><span>Data Structures</span></a></li>
<li class="current"><a href="files.html"><span>Files</span></a></li>
</ul>
</div>
<div id="navrow2" class="tabs2">
<ul class="tablist">
<li><a href="files.html"><span>File List</span></a></li>
<li><a href="globals.html"><span>Globals</span></a></li>
</ul>
</div>
</div>
<div id="side-nav" class="ui-resizable side-nav-resizable">
<div id="nav-tree">
<div id="nav-tree-contents">
</div>
</div>
<div id="splitbar" style="-moz-user-select:none;"
class="ui-resizable-handle">
</div>
</div>
<script type="text/javascript">
initNavTree('ngram__model__dmp_8c.html','');
</script>
<div id="doc-content">
<div class="header">
<div class="headertitle">
<h1>src/libsphinxbase/lm/ngram_model_dmp.c</h1> </div>
</div>
<div class="contents">
<div class="fragment"><pre class="fragment"><a name="l00001"></a>00001 <span class="comment">/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */</span>
<a name="l00002"></a>00002 <span class="comment">/* ====================================================================</span>
<a name="l00003"></a>00003 <span class="comment"> * Copyright (c) 1999-2007 Carnegie Mellon University. All rights</span>
<a name="l00004"></a>00004 <span class="comment"> * reserved.</span>
<a name="l00005"></a>00005 <span class="comment"> *</span>
<a name="l00006"></a>00006 <span class="comment"> * Redistribution and use in source and binary forms, with or without</span>
<a name="l00007"></a>00007 <span class="comment"> * modification, are permitted provided that the following conditions</span>
<a name="l00008"></a>00008 <span class="comment"> * are met:</span>
<a name="l00009"></a>00009 <span class="comment"> *</span>
<a name="l00010"></a>00010 <span class="comment"> * 1. Redistributions of source code must retain the above copyright</span>
<a name="l00011"></a>00011 <span class="comment"> * notice, this list of conditions and the following disclaimer. </span>
<a name="l00012"></a>00012 <span class="comment"> *</span>
<a name="l00013"></a>00013 <span class="comment"> * 2. Redistributions in binary form must reproduce the above copyright</span>
<a name="l00014"></a>00014 <span class="comment"> * notice, this list of conditions and the following disclaimer in</span>
<a name="l00015"></a>00015 <span class="comment"> * the documentation and/or other materials provided with the</span>
<a name="l00016"></a>00016 <span class="comment"> * distribution.</span>
<a name="l00017"></a>00017 <span class="comment"> *</span>
<a name="l00018"></a>00018 <span class="comment"> * This work was supported in part by funding from the Defense Advanced </span>
<a name="l00019"></a>00019 <span class="comment"> * Research Projects Agency and the National Science Foundation of the </span>
<a name="l00020"></a>00020 <span class="comment"> * United States of America, and the CMU Sphinx Speech Consortium.</span>
<a name="l00021"></a>00021 <span class="comment"> *</span>
<a name="l00022"></a>00022 <span class="comment"> * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND </span>
<a name="l00023"></a>00023 <span class="comment"> * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, </span>
<a name="l00024"></a>00024 <span class="comment"> * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR</span>
<a name="l00025"></a>00025 <span class="comment"> * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY</span>
<a name="l00026"></a>00026 <span class="comment"> * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</span>
<a name="l00027"></a>00027 <span class="comment"> * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT </span>
<a name="l00028"></a>00028 <span class="comment"> * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, </span>
<a name="l00029"></a>00029 <span class="comment"> * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY </span>
<a name="l00030"></a>00030 <span class="comment"> * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT </span>
<a name="l00031"></a>00031 <span class="comment"> * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE </span>
<a name="l00032"></a>00032 <span class="comment"> * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</span>
<a name="l00033"></a>00033 <span class="comment"> *</span>
<a name="l00034"></a>00034 <span class="comment"> * ====================================================================</span>
<a name="l00035"></a>00035 <span class="comment"> *</span>
<a name="l00036"></a>00036 <span class="comment"> */</span>
<a name="l00037"></a>00037 <span class="comment">/*</span>
<a name="l00038"></a>00038 <span class="comment"> * \file ngram_model_dmp.c DMP format language models</span>
<a name="l00039"></a>00039 <span class="comment"> *</span>
<a name="l00040"></a>00040 <span class="comment"> * Author: David Huggins-Daines <dhuggins@cs.cmu.edu></span>
<a name="l00041"></a>00041 <span class="comment"> */</span>
<a name="l00042"></a>00042
<a name="l00043"></a>00043 <span class="preprocessor">#include <assert.h></span>
<a name="l00044"></a>00044 <span class="preprocessor">#include <stdio.h></span>
<a name="l00045"></a>00045 <span class="preprocessor">#include <string.h></span>
<a name="l00046"></a>00046 <span class="preprocessor">#include <stdlib.h></span>
<a name="l00047"></a>00047 <span class="preprocessor">#include <limits.h></span>
<a name="l00048"></a>00048
<a name="l00049"></a>00049 <span class="preprocessor">#include "sphinxbase/ckd_alloc.h"</span>
<a name="l00050"></a>00050 <span class="preprocessor">#include "sphinxbase/pio.h"</span>
<a name="l00051"></a>00051 <span class="preprocessor">#include "sphinxbase/err.h"</span>
<a name="l00052"></a>00052 <span class="preprocessor">#include "sphinxbase/byteorder.h"</span>
<a name="l00053"></a>00053 <span class="preprocessor">#include "sphinxbase/listelem_alloc.h"</span>
<a name="l00054"></a>00054
<a name="l00055"></a>00055 <span class="preprocessor">#include "ngram_model_dmp.h"</span>
<a name="l00056"></a>00056
<a name="l00057"></a>00057 <span class="keyword">static</span> <span class="keyword">const</span> <span class="keywordtype">char</span> darpa_hdr[] = <span class="stringliteral">"Darpa Trigram LM"</span>;
<a name="l00058"></a>00058 <span class="keyword">static</span> <a class="code" href="structngram__funcs__s.html" title="Implementation-specific functions for operating on ngram_model_t objects.">ngram_funcs_t</a> ngram_model_dmp_funcs;
<a name="l00059"></a>00059
<a name="l00060"></a>00060 <span class="preprocessor">#define TSEG_BASE(m,b) ((m)->lm3g.tseg_base[(b)>>LOG_BG_SEG_SZ])</span>
<a name="l00061"></a>00061 <span class="preprocessor"></span><span class="preprocessor">#define FIRST_BG(m,u) ((m)->lm3g.unigrams[u].bigrams)</span>
<a name="l00062"></a>00062 <span class="preprocessor"></span><span class="preprocessor">#define FIRST_TG(m,b) (TSEG_BASE((m),(b))+((m)->lm3g.bigrams[b].trigrams))</span>
<a name="l00063"></a>00063 <span class="preprocessor"></span>
<a name="l00064"></a>00064 <span class="keyword">static</span> <a class="code" href="structunigram__s.html" title="Unigram structure (common among all lm3g implementations)">unigram_t</a> *
<a name="l00065"></a>00065 new_unigram_table(int32 n_ug)
<a name="l00066"></a>00066 {
<a name="l00067"></a>00067 <a class="code" href="structunigram__s.html" title="Unigram structure (common among all lm3g implementations)">unigram_t</a> *table;
<a name="l00068"></a>00068 int32 i;
<a name="l00069"></a>00069
<a name="l00070"></a>00070 table = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_ug, <span class="keyword">sizeof</span>(<a class="code" href="structunigram__s.html" title="Unigram structure (common among all lm3g implementations)">unigram_t</a>));
<a name="l00071"></a>00071 <span class="keywordflow">for</span> (i = 0; i < n_ug; i++) {
<a name="l00072"></a>00072 table[i].<a class="code" href="structunigram__s.html#a488db9623272838a933cd4b768409fea" title="Unigram probability.">prob1</a>.f = -99.0;
<a name="l00073"></a>00073 table[i].<a class="code" href="structunigram__s.html#ad33b4af5b40a8d13ffae932bab003df6" title="Unigram backoff weight.">bo_wt1</a>.f = -99.0;
<a name="l00074"></a>00074 }
<a name="l00075"></a>00075 <span class="keywordflow">return</span> table;
<a name="l00076"></a>00076 }
<a name="l00077"></a>00077
<a name="l00078"></a>00078 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *
<a name="l00079"></a>00079 ngram_model_dmp_read(<a class="code" href="structcmd__ln__t.html" title="Opaque structure used to hold the results of command-line parsing.">cmd_ln_t</a> *config,
<a name="l00080"></a>00080 <span class="keyword">const</span> <span class="keywordtype">char</span> *file_name,
<a name="l00081"></a>00081 <a class="code" href="structlogmath__s.html">logmath_t</a> *lmath)
<a name="l00082"></a>00082 {
<a name="l00083"></a>00083 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base;
<a name="l00084"></a>00084 <a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *model;
<a name="l00085"></a>00085 FILE *fp;
<a name="l00086"></a>00086 <span class="keywordtype">int</span> do_mmap, do_swap;
<a name="l00087"></a>00087 int32 is_pipe;
<a name="l00088"></a>00088 int32 i, j, k, vn, n, ts;
<a name="l00089"></a>00089 int32 n_unigram;
<a name="l00090"></a>00090 int32 n_bigram;
<a name="l00091"></a>00091 int32 n_trigram;
<a name="l00092"></a>00092 <span class="keywordtype">char</span> str[1024];
<a name="l00093"></a>00093 <a class="code" href="structunigram__s.html" title="Unigram structure (common among all lm3g implementations)">unigram_t</a> *ugptr;
<a name="l00094"></a>00094 <a class="code" href="structbigram__s.html" title="Bigram structure.">bigram_t</a> *bgptr;
<a name="l00095"></a>00095 <a class="code" href="structtrigram__s.html" title="Trigram structure.">trigram_t</a> *tgptr;
<a name="l00096"></a>00096 <span class="keywordtype">char</span> *tmp_word_str;
<a name="l00097"></a>00097 <span class="keywordtype">char</span> *map_base = NULL;
<a name="l00098"></a>00098 <span class="keywordtype">size_t</span> offset = 0, filesize;
<a name="l00099"></a>00099
<a name="l00100"></a>00100 base = NULL;
<a name="l00101"></a>00101 do_mmap = FALSE;
<a name="l00102"></a>00102 <span class="keywordflow">if</span> (config)
<a name="l00103"></a>00103 do_mmap = <a class="code" href="cmd__ln_8h.html#a159e691c95089689cf9a8f85a67830a6" title="Retrieve a boolean value from a command-line object.">cmd_ln_boolean_r</a>(config, <span class="stringliteral">"-mmap"</span>);
<a name="l00104"></a>00104
<a name="l00105"></a>00105 <span class="keywordflow">if</span> ((fp = <a class="code" href="pio_8h.html#aa3d71506049eb49cf03eff1b89ef281f" title="Like fopen, but use popen and zcat if it is determined that &quot;file&quot; is compressed (i...">fopen_comp</a>(file_name, <span class="stringliteral">"rb"</span>, &is_pipe)) == NULL) {
<a name="l00106"></a>00106 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Dump file %s not found\n"</span>, file_name);
<a name="l00107"></a>00107 <span class="keywordflow">goto</span> error_out;
<a name="l00108"></a>00108 }
<a name="l00109"></a>00109
<a name="l00110"></a>00110 <span class="keywordflow">if</span> (is_pipe && do_mmap) {
<a name="l00111"></a>00111 <a class="code" href="err_8h.html#a6a794bec721b555ac1f2167f9e12f662" title="Print warning information to standard error stream.">E_WARN</a>(<span class="stringliteral">"Dump file is compressed, will not use memory-mapped I/O\n"</span>);
<a name="l00112"></a>00112 do_mmap = 0;
<a name="l00113"></a>00113 }
<a name="l00114"></a>00114
<a name="l00115"></a>00115 do_swap = FALSE;
<a name="l00116"></a>00116 <span class="keywordflow">if</span> (fread(&k, <span class="keyword">sizeof</span>(k), 1, fp) != 1)
<a name="l00117"></a>00117 <span class="keywordflow">goto</span> error_out;
<a name="l00118"></a>00118 <span class="keywordflow">if</span> (k != strlen(darpa_hdr)+1) {
<a name="l00119"></a>00119 SWAP_INT32(&k);
<a name="l00120"></a>00120 <span class="keywordflow">if</span> (k != strlen(darpa_hdr)+1) {
<a name="l00121"></a>00121 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Wrong magic header size number %x: %s is not a dump file\n"</span>, k, file_name);
<a name="l00122"></a>00122 <span class="keywordflow">goto</span> error_out;
<a name="l00123"></a>00123 }
<a name="l00124"></a>00124 do_swap = 1;
<a name="l00125"></a>00125 }
<a name="l00126"></a>00126 <span class="keywordflow">if</span> (fread(str, 1, k, fp) != (<span class="keywordtype">size_t</span>) k) {
<a name="l00127"></a>00127 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Cannot read header\n"</span>);
<a name="l00128"></a>00128 <span class="keywordflow">goto</span> error_out;
<a name="l00129"></a>00129 }
<a name="l00130"></a>00130 <span class="keywordflow">if</span> (strncmp(str, darpa_hdr, k) != 0) {
<a name="l00131"></a>00131 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Wrong header %s: %s is not a dump file\n"</span>, darpa_hdr);
<a name="l00132"></a>00132 <span class="keywordflow">goto</span> error_out;
<a name="l00133"></a>00133 }
<a name="l00134"></a>00134
<a name="l00135"></a>00135 <span class="keywordflow">if</span> (do_mmap) {
<a name="l00136"></a>00136 <span class="keywordflow">if</span> (do_swap) {
<a name="l00137"></a>00137 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>
<a name="l00138"></a>00138 (<span class="stringliteral">"Byteswapping required, will not use memory-mapped I/O for LM file\n"</span>);
<a name="l00139"></a>00139 do_mmap = 0;
<a name="l00140"></a>00140 }
<a name="l00141"></a>00141 <span class="keywordflow">else</span> {
<a name="l00142"></a>00142 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"Will use memory-mapped I/O for LM file\n"</span>);
<a name="l00143"></a>00143 <span class="preprocessor">#ifdef __ADSPBLACKFIN__ </span><span class="comment">/* This is true for both VisualDSP++ and uClinux. */</span>
<a name="l00144"></a>00144 <a class="code" href="err_8h.html#a1a4495946ab2449d61108fe829a94613" title="Exit with non-zero status after error message.">E_FATAL</a>(<span class="stringliteral">"memory mapping is not supported at the moment."</span>);
<a name="l00145"></a>00145 <span class="preprocessor">#else</span>
<a name="l00146"></a>00146 <span class="preprocessor"></span><span class="preprocessor">#endif</span>
<a name="l00147"></a>00147 <span class="preprocessor"></span> }
<a name="l00148"></a>00148 }
<a name="l00149"></a>00149
<a name="l00150"></a>00150 <span class="keywordflow">if</span> (fread(&k, <span class="keyword">sizeof</span>(k), 1, fp) != 1)
<a name="l00151"></a>00151 <span class="keywordflow">goto</span> error_out;
<a name="l00152"></a>00152 <span class="keywordflow">if</span> (do_swap) SWAP_INT32(&k);
<a name="l00153"></a>00153 <span class="keywordflow">if</span> (fread(str, 1, k, fp) != (<span class="keywordtype">size_t</span>) k) {
<a name="l00154"></a>00154 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Cannot read LM filename in header\n"</span>);
<a name="l00155"></a>00155 <span class="keywordflow">goto</span> error_out;
<a name="l00156"></a>00156 }
<a name="l00157"></a>00157
<a name="l00158"></a>00158 <span class="comment">/* read version#, if present (must be <= 0) */</span>
<a name="l00159"></a>00159 <span class="keywordflow">if</span> (fread(&vn, <span class="keyword">sizeof</span>(vn), 1, fp) != 1)
<a name="l00160"></a>00160 <span class="keywordflow">goto</span> error_out;
<a name="l00161"></a>00161 <span class="keywordflow">if</span> (do_swap) SWAP_INT32(&vn);
<a name="l00162"></a>00162 <span class="keywordflow">if</span> (vn <= 0) {
<a name="l00163"></a>00163 <span class="comment">/* read and don't compare timestamps (we don't care) */</span>
<a name="l00164"></a>00164 <span class="keywordflow">if</span> (fread(&ts, <span class="keyword">sizeof</span>(ts), 1, fp) != 1)
<a name="l00165"></a>00165 <span class="keywordflow">goto</span> error_out;
<a name="l00166"></a>00166 <span class="keywordflow">if</span> (do_swap) SWAP_INT32(&ts);
<a name="l00167"></a>00167
<a name="l00168"></a>00168 <span class="comment">/* read and skip format description */</span>
<a name="l00169"></a>00169 <span class="keywordflow">for</span> (;;) {
<a name="l00170"></a>00170 <span class="keywordflow">if</span> (fread(&k, <span class="keyword">sizeof</span>(k), 1, fp) != 1)
<a name="l00171"></a>00171 <span class="keywordflow">goto</span> error_out;
<a name="l00172"></a>00172 <span class="keywordflow">if</span> (do_swap) SWAP_INT32(&k);
<a name="l00173"></a>00173 <span class="keywordflow">if</span> (k == 0)
<a name="l00174"></a>00174 <span class="keywordflow">break</span>;
<a name="l00175"></a>00175 <span class="keywordflow">if</span> (fread(str, 1, k, fp) != (<span class="keywordtype">size_t</span>) k) {
<a name="l00176"></a>00176 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"fread(word) failed\n"</span>);
<a name="l00177"></a>00177 <span class="keywordflow">goto</span> error_out;
<a name="l00178"></a>00178 }
<a name="l00179"></a>00179 }
<a name="l00180"></a>00180 <span class="comment">/* read model->ucount */</span>
<a name="l00181"></a>00181 <span class="keywordflow">if</span> (fread(&n_unigram, <span class="keyword">sizeof</span>(n_unigram), 1, fp) != 1)
<a name="l00182"></a>00182 <span class="keywordflow">goto</span> error_out;
<a name="l00183"></a>00183 <span class="keywordflow">if</span> (do_swap) SWAP_INT32(&n_unigram);
<a name="l00184"></a>00184 }
<a name="l00185"></a>00185 <span class="keywordflow">else</span> {
<a name="l00186"></a>00186 n_unigram = vn;
<a name="l00187"></a>00187 }
<a name="l00188"></a>00188
<a name="l00189"></a>00189 <span class="comment">/* read model->bcount, tcount */</span>
<a name="l00190"></a>00190 <span class="keywordflow">if</span> (fread(&n_bigram, <span class="keyword">sizeof</span>(n_bigram), 1, fp) != 1)
<a name="l00191"></a>00191 <span class="keywordflow">goto</span> error_out;
<a name="l00192"></a>00192 <span class="keywordflow">if</span> (do_swap) SWAP_INT32(&n_bigram);
<a name="l00193"></a>00193 <span class="keywordflow">if</span> (fread(&n_trigram, <span class="keyword">sizeof</span>(n_trigram), 1, fp) != 1)
<a name="l00194"></a>00194 <span class="keywordflow">goto</span> error_out;
<a name="l00195"></a>00195 <span class="keywordflow">if</span> (do_swap) SWAP_INT32(&n_trigram);
<a name="l00196"></a>00196 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"ngrams 1=%d, 2=%d, 3=%d\n"</span>, n_unigram, n_bigram, n_trigram);
<a name="l00197"></a>00197
<a name="l00198"></a>00198 <span class="comment">/* Allocate space for LM, including initial OOVs and placeholders; initialize it */</span>
<a name="l00199"></a>00199 model = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(1, <span class="keyword">sizeof</span>(*model));
<a name="l00200"></a>00200 base = &model-><a class="code" href="structngram__model__dmp__s.html#afd4571dc9702255aed667b5de62e5332" title="Base ngram_model_t structure.">base</a>;
<a name="l00201"></a>00201 <span class="keywordflow">if</span> (n_trigram > 0)
<a name="l00202"></a>00202 n = 3;
<a name="l00203"></a>00203 <span class="keywordflow">else</span> <span class="keywordflow">if</span> (n_bigram > 0)
<a name="l00204"></a>00204 n = 2;
<a name="l00205"></a>00205 <span class="keywordflow">else</span>
<a name="l00206"></a>00206 n = 1;
<a name="l00207"></a>00207 ngram_model_init(base, &ngram_model_dmp_funcs, lmath, n, n_unigram);
<a name="l00208"></a>00208 base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0] = n_unigram;
<a name="l00209"></a>00209 base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1] = n_bigram;
<a name="l00210"></a>00210 base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[2] = n_trigram;
<a name="l00211"></a>00211
<a name="l00212"></a>00212 <span class="comment">/* read unigrams (always in memory, as they contain dictionary</span>
<a name="l00213"></a>00213 <span class="comment"> * mappings that can't be precomputed, and also could have OOVs added) */</span>
<a name="l00214"></a>00214 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.unigrams = new_unigram_table(n_unigram + 1);
<a name="l00215"></a>00215 ugptr = model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.unigrams;
<a name="l00216"></a>00216 <span class="keywordflow">for</span> (i = 0; i <= n_unigram; ++i) {
<a name="l00217"></a>00217 <span class="comment">/* Skip over the mapping ID, we don't care about it. */</span>
<a name="l00218"></a>00218 <span class="keywordflow">if</span> (fread(ugptr, <span class="keyword">sizeof</span>(int32), 1, fp) != 1) {
<a name="l00219"></a>00219 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"fread(mapid[%d]) failed\n"</span>, i);
<a name="l00220"></a>00220 <span class="keywordflow">goto</span> error_out;
<a name="l00221"></a>00221 }
<a name="l00222"></a>00222 <span class="comment">/* Read the actual unigram structure. */</span>
<a name="l00223"></a>00223 <span class="keywordflow">if</span> (fread(ugptr, <span class="keyword">sizeof</span>(<a class="code" href="structunigram__s.html" title="Unigram structure (common among all lm3g implementations)">unigram_t</a>), 1, fp) != 1) {
<a name="l00224"></a>00224 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"fread(unigrams) failed\n"</span>);
<a name="l00225"></a>00225 <a class="code" href="ngram__model_8h.html#aec73d28e7285e539a0b44a7ac0cbe489" title="Release memory associated with an N-Gram model.">ngram_model_free</a>(base);
<a name="l00226"></a>00226 <a class="code" href="pio_8h.html#a87592c3a2d0a00eed9eda014950beb65" title="Close a file opened using fopen_comp.">fclose_comp</a>(fp, is_pipe);
<a name="l00227"></a>00227 <span class="keywordflow">return</span> NULL;
<a name="l00228"></a>00228 }
<a name="l00229"></a>00229 <span class="comment">/* Byte swap if necessary. */</span>
<a name="l00230"></a>00230 <span class="keywordflow">if</span> (do_swap) {
<a name="l00231"></a>00231 SWAP_INT32(&ugptr-><a class="code" href="structunigram__s.html#a488db9623272838a933cd4b768409fea" title="Unigram probability.">prob1</a>.l);
<a name="l00232"></a>00232 SWAP_INT32(&ugptr-><a class="code" href="structunigram__s.html#ad33b4af5b40a8d13ffae932bab003df6" title="Unigram backoff weight.">bo_wt1</a>.l);
<a name="l00233"></a>00233 SWAP_INT32(&ugptr-><a class="code" href="structunigram__s.html#ae148f631c0d9851b14bb9cb31c0c061d" title="Index of 1st entry in lm_t.bigrams[].">bigrams</a>);
<a name="l00234"></a>00234 }
<a name="l00235"></a>00235 <span class="comment">/* Convert values to log. */</span>
<a name="l00236"></a>00236 ugptr-><a class="code" href="structunigram__s.html#a488db9623272838a933cd4b768409fea" title="Unigram probability.">prob1</a>.l = <a class="code" href="logmath_8h.html#aacb4dddeed63a61fb927915f7e3a642e" title="Convert base 10 log (in floating point) to integer log in base B.">logmath_log10_to_log</a>(lmath, ugptr-><a class="code" href="structunigram__s.html#a488db9623272838a933cd4b768409fea" title="Unigram probability.">prob1</a>.f);
<a name="l00237"></a>00237 ugptr-><a class="code" href="structunigram__s.html#ad33b4af5b40a8d13ffae932bab003df6" title="Unigram backoff weight.">bo_wt1</a>.l = <a class="code" href="logmath_8h.html#aacb4dddeed63a61fb927915f7e3a642e" title="Convert base 10 log (in floating point) to integer log in base B.">logmath_log10_to_log</a>(lmath, ugptr-><a class="code" href="structunigram__s.html#ad33b4af5b40a8d13ffae932bab003df6" title="Unigram backoff weight.">bo_wt1</a>.f);
<a name="l00238"></a>00238 <a class="code" href="err_8h.html#af46f94d0e21f22f1153f8f1cd9a372d6" title="Print debugging information to standard error stream.">E_DEBUG</a>(2, (<span class="stringliteral">"ug %d: prob %d bo %d bigrams %d\n"</span>,
<a name="l00239"></a>00239 i, ugptr-><a class="code" href="structunigram__s.html#a488db9623272838a933cd4b768409fea" title="Unigram probability.">prob1</a>.l, ugptr-><a class="code" href="structunigram__s.html#ad33b4af5b40a8d13ffae932bab003df6" title="Unigram backoff weight.">bo_wt1</a>.l, ugptr-><a class="code" href="structunigram__s.html#ae148f631c0d9851b14bb9cb31c0c061d" title="Index of 1st entry in lm_t.bigrams[].">bigrams</a>));
<a name="l00240"></a>00240 ++ugptr;
<a name="l00241"></a>00241 }
<a name="l00242"></a>00242 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = LM.unigrams(+trailer) read\n"</span>, n_unigram);
<a name="l00243"></a>00243
<a name="l00244"></a>00244 <span class="comment">/* Now mmap() the file and read in the rest of the (read-only) stuff. */</span>
<a name="l00245"></a>00245 <span class="keywordflow">if</span> (do_mmap) {
<a name="l00246"></a>00246 offset = ftell(fp);
<a name="l00247"></a>00247 fseek(fp, 0, SEEK_END);
<a name="l00248"></a>00248 filesize = ftell(fp);
<a name="l00249"></a>00249 fseek(fp, offset, SEEK_SET);
<a name="l00250"></a>00250
<a name="l00251"></a>00251 <span class="comment">/* Check for improper word alignment. */</span>
<a name="l00252"></a>00252 <span class="keywordflow">if</span> (offset & 0x3) {
<a name="l00253"></a>00253 <a class="code" href="err_8h.html#a6a794bec721b555ac1f2167f9e12f662" title="Print warning information to standard error stream.">E_WARN</a>(<span class="stringliteral">"-mmap specified, but tseg_base is not word-aligned. Will not memory-map.\n"</span>);
<a name="l00254"></a>00254 do_mmap = FALSE;
<a name="l00255"></a>00255 }
<a name="l00256"></a>00256 <span class="keywordflow">else</span> {
<a name="l00257"></a>00257 model-><a class="code" href="structngram__model__dmp__s.html#ae86e39a2c9e3078f0d9cffc6cf384702" title="mmap() of dump file (or NULL if none)">dump_mmap</a> = <a class="code" href="mmio_8h.html#ae3367a51a50a6108178ecfdd6c983c61" title="Memory-map a file for reading.">mmio_file_read</a>(file_name);
<a name="l00258"></a>00258 <span class="keywordflow">if</span> (model-><a class="code" href="structngram__model__dmp__s.html#ae86e39a2c9e3078f0d9cffc6cf384702" title="mmap() of dump file (or NULL if none)">dump_mmap</a> == NULL) {
<a name="l00259"></a>00259 do_mmap = FALSE;
<a name="l00260"></a>00260 }
<a name="l00261"></a>00261 <span class="keywordflow">else</span> {
<a name="l00262"></a>00262 map_base = <a class="code" href="mmio_8h.html#a5e6d8bf5cd7785563abc18c70a31ce0d" title="Get a pointer to the memory mapped for a file.">mmio_file_ptr</a>(model-><a class="code" href="structngram__model__dmp__s.html#ae86e39a2c9e3078f0d9cffc6cf384702" title="mmap() of dump file (or NULL if none)">dump_mmap</a>);
<a name="l00263"></a>00263 }
<a name="l00264"></a>00264 }
<a name="l00265"></a>00265 }
<a name="l00266"></a>00266
<a name="l00267"></a>00267 <span class="keywordflow">if</span> (n_bigram > 0) {
<a name="l00268"></a>00268 <span class="comment">/* read bigrams */</span>
<a name="l00269"></a>00269 <span class="keywordflow">if</span> (do_mmap) {
<a name="l00270"></a>00270 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.bigrams = (<a class="code" href="structbigram__s.html" title="Bigram structure.">bigram_t</a> *) (map_base + offset);
<a name="l00271"></a>00271 offset += (n_bigram + 1) * <span class="keyword">sizeof</span>(<a class="code" href="structbigram__s.html" title="Bigram structure.">bigram_t</a>);
<a name="l00272"></a>00272 }
<a name="l00273"></a>00273 <span class="keywordflow">else</span> {
<a name="l00274"></a>00274 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.bigrams =
<a name="l00275"></a>00275 <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_bigram + 1, <span class="keyword">sizeof</span>(<a class="code" href="structbigram__s.html" title="Bigram structure.">bigram_t</a>));
<a name="l00276"></a>00276 <span class="keywordflow">if</span> (fread(model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.bigrams, <span class="keyword">sizeof</span>(<a class="code" href="structbigram__s.html" title="Bigram structure.">bigram_t</a>), n_bigram + 1, fp)
<a name="l00277"></a>00277 != (<span class="keywordtype">size_t</span>) n_bigram + 1) {
<a name="l00278"></a>00278 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"fread(bigrams) failed\n"</span>);
<a name="l00279"></a>00279 <span class="keywordflow">goto</span> error_out;
<a name="l00280"></a>00280 }
<a name="l00281"></a>00281 <span class="keywordflow">if</span> (do_swap) {
<a name="l00282"></a>00282 <span class="keywordflow">for</span> (i = 0, bgptr = model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.bigrams; i <= n_bigram;
<a name="l00283"></a>00283 i++, bgptr++) {
<a name="l00284"></a>00284 SWAP_INT16(&bgptr-><a class="code" href="structbigram__s.html#af66faad954d56f5ceaf6ba47cdcd9b1f" title="Index of unigram entry for this.">wid</a>);
<a name="l00285"></a>00285 SWAP_INT16(&bgptr-><a class="code" href="structbigram__s.html#a769b8a54f864ac4844e37cfa17ed1af0" title="Index into array of actual bigram probs.">prob2</a>);
<a name="l00286"></a>00286 SWAP_INT16(&bgptr-><a class="code" href="structbigram__s.html#a410991054e81251285cbc209964cdaf4" title="Index into array of actual bigram backoff wts.">bo_wt2</a>);
<a name="l00287"></a>00287 SWAP_INT16(&bgptr-><a class="code" href="structbigram__s.html#a11c1cb1d7c40de624f09c372b36a6857" title="Index of 1st entry in lm_t.trigrams[], RELATIVE TO its segment base (see above)">trigrams</a>);
<a name="l00288"></a>00288 }
<a name="l00289"></a>00289 }
<a name="l00290"></a>00290 }
<a name="l00291"></a>00291 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = LM.bigrams(+trailer) read\n"</span>, n_bigram);
<a name="l00292"></a>00292 }
<a name="l00293"></a>00293
<a name="l00294"></a>00294 <span class="comment">/* read trigrams */</span>
<a name="l00295"></a>00295 <span class="keywordflow">if</span> (n_trigram > 0) {
<a name="l00296"></a>00296 <span class="keywordflow">if</span> (do_mmap) {
<a name="l00297"></a>00297 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.trigrams = (<a class="code" href="structtrigram__s.html" title="Trigram structure.">trigram_t</a> *) (map_base + offset);
<a name="l00298"></a>00298 offset += n_trigram * <span class="keyword">sizeof</span>(<a class="code" href="structtrigram__s.html" title="Trigram structure.">trigram_t</a>);
<a name="l00299"></a>00299 }
<a name="l00300"></a>00300 <span class="keywordflow">else</span> {
<a name="l00301"></a>00301 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.trigrams =
<a name="l00302"></a>00302 <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_trigram, <span class="keyword">sizeof</span>(<a class="code" href="structtrigram__s.html" title="Trigram structure.">trigram_t</a>));
<a name="l00303"></a>00303 <span class="keywordflow">if</span> (fread
<a name="l00304"></a>00304 (model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.trigrams, <span class="keyword">sizeof</span>(<a class="code" href="structtrigram__s.html" title="Trigram structure.">trigram_t</a>), n_trigram, fp)
<a name="l00305"></a>00305 != (<span class="keywordtype">size_t</span>) n_trigram) {
<a name="l00306"></a>00306 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"fread(trigrams) failed\n"</span>);
<a name="l00307"></a>00307 <span class="keywordflow">goto</span> error_out;
<a name="l00308"></a>00308 }
<a name="l00309"></a>00309 <span class="keywordflow">if</span> (do_swap) {
<a name="l00310"></a>00310 <span class="keywordflow">for</span> (i = 0, tgptr = model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.trigrams; i < n_trigram;
<a name="l00311"></a>00311 i++, tgptr++) {
<a name="l00312"></a>00312 SWAP_INT16(&tgptr-><a class="code" href="structtrigram__s.html#a492e1fedb16fa11d26a941177c1305be" title="Index of unigram entry for this.">wid</a>);
<a name="l00313"></a>00313 SWAP_INT16(&tgptr-><a class="code" href="structtrigram__s.html#a52f3fad42f7c653a469af0c1b75988a9" title="Index into array of actual trigram probs.">prob3</a>);
<a name="l00314"></a>00314 }
<a name="l00315"></a>00315 }
<a name="l00316"></a>00316 }
<a name="l00317"></a>00317 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = LM.trigrams read\n"</span>, n_trigram);
<a name="l00318"></a>00318 <span class="comment">/* Initialize tginfo */</span>
<a name="l00319"></a>00319 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a9be0c8197334e3ef632e9e3abdad6a4f" title="tginfo[lw2] is head of linked list of trigram information for some cached subset of bigrams (*...">tginfo</a> = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_unigram, <span class="keyword">sizeof</span>(<a class="code" href="structtginfo__s.html" title="Trigram information cache.">tginfo_t</a> *));
<a name="l00320"></a>00320 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a2c96ba8aa34632e4d42908f9384d1b01" title="List element allocator for tginfo.">le</a> = <a class="code" href="listelem__alloc_8h.html#a4b08b49eaa74cbe9a3c95170cee78de7" title="Initialize and return a list element allocator.">listelem_alloc_init</a>(<span class="keyword">sizeof</span>(<a class="code" href="structtginfo__s.html" title="Trigram information cache.">tginfo_t</a>));
<a name="l00321"></a>00321 }
<a name="l00322"></a>00322
<a name="l00323"></a>00323 <span class="keywordflow">if</span> (n_bigram > 0) {
<a name="l00324"></a>00324 <span class="comment">/* read n_prob2 and prob2 array (in memory) */</span>
<a name="l00325"></a>00325 <span class="keywordflow">if</span> (do_mmap)
<a name="l00326"></a>00326 fseek(fp, offset, SEEK_SET);
<a name="l00327"></a>00327 <span class="keywordflow">if</span> (fread(&k, <span class="keyword">sizeof</span>(k), 1, fp) != 1)
<a name="l00328"></a>00328 <span class="keywordflow">goto</span> error_out;
<a name="l00329"></a>00329 <span class="keywordflow">if</span> (do_swap) SWAP_INT32(&k);
<a name="l00330"></a>00330 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a273e6ea4c39d1a563cc59f00b4b6ee98" title="prob2 size">n_prob2</a> = k;
<a name="l00331"></a>00331 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a34acf8c1ffaa4bb712ce1196eea59678" title="Table of actual bigram probs.">prob2</a> = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(k, <span class="keyword">sizeof</span>(*model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a34acf8c1ffaa4bb712ce1196eea59678" title="Table of actual bigram probs.">prob2</a>));
<a name="l00332"></a>00332 <span class="keywordflow">if</span> (fread(model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a34acf8c1ffaa4bb712ce1196eea59678" title="Table of actual bigram probs.">prob2</a>, <span class="keyword">sizeof</span>(*model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a34acf8c1ffaa4bb712ce1196eea59678" title="Table of actual bigram probs.">prob2</a>), k, fp) != (<span class="keywordtype">size_t</span>) k) {
<a name="l00333"></a>00333 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"fread(prob2) failed\n"</span>);
<a name="l00334"></a>00334 <span class="keywordflow">goto</span> error_out;
<a name="l00335"></a>00335 }
<a name="l00336"></a>00336 <span class="keywordflow">for</span> (i = 0; i < k; i++) {
<a name="l00337"></a>00337 <span class="keywordflow">if</span> (do_swap)
<a name="l00338"></a>00338 SWAP_INT32(&model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a34acf8c1ffaa4bb712ce1196eea59678" title="Table of actual bigram probs.">prob2</a>[i].l);
<a name="l00339"></a>00339 <span class="comment">/* Convert values to log. */</span>
<a name="l00340"></a>00340 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a34acf8c1ffaa4bb712ce1196eea59678" title="Table of actual bigram probs.">prob2</a>[i].l = <a class="code" href="logmath_8h.html#aacb4dddeed63a61fb927915f7e3a642e" title="Convert base 10 log (in floating point) to integer log in base B.">logmath_log10_to_log</a>(lmath, model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a34acf8c1ffaa4bb712ce1196eea59678" title="Table of actual bigram probs.">prob2</a>[i].f);
<a name="l00341"></a>00341 }
<a name="l00342"></a>00342 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = LM.prob2 entries read\n"</span>, k);
<a name="l00343"></a>00343 }
<a name="l00344"></a>00344
<a name="l00345"></a>00345 <span class="comment">/* read n_bo_wt2 and bo_wt2 array (in memory) */</span>
<a name="l00346"></a>00346 <span class="keywordflow">if</span> (base-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> > 2) {
<a name="l00347"></a>00347 <span class="keywordflow">if</span> (fread(&k, <span class="keyword">sizeof</span>(k), 1, fp) != 1)
<a name="l00348"></a>00348 <span class="keywordflow">goto</span> error_out;
<a name="l00349"></a>00349 <span class="keywordflow">if</span> (do_swap) SWAP_INT32(&k);
<a name="l00350"></a>00350 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#ad4bbdd65d13712fe653afe8b1de9b096" title="bo_wt2 size">n_bo_wt2</a> = k;
<a name="l00351"></a>00351 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a7ee629aa1b8e88529127cf4da470d80f" title="Table of actual bigram backoff weights.">bo_wt2</a> = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(k, <span class="keyword">sizeof</span>(*model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a7ee629aa1b8e88529127cf4da470d80f" title="Table of actual bigram backoff weights.">bo_wt2</a>));
<a name="l00352"></a>00352 <span class="keywordflow">if</span> (fread(model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a7ee629aa1b8e88529127cf4da470d80f" title="Table of actual bigram backoff weights.">bo_wt2</a>, <span class="keyword">sizeof</span>(*model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a7ee629aa1b8e88529127cf4da470d80f" title="Table of actual bigram backoff weights.">bo_wt2</a>), k, fp) != (<span class="keywordtype">size_t</span>) k) {
<a name="l00353"></a>00353 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"fread(bo_wt2) failed\n"</span>);
<a name="l00354"></a>00354 <span class="keywordflow">goto</span> error_out;
<a name="l00355"></a>00355 }
<a name="l00356"></a>00356 <span class="keywordflow">for</span> (i = 0; i < k; i++) {
<a name="l00357"></a>00357 <span class="keywordflow">if</span> (do_swap)
<a name="l00358"></a>00358 SWAP_INT32(&model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a7ee629aa1b8e88529127cf4da470d80f" title="Table of actual bigram backoff weights.">bo_wt2</a>[i].l);
<a name="l00359"></a>00359 <span class="comment">/* Convert values to log. */</span>
<a name="l00360"></a>00360 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a7ee629aa1b8e88529127cf4da470d80f" title="Table of actual bigram backoff weights.">bo_wt2</a>[i].l = <a class="code" href="logmath_8h.html#aacb4dddeed63a61fb927915f7e3a642e" title="Convert base 10 log (in floating point) to integer log in base B.">logmath_log10_to_log</a>(lmath, model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a7ee629aa1b8e88529127cf4da470d80f" title="Table of actual bigram backoff weights.">bo_wt2</a>[i].f);
<a name="l00361"></a>00361 }
<a name="l00362"></a>00362 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = LM.bo_wt2 entries read\n"</span>, k);
<a name="l00363"></a>00363 }
<a name="l00364"></a>00364
<a name="l00365"></a>00365 <span class="comment">/* read n_prob3 and prob3 array (in memory) */</span>
<a name="l00366"></a>00366 <span class="keywordflow">if</span> (base-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> > 2) {
<a name="l00367"></a>00367 <span class="keywordflow">if</span> (fread(&k, <span class="keyword">sizeof</span>(k), 1, fp) != 1)
<a name="l00368"></a>00368 <span class="keywordflow">goto</span> error_out;
<a name="l00369"></a>00369 <span class="keywordflow">if</span> (do_swap) SWAP_INT32(&k);
<a name="l00370"></a>00370 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#add28369f51e657ee54deed5291c84d09" title="prob3 size">n_prob3</a> = k;
<a name="l00371"></a>00371 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#adc9ed2ad1f2daefdd40713a9dd371673" title="Table of actual trigram probs.">prob3</a> = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(k, <span class="keyword">sizeof</span>(*model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#adc9ed2ad1f2daefdd40713a9dd371673" title="Table of actual trigram probs.">prob3</a>));
<a name="l00372"></a>00372 <span class="keywordflow">if</span> (fread(model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#adc9ed2ad1f2daefdd40713a9dd371673" title="Table of actual trigram probs.">prob3</a>, <span class="keyword">sizeof</span>(*model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#adc9ed2ad1f2daefdd40713a9dd371673" title="Table of actual trigram probs.">prob3</a>), k, fp) != (<span class="keywordtype">size_t</span>) k) {
<a name="l00373"></a>00373 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"fread(prob3) failed\n"</span>);
<a name="l00374"></a>00374 <span class="keywordflow">goto</span> error_out;
<a name="l00375"></a>00375 }
<a name="l00376"></a>00376 <span class="keywordflow">for</span> (i = 0; i < k; i++) {
<a name="l00377"></a>00377 <span class="keywordflow">if</span> (do_swap)
<a name="l00378"></a>00378 SWAP_INT32(&model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#adc9ed2ad1f2daefdd40713a9dd371673" title="Table of actual trigram probs.">prob3</a>[i].l);
<a name="l00379"></a>00379 <span class="comment">/* Convert values to log. */</span>
<a name="l00380"></a>00380 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#adc9ed2ad1f2daefdd40713a9dd371673" title="Table of actual trigram probs.">prob3</a>[i].l = <a class="code" href="logmath_8h.html#aacb4dddeed63a61fb927915f7e3a642e" title="Convert base 10 log (in floating point) to integer log in base B.">logmath_log10_to_log</a>(lmath, model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#adc9ed2ad1f2daefdd40713a9dd371673" title="Table of actual trigram probs.">prob3</a>[i].f);
<a name="l00381"></a>00381 }
<a name="l00382"></a>00382 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = LM.prob3 entries read\n"</span>, k);
<a name="l00383"></a>00383 }
<a name="l00384"></a>00384
<a name="l00385"></a>00385 <span class="comment">/* read tseg_base size and tseg_base */</span>
<a name="l00386"></a>00386 <span class="keywordflow">if</span> (do_mmap)
<a name="l00387"></a>00387 offset = ftell(fp);
<a name="l00388"></a>00388 <span class="keywordflow">if</span> (n_trigram > 0) {
<a name="l00389"></a>00389 <span class="keywordflow">if</span> (do_mmap) {
<a name="l00390"></a>00390 memcpy(&k, map_base + offset, <span class="keyword">sizeof</span>(k));
<a name="l00391"></a>00391 offset += <span class="keyword">sizeof</span>(int32);
<a name="l00392"></a>00392 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a> = (int32 *) (map_base + offset);
<a name="l00393"></a>00393 offset += k * <span class="keyword">sizeof</span>(int32);
<a name="l00394"></a>00394 }
<a name="l00395"></a>00395 <span class="keywordflow">else</span> {
<a name="l00396"></a>00396 k = (n_bigram + 1) / BG_SEG_SZ + 1;
<a name="l00397"></a>00397 <span class="keywordflow">if</span> (fread(&k, <span class="keyword">sizeof</span>(k), 1, fp) != 1)
<a name="l00398"></a>00398 <span class="keywordflow">goto</span> error_out;
<a name="l00399"></a>00399 <span class="keywordflow">if</span> (do_swap) SWAP_INT32(&k);
<a name="l00400"></a>00400 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a> = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(k, <span class="keyword">sizeof</span>(int32));
<a name="l00401"></a>00401 <span class="keywordflow">if</span> (fread(model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>, <span class="keyword">sizeof</span>(int32), k, fp) !=
<a name="l00402"></a>00402 (<span class="keywordtype">size_t</span>) k) {
<a name="l00403"></a>00403 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"fread(tseg_base) failed\n"</span>);
<a name="l00404"></a>00404 <span class="keywordflow">goto</span> error_out;
<a name="l00405"></a>00405 }
<a name="l00406"></a>00406 <span class="keywordflow">if</span> (do_swap)
<a name="l00407"></a>00407 <span class="keywordflow">for</span> (i = 0; i < k; i++)
<a name="l00408"></a>00408 SWAP_INT32(&model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>[i]);
<a name="l00409"></a>00409 }
<a name="l00410"></a>00410 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = LM.tseg_base entries read\n"</span>, k);
<a name="l00411"></a>00411 }
<a name="l00412"></a>00412
<a name="l00413"></a>00413 <span class="comment">/* read ascii word strings */</span>
<a name="l00414"></a>00414 <span class="keywordflow">if</span> (do_mmap) {
<a name="l00415"></a>00415 memcpy(&k, map_base + offset, <span class="keyword">sizeof</span>(k));
<a name="l00416"></a>00416 offset += <span class="keyword">sizeof</span>(int32);
<a name="l00417"></a>00417 tmp_word_str = (<span class="keywordtype">char</span> *) (map_base + offset);
<a name="l00418"></a>00418 offset += k;
<a name="l00419"></a>00419 }
<a name="l00420"></a>00420 <span class="keywordflow">else</span> {
<a name="l00421"></a>00421 base-><a class="code" href="structngram__model__s.html#a78a3253febced2cae4732044da466ee6" title="Are word strings writable?">writable</a> = TRUE;
<a name="l00422"></a>00422 <span class="keywordflow">if</span> (fread(&k, <span class="keyword">sizeof</span>(k), 1, fp) != 1)
<a name="l00423"></a>00423 <span class="keywordflow">goto</span> error_out;
<a name="l00424"></a>00424 <span class="keywordflow">if</span> (do_swap) SWAP_INT32(&k);
<a name="l00425"></a>00425 tmp_word_str = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(k, 1);
<a name="l00426"></a>00426 <span class="keywordflow">if</span> (fread(tmp_word_str, 1, k, fp) != (<span class="keywordtype">size_t</span>) k) {
<a name="l00427"></a>00427 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"fread(word-string) failed\n"</span>);
<a name="l00428"></a>00428 <span class="keywordflow">goto</span> error_out;
<a name="l00429"></a>00429 }
<a name="l00430"></a>00430 }
<a name="l00431"></a>00431
<a name="l00432"></a>00432 <span class="comment">/* First make sure string just read contains n_counts[0] words (PARANOIA!!) */</span>
<a name="l00433"></a>00433 <span class="keywordflow">for</span> (i = 0, j = 0; i < k; i++)
<a name="l00434"></a>00434 <span class="keywordflow">if</span> (tmp_word_str[i] == <span class="charliteral">'\0'</span>)
<a name="l00435"></a>00435 j++;
<a name="l00436"></a>00436 <span class="keywordflow">if</span> (j != n_unigram) {
<a name="l00437"></a>00437 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Error reading word strings (%d doesn't match n_unigrams %d)\n"</span>,
<a name="l00438"></a>00438 j, n_unigram);
<a name="l00439"></a>00439 <span class="keywordflow">goto</span> error_out;
<a name="l00440"></a>00440 }
<a name="l00441"></a>00441
<a name="l00442"></a>00442 <span class="comment">/* Break up string just read into words */</span>
<a name="l00443"></a>00443 <span class="keywordflow">if</span> (do_mmap) {
<a name="l00444"></a>00444 j = 0;
<a name="l00445"></a>00445 <span class="keywordflow">for</span> (i = 0; i < n_unigram; i++) {
<a name="l00446"></a>00446 base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i] = tmp_word_str + j;
<a name="l00447"></a>00447 <span class="keywordflow">if</span> (<a class="code" href="hash__table_8h.html#aebfe63c3869c271b125a8413ee384412" title="Try to add a new entry with given key and associated value to hash table h.">hash_table_enter</a>(base-><a class="code" href="structngram__model__s.html#a75567419a8002ef6e916c81f5d9ee9ed" title="Mapping of unigram names to word IDs.">wid</a>, base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i],
<a name="l00448"></a>00448 (<span class="keywordtype">void</span> *)(<span class="keywordtype">long</span>)i) != (<span class="keywordtype">void</span> *)(<span class="keywordtype">long</span>)i) {
<a name="l00449"></a>00449 <a class="code" href="err_8h.html#a6a794bec721b555ac1f2167f9e12f662" title="Print warning information to standard error stream.">E_WARN</a>(<span class="stringliteral">"Duplicate word in dictionary: %s\n"</span>, base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i]);
<a name="l00450"></a>00450 }
<a name="l00451"></a>00451 j += strlen(base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i]) + 1;
<a name="l00452"></a>00452 }
<a name="l00453"></a>00453 }
<a name="l00454"></a>00454 <span class="keywordflow">else</span> {
<a name="l00455"></a>00455 j = 0;
<a name="l00456"></a>00456 <span class="keywordflow">for</span> (i = 0; i < n_unigram; i++) {
<a name="l00457"></a>00457 base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i] = <a class="code" href="ckd__alloc_8h.html#ad313f92478859f9e4ea99d0f6e78c393" title="Macro for __ckd_salloc__.">ckd_salloc</a>(tmp_word_str + j);
<a name="l00458"></a>00458 <span class="keywordflow">if</span> (<a class="code" href="hash__table_8h.html#aebfe63c3869c271b125a8413ee384412" title="Try to add a new entry with given key and associated value to hash table h.">hash_table_enter</a>(base-><a class="code" href="structngram__model__s.html#a75567419a8002ef6e916c81f5d9ee9ed" title="Mapping of unigram names to word IDs.">wid</a>, base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i],
<a name="l00459"></a>00459 (<span class="keywordtype">void</span> *)(<span class="keywordtype">long</span>)i) != (<span class="keywordtype">void</span> *)(<span class="keywordtype">long</span>)i) {
<a name="l00460"></a>00460 <a class="code" href="err_8h.html#a6a794bec721b555ac1f2167f9e12f662" title="Print warning information to standard error stream.">E_WARN</a>(<span class="stringliteral">"Duplicate word in dictionary: %s\n"</span>, base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i]);
<a name="l00461"></a>00461 }
<a name="l00462"></a>00462 j += strlen(base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i]) + 1;
<a name="l00463"></a>00463 }
<a name="l00464"></a>00464 free(tmp_word_str);
<a name="l00465"></a>00465 }
<a name="l00466"></a>00466 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = ascii word strings read\n"</span>, i);
<a name="l00467"></a>00467
<a name="l00468"></a>00468 <a class="code" href="pio_8h.html#a87592c3a2d0a00eed9eda014950beb65" title="Close a file opened using fopen_comp.">fclose_comp</a>(fp, is_pipe);
<a name="l00469"></a>00469 <span class="keywordflow">return</span> base;
<a name="l00470"></a>00470
<a name="l00471"></a>00471 error_out:
<a name="l00472"></a>00472 <span class="keywordflow">if</span> (fp)
<a name="l00473"></a>00473 <a class="code" href="pio_8h.html#a87592c3a2d0a00eed9eda014950beb65" title="Close a file opened using fopen_comp.">fclose_comp</a>(fp, is_pipe);
<a name="l00474"></a>00474 <a class="code" href="ngram__model_8h.html#aec73d28e7285e539a0b44a7ac0cbe489" title="Release memory associated with an N-Gram model.">ngram_model_free</a>(base);
<a name="l00475"></a>00475 <span class="keywordflow">return</span> NULL;
<a name="l00476"></a>00476 }
<a name="l00477"></a>00477
<a name="l00478"></a>00478 <a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *
<a name="l00479"></a>00479 ngram_model_dmp_build(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base)
<a name="l00480"></a>00480 {
<a name="l00481"></a>00481 <a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *model;
<a name="l00482"></a>00482 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *newbase;
<a name="l00483"></a>00483 <a class="code" href="structngram__iter__s.html" title="Base iterator structure for N-grams.">ngram_iter_t</a> *itor;
<a name="l00484"></a>00484 <a class="code" href="structsorted__list__t.html" title="The sorted list.">sorted_list_t</a> sorted_prob2;
<a name="l00485"></a>00485 <a class="code" href="structsorted__list__t.html" title="The sorted list.">sorted_list_t</a> sorted_bo_wt2;
<a name="l00486"></a>00486 <a class="code" href="structsorted__list__t.html" title="The sorted list.">sorted_list_t</a> sorted_prob3;
<a name="l00487"></a>00487 <a class="code" href="structbigram__s.html" title="Bigram structure.">bigram_t</a> *bgptr;
<a name="l00488"></a>00488 <a class="code" href="structtrigram__s.html" title="Trigram structure.">trigram_t</a> *tgptr;
<a name="l00489"></a>00489 <span class="keywordtype">int</span> i, bgcount, tgcount, seg;
<a name="l00490"></a>00490
<a name="l00491"></a>00491 <span class="keywordflow">if</span> (base-><a class="code" href="structngram__model__s.html#ad3d9d8ad9773f958a89534220eda6fb9" title="Implementation-specific methods.">funcs</a> == &ngram_model_dmp_funcs) {
<a name="l00492"></a>00492 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"Using existing DMP model.\n"</span>);
<a name="l00493"></a>00493 <span class="keywordflow">return</span> (<a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *)<a class="code" href="ngram__model_8h.html#a046e6ff8cd8787e412400534a9649a81" title="Retain ownership of an N-Gram model.">ngram_model_retain</a>(base);
<a name="l00494"></a>00494 }
<a name="l00495"></a>00495
<a name="l00496"></a>00496 <span class="comment">/* Initialize new base model structure with params from base. */</span>
<a name="l00497"></a>00497 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"Building DMP model...\n"</span>);
<a name="l00498"></a>00498 model = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(1, <span class="keyword">sizeof</span>(*model));
<a name="l00499"></a>00499 newbase = &model-><a class="code" href="structngram__model__dmp__s.html#afd4571dc9702255aed667b5de62e5332" title="Base ngram_model_t structure.">base</a>;
<a name="l00500"></a>00500 ngram_model_init(newbase, &ngram_model_dmp_funcs,
<a name="l00501"></a>00501 <a class="code" href="logmath_8h.html#a1c1b2ba3b137a39e9e835a8f3e27d381" title="Retain ownership of a log table.">logmath_retain</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>),
<a name="l00502"></a>00502 base-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a>, base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0]);
<a name="l00503"></a>00503 <span class="comment">/* Copy N-gram counts over. */</span>
<a name="l00504"></a>00504 memcpy(newbase-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>, base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>,
<a name="l00505"></a>00505 base-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> * <span class="keyword">sizeof</span>(*base-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>));
<a name="l00506"></a>00506 <span class="comment">/* Make sure word strings are freed. */</span>
<a name="l00507"></a>00507 newbase-><a class="code" href="structngram__model__s.html#a78a3253febced2cae4732044da466ee6" title="Are word strings writable?">writable</a> = TRUE;
<a name="l00508"></a>00508 <span class="comment">/* Initialize unigram table and string table. */</span>
<a name="l00509"></a>00509 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.unigrams = new_unigram_table(newbase-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0] + 1);
<a name="l00510"></a>00510 <span class="keywordflow">for</span> (itor = <a class="code" href="ngram__model_8h.html#a12683dda2253dc45680102f02fbdb1e2" title="Iterate over all M-grams.">ngram_model_mgrams</a>(base, 0); itor;
<a name="l00511"></a>00511 itor = <a class="code" href="ngram__model_8h.html#a3a2b285c01393b3ebddaec1fefed11a4" title="Advance an M-gram iterator.">ngram_iter_next</a>(itor)) {
<a name="l00512"></a>00512 int32 prob1, bo_wt1;
<a name="l00513"></a>00513 int32 <span class="keyword">const</span> *wids;
<a name="l00514"></a>00514
<a name="l00515"></a>00515 <span class="comment">/* Can't guarantee they will go in unigram order, so just to</span>
<a name="l00516"></a>00516 <span class="comment"> * be correct, we do this... */</span>
<a name="l00517"></a>00517 wids = <a class="code" href="ngram__model_8h.html#a240c738781daa226a2fc13395dbdb514" title="Get information from the current M-gram in an iterator.">ngram_iter_get</a>(itor, &prob1, &bo_wt1);
<a name="l00518"></a>00518 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.unigrams[wids[0]].<a class="code" href="structunigram__s.html#a488db9623272838a933cd4b768409fea" title="Unigram probability.">prob1</a>.l = prob1;
<a name="l00519"></a>00519 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.unigrams[wids[0]].<a class="code" href="structunigram__s.html#ad33b4af5b40a8d13ffae932bab003df6" title="Unigram backoff weight.">bo_wt1</a>.l = bo_wt1;
<a name="l00520"></a>00520 newbase-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wids[0]] = <a class="code" href="ckd__alloc_8h.html#ad313f92478859f9e4ea99d0f6e78c393" title="Macro for __ckd_salloc__.">ckd_salloc</a>(<a class="code" href="ngram__model_8h.html#a96e36290a005c03464ea6c637ccde2f5" title="Look up word string for numerical word ID.">ngram_word</a>(base, wids[0]));
<a name="l00521"></a>00521 <span class="keywordflow">if</span> ((<a class="code" href="hash__table_8h.html#a393c56322e54607a48e6bc61169d92bf" title="Add a 32-bit integer value to a hash table.">hash_table_enter_int32</a>(newbase-><a class="code" href="structngram__model__s.html#a75567419a8002ef6e916c81f5d9ee9ed" title="Mapping of unigram names to word IDs.">wid</a>,
<a name="l00522"></a>00522 newbase-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wids[0]], wids[0]))
<a name="l00523"></a>00523 != wids[0]) {
<a name="l00524"></a>00524 <a class="code" href="err_8h.html#a6a794bec721b555ac1f2167f9e12f662" title="Print warning information to standard error stream.">E_WARN</a>(<span class="stringliteral">"Duplicate word in dictionary: %s\n"</span>, newbase-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wids[0]]);
<a name="l00525"></a>00525 }
<a name="l00526"></a>00526 }
<a name="l00527"></a>00527 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = #unigrams created\n"</span>, newbase-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0]);
<a name="l00528"></a>00528
<a name="l00529"></a>00529 <span class="keywordflow">if</span> (newbase-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> < 2)
<a name="l00530"></a>00530 <span class="keywordflow">return</span> model;
<a name="l00531"></a>00531
<a name="l00532"></a>00532 <span class="comment">/* Construct quantized probability table for bigrams and</span>
<a name="l00533"></a>00533 <span class="comment"> * (optionally) trigrams. Hesitate to use the "sorted list" thing</span>
<a name="l00534"></a>00534 <span class="comment"> * since it isn't so useful, but it's there already. */</span>
<a name="l00535"></a>00535 init_sorted_list(&sorted_prob2);
<a name="l00536"></a>00536 <span class="keywordflow">if</span> (newbase-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> > 2) {
<a name="l00537"></a>00537 init_sorted_list(&sorted_bo_wt2);
<a name="l00538"></a>00538 init_sorted_list(&sorted_prob3);
<a name="l00539"></a>00539 }
<a name="l00540"></a>00540 <span class="comment">/* Construct bigram and trigram arrays. */</span>
<a name="l00541"></a>00541 bgptr = model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.bigrams = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(newbase-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1] + 1, <span class="keyword">sizeof</span>(<a class="code" href="structbigram__s.html" title="Bigram structure.">bigram_t</a>));
<a name="l00542"></a>00542 <span class="keywordflow">if</span> (newbase-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> > 2) {
<a name="l00543"></a>00543 tgptr = model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.trigrams = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(newbase-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[2], <span class="keyword">sizeof</span>(<a class="code" href="structtrigram__s.html" title="Trigram structure.">trigram_t</a>));
<a name="l00544"></a>00544 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a> =
<a name="l00545"></a>00545 <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>((newbase-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1] + 1) / BG_SEG_SZ + 1, <span class="keyword">sizeof</span>(int32));
<a name="l00546"></a>00546 }
<a name="l00547"></a>00547 <span class="keywordflow">else</span>
<a name="l00548"></a>00548 tgptr = NULL;
<a name="l00549"></a>00549 <span class="comment">/* Since bigrams and trigrams have to be contiguous with others</span>
<a name="l00550"></a>00550 <span class="comment"> * with the same N-1-gram, we traverse them in depth-first order</span>
<a name="l00551"></a>00551 <span class="comment"> * to build the bigram and trigram arrays. */</span>
<a name="l00552"></a>00552 <span class="keywordflow">for</span> (i = 0; i < newbase-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0]; ++i) {
<a name="l00553"></a>00553 <a class="code" href="structngram__iter__s.html" title="Base iterator structure for N-grams.">ngram_iter_t</a> *uitor;
<a name="l00554"></a>00554 bgcount = bgptr - model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.bigrams;
<a name="l00555"></a>00555 <span class="comment">/* First bigram index (same as next if no bigrams...) */</span>
<a name="l00556"></a>00556 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.unigrams[i].<a class="code" href="structunigram__s.html#ae148f631c0d9851b14bb9cb31c0c061d" title="Index of 1st entry in lm_t.bigrams[].">bigrams</a> = bgcount;
<a name="l00557"></a>00557 <a class="code" href="err_8h.html#af46f94d0e21f22f1153f8f1cd9a372d6" title="Print debugging information to standard error stream.">E_DEBUG</a>(2, (<span class="stringliteral">"unigram %d: %s => bigram %d\n"</span>, i, newbase-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i], bgcount));
<a name="l00558"></a>00558 <span class="comment">/* All bigrams corresponding to unigram i */</span>
<a name="l00559"></a>00559 uitor = <a class="code" href="ngram__model_8h.html#a461c6e5914ce463422dfeaeee377e024" title="Get an iterator over M-grams pointing to the specified M-gram.">ngram_ng_iter</a>(base, i, NULL, 0);
<a name="l00560"></a>00560 <span class="keywordflow">for</span> (itor = <a class="code" href="ngram__model_8h.html#ae85f41e2defc5b65b12026d29cd4fdaa" title="Iterate over all M-gram successors of an M-1-gram.">ngram_iter_successors</a>(uitor);
<a name="l00561"></a>00561 itor; ++bgptr, itor = <a class="code" href="ngram__model_8h.html#a3a2b285c01393b3ebddaec1fefed11a4" title="Advance an M-gram iterator.">ngram_iter_next</a>(itor)) {
<a name="l00562"></a>00562 int32 prob2, bo_wt2;
<a name="l00563"></a>00563 int32 <span class="keyword">const</span> *wids;
<a name="l00564"></a>00564 <a class="code" href="structngram__iter__s.html" title="Base iterator structure for N-grams.">ngram_iter_t</a> *titor;
<a name="l00565"></a>00565
<a name="l00566"></a>00566 wids = <a class="code" href="ngram__model_8h.html#a240c738781daa226a2fc13395dbdb514" title="Get information from the current M-gram in an iterator.">ngram_iter_get</a>(itor, &prob2, &bo_wt2);
<a name="l00567"></a>00567
<a name="l00568"></a>00568 assert (bgptr - model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.bigrams < newbase-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1]);
<a name="l00569"></a>00569
<a name="l00570"></a>00570 bgptr-><a class="code" href="structbigram__s.html#af66faad954d56f5ceaf6ba47cdcd9b1f" title="Index of unigram entry for this.">wid</a> = wids[1];
<a name="l00571"></a>00571 bgptr-><a class="code" href="structbigram__s.html#a769b8a54f864ac4844e37cfa17ed1af0" title="Index into array of actual bigram probs.">prob2</a> = sorted_id(&sorted_prob2, &prob2);
<a name="l00572"></a>00572 <span class="keywordflow">if</span> (newbase-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> > 2) {
<a name="l00573"></a>00573 tgcount = (tgptr - model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.trigrams);
<a name="l00574"></a>00574 bgcount = (bgptr - model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.bigrams);
<a name="l00575"></a>00575
<a name="l00576"></a>00576 <span class="comment">/* Backoff weight (only if there are trigrams...) */</span>
<a name="l00577"></a>00577 bgptr-><a class="code" href="structbigram__s.html#a410991054e81251285cbc209964cdaf4" title="Index into array of actual bigram backoff wts.">bo_wt2</a> = sorted_id(&sorted_bo_wt2, &bo_wt2);
<a name="l00578"></a>00578
<a name="l00579"></a>00579 <span class="comment">/* Find bigram segment for this bigram (this isn't</span>
<a name="l00580"></a>00580 <span class="comment"> * used unless there are trigrams) */</span>
<a name="l00581"></a>00581 seg = bgcount >> LOG_BG_SEG_SZ;
<a name="l00582"></a>00582 <span class="comment">/* If we just crossed a bigram segment boundary, then</span>
<a name="l00583"></a>00583 <span class="comment"> * point tseg_base for the new segment to the current</span>
<a name="l00584"></a>00584 <span class="comment"> * trigram pointer. */</span>
<a name="l00585"></a>00585 <span class="keywordflow">if</span> (seg != (bgcount - 1) >> LOG_BG_SEG_SZ)
<a name="l00586"></a>00586 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>[seg] = tgcount;
<a name="l00587"></a>00587 <span class="comment">/* Now calculate the trigram offset. */</span>
<a name="l00588"></a>00588 bgptr-><a class="code" href="structbigram__s.html#a11c1cb1d7c40de624f09c372b36a6857" title="Index of 1st entry in lm_t.trigrams[], RELATIVE TO its segment base (see above)">trigrams</a> = tgcount - model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>[seg];
<a name="l00589"></a>00589 <a class="code" href="err_8h.html#af46f94d0e21f22f1153f8f1cd9a372d6" title="Print debugging information to standard error stream.">E_DEBUG</a>(2, (<span class="stringliteral">"bigram %d %s %s => trigram %d:%d\n"</span>,
<a name="l00590"></a>00590 bgcount,
<a name="l00591"></a>00591 newbase-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wids[0]],
<a name="l00592"></a>00592 newbase-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wids[1]],
<a name="l00593"></a>00593 seg, bgptr-><a class="code" href="structbigram__s.html#a11c1cb1d7c40de624f09c372b36a6857" title="Index of 1st entry in lm_t.trigrams[], RELATIVE TO its segment base (see above)">trigrams</a>));
<a name="l00594"></a>00594
<a name="l00595"></a>00595 <span class="comment">/* And fill in successors' trigram info. */</span>
<a name="l00596"></a>00596 <span class="keywordflow">for</span> (titor = <a class="code" href="ngram__model_8h.html#ae85f41e2defc5b65b12026d29cd4fdaa" title="Iterate over all M-gram successors of an M-1-gram.">ngram_iter_successors</a>(itor);
<a name="l00597"></a>00597 titor; ++tgptr, titor = <a class="code" href="ngram__model_8h.html#a3a2b285c01393b3ebddaec1fefed11a4" title="Advance an M-gram iterator.">ngram_iter_next</a>(titor)) {
<a name="l00598"></a>00598 int32 prob3, dummy;
<a name="l00599"></a>00599
<a name="l00600"></a>00600 assert(tgptr - model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.trigrams < newbase-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[2]);
<a name="l00601"></a>00601 wids = <a class="code" href="ngram__model_8h.html#a240c738781daa226a2fc13395dbdb514" title="Get information from the current M-gram in an iterator.">ngram_iter_get</a>(titor, &prob3, &dummy);
<a name="l00602"></a>00602 tgptr-><a class="code" href="structtrigram__s.html#a492e1fedb16fa11d26a941177c1305be" title="Index of unigram entry for this.">wid</a> = wids[2];
<a name="l00603"></a>00603 tgptr-><a class="code" href="structtrigram__s.html#a52f3fad42f7c653a469af0c1b75988a9" title="Index into array of actual trigram probs.">prob3</a> = sorted_id(&sorted_prob3, &prob3);
<a name="l00604"></a>00604 <a class="code" href="err_8h.html#af46f94d0e21f22f1153f8f1cd9a372d6" title="Print debugging information to standard error stream.">E_DEBUG</a>(2, (<span class="stringliteral">"trigram %d %s %s %s => prob %d\n"</span>,
<a name="l00605"></a>00605 tgcount,
<a name="l00606"></a>00606 newbase-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wids[0]],
<a name="l00607"></a>00607 newbase-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wids[1]],
<a name="l00608"></a>00608 newbase-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wids[2]],
<a name="l00609"></a>00609 tgptr-><a class="code" href="structtrigram__s.html#a52f3fad42f7c653a469af0c1b75988a9" title="Index into array of actual trigram probs.">prob3</a>));
<a name="l00610"></a>00610 }
<a name="l00611"></a>00611 }
<a name="l00612"></a>00612 }
<a name="l00613"></a>00613 <a class="code" href="ngram__model_8h.html#ac9f746c8a5db78ef8b2fb7c312be4a22" title="Terminate an M-gram iterator.">ngram_iter_free</a>(uitor);
<a name="l00614"></a>00614 }
<a name="l00615"></a>00615 <span class="comment">/* Add sentinal unigram and bigram records. */</span>
<a name="l00616"></a>00616 bgcount = bgptr - model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.bigrams;
<a name="l00617"></a>00617 tgcount = tgptr - model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.trigrams;
<a name="l00618"></a>00618 seg = bgcount >> LOG_BG_SEG_SZ;
<a name="l00619"></a>00619 <span class="keywordflow">if</span> (seg != (bgcount - 1) >> LOG_BG_SEG_SZ)
<a name="l00620"></a>00620 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>[seg] = tgcount;
<a name="l00621"></a>00621 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.unigrams[i].<a class="code" href="structunigram__s.html#ae148f631c0d9851b14bb9cb31c0c061d" title="Index of 1st entry in lm_t.bigrams[].">bigrams</a> = bgcount;
<a name="l00622"></a>00622 <span class="keywordflow">if</span> (newbase-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> > 2)
<a name="l00623"></a>00623 bgptr-><a class="code" href="structbigram__s.html#a11c1cb1d7c40de624f09c372b36a6857" title="Index of 1st entry in lm_t.trigrams[], RELATIVE TO its segment base (see above)">trigrams</a> = tgcount - model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>[seg];
<a name="l00624"></a>00624
<a name="l00625"></a>00625 <span class="comment">/* Now create probability tables. */</span>
<a name="l00626"></a>00626 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a273e6ea4c39d1a563cc59f00b4b6ee98" title="prob2 size">n_prob2</a> = sorted_prob2.<a class="code" href="structsorted__list__t.html#aa7468ec9a2fe7c61d2bc76ba43c575ce" title="first free element in list">free</a>;
<a name="l00627"></a>00627 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a34acf8c1ffaa4bb712ce1196eea59678" title="Table of actual bigram probs.">prob2</a> = vals_in_sorted_list(&sorted_prob2);
<a name="l00628"></a>00628 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = #bigrams created\n"</span>, newbase-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1]);
<a name="l00629"></a>00629 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = #prob2 entries\n"</span>, model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a273e6ea4c39d1a563cc59f00b4b6ee98" title="prob2 size">n_prob2</a>);
<a name="l00630"></a>00630 free_sorted_list(&sorted_prob2);
<a name="l00631"></a>00631 <span class="keywordflow">if</span> (newbase-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> > 2) {
<a name="l00632"></a>00632 <span class="comment">/* Create trigram bo-wts array. */</span>
<a name="l00633"></a>00633 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#ad4bbdd65d13712fe653afe8b1de9b096" title="bo_wt2 size">n_bo_wt2</a> = sorted_bo_wt2.<a class="code" href="structsorted__list__t.html#aa7468ec9a2fe7c61d2bc76ba43c575ce" title="first free element in list">free</a>;
<a name="l00634"></a>00634 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a7ee629aa1b8e88529127cf4da470d80f" title="Table of actual bigram backoff weights.">bo_wt2</a> = vals_in_sorted_list(&sorted_bo_wt2);
<a name="l00635"></a>00635 free_sorted_list(&sorted_bo_wt2);
<a name="l00636"></a>00636 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = #bo_wt2 entries\n"</span>, model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#ad4bbdd65d13712fe653afe8b1de9b096" title="bo_wt2 size">n_bo_wt2</a>);
<a name="l00637"></a>00637 <span class="comment">/* Create trigram probability table. */</span>
<a name="l00638"></a>00638 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#add28369f51e657ee54deed5291c84d09" title="prob3 size">n_prob3</a> = sorted_prob3.<a class="code" href="structsorted__list__t.html#aa7468ec9a2fe7c61d2bc76ba43c575ce" title="first free element in list">free</a>;
<a name="l00639"></a>00639 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#adc9ed2ad1f2daefdd40713a9dd371673" title="Table of actual trigram probs.">prob3</a> = vals_in_sorted_list(&sorted_prob3);
<a name="l00640"></a>00640 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = #trigrams created\n"</span>, newbase-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[2]);
<a name="l00641"></a>00641 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"%8d = #prob3 entries\n"</span>, model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#add28369f51e657ee54deed5291c84d09" title="prob3 size">n_prob3</a>);
<a name="l00642"></a>00642 free_sorted_list(&sorted_prob3);
<a name="l00643"></a>00643 <span class="comment">/* Initialize tginfo */</span>
<a name="l00644"></a>00644 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a9be0c8197334e3ef632e9e3abdad6a4f" title="tginfo[lw2] is head of linked list of trigram information for some cached subset of bigrams (*...">tginfo</a> = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(newbase-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0], <span class="keyword">sizeof</span>(<a class="code" href="structtginfo__s.html" title="Trigram information cache.">tginfo_t</a> *));
<a name="l00645"></a>00645 model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a2c96ba8aa34632e4d42908f9384d1b01" title="List element allocator for tginfo.">le</a> = <a class="code" href="listelem__alloc_8h.html#a4b08b49eaa74cbe9a3c95170cee78de7" title="Initialize and return a list element allocator.">listelem_alloc_init</a>(<span class="keyword">sizeof</span>(<a class="code" href="structtginfo__s.html" title="Trigram information cache.">tginfo_t</a>));
<a name="l00646"></a>00646 }
<a name="l00647"></a>00647
<a name="l00648"></a>00648 <span class="keywordflow">return</span> model;
<a name="l00649"></a>00649 }
<a name="l00650"></a>00650
<a name="l00651"></a>00651 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00652"></a>00652 fwrite_int32(FILE *fh, int32 val)
<a name="l00653"></a>00653 {
<a name="l00654"></a>00654 fwrite(&val, 4, 1, fh);
<a name="l00655"></a>00655 }
<a name="l00656"></a>00656
<a name="l00657"></a>00657 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00658"></a>00658 fwrite_ug(FILE *fh, <a class="code" href="structunigram__s.html" title="Unigram structure (common among all lm3g implementations)">unigram_t</a> *ug, <a class="code" href="structlogmath__s.html">logmath_t</a> *lmath)
<a name="l00659"></a>00659 {
<a name="l00660"></a>00660 int32 bogus = -1;
<a name="l00661"></a>00661 float32 log10val;
<a name="l00662"></a>00662
<a name="l00663"></a>00663 <span class="comment">/* Bogus dictionary mapping field. */</span>
<a name="l00664"></a>00664 fwrite(&bogus, 4, 1, fh);
<a name="l00665"></a>00665 <span class="comment">/* Convert values to log10. */</span>
<a name="l00666"></a>00666 log10val = <a class="code" href="logmath_8h.html#a7c17cb624003975e84fbd141ca6e2e06" title="Convert integer log in base B to base 10 log (in floating point).">logmath_log_to_log10</a>(lmath, ug-><a class="code" href="structunigram__s.html#a488db9623272838a933cd4b768409fea" title="Unigram probability.">prob1</a>.l);
<a name="l00667"></a>00667 fwrite(&log10val, 4, 1, fh);
<a name="l00668"></a>00668 log10val = <a class="code" href="logmath_8h.html#a7c17cb624003975e84fbd141ca6e2e06" title="Convert integer log in base B to base 10 log (in floating point).">logmath_log_to_log10</a>(lmath, ug-><a class="code" href="structunigram__s.html#ad33b4af5b40a8d13ffae932bab003df6" title="Unigram backoff weight.">bo_wt1</a>.l);
<a name="l00669"></a>00669 fwrite(&log10val, 4, 1, fh);
<a name="l00670"></a>00670 fwrite_int32(fh, ug-><a class="code" href="structunigram__s.html#ae148f631c0d9851b14bb9cb31c0c061d" title="Index of 1st entry in lm_t.bigrams[].">bigrams</a>);
<a name="l00671"></a>00671 }
<a name="l00672"></a>00672
<a name="l00673"></a>00673 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00674"></a>00674 fwrite_bg(FILE *fh, <a class="code" href="structbigram__s.html" title="Bigram structure.">bigram_t</a> *bg)
<a name="l00675"></a>00675 {
<a name="l00676"></a>00676 fwrite(bg, <span class="keyword">sizeof</span>(*bg), 1, fh);
<a name="l00677"></a>00677 }
<a name="l00678"></a>00678
<a name="l00679"></a>00679 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00680"></a>00680 fwrite_tg(FILE *fh, <a class="code" href="structtrigram__s.html" title="Trigram structure.">trigram_t</a> *tg)
<a name="l00681"></a>00681 {
<a name="l00682"></a>00682 fwrite(tg, <span class="keyword">sizeof</span>(*tg), 1, fh);
<a name="l00683"></a>00683 }
<a name="l00684"></a>00684
<a name="l00687"></a>00687 <span class="keyword">static</span> <span class="keywordtype">char</span> <span class="keyword">const</span> *fmtdesc[] = {
<a name="l00688"></a>00688 <span class="stringliteral">"BEGIN FILE FORMAT DESCRIPTION"</span>,
<a name="l00689"></a>00689 <span class="stringliteral">"Header string length (int32) and string (including trailing 0)"</span>,
<a name="l00690"></a>00690 <span class="stringliteral">"Original LM filename string-length (int32) and filename (including trailing 0)"</span>,
<a name="l00691"></a>00691 <span class="stringliteral">"(int32) version number (present iff value <= 0)"</span>,
<a name="l00692"></a>00692 <span class="stringliteral">"(int32) original LM file modification timestamp (iff version# present)"</span>,
<a name="l00693"></a>00693 <span class="stringliteral">"(int32) string-length and string (including trailing 0) (iff version# present)"</span>,
<a name="l00694"></a>00694 <span class="stringliteral">"... previous entry continued any number of times (iff version# present)"</span>,
<a name="l00695"></a>00695 <span class="stringliteral">"(int32) 0 (terminating sequence of strings) (iff version# present)"</span>,
<a name="l00696"></a>00696 <span class="stringliteral">"(int32) log_bg_seg_sz (present iff different from default value of LOG2_BG_SEG_SZ)"</span>,
<a name="l00697"></a>00697 <span class="stringliteral">"(int32) lm_t.ucount (must be > 0)"</span>,
<a name="l00698"></a>00698 <span class="stringliteral">"(int32) lm_t.bcount"</span>,
<a name="l00699"></a>00699 <span class="stringliteral">"(int32) lm_t.tcount"</span>,
<a name="l00700"></a>00700 <span class="stringliteral">"lm_t.ucount+1 unigrams (including sentinel)"</span>,
<a name="l00701"></a>00701 <span class="stringliteral">"lm_t.bcount+1 bigrams (including sentinel 64 bits (bg_t) each if version=-1/-2, 128 bits (bg32_t) each if version=-3"</span>,
<a name="l00702"></a>00702 <span class="stringliteral">"lm_t.tcount trigrams (present iff lm_t.tcount > 0 32 bits (tg_t) each if version=-1/-2, 64 bits (tg32_t) each if version=-3)"</span>,
<a name="l00703"></a>00703 <span class="stringliteral">"(int32) lm_t.n_prob2"</span>,
<a name="l00704"></a>00704 <span class="stringliteral">"(int32) lm_t.prob2[]"</span>,
<a name="l00705"></a>00705 <span class="stringliteral">"(int32) lm_t.n_bo_wt2 (present iff lm_t.tcount > 0)"</span>,
<a name="l00706"></a>00706 <span class="stringliteral">"(int32) lm_t.bo_wt2[] (present iff lm_t.tcount > 0)"</span>,
<a name="l00707"></a>00707 <span class="stringliteral">"(int32) lm_t.n_prob3 (present iff lm_t.tcount > 0)"</span>,
<a name="l00708"></a>00708 <span class="stringliteral">"(int32) lm_t.prob3[] (present iff lm_t.tcount > 0)"</span>,
<a name="l00709"></a>00709 <span class="stringliteral">"(int32) (lm_t.bcount+1)/BG_SEG_SZ+1 (present iff lm_t.tcount > 0)"</span>,
<a name="l00710"></a>00710 <span class="stringliteral">"(int32) lm_t.tseg_base[] (present iff lm_t.tcount > 0)"</span>,
<a name="l00711"></a>00711 <span class="stringliteral">"(int32) Sum(all word string-lengths, including trailing 0 for each)"</span>,
<a name="l00712"></a>00712 <span class="stringliteral">"All word strings (including trailing 0 for each)"</span>,
<a name="l00713"></a>00713 <span class="stringliteral">"END FILE FORMAT DESCRIPTION"</span>,
<a name="l00714"></a>00714 NULL,
<a name="l00715"></a>00715 };
<a name="l00716"></a>00716
<a name="l00717"></a>00717 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00718"></a>00718 ngram_model_dmp_write_header(FILE * fh)
<a name="l00719"></a>00719 {
<a name="l00720"></a>00720 int32 k;
<a name="l00721"></a>00721 k = strlen(darpa_hdr) + 1;
<a name="l00722"></a>00722 fwrite_int32(fh, k);
<a name="l00723"></a>00723 fwrite(darpa_hdr, 1, k, fh);
<a name="l00724"></a>00724 }
<a name="l00725"></a>00725
<a name="l00726"></a>00726 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00727"></a>00727 ngram_model_dmp_write_lm_filename(FILE * fh, <span class="keyword">const</span> <span class="keywordtype">char</span> *lmfile)
<a name="l00728"></a>00728 {
<a name="l00729"></a>00729 int32 k;
<a name="l00730"></a>00730
<a name="l00731"></a>00731 k = strlen(lmfile) + 1;
<a name="l00732"></a>00732 fwrite_int32(fh, k);
<a name="l00733"></a>00733 fwrite(lmfile, 1, k, fh);
<a name="l00734"></a>00734 }
<a name="l00735"></a>00735
<a name="l00736"></a>00736 <span class="preprocessor">#define LMDMP_VERSION_TG_16BIT -1 </span>
<a name="l00740"></a>00740 <span class="preprocessor">static void</span>
<a name="l00741"></a>00741 <span class="preprocessor"></span>ngram_model_dmp_write_version(FILE * fh, int32 mtime)
<a name="l00742"></a>00742 {
<a name="l00743"></a>00743 fwrite_int32(fh, LMDMP_VERSION_TG_16BIT); <span class="comment">/* version # */</span>
<a name="l00744"></a>00744 fwrite_int32(fh, mtime);
<a name="l00745"></a>00745 }
<a name="l00746"></a>00746
<a name="l00747"></a>00747 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00748"></a>00748 ngram_model_dmp_write_ngram_counts(FILE * fh, <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *model)
<a name="l00749"></a>00749 {
<a name="l00750"></a>00750 fwrite_int32(fh, model-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0]);
<a name="l00751"></a>00751 fwrite_int32(fh, model-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1]);
<a name="l00752"></a>00752 fwrite_int32(fh, model-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[2]);
<a name="l00753"></a>00753 }
<a name="l00754"></a>00754
<a name="l00755"></a>00755 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00756"></a>00756 ngram_model_dmp_write_fmtdesc(FILE * fh)
<a name="l00757"></a>00757 {
<a name="l00758"></a>00758 int32 i, k;
<a name="l00759"></a>00759 <span class="keywordtype">long</span> pos;
<a name="l00760"></a>00760
<a name="l00761"></a>00761 <span class="comment">/* Write file format description into header */</span>
<a name="l00762"></a>00762 <span class="keywordflow">for</span> (i = 0; fmtdesc[i] != NULL; i++) {
<a name="l00763"></a>00763 k = strlen(fmtdesc[i]) + 1;
<a name="l00764"></a>00764 fwrite_int32(fh, k);
<a name="l00765"></a>00765 fwrite(fmtdesc[i], 1, k, fh);
<a name="l00766"></a>00766 }
<a name="l00767"></a>00767 <span class="comment">/* Pad it out in order to achieve 32-bit alignment */</span>
<a name="l00768"></a>00768 pos = ftell(fh);
<a name="l00769"></a>00769 k = pos & 3;
<a name="l00770"></a>00770 <span class="keywordflow">if</span> (k) {
<a name="l00771"></a>00771 fwrite_int32(fh, 4-k);
<a name="l00772"></a>00772 fwrite(<span class="stringliteral">"!!!!"</span>, 1, 4-k, fh);
<a name="l00773"></a>00773 }
<a name="l00774"></a>00774 fwrite_int32(fh, 0);
<a name="l00775"></a>00775 }
<a name="l00776"></a>00776
<a name="l00777"></a>00777 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00778"></a>00778 ngram_model_dmp_write_unigram(FILE *fh, <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *model)
<a name="l00779"></a>00779 {
<a name="l00780"></a>00780 <a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *lm = (<a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *)model;
<a name="l00781"></a>00781 int32 i;
<a name="l00782"></a>00782
<a name="l00783"></a>00783 <span class="keywordflow">for</span> (i = 0; i <= model-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0]; i++) {
<a name="l00784"></a>00784 fwrite_ug(fh, &(lm-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.unigrams[i]), model-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>);
<a name="l00785"></a>00785 }
<a name="l00786"></a>00786 }
<a name="l00787"></a>00787
<a name="l00788"></a>00788
<a name="l00789"></a>00789 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00790"></a>00790 ngram_model_dmp_write_bigram(FILE *fh, <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *model)
<a name="l00791"></a>00791 {
<a name="l00792"></a>00792 <a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *lm = (<a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *)model;
<a name="l00793"></a>00793 int32 i;
<a name="l00794"></a>00794
<a name="l00795"></a>00795 <span class="keywordflow">for</span> (i = 0; i <= model-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1]; i++) {
<a name="l00796"></a>00796 fwrite_bg(fh, &(lm-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.bigrams[i]));
<a name="l00797"></a>00797 }
<a name="l00798"></a>00798
<a name="l00799"></a>00799 }
<a name="l00800"></a>00800
<a name="l00801"></a>00801 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00802"></a>00802 ngram_model_dmp_write_trigram(FILE *fh, <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *model)
<a name="l00803"></a>00803 {
<a name="l00804"></a>00804 <a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *lm = (<a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *)model;
<a name="l00805"></a>00805 int32 i;
<a name="l00806"></a>00806
<a name="l00807"></a>00807 <span class="keywordflow">for</span> (i = 0; i < model-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[2]; i++) {
<a name="l00808"></a>00808 fwrite_tg(fh, &(lm-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.trigrams[i]));
<a name="l00809"></a>00809 }
<a name="l00810"></a>00810 }
<a name="l00811"></a>00811
<a name="l00812"></a>00812 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00813"></a>00813 ngram_model_dmp_write_bgprob(FILE *fh, <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *model)
<a name="l00814"></a>00814 {
<a name="l00815"></a>00815 <a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *lm = (<a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *)model;
<a name="l00816"></a>00816 int32 i;
<a name="l00817"></a>00817
<a name="l00818"></a>00818 fwrite_int32(fh, lm-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a273e6ea4c39d1a563cc59f00b4b6ee98" title="prob2 size">n_prob2</a>);
<a name="l00819"></a>00819 <span class="keywordflow">for</span> (i = 0; i < lm-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a273e6ea4c39d1a563cc59f00b4b6ee98" title="prob2 size">n_prob2</a>; i++) {
<a name="l00820"></a>00820 float32 log10val = <a class="code" href="logmath_8h.html#a7c17cb624003975e84fbd141ca6e2e06" title="Convert integer log in base B to base 10 log (in floating point).">logmath_log_to_log10</a>(model-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, lm-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a34acf8c1ffaa4bb712ce1196eea59678" title="Table of actual bigram probs.">prob2</a>[i].l);
<a name="l00821"></a>00821 fwrite(&log10val, 4, 1, fh);
<a name="l00822"></a>00822 }
<a name="l00823"></a>00823 }
<a name="l00824"></a>00824
<a name="l00825"></a>00825 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00826"></a>00826 ngram_model_dmp_write_tgbowt(FILE *fh, <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *model)
<a name="l00827"></a>00827 {
<a name="l00828"></a>00828 <a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *lm = (<a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *)model;
<a name="l00829"></a>00829 int32 i;
<a name="l00830"></a>00830
<a name="l00831"></a>00831 fwrite_int32(fh, lm-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#ad4bbdd65d13712fe653afe8b1de9b096" title="bo_wt2 size">n_bo_wt2</a>);
<a name="l00832"></a>00832 <span class="keywordflow">for</span> (i = 0; i < lm-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#ad4bbdd65d13712fe653afe8b1de9b096" title="bo_wt2 size">n_bo_wt2</a>; i++) {
<a name="l00833"></a>00833 float32 log10val = <a class="code" href="logmath_8h.html#a7c17cb624003975e84fbd141ca6e2e06" title="Convert integer log in base B to base 10 log (in floating point).">logmath_log_to_log10</a>(model-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, lm-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a7ee629aa1b8e88529127cf4da470d80f" title="Table of actual bigram backoff weights.">bo_wt2</a>[i].l);
<a name="l00834"></a>00834 fwrite(&log10val, 4, 1, fh);
<a name="l00835"></a>00835 }
<a name="l00836"></a>00836 }
<a name="l00837"></a>00837
<a name="l00838"></a>00838 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00839"></a>00839 ngram_model_dmp_write_tgprob(FILE *fh, <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *model)
<a name="l00840"></a>00840 {
<a name="l00841"></a>00841 <a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *lm = (<a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *)model;
<a name="l00842"></a>00842 int32 i;
<a name="l00843"></a>00843
<a name="l00844"></a>00844 fwrite_int32(fh, lm-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#add28369f51e657ee54deed5291c84d09" title="prob3 size">n_prob3</a>);
<a name="l00845"></a>00845 <span class="keywordflow">for</span> (i = 0; i < lm-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#add28369f51e657ee54deed5291c84d09" title="prob3 size">n_prob3</a>; i++) {
<a name="l00846"></a>00846 float32 log10val = <a class="code" href="logmath_8h.html#a7c17cb624003975e84fbd141ca6e2e06" title="Convert integer log in base B to base 10 log (in floating point).">logmath_log_to_log10</a>(model-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, lm-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#adc9ed2ad1f2daefdd40713a9dd371673" title="Table of actual trigram probs.">prob3</a>[i].l);
<a name="l00847"></a>00847 fwrite(&log10val, 4, 1, fh);
<a name="l00848"></a>00848 }
<a name="l00849"></a>00849 }
<a name="l00850"></a>00850
<a name="l00851"></a>00851 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00852"></a>00852 ngram_model_dmp_write_tg_segbase(FILE *fh, <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *model)
<a name="l00853"></a>00853 {
<a name="l00854"></a>00854 <a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *lm = (<a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *)model;
<a name="l00855"></a>00855 int32 i, k;
<a name="l00856"></a>00856
<a name="l00857"></a>00857 k = (model-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[1] + 1) / BG_SEG_SZ + 1;
<a name="l00858"></a>00858 fwrite_int32(fh, k);
<a name="l00859"></a>00859 <span class="keywordflow">for</span> (i = 0; i < k; i++)
<a name="l00860"></a>00860 fwrite_int32(fh, lm-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>[i]);
<a name="l00861"></a>00861 }
<a name="l00862"></a>00862
<a name="l00863"></a>00863 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00864"></a>00864 ngram_model_dmp_write_wordstr(FILE *fh, <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *model)
<a name="l00865"></a>00865 {
<a name="l00866"></a>00866 int32 i, k;
<a name="l00867"></a>00867
<a name="l00868"></a>00868 k = 0;
<a name="l00869"></a>00869 <span class="keywordflow">for</span> (i = 0; i < model-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0]; i++)
<a name="l00870"></a>00870 k += strlen(model-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i]) + 1;
<a name="l00871"></a>00871 fwrite_int32(fh, k);
<a name="l00872"></a>00872 <span class="keywordflow">for</span> (i = 0; i < model-><a class="code" href="structngram__model__s.html#a9dcba9b49cc1cd189b257e5838da0eee" title="Counts for 1, 2, 3, ...">n_counts</a>[0]; i++)
<a name="l00873"></a>00873 fwrite(model-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i], 1,
<a name="l00874"></a>00874 strlen(model-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i]) + 1, fh);
<a name="l00875"></a>00875 }
<a name="l00876"></a>00876
<a name="l00877"></a>00877 <span class="keywordtype">int</span>
<a name="l00878"></a>00878 ngram_model_dmp_write(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base,
<a name="l00879"></a>00879 <span class="keyword">const</span> <span class="keywordtype">char</span> *file_name)
<a name="l00880"></a>00880 {
<a name="l00881"></a>00881 <a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *model;
<a name="l00882"></a>00882 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *newbase;
<a name="l00883"></a>00883 FILE *fh;
<a name="l00884"></a>00884
<a name="l00885"></a>00885 <span class="comment">/* First, construct a DMP model from the base model. */</span>
<a name="l00886"></a>00886 model = ngram_model_dmp_build(base);
<a name="l00887"></a>00887 newbase = &model-><a class="code" href="structngram__model__dmp__s.html#afd4571dc9702255aed667b5de62e5332" title="Base ngram_model_t structure.">base</a>;
<a name="l00888"></a>00888
<a name="l00889"></a>00889 <span class="comment">/* Now write it, confident in the knowledge that it's the right</span>
<a name="l00890"></a>00890 <span class="comment"> * kind of language model internally. */</span>
<a name="l00891"></a>00891 <span class="keywordflow">if</span> ((fh = fopen(file_name, <span class="stringliteral">"wb"</span>)) == NULL) {
<a name="l00892"></a>00892 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Cannot create file %s\n"</span>, file_name);
<a name="l00893"></a>00893 <span class="keywordflow">return</span> -1;
<a name="l00894"></a>00894 }
<a name="l00895"></a>00895 ngram_model_dmp_write_header(fh);
<a name="l00896"></a>00896 ngram_model_dmp_write_lm_filename(fh, file_name);
<a name="l00897"></a>00897 ngram_model_dmp_write_version(fh, 0);
<a name="l00898"></a>00898 ngram_model_dmp_write_fmtdesc(fh);
<a name="l00899"></a>00899 ngram_model_dmp_write_ngram_counts(fh, newbase);
<a name="l00900"></a>00900 ngram_model_dmp_write_unigram(fh, newbase);
<a name="l00901"></a>00901 <span class="keywordflow">if</span> (newbase-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> > 1) {
<a name="l00902"></a>00902 ngram_model_dmp_write_bigram(fh, newbase);
<a name="l00903"></a>00903 <span class="keywordflow">if</span> (newbase-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> > 2) {
<a name="l00904"></a>00904 ngram_model_dmp_write_trigram(fh, newbase);
<a name="l00905"></a>00905 }
<a name="l00906"></a>00906 ngram_model_dmp_write_bgprob(fh, newbase);
<a name="l00907"></a>00907 <span class="keywordflow">if</span> (newbase-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> > 2) {
<a name="l00908"></a>00908 ngram_model_dmp_write_tgbowt(fh, newbase);
<a name="l00909"></a>00909 ngram_model_dmp_write_tgprob(fh, newbase);
<a name="l00910"></a>00910 ngram_model_dmp_write_tg_segbase(fh, newbase);
<a name="l00911"></a>00911 }
<a name="l00912"></a>00912 }
<a name="l00913"></a>00913 ngram_model_dmp_write_wordstr(fh, newbase);
<a name="l00914"></a>00914 <a class="code" href="ngram__model_8h.html#aec73d28e7285e539a0b44a7ac0cbe489" title="Release memory associated with an N-Gram model.">ngram_model_free</a>(newbase);
<a name="l00915"></a>00915
<a name="l00916"></a>00916 <span class="keywordflow">return</span> fclose(fh);
<a name="l00917"></a>00917 }
<a name="l00918"></a>00918
<a name="l00919"></a>00919 <span class="keyword">static</span> <span class="keywordtype">int</span>
<a name="l00920"></a>00920 ngram_model_dmp_apply_weights(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base, float32 lw,
<a name="l00921"></a>00921 float32 wip, float32 uw)
<a name="l00922"></a>00922 {
<a name="l00923"></a>00923 <a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *model = (<a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *)base;
<a name="l00924"></a>00924 lm3g_apply_weights(base, &model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>, lw, wip, uw);
<a name="l00925"></a>00925 <span class="keywordflow">return</span> 0;
<a name="l00926"></a>00926 }
<a name="l00927"></a>00927
<a name="l00928"></a>00928 <span class="comment">/* Lousy "templating" for things that are largely the same in DMP and</span>
<a name="l00929"></a>00929 <span class="comment"> * ARPA models, except for the bigram and trigram types and some</span>
<a name="l00930"></a>00930 <span class="comment"> * names. */</span>
<a name="l00931"></a>00931 <span class="preprocessor">#define NGRAM_MODEL_TYPE ngram_model_dmp_t</span>
<a name="l00932"></a>00932 <span class="preprocessor"></span><span class="preprocessor">#include "lm3g_templates.c"</span>
<a name="l00933"></a>00933
<a name="l00934"></a>00934 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00935"></a>00935 ngram_model_dmp_free(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base)
<a name="l00936"></a>00936 {
<a name="l00937"></a>00937 <a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *model = (<a class="code" href="structngram__model__dmp__s.html" title="Subclass of ngram_model for DMP file reading.">ngram_model_dmp_t</a> *)base;
<a name="l00938"></a>00938
<a name="l00939"></a>00939 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.unigrams);
<a name="l00940"></a>00940 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a34acf8c1ffaa4bb712ce1196eea59678" title="Table of actual bigram probs.">prob2</a>);
<a name="l00941"></a>00941 <span class="keywordflow">if</span> (model-><a class="code" href="structngram__model__dmp__s.html#ae86e39a2c9e3078f0d9cffc6cf384702" title="mmap() of dump file (or NULL if none)">dump_mmap</a>) {
<a name="l00942"></a>00942 <a class="code" href="mmio_8h.html#a341a9c1cc8a3c4bddfd2d29a1b0993f6" title="Unmap a file, releasing memory associated with it.">mmio_file_unmap</a>(model-><a class="code" href="structngram__model__dmp__s.html#ae86e39a2c9e3078f0d9cffc6cf384702" title="mmap() of dump file (or NULL if none)">dump_mmap</a>);
<a name="l00943"></a>00943 }
<a name="l00944"></a>00944 <span class="keywordflow">else</span> {
<a name="l00945"></a>00945 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.bigrams);
<a name="l00946"></a>00946 <span class="keywordflow">if</span> (base-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> > 2) {
<a name="l00947"></a>00947 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.trigrams);
<a name="l00948"></a>00948 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a6cec34be6a748a9cece94ae3dfc1e8d7" title="tseg_base[i&gt;&gt;LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i&gt;&gt;LOG_BG_SEG_SZ)">tseg_base</a>);
<a name="l00949"></a>00949 }
<a name="l00950"></a>00950 }
<a name="l00951"></a>00951 <span class="keywordflow">if</span> (base-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> > 2) {
<a name="l00952"></a>00952 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#a7ee629aa1b8e88529127cf4da470d80f" title="Table of actual bigram backoff weights.">bo_wt2</a>);
<a name="l00953"></a>00953 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>.<a class="code" href="structlm3g__model__s.html#adc9ed2ad1f2daefdd40713a9dd371673" title="Table of actual trigram probs.">prob3</a>);
<a name="l00954"></a>00954 }
<a name="l00955"></a>00955
<a name="l00956"></a>00956 lm3g_tginfo_free(base, &model-><a class="code" href="structngram__model__dmp__s.html#af889dddcba4b473e782f4a5a11ca47bd" title="Common lm3g_model_t structure.">lm3g</a>);
<a name="l00957"></a>00957 }
<a name="l00958"></a>00958
<a name="l00959"></a>00959 <span class="keyword">static</span> <a class="code" href="structngram__funcs__s.html" title="Implementation-specific functions for operating on ngram_model_t objects.">ngram_funcs_t</a> ngram_model_dmp_funcs = {
<a name="l00960"></a>00960 ngram_model_dmp_free, <span class="comment">/* free */</span>
<a name="l00961"></a>00961 ngram_model_dmp_apply_weights, <span class="comment">/* apply_weights */</span>
<a name="l00962"></a>00962 lm3g_template_score, <span class="comment">/* score */</span>
<a name="l00963"></a>00963 lm3g_template_raw_score, <span class="comment">/* raw_score */</span>
<a name="l00964"></a>00964 lm3g_template_add_ug, <span class="comment">/* add_ug */</span>
<a name="l00965"></a>00965 lm3g_template_flush, <span class="comment">/* flush */</span>
<a name="l00966"></a>00966 lm3g_template_iter, <span class="comment">/* iter */</span>
<a name="l00967"></a>00967 lm3g_template_mgrams, <span class="comment">/* mgrams */</span>
<a name="l00968"></a>00968 lm3g_template_successors, <span class="comment">/* successors */</span>
<a name="l00969"></a>00969 lm3g_template_iter_get, <span class="comment">/* iter_get */</span>
<a name="l00970"></a>00970 lm3g_template_iter_next, <span class="comment">/* iter_next */</span>
<a name="l00971"></a>00971 lm3g_template_iter_free <span class="comment">/* iter_free */</span>
<a name="l00972"></a>00972 };
</pre></div></div>
</div>
<div id="nav-path" class="navpath">
<ul>
<li class="navelem"><b>ngram_model_dmp.c</b> </li>
<li class="footer">Generated on Tue Apr 19 2011 for SphinxBase by 
<a href="http://www.doxygen.org/index.html">
<img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.7.3 </li>
</ul>
</div>
</body>
</html>
distrib
>
Fedora
>
14
>
x86_64
>
media
>
updates
>
by-pkgid
>
0b420d0fce195cf4115dc6a3be5c2da2
>
files
>
299
