<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<title>SphinxBase: src/libsphinxbase/lm/ngram_model_set.c Source File</title>
<link href="tabs.css" rel="stylesheet" type="text/css"/>
<link href="navtree.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="navtree.js"></script>
<script type="text/javascript" src="resize.js"></script>
<script type="text/javascript">
$(document).ready(initResizable);
</script>
<link href="doxygen.css" rel="stylesheet" type="text/css"/>
</head>
<body>
<!-- Generated by Doxygen 1.7.3 -->
<div id="top">
<div id="titlearea">
<table cellspacing="0" cellpadding="0">
<tbody>
<tr style="height: 56px;">
<td style="padding-left: 0.5em;">
<div id="projectname">SphinxBase <span id="projectnumber">0.6</span></div>
</td>
</tr>
</tbody>
</table>
</div>
<div id="navrow1" class="tabs">
<ul class="tablist">
<li><a href="index.html"><span>Main Page</span></a></li>
<li><a href="pages.html"><span>Related Pages</span></a></li>
<li><a href="annotated.html"><span>Data Structures</span></a></li>
<li class="current"><a href="files.html"><span>Files</span></a></li>
</ul>
</div>
<div id="navrow2" class="tabs2">
<ul class="tablist">
<li><a href="files.html"><span>File List</span></a></li>
<li><a href="globals.html"><span>Globals</span></a></li>
</ul>
</div>
</div>
<div id="side-nav" class="ui-resizable side-nav-resizable">
<div id="nav-tree">
<div id="nav-tree-contents">
</div>
</div>
<div id="splitbar" style="-moz-user-select:none;"
class="ui-resizable-handle">
</div>
</div>
<script type="text/javascript">
initNavTree('ngram__model__set_8c.html','');
</script>
<div id="doc-content">
<div class="header">
<div class="headertitle">
<h1>src/libsphinxbase/lm/ngram_model_set.c</h1> </div>
</div>
<div class="contents">
<a href="ngram__model__set_8c.html">Go to the documentation of this file.</a><div class="fragment"><pre class="fragment"><a name="l00001"></a>00001 <span class="comment">/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */</span>
<a name="l00002"></a>00002 <span class="comment">/* ====================================================================</span>
<a name="l00003"></a>00003 <span class="comment"> * Copyright (c) 2008 Carnegie Mellon University. All rights</span>
<a name="l00004"></a>00004 <span class="comment"> * reserved.</span>
<a name="l00005"></a>00005 <span class="comment"> *</span>
<a name="l00006"></a>00006 <span class="comment"> * Redistribution and use in source and binary forms, with or without</span>
<a name="l00007"></a>00007 <span class="comment"> * modification, are permitted provided that the following conditions</span>
<a name="l00008"></a>00008 <span class="comment"> * are met:</span>
<a name="l00009"></a>00009 <span class="comment"> *</span>
<a name="l00010"></a>00010 <span class="comment"> * 1. Redistributions of source code must retain the above copyright</span>
<a name="l00011"></a>00011 <span class="comment"> * notice, this list of conditions and the following disclaimer. </span>
<a name="l00012"></a>00012 <span class="comment"> *</span>
<a name="l00013"></a>00013 <span class="comment"> * 2. Redistributions in binary form must reproduce the above copyright</span>
<a name="l00014"></a>00014 <span class="comment"> * notice, this list of conditions and the following disclaimer in</span>
<a name="l00015"></a>00015 <span class="comment"> * the documentation and/or other materials provided with the</span>
<a name="l00016"></a>00016 <span class="comment"> * distribution.</span>
<a name="l00017"></a>00017 <span class="comment"> *</span>
<a name="l00018"></a>00018 <span class="comment"> * This work was supported in part by funding from the Defense Advanced </span>
<a name="l00019"></a>00019 <span class="comment"> * Research Projects Agency and the National Science Foundation of the </span>
<a name="l00020"></a>00020 <span class="comment"> * United States of America, and the CMU Sphinx Speech Consortium.</span>
<a name="l00021"></a>00021 <span class="comment"> *</span>
<a name="l00022"></a>00022 <span class="comment"> * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND </span>
<a name="l00023"></a>00023 <span class="comment"> * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, </span>
<a name="l00024"></a>00024 <span class="comment"> * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR</span>
<a name="l00025"></a>00025 <span class="comment"> * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY</span>
<a name="l00026"></a>00026 <span class="comment"> * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</span>
<a name="l00027"></a>00027 <span class="comment"> * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT </span>
<a name="l00028"></a>00028 <span class="comment"> * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, </span>
<a name="l00029"></a>00029 <span class="comment"> * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY </span>
<a name="l00030"></a>00030 <span class="comment"> * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT </span>
<a name="l00031"></a>00031 <span class="comment"> * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE </span>
<a name="l00032"></a>00032 <span class="comment"> * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</span>
<a name="l00033"></a>00033 <span class="comment"> *</span>
<a name="l00034"></a>00034 <span class="comment"> * ====================================================================</span>
<a name="l00035"></a>00035 <span class="comment"> *</span>
<a name="l00036"></a>00036 <span class="comment"> */</span>
<a name="l00042"></a>00042 <span class="preprocessor">#include <string.h></span>
<a name="l00043"></a>00043 <span class="preprocessor">#include <stdlib.h></span>
<a name="l00044"></a>00044
<a name="l00045"></a>00045 <span class="preprocessor">#include "sphinxbase/err.h"</span>
<a name="l00046"></a>00046 <span class="preprocessor">#include "sphinxbase/ckd_alloc.h"</span>
<a name="l00047"></a>00047 <span class="preprocessor">#include "sphinxbase/strfuncs.h"</span>
<a name="l00048"></a>00048 <span class="preprocessor">#include "sphinxbase/filename.h"</span>
<a name="l00049"></a>00049
<a name="l00050"></a>00050 <span class="preprocessor">#include "<a class="code" href="ngram__model__set_8h.html" title="Set of language models.">ngram_model_set.h</a>"</span>
<a name="l00051"></a>00051
<a name="l00052"></a>00052 <span class="keyword">static</span> <a class="code" href="structngram__funcs__s.html" title="Implementation-specific functions for operating on ngram_model_t objects.">ngram_funcs_t</a> ngram_model_set_funcs;
<a name="l00053"></a>00053
<a name="l00054"></a>00054 <span class="keyword">static</span> <span class="keywordtype">int</span>
<a name="l00055"></a>00055 my_compare(<span class="keyword">const</span> <span class="keywordtype">void</span> *a, <span class="keyword">const</span> <span class="keywordtype">void</span> *b)
<a name="l00056"></a>00056 {
<a name="l00057"></a>00057 <span class="comment">/* Make sure <UNK> floats to the beginning. */</span>
<a name="l00058"></a>00058 <span class="keywordflow">if</span> (strcmp(*(<span class="keywordtype">char</span> * <span class="keyword">const</span> *)a, <span class="stringliteral">"<UNK>"</span>) == 0)
<a name="l00059"></a>00059 <span class="keywordflow">return</span> -1;
<a name="l00060"></a>00060 <span class="keywordflow">else</span> <span class="keywordflow">if</span> (strcmp(*(<span class="keywordtype">char</span> * <span class="keyword">const</span> *)b, <span class="stringliteral">"<UNK>"</span>) == 0)
<a name="l00061"></a>00061 <span class="keywordflow">return</span> 1;
<a name="l00062"></a>00062 <span class="keywordflow">else</span>
<a name="l00063"></a>00063 <span class="keywordflow">return</span> strcmp(*(<span class="keywordtype">char</span> * <span class="keyword">const</span> *)a, *(<span class="keywordtype">char</span> * <span class="keyword">const</span> *)b);
<a name="l00064"></a>00064 }
<a name="l00065"></a>00065
<a name="l00066"></a>00066 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00067"></a>00067 build_widmap(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base, <a class="code" href="structlogmath__s.html">logmath_t</a> *lmath, int32 n)
<a name="l00068"></a>00068 {
<a name="l00069"></a>00069 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00070"></a>00070 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> **models = <span class="keyword">set</span>->lms;
<a name="l00071"></a>00071 <a class="code" href="structhash__table__t.html">hash_table_t</a> *vocab;
<a name="l00072"></a>00072 <a class="code" href="structgnode__s.html" title="A node in a generic list.">glist_t</a> hlist;
<a name="l00073"></a>00073 <a class="code" href="structgnode__s.html" title="A node in a generic list.">gnode_t</a> *gn;
<a name="l00074"></a>00074 int32 i;
<a name="l00075"></a>00075
<a name="l00076"></a>00076 <span class="comment">/* Construct a merged vocabulary and a set of word-ID mappings. */</span>
<a name="l00077"></a>00077 vocab = <a class="code" href="hash__table_8h.html#a56d93e8c03e066b77377ac6eab50cfae" title="Allocate a new hash table for a given expected size.">hash_table_new</a>(models[0]->n_words, FALSE);
<a name="l00078"></a>00078 <span class="comment">/* Create the set of merged words. */</span>
<a name="l00079"></a>00079 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models; ++i) {
<a name="l00080"></a>00080 int32 j;
<a name="l00081"></a>00081 <span class="keywordflow">for</span> (j = 0; j < models[i]-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a>; ++j) {
<a name="l00082"></a>00082 <span class="comment">/* Ignore collisions. */</span>
<a name="l00083"></a>00083 (void)<a class="code" href="hash__table_8h.html#a393c56322e54607a48e6bc61169d92bf" title="Add a 32-bit integer value to a hash table.">hash_table_enter_int32</a>(vocab, models[i]->word_str[j], j);
<a name="l00084"></a>00084 }
<a name="l00085"></a>00085 }
<a name="l00086"></a>00086 <span class="comment">/* Create the array of words, then sort it. */</span>
<a name="l00087"></a>00087 <span class="keywordflow">if</span> (<a class="code" href="hash__table_8h.html#a9a1e5ed410eb96f514b00fdce770fbd7" title="Look up a key in a hash table and optionally return the associated value.">hash_table_lookup</a>(vocab, <span class="stringliteral">"<UNK>"</span>, NULL) != 0)
<a name="l00088"></a>00088 (<span class="keywordtype">void</span>)<a class="code" href="hash__table_8h.html#a393c56322e54607a48e6bc61169d92bf" title="Add a 32-bit integer value to a hash table.">hash_table_enter_int32</a>(vocab, <span class="stringliteral">"<UNK>"</span>, 0);
<a name="l00089"></a>00089 <span class="comment">/* Now we know the number of unigrams, initialize the base model. */</span>
<a name="l00090"></a>00090 ngram_model_init(base, &ngram_model_set_funcs, lmath, n, hash_table_inuse(vocab));
<a name="l00091"></a>00091 base-><a class="code" href="structngram__model__s.html#a78a3253febced2cae4732044da466ee6" title="Are word strings writable?">writable</a> = FALSE; <span class="comment">/* We will reuse the pointers from the submodels. */</span>
<a name="l00092"></a>00092 i = 0;
<a name="l00093"></a>00093 hlist = <a class="code" href="hash__table_8h.html#a61f59389f05d8871003da4692a9c2acc" title="Build a glist of valid hash_entry_t pointers from the given hash table.">hash_table_tolist</a>(vocab, NULL);
<a name="l00094"></a>00094 <span class="keywordflow">for</span> (gn = hlist; gn; gn = gnode_next(gn)) {
<a name="l00095"></a>00095 <a class="code" href="structhash__entry__s.html" title="A note by ARCHAN at 20050510: Technically what we use is so-called &quot;hash table with buckets&quot...">hash_entry_t</a> *ent = <a class="code" href="glist_8h.html#ace56682f14d84cc456c805d26fd86734" title="Head of a list of gnodes.">gnode_ptr</a>(gn);
<a name="l00096"></a>00096 base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i++] = (<span class="keywordtype">char</span> *)ent->key;
<a name="l00097"></a>00097 }
<a name="l00098"></a>00098 <a class="code" href="glist_8h.html#a45380e15d2c33afc554fd60a8828580c" title="Free the given generic list; user-defined data contained within is not automatically freed...">glist_free</a>(hlist);
<a name="l00099"></a>00099 qsort(base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>, base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a>, <span class="keyword">sizeof</span>(*base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>), my_compare);
<a name="l00100"></a>00100
<a name="l00101"></a>00101 <span class="comment">/* Now create the word ID mappings. */</span>
<a name="l00102"></a>00102 <span class="keywordflow">if</span> (set->widmap)
<a name="l00103"></a>00103 <a class="code" href="ckd__alloc_8h.html#a8246c071ac12e98b7e1df9fe6da15aef" title="Free a 2-D array (ptr) previously allocated by ckd_calloc_2d.">ckd_free_2d</a>((<span class="keywordtype">void</span> **)set->widmap);
<a name="l00104"></a>00104 <span class="keyword">set</span>->widmap = (int32 **) <a class="code" href="ckd__alloc_8h.html#a949e7b50fcd9697b1563fa57f50e9c4f" title="Macro for __ckd_calloc_2d__.">ckd_calloc_2d</a>(base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a>, set->n_models,
<a name="l00105"></a>00105 <span class="keyword">sizeof</span>(**set->widmap));
<a name="l00106"></a>00106 <span class="keywordflow">for</span> (i = 0; i < base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a>; ++i) {
<a name="l00107"></a>00107 int32 j;
<a name="l00108"></a>00108 <span class="comment">/* Also create the master wid mapping. */</span>
<a name="l00109"></a>00109 (void)<a class="code" href="hash__table_8h.html#a393c56322e54607a48e6bc61169d92bf" title="Add a 32-bit integer value to a hash table.">hash_table_enter_int32</a>(base-><a class="code" href="structngram__model__s.html#a75567419a8002ef6e916c81f5d9ee9ed" title="Mapping of unigram names to word IDs.">wid</a>, base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i], i);
<a name="l00110"></a>00110 <span class="comment">/* printf("%s: %d => ", base->word_str[i], i); */</span>
<a name="l00111"></a>00111 <span class="keywordflow">for</span> (j = 0; j < <span class="keyword">set</span>->n_models; ++j) {
<a name="l00112"></a>00112 <span class="keyword">set</span>->widmap[i][j] = <a class="code" href="ngram__model_8h.html#ad03d4355d4ea659815dc25bce8d83880" title="Look up numerical word ID.">ngram_wid</a>(models[j], base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i]);
<a name="l00113"></a>00113 <span class="comment">/* printf("%d ", set->widmap[i][j]); */</span>
<a name="l00114"></a>00114 }
<a name="l00115"></a>00115 <span class="comment">/* printf("\n"); */</span>
<a name="l00116"></a>00116 }
<a name="l00117"></a>00117 <a class="code" href="hash__table_8h.html#a0a588c22946f8cc16328973035ed19e3" title="Free the specified hash table; the caller is responsible for freeing the key strings pointed to by th...">hash_table_free</a>(vocab);
<a name="l00118"></a>00118 }
<a name="l00119"></a>00119
<a name="l00120"></a>00120 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *
<a name="l00121"></a><a class="code" href="ngram__model__set_8c.html#aa4cc619035ac8091759c07d6a3b2fb7c">00121</a> <a class="code" href="ngram__model_8h.html#a02310e01a48928e1d430aef0ea3c9a99" title="Create a set of language models sharing a common space of word IDs.">ngram_model_set_init</a>(<a class="code" href="structcmd__ln__t.html" title="Opaque structure used to hold the results of command-line parsing.">cmd_ln_t</a> *config,
<a name="l00122"></a>00122 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> **models,
<a name="l00123"></a>00123 <span class="keywordtype">char</span> **names,
<a name="l00124"></a>00124 <span class="keyword">const</span> float32 *weights,
<a name="l00125"></a>00125 int32 n_models)
<a name="l00126"></a>00126 {
<a name="l00127"></a>00127 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *model;
<a name="l00128"></a>00128 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base;
<a name="l00129"></a>00129 <a class="code" href="structlogmath__s.html">logmath_t</a> *lmath;
<a name="l00130"></a>00130 int32 i, n;
<a name="l00131"></a>00131
<a name="l00132"></a>00132 <span class="keywordflow">if</span> (n_models == 0) <span class="comment">/* WTF */</span>
<a name="l00133"></a>00133 <span class="keywordflow">return</span> NULL;
<a name="l00134"></a>00134
<a name="l00135"></a>00135 <span class="comment">/* Do consistency checking on the models. They must all use the</span>
<a name="l00136"></a>00136 <span class="comment"> * same logbase and shift. */</span>
<a name="l00137"></a>00137 lmath = models[0]-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>;
<a name="l00138"></a>00138 <span class="keywordflow">for</span> (i = 1; i < n_models; ++i) {
<a name="l00139"></a>00139 <span class="keywordflow">if</span> (<a class="code" href="logmath_8h.html#a6114206ec0321d7015c42fc7b81cb83e" title="Get the log base.">logmath_get_base</a>(models[i]->lmath) != <a class="code" href="logmath_8h.html#a6114206ec0321d7015c42fc7b81cb83e" title="Get the log base.">logmath_get_base</a>(lmath)
<a name="l00140"></a>00140 || <a class="code" href="logmath_8h.html#aed009aca9736612bebdda57444ec63a6" title="Get the shift of the values in a log table.">logmath_get_shift</a>(models[i]->lmath) != <a class="code" href="logmath_8h.html#aed009aca9736612bebdda57444ec63a6" title="Get the shift of the values in a log table.">logmath_get_shift</a>(lmath)) {
<a name="l00141"></a>00141 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Log-math parameters don't match, will not create LM set\n"</span>);
<a name="l00142"></a>00142 <span class="keywordflow">return</span> NULL;
<a name="l00143"></a>00143 }
<a name="l00144"></a>00144 }
<a name="l00145"></a>00145
<a name="l00146"></a>00146 <span class="comment">/* Allocate the combined model, initialize it. */</span>
<a name="l00147"></a>00147 model = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(1, <span class="keyword">sizeof</span>(*model));
<a name="l00148"></a>00148 base = &model-><a class="code" href="structngram__model__set__s.html#a3b88f17aa148ceac4f32dd4cbf5f4f81" title="Base ngram_model_t structure.">base</a>;
<a name="l00149"></a>00149 model-><a class="code" href="structngram__model__set__s.html#a6c28858d5631a9c9dbc7b2c9583f5c5a" title="Number of models in this set.">n_models</a> = n_models;
<a name="l00150"></a>00150 model-><a class="code" href="structngram__model__set__s.html#a862d4c4feb5a903101f4a0fdc5ba8a62" title="Language models in this set.">lms</a> = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_models, <span class="keyword">sizeof</span>(*model-><a class="code" href="structngram__model__set__s.html#a862d4c4feb5a903101f4a0fdc5ba8a62" title="Language models in this set.">lms</a>));
<a name="l00151"></a>00151 model-><a class="code" href="structngram__model__set__s.html#aff5e13c45decde4c5bf30d8aa2b1c7d9" title="Names for language models.">names</a> = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_models, <span class="keyword">sizeof</span>(*model-><a class="code" href="structngram__model__set__s.html#aff5e13c45decde4c5bf30d8aa2b1c7d9" title="Names for language models.">names</a>));
<a name="l00152"></a>00152 <span class="comment">/* Initialize weights to a uniform distribution */</span>
<a name="l00153"></a>00153 model-><a class="code" href="structngram__model__set__s.html#a11068bb0029c788c506bec4507dd6d23" title="Log interpolation weights.">lweights</a> = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_models, <span class="keyword">sizeof</span>(*model-><a class="code" href="structngram__model__set__s.html#a11068bb0029c788c506bec4507dd6d23" title="Log interpolation weights.">lweights</a>));
<a name="l00154"></a>00154 {
<a name="l00155"></a>00155 int32 uniform = <a class="code" href="logmath_8h.html#aebb4711268322fa7aec31e5798fe7e90" title="Convert linear floating point number to integer log in base B.">logmath_log</a>(lmath, 1.0/n_models);
<a name="l00156"></a>00156 <span class="keywordflow">for</span> (i = 0; i < n_models; ++i)
<a name="l00157"></a>00157 model-><a class="code" href="structngram__model__set__s.html#a11068bb0029c788c506bec4507dd6d23" title="Log interpolation weights.">lweights</a>[i] = uniform;
<a name="l00158"></a>00158 }
<a name="l00159"></a>00159 <span class="comment">/* Default to interpolate if weights were given. */</span>
<a name="l00160"></a>00160 <span class="keywordflow">if</span> (weights)
<a name="l00161"></a>00161 model-><a class="code" href="structngram__model__set__s.html#a031e7d7dcf1667f69e1cb980f45623cb" title="Currently selected model, or -1 for none.">cur</a> = -1;
<a name="l00162"></a>00162
<a name="l00163"></a>00163 n = 0;
<a name="l00164"></a>00164 <span class="keywordflow">for</span> (i = 0; i < n_models; ++i) {
<a name="l00165"></a>00165 model-><a class="code" href="structngram__model__set__s.html#a862d4c4feb5a903101f4a0fdc5ba8a62" title="Language models in this set.">lms</a>[i] = models[i];
<a name="l00166"></a>00166 model-><a class="code" href="structngram__model__set__s.html#aff5e13c45decde4c5bf30d8aa2b1c7d9" title="Names for language models.">names</a>[i] = <a class="code" href="ckd__alloc_8h.html#ad313f92478859f9e4ea99d0f6e78c393" title="Macro for __ckd_salloc__.">ckd_salloc</a>(names[i]);
<a name="l00167"></a>00167 <span class="keywordflow">if</span> (weights)
<a name="l00168"></a>00168 model-><a class="code" href="structngram__model__set__s.html#a11068bb0029c788c506bec4507dd6d23" title="Log interpolation weights.">lweights</a>[i] = <a class="code" href="logmath_8h.html#aebb4711268322fa7aec31e5798fe7e90" title="Convert linear floating point number to integer log in base B.">logmath_log</a>(lmath, weights[i]);
<a name="l00169"></a>00169 <span class="comment">/* N is the maximum of all merged models. */</span>
<a name="l00170"></a>00170 <span class="keywordflow">if</span> (models[i]->n > n)
<a name="l00171"></a>00171 n = models[i]-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a>;
<a name="l00172"></a>00172 }
<a name="l00173"></a>00173 <span class="comment">/* Allocate the history mapping table. */</span>
<a name="l00174"></a>00174 model-><a class="code" href="structngram__model__set__s.html#af34ca232338c2d8c7dc8d13869ac09fd" title="Word ID mapping for N-Gram history.">maphist</a> = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n - 1, <span class="keyword">sizeof</span>(*model-><a class="code" href="structngram__model__set__s.html#af34ca232338c2d8c7dc8d13869ac09fd" title="Word ID mapping for N-Gram history.">maphist</a>));
<a name="l00175"></a>00175
<a name="l00176"></a>00176 <span class="comment">/* Now build the word-ID mapping and merged vocabulary. */</span>
<a name="l00177"></a>00177 build_widmap(base, lmath, n);
<a name="l00178"></a>00178 <span class="keywordflow">return</span> base;
<a name="l00179"></a>00179 }
<a name="l00180"></a>00180
<a name="l00181"></a>00181 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *
<a name="l00182"></a><a class="code" href="ngram__model__set_8c.html#a1389a90bf32110fd7e56a73b6d39fa07">00182</a> <a class="code" href="ngram__model_8h.html#afb7b13dbb4a3149b2fa9f7b15dbe35e8" title="Read a set of language models from a control file.">ngram_model_set_read</a>(<a class="code" href="structcmd__ln__t.html" title="Opaque structure used to hold the results of command-line parsing.">cmd_ln_t</a> *config,
<a name="l00183"></a>00183 <span class="keyword">const</span> <span class="keywordtype">char</span> *lmctlfile,
<a name="l00184"></a>00184 <a class="code" href="structlogmath__s.html">logmath_t</a> *lmath)
<a name="l00185"></a>00185 {
<a name="l00186"></a>00186 FILE *ctlfp;
<a name="l00187"></a>00187 <a class="code" href="structgnode__s.html" title="A node in a generic list.">glist_t</a> lms = NULL;
<a name="l00188"></a>00188 <a class="code" href="structgnode__s.html" title="A node in a generic list.">glist_t</a> lmnames = NULL;
<a name="l00189"></a>00189 __BIGSTACKVARIABLE__ <span class="keywordtype">char</span> str[1024];
<a name="l00190"></a>00190 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *<span class="keyword">set</span> = NULL;
<a name="l00191"></a>00191 <a class="code" href="structhash__table__t.html">hash_table_t</a> *classes;
<a name="l00192"></a>00192 <span class="keywordtype">char</span> *basedir, *c;
<a name="l00193"></a>00193
<a name="l00194"></a>00194 <span class="comment">/* Read all the class definition files to accumulate a mapping of</span>
<a name="l00195"></a>00195 <span class="comment"> * classnames to definitions. */</span>
<a name="l00196"></a>00196 classes = <a class="code" href="hash__table_8h.html#a56d93e8c03e066b77377ac6eab50cfae" title="Allocate a new hash table for a given expected size.">hash_table_new</a>(0, FALSE);
<a name="l00197"></a>00197 <span class="keywordflow">if</span> ((ctlfp = fopen(lmctlfile, <span class="stringliteral">"r"</span>)) == NULL) {
<a name="l00198"></a>00198 <a class="code" href="err_8h.html#a54ffbfe898d74595c586a1f48f32ef03" title="Print error text; Call perror(&quot;&quot;);.">E_ERROR_SYSTEM</a>(<span class="stringliteral">"Failed to open %s"</span>, lmctlfile);
<a name="l00199"></a>00199 <span class="keywordflow">return</span> NULL;
<a name="l00200"></a>00200 }
<a name="l00201"></a>00201
<a name="l00202"></a>00202 <span class="comment">/* Try to find the base directory to append to relative paths in</span>
<a name="l00203"></a>00203 <span class="comment"> * the lmctl file. */</span>
<a name="l00204"></a>00204 <span class="keywordflow">if</span> ((c = strrchr(lmctlfile, <span class="charliteral">'/'</span>)) || (c = strrchr(lmctlfile, <span class="charliteral">'\\'</span>))) {
<a name="l00205"></a>00205 <span class="comment">/* Include the trailing slash. */</span>
<a name="l00206"></a>00206 basedir = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(c - lmctlfile + 2, 1);
<a name="l00207"></a>00207 memcpy(basedir, lmctlfile, c - lmctlfile + 1);
<a name="l00208"></a>00208 }
<a name="l00209"></a>00209 <span class="keywordflow">else</span> {
<a name="l00210"></a>00210 basedir = NULL;
<a name="l00211"></a>00211 }
<a name="l00212"></a>00212 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"Reading LM control file '%s'\n"</span>, lmctlfile);
<a name="l00213"></a>00213 <span class="keywordflow">if</span> (basedir)
<a name="l00214"></a>00214 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"Will prepend '%s' to unqualified paths\n"</span>, basedir);
<a name="l00215"></a>00215
<a name="l00216"></a>00216 <span class="keywordflow">if</span> (fscanf(ctlfp, <span class="stringliteral">"%1023s"</span>, str) == 1) {
<a name="l00217"></a>00217 <span class="keywordflow">if</span> (strcmp(str, <span class="stringliteral">"{"</span>) == 0) {
<a name="l00218"></a>00218 <span class="comment">/* Load LMclass files */</span>
<a name="l00219"></a>00219 <span class="keywordflow">while</span> ((fscanf(ctlfp, <span class="stringliteral">"%1023s"</span>, str) == 1)
<a name="l00220"></a>00220 && (strcmp(str, <span class="stringliteral">"}"</span>) != 0)) {
<a name="l00221"></a>00221 <span class="keywordtype">char</span> *deffile;
<a name="l00222"></a>00222 <span class="keywordflow">if</span> (basedir && !<a class="code" href="filename_8h.html#ac59add2db73b33e81b354de114268e7a" title="Test whether a pathname is absolute for the current OS.">path_is_absolute</a>(str))
<a name="l00223"></a>00223 deffile = <a class="code" href="strfuncs_8h.html#a317522f23d291311e366de34ef86b777" title="Concatenate a NULL-terminated argument list of strings, returning a newly allocated string...">string_join</a>(basedir, str, NULL);
<a name="l00224"></a>00224 <span class="keywordflow">else</span>
<a name="l00225"></a>00225 deffile = <a class="code" href="ckd__alloc_8h.html#ad313f92478859f9e4ea99d0f6e78c393" title="Macro for __ckd_salloc__.">ckd_salloc</a>(str);
<a name="l00226"></a>00226 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"Reading classdef from '%s'\n"</span>, deffile);
<a name="l00227"></a>00227 <span class="keywordflow">if</span> (read_classdef_file(classes, deffile) < 0) {
<a name="l00228"></a>00228 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(deffile);
<a name="l00229"></a>00229 <span class="keywordflow">goto</span> error_out;
<a name="l00230"></a>00230 }
<a name="l00231"></a>00231 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(deffile);
<a name="l00232"></a>00232 }
<a name="l00233"></a>00233
<a name="l00234"></a>00234 <span class="keywordflow">if</span> (strcmp(str, <span class="stringliteral">"}"</span>) != 0) {
<a name="l00235"></a>00235 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Unexpected EOF in %s\n"</span>, lmctlfile);
<a name="l00236"></a>00236 <span class="keywordflow">goto</span> error_out;
<a name="l00237"></a>00237 }
<a name="l00238"></a>00238
<a name="l00239"></a>00239 <span class="comment">/* This might be the first LM name. */</span>
<a name="l00240"></a>00240 <span class="keywordflow">if</span> (fscanf(ctlfp, <span class="stringliteral">"%1023s"</span>, str) != 1)
<a name="l00241"></a>00241 str[0] = <span class="charliteral">'\0'</span>;
<a name="l00242"></a>00242 }
<a name="l00243"></a>00243 }
<a name="l00244"></a>00244 <span class="keywordflow">else</span>
<a name="l00245"></a>00245 str[0] = <span class="charliteral">'\0'</span>;
<a name="l00246"></a>00246
<a name="l00247"></a>00247 <span class="comment">/* Read in one LM at a time and add classes to them as necessary. */</span>
<a name="l00248"></a>00248 <span class="keywordflow">while</span> (str[0] != <span class="charliteral">'\0'</span>) {
<a name="l00249"></a>00249 <span class="keywordtype">char</span> *lmfile;
<a name="l00250"></a>00250 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *lm;
<a name="l00251"></a>00251
<a name="l00252"></a>00252 <span class="keywordflow">if</span> (basedir && str[0] != <span class="charliteral">'/'</span> && str[0] != <span class="charliteral">'\\'</span>)
<a name="l00253"></a>00253 lmfile = <a class="code" href="strfuncs_8h.html#a317522f23d291311e366de34ef86b777" title="Concatenate a NULL-terminated argument list of strings, returning a newly allocated string...">string_join</a>(basedir, str, NULL);
<a name="l00254"></a>00254 <span class="keywordflow">else</span>
<a name="l00255"></a>00255 lmfile = <a class="code" href="ckd__alloc_8h.html#ad313f92478859f9e4ea99d0f6e78c393" title="Macro for __ckd_salloc__.">ckd_salloc</a>(str);
<a name="l00256"></a>00256 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"Reading lm from '%s'\n"</span>, lmfile);
<a name="l00257"></a>00257 lm = <a class="code" href="ngram__model_8h.html#ab0c840f2bdfc38cea08bb70054f76624" title="Read an N-Gram model from a file on disk.">ngram_model_read</a>(config, lmfile, <a class="code" href="ngram__model_8h.html#a406c0d64c15a9d1749d07c8ab6e0ae74a441701bf8ae0a2b79716feb31b5f257a" title="Determine file type automatically.">NGRAM_AUTO</a>, lmath);
<a name="l00258"></a>00258 <span class="keywordflow">if</span> (lm == NULL) {
<a name="l00259"></a>00259 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(lmfile);
<a name="l00260"></a>00260 <span class="keywordflow">goto</span> error_out;
<a name="l00261"></a>00261 }
<a name="l00262"></a>00262 <span class="keywordflow">if</span> (fscanf(ctlfp, <span class="stringliteral">"%1023s"</span>, str) != 1) {
<a name="l00263"></a>00263 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"LMname missing after LMFileName '%s'\n"</span>, lmfile);
<a name="l00264"></a>00264 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(lmfile);
<a name="l00265"></a>00265 <span class="keywordflow">goto</span> error_out;
<a name="l00266"></a>00266 }
<a name="l00267"></a>00267 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(lmfile);
<a name="l00268"></a>00268 lms = <a class="code" href="glist_8h.html#a77a9c20b7df5a289477af405ab778377" title="Create and prepend a new list node, with the given user-defined data, at the HEAD of the given generi...">glist_add_ptr</a>(lms, lm);
<a name="l00269"></a>00269 lmnames = <a class="code" href="glist_8h.html#a77a9c20b7df5a289477af405ab778377" title="Create and prepend a new list node, with the given user-defined data, at the HEAD of the given generi...">glist_add_ptr</a>(lmnames, <a class="code" href="ckd__alloc_8h.html#ad313f92478859f9e4ea99d0f6e78c393" title="Macro for __ckd_salloc__.">ckd_salloc</a>(str));
<a name="l00270"></a>00270
<a name="l00271"></a>00271 <span class="keywordflow">if</span> (fscanf(ctlfp, <span class="stringliteral">"%1023s"</span>, str) == 1) {
<a name="l00272"></a>00272 <span class="keywordflow">if</span> (strcmp(str, <span class="stringliteral">"{"</span>) == 0) {
<a name="l00273"></a>00273 <span class="comment">/* LM uses classes; read their names */</span>
<a name="l00274"></a>00274 <span class="keywordflow">while</span> ((fscanf(ctlfp, <span class="stringliteral">"%1023s"</span>, str) == 1) &&
<a name="l00275"></a>00275 (strcmp(str, <span class="stringliteral">"}"</span>) != 0)) {
<a name="l00276"></a>00276 <span class="keywordtype">void</span> *val;
<a name="l00277"></a>00277 <a class="code" href="structclassdef__s.html" title="One class definition from a classdef file.">classdef_t</a> *classdef;
<a name="l00278"></a>00278
<a name="l00279"></a>00279 <span class="keywordflow">if</span> (<a class="code" href="hash__table_8h.html#a9a1e5ed410eb96f514b00fdce770fbd7" title="Look up a key in a hash table and optionally return the associated value.">hash_table_lookup</a>(classes, str, &val) == -1) {
<a name="l00280"></a>00280 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Unknown class %s in control file\n"</span>, str);
<a name="l00281"></a>00281 <span class="keywordflow">goto</span> error_out;
<a name="l00282"></a>00282 }
<a name="l00283"></a>00283 classdef = val;
<a name="l00284"></a>00284 <span class="keywordflow">if</span> (<a class="code" href="ngram__model_8h.html#a0481b03a2bf03ef0a197da0a1f7d1caa" title="Add a new class to a language model.">ngram_model_add_class</a>(lm, str, 1.0,
<a name="l00285"></a>00285 classdef->words, classdef->weights,
<a name="l00286"></a>00286 classdef->n_words) < 0) {
<a name="l00287"></a>00287 <span class="keywordflow">goto</span> error_out;
<a name="l00288"></a>00288 }
<a name="l00289"></a>00289 <a class="code" href="err_8h.html#ac3c705943d946708cea0a1443be1c853" title="Print logging information to standard error stream.">E_INFO</a>(<span class="stringliteral">"Added class %s containing %d words\n"</span>,
<a name="l00290"></a>00290 str, classdef->n_words);
<a name="l00291"></a>00291 }
<a name="l00292"></a>00292 <span class="keywordflow">if</span> (strcmp(str, <span class="stringliteral">"}"</span>) != 0) {
<a name="l00293"></a>00293 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Unexpected EOF in %s\n"</span>, lmctlfile);
<a name="l00294"></a>00294 <span class="keywordflow">goto</span> error_out;
<a name="l00295"></a>00295 }
<a name="l00296"></a>00296 <span class="keywordflow">if</span> (fscanf(ctlfp, <span class="stringliteral">"%1023s"</span>, str) != 1)
<a name="l00297"></a>00297 str[0] = <span class="charliteral">'\0'</span>;
<a name="l00298"></a>00298 }
<a name="l00299"></a>00299 }
<a name="l00300"></a>00300 <span class="keywordflow">else</span>
<a name="l00301"></a>00301 str[0] = <span class="charliteral">'\0'</span>;
<a name="l00302"></a>00302 }
<a name="l00303"></a>00303 fclose(ctlfp);
<a name="l00304"></a>00304
<a name="l00305"></a>00305 <span class="comment">/* Now construct arrays out of lms and lmnames, and build an</span>
<a name="l00306"></a>00306 <span class="comment"> * ngram_model_set. */</span>
<a name="l00307"></a>00307 lms = <a class="code" href="glist_8h.html#a399a2a093c6c4ce1012762e4c25c8185" title="Reverse the order of the given glist.">glist_reverse</a>(lms);
<a name="l00308"></a>00308 lmnames = <a class="code" href="glist_8h.html#a399a2a093c6c4ce1012762e4c25c8185" title="Reverse the order of the given glist.">glist_reverse</a>(lmnames);
<a name="l00309"></a>00309 {
<a name="l00310"></a>00310 int32 n_models;
<a name="l00311"></a>00311 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> **lm_array;
<a name="l00312"></a>00312 <span class="keywordtype">char</span> **name_array;
<a name="l00313"></a>00313 <a class="code" href="structgnode__s.html" title="A node in a generic list.">gnode_t</a> *lm_node, *name_node;
<a name="l00314"></a>00314 int32 i;
<a name="l00315"></a>00315
<a name="l00316"></a>00316 n_models = <a class="code" href="glist_8h.html#aeb046e39c540d2f5f792119ea0d24c48" title="Count the number of element in a given link list.">glist_count</a>(lms);
<a name="l00317"></a>00317 lm_array = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_models, <span class="keyword">sizeof</span>(*lm_array));
<a name="l00318"></a>00318 name_array = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_models, <span class="keyword">sizeof</span>(*name_array));
<a name="l00319"></a>00319 lm_node = lms;
<a name="l00320"></a>00320 name_node = lmnames;
<a name="l00321"></a>00321 <span class="keywordflow">for</span> (i = 0; i < n_models; ++i) {
<a name="l00322"></a>00322 lm_array[i] = <a class="code" href="glist_8h.html#ace56682f14d84cc456c805d26fd86734" title="Head of a list of gnodes.">gnode_ptr</a>(lm_node);
<a name="l00323"></a>00323 name_array[i] = <a class="code" href="glist_8h.html#ace56682f14d84cc456c805d26fd86734" title="Head of a list of gnodes.">gnode_ptr</a>(name_node);
<a name="l00324"></a>00324 lm_node = gnode_next(lm_node);
<a name="l00325"></a>00325 name_node = gnode_next(name_node);
<a name="l00326"></a>00326 }
<a name="l00327"></a>00327 <span class="keyword">set</span> = <a class="code" href="ngram__model_8h.html#a02310e01a48928e1d430aef0ea3c9a99" title="Create a set of language models sharing a common space of word IDs.">ngram_model_set_init</a>(config, lm_array, name_array,
<a name="l00328"></a>00328 NULL, n_models);
<a name="l00329"></a>00329 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(lm_array);
<a name="l00330"></a>00330 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(name_array);
<a name="l00331"></a>00331 }
<a name="l00332"></a>00332 error_out:
<a name="l00333"></a>00333 {
<a name="l00334"></a>00334 <a class="code" href="structgnode__s.html" title="A node in a generic list.">gnode_t</a> *gn;
<a name="l00335"></a>00335 <a class="code" href="structgnode__s.html" title="A node in a generic list.">glist_t</a> hlist;
<a name="l00336"></a>00336
<a name="l00337"></a>00337 <span class="keywordflow">if</span> (<span class="keyword">set</span> == NULL) {
<a name="l00338"></a>00338 <span class="keywordflow">for</span> (gn = lms; gn; gn = gnode_next(gn)) {
<a name="l00339"></a>00339 <a class="code" href="ngram__model_8h.html#aec73d28e7285e539a0b44a7ac0cbe489" title="Release memory associated with an N-Gram model.">ngram_model_free</a>(<a class="code" href="glist_8h.html#ace56682f14d84cc456c805d26fd86734" title="Head of a list of gnodes.">gnode_ptr</a>(gn));
<a name="l00340"></a>00340 }
<a name="l00341"></a>00341 }
<a name="l00342"></a>00342 <a class="code" href="glist_8h.html#a45380e15d2c33afc554fd60a8828580c" title="Free the given generic list; user-defined data contained within is not automatically freed...">glist_free</a>(lms);
<a name="l00343"></a>00343 <span class="keywordflow">for</span> (gn = lmnames; gn; gn = gnode_next(gn)) {
<a name="l00344"></a>00344 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(<a class="code" href="glist_8h.html#ace56682f14d84cc456c805d26fd86734" title="Head of a list of gnodes.">gnode_ptr</a>(gn));
<a name="l00345"></a>00345 }
<a name="l00346"></a>00346 <a class="code" href="glist_8h.html#a45380e15d2c33afc554fd60a8828580c" title="Free the given generic list; user-defined data contained within is not automatically freed...">glist_free</a>(lmnames);
<a name="l00347"></a>00347 hlist = <a class="code" href="hash__table_8h.html#a61f59389f05d8871003da4692a9c2acc" title="Build a glist of valid hash_entry_t pointers from the given hash table.">hash_table_tolist</a>(classes, NULL);
<a name="l00348"></a>00348 <span class="keywordflow">for</span> (gn = hlist; gn; gn = gnode_next(gn)) {
<a name="l00349"></a>00349 <a class="code" href="structhash__entry__s.html" title="A note by ARCHAN at 20050510: Technically what we use is so-called &quot;hash table with buckets&quot...">hash_entry_t</a> *he = <a class="code" href="glist_8h.html#ace56682f14d84cc456c805d26fd86734" title="Head of a list of gnodes.">gnode_ptr</a>(gn);
<a name="l00350"></a>00350 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>((<span class="keywordtype">char</span> *)he->key);
<a name="l00351"></a>00351 classdef_free(he-><a class="code" href="structhash__entry__s.html#a0d57012963084fed93886681108aa636" title="Key-length; the key string does not have to be a C-style NULL terminated string; it can have arbitrar...">val</a>);
<a name="l00352"></a>00352 }
<a name="l00353"></a>00353 <a class="code" href="glist_8h.html#a45380e15d2c33afc554fd60a8828580c" title="Free the given generic list; user-defined data contained within is not automatically freed...">glist_free</a>(hlist);
<a name="l00354"></a>00354 <a class="code" href="hash__table_8h.html#a0a588c22946f8cc16328973035ed19e3" title="Free the specified hash table; the caller is responsible for freeing the key strings pointed to by th...">hash_table_free</a>(classes);
<a name="l00355"></a>00355 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(basedir);
<a name="l00356"></a>00356 }
<a name="l00357"></a>00357 <span class="keywordflow">return</span> <span class="keyword">set</span>;
<a name="l00358"></a>00358 }
<a name="l00359"></a>00359
<a name="l00360"></a>00360 int32
<a name="l00361"></a><a class="code" href="ngram__model__set_8c.html#afa38d4e0f6cd64cc2875000fcef487f0">00361</a> <a class="code" href="ngram__model_8h.html#a22b8f5f49231b4aef3d5a166ea7155b6" title="Returns the number of language models in a set.">ngram_model_set_count</a>(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base)
<a name="l00362"></a>00362 {
<a name="l00363"></a>00363 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00364"></a>00364 <span class="keywordflow">return</span> <span class="keyword">set</span>->n_models;
<a name="l00365"></a>00365 }
<a name="l00366"></a>00366
<a name="l00367"></a>00367 <a class="code" href="structngram__model__set__iter__s.html" title="Iterator over a model set.">ngram_model_set_iter_t</a> *
<a name="l00368"></a><a class="code" href="ngram__model__set_8c.html#a5a1593344017a1210569beb63e4089d3">00368</a> <a class="code" href="ngram__model_8h.html#a880c7768b082a124e1b6f6f1a764b1e3" title="Begin iterating over language models in a set.">ngram_model_set_iter</a>(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base)
<a name="l00369"></a>00369 {
<a name="l00370"></a>00370 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00371"></a>00371 <a class="code" href="structngram__model__set__iter__s.html" title="Iterator over a model set.">ngram_model_set_iter_t</a> *itor;
<a name="l00372"></a>00372
<a name="l00373"></a>00373 <span class="keywordflow">if</span> (<span class="keyword">set</span> == NULL || set->n_models == 0)
<a name="l00374"></a>00374 <span class="keywordflow">return</span> NULL;
<a name="l00375"></a>00375 itor = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(1, <span class="keyword">sizeof</span>(*itor));
<a name="l00376"></a>00376 itor->set = <span class="keyword">set</span>;
<a name="l00377"></a>00377 <span class="keywordflow">return</span> itor;
<a name="l00378"></a>00378 }
<a name="l00379"></a>00379
<a name="l00380"></a>00380 <a class="code" href="structngram__model__set__iter__s.html" title="Iterator over a model set.">ngram_model_set_iter_t</a> *
<a name="l00381"></a><a class="code" href="ngram__model__set_8c.html#a13893e41e56962f6f237aa1305337b7a">00381</a> <a class="code" href="ngram__model_8h.html#a038a174f17632703cffce02611149265" title="Move to the next language model in a set.">ngram_model_set_iter_next</a>(<a class="code" href="structngram__model__set__iter__s.html" title="Iterator over a model set.">ngram_model_set_iter_t</a> *itor)
<a name="l00382"></a>00382 {
<a name="l00383"></a>00383 <span class="keywordflow">if</span> (++itor->cur == itor->set-><a class="code" href="structngram__model__set__s.html#a6c28858d5631a9c9dbc7b2c9583f5c5a" title="Number of models in this set.">n_models</a>) {
<a name="l00384"></a>00384 <a class="code" href="ngram__model_8h.html#ae5b58f360ab6432528ffaead1d2b2a2d" title="Finish iteration over a langauge model set.">ngram_model_set_iter_free</a>(itor);
<a name="l00385"></a>00385 <span class="keywordflow">return</span> NULL;
<a name="l00386"></a>00386 }
<a name="l00387"></a>00387 <span class="keywordflow">return</span> itor;
<a name="l00388"></a>00388 }
<a name="l00389"></a>00389
<a name="l00390"></a>00390 <span class="keywordtype">void</span>
<a name="l00391"></a><a class="code" href="ngram__model__set_8c.html#a532787b5b709d0c5f0ad57e3df0d1d73">00391</a> <a class="code" href="ngram__model_8h.html#ae5b58f360ab6432528ffaead1d2b2a2d" title="Finish iteration over a langauge model set.">ngram_model_set_iter_free</a>(<a class="code" href="structngram__model__set__iter__s.html" title="Iterator over a model set.">ngram_model_set_iter_t</a> *itor)
<a name="l00392"></a>00392 {
<a name="l00393"></a>00393 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(itor);
<a name="l00394"></a>00394 }
<a name="l00395"></a>00395
<a name="l00396"></a>00396 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *
<a name="l00397"></a><a class="code" href="ngram__model__set_8c.html#abec0455e59945b910d86c2f825ab0894">00397</a> <a class="code" href="ngram__model_8h.html#a438592444037281175150e3cf1543052" title="Get language model and associated name from an iterator.">ngram_model_set_iter_model</a>(<a class="code" href="structngram__model__set__iter__s.html" title="Iterator over a model set.">ngram_model_set_iter_t</a> *itor,
<a name="l00398"></a>00398 <span class="keywordtype">char</span> <span class="keyword">const</span> **lmname)
<a name="l00399"></a>00399 {
<a name="l00400"></a>00400 <span class="keywordflow">if</span> (lmname) *lmname = itor->set-><a class="code" href="structngram__model__set__s.html#aff5e13c45decde4c5bf30d8aa2b1c7d9" title="Names for language models.">names</a>[itor->cur];
<a name="l00401"></a>00401 <span class="keywordflow">return</span> itor->set-><a class="code" href="structngram__model__set__s.html#a862d4c4feb5a903101f4a0fdc5ba8a62" title="Language models in this set.">lms</a>[itor->cur];
<a name="l00402"></a>00402 }
<a name="l00403"></a>00403
<a name="l00404"></a>00404 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *
<a name="l00405"></a><a class="code" href="ngram__model__set_8c.html#a3e06e5d47a9c35186644cc1310924630">00405</a> <a class="code" href="ngram__model_8h.html#a1797a53b95a2a1ca2a6be732918d9fa0" title="Look up a language model by name from a set.">ngram_model_set_lookup</a>(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base,
<a name="l00406"></a>00406 <span class="keyword">const</span> <span class="keywordtype">char</span> *name)
<a name="l00407"></a>00407 {
<a name="l00408"></a>00408 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00409"></a>00409 int32 i;
<a name="l00410"></a>00410
<a name="l00411"></a>00411 <span class="keywordflow">if</span> (name == NULL) {
<a name="l00412"></a>00412 <span class="keywordflow">if</span> (set->cur == -1)
<a name="l00413"></a>00413 <span class="keywordflow">return</span> NULL;
<a name="l00414"></a>00414 <span class="keywordflow">else</span>
<a name="l00415"></a>00415 <span class="keywordflow">return</span> <span class="keyword">set</span>->lms[<span class="keyword">set</span>->cur];
<a name="l00416"></a>00416 }
<a name="l00417"></a>00417
<a name="l00418"></a>00418 <span class="comment">/* There probably won't be very many submodels. */</span>
<a name="l00419"></a>00419 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models; ++i)
<a name="l00420"></a>00420 <span class="keywordflow">if</span> (0 == strcmp(set->names[i], name))
<a name="l00421"></a>00421 <span class="keywordflow">break</span>;
<a name="l00422"></a>00422 <span class="keywordflow">if</span> (i == set->n_models)
<a name="l00423"></a>00423 <span class="keywordflow">return</span> NULL;
<a name="l00424"></a>00424 <span class="keywordflow">return</span> <span class="keyword">set</span>->lms[i];
<a name="l00425"></a>00425 }
<a name="l00426"></a>00426
<a name="l00427"></a>00427 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *
<a name="l00428"></a><a class="code" href="ngram__model__set_8c.html#a2cbeda856dd655c80f8dc6ba91f5d20b">00428</a> <a class="code" href="ngram__model_8h.html#a5e7a3fd20d32783ce07b760781f658f8" title="Select a single language model from a set for scoring.">ngram_model_set_select</a>(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base,
<a name="l00429"></a>00429 <span class="keyword">const</span> <span class="keywordtype">char</span> *name)
<a name="l00430"></a>00430 {
<a name="l00431"></a>00431 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00432"></a>00432 int32 i;
<a name="l00433"></a>00433
<a name="l00434"></a>00434 <span class="comment">/* There probably won't be very many submodels. */</span>
<a name="l00435"></a>00435 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models; ++i)
<a name="l00436"></a>00436 <span class="keywordflow">if</span> (0 == strcmp(set->names[i], name))
<a name="l00437"></a>00437 <span class="keywordflow">break</span>;
<a name="l00438"></a>00438 <span class="keywordflow">if</span> (i == set->n_models)
<a name="l00439"></a>00439 <span class="keywordflow">return</span> NULL;
<a name="l00440"></a>00440 <span class="keyword">set</span>->cur = i;
<a name="l00441"></a>00441 <span class="keywordflow">return</span> <span class="keyword">set</span>->lms[<span class="keyword">set</span>->cur];
<a name="l00442"></a>00442 }
<a name="l00443"></a>00443
<a name="l00444"></a>00444 <span class="keyword">const</span> <span class="keywordtype">char</span> *
<a name="l00445"></a><a class="code" href="ngram__model__set_8c.html#a3a4f2bb40d10e9420e6291e2706b6865">00445</a> <a class="code" href="ngram__model_8h.html#a95a4da1562cb6d67a01a733bd0bbb436" title="Get the current language model name, if any.">ngram_model_set_current</a>(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base)
<a name="l00446"></a>00446 {
<a name="l00447"></a>00447 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00448"></a>00448
<a name="l00449"></a>00449 <span class="keywordflow">if</span> (set->cur == -1)
<a name="l00450"></a>00450 <span class="keywordflow">return</span> NULL;
<a name="l00451"></a>00451 <span class="keywordflow">else</span>
<a name="l00452"></a>00452 <span class="keywordflow">return</span> <span class="keyword">set</span>->names[<span class="keyword">set</span>->cur];
<a name="l00453"></a>00453 }
<a name="l00454"></a>00454
<a name="l00455"></a>00455 int32
<a name="l00456"></a><a class="code" href="ngram__model__set_8c.html#aa6239b58f333177839a15de79bae6146">00456</a> <a class="code" href="ngram__model_8h.html#af783553aa7527b7544d093aaea74d97a" title="Query the word-ID mapping for the current language model.">ngram_model_set_current_wid</a>(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base,
<a name="l00457"></a>00457 int32 set_wid)
<a name="l00458"></a>00458 {
<a name="l00459"></a>00459 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00460"></a>00460
<a name="l00461"></a>00461 <span class="keywordflow">if</span> (set->cur == -1 || set_wid >= base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a>)
<a name="l00462"></a>00462 <span class="keywordflow">return</span> <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>;
<a name="l00463"></a>00463 <span class="keywordflow">else</span>
<a name="l00464"></a>00464 <span class="keywordflow">return</span> <span class="keyword">set</span>->widmap[set_wid][<span class="keyword">set</span>->cur];
<a name="l00465"></a>00465 }
<a name="l00466"></a>00466
<a name="l00467"></a>00467 int32
<a name="l00468"></a><a class="code" href="ngram__model__set_8c.html#af30197146b9b2d6c143914240dee3c04">00468</a> <a class="code" href="ngram__model_8h.html#ad87975912cfe1ac5d0c3ebe12de79209" title="Test whether a word ID corresponds to a known word in the current state of the language model set...">ngram_model_set_known_wid</a>(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base,
<a name="l00469"></a>00469 int32 set_wid)
<a name="l00470"></a>00470 {
<a name="l00471"></a>00471 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00472"></a>00472
<a name="l00473"></a>00473 <span class="keywordflow">if</span> (set_wid >= base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a>)
<a name="l00474"></a>00474 <span class="keywordflow">return</span> FALSE;
<a name="l00475"></a>00475 <span class="keywordflow">else</span> <span class="keywordflow">if</span> (set->cur == -1) {
<a name="l00476"></a>00476 int32 i;
<a name="l00477"></a>00477 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models; ++i) {
<a name="l00478"></a>00478 <span class="keywordflow">if</span> (set->widmap[set_wid][i] != <a class="code" href="ngram__model_8h.html#a1469e9e1c8516a77c9ac1e248a61ef4e" title="Get the unknown word ID for a language model.">ngram_unknown_wid</a>(set->lms[i]))
<a name="l00479"></a>00479 <span class="keywordflow">return</span> TRUE;
<a name="l00480"></a>00480 }
<a name="l00481"></a>00481 <span class="keywordflow">return</span> FALSE;
<a name="l00482"></a>00482 }
<a name="l00483"></a>00483 <span class="keywordflow">else</span>
<a name="l00484"></a>00484 <span class="keywordflow">return</span> (set->widmap[set_wid][set->cur]
<a name="l00485"></a>00485 != <a class="code" href="ngram__model_8h.html#a1469e9e1c8516a77c9ac1e248a61ef4e" title="Get the unknown word ID for a language model.">ngram_unknown_wid</a>(set->lms[set->cur]));
<a name="l00486"></a>00486 }
<a name="l00487"></a>00487
<a name="l00488"></a>00488 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *
<a name="l00489"></a><a class="code" href="ngram__model__set_8c.html#a3a926714c60e2b8308d4c88d11d17d7e">00489</a> <a class="code" href="ngram__model_8h.html#a8a4c853e77c6b48face3a781480e77d6" title="Set interpolation weights for a set and enables interpolation.">ngram_model_set_interp</a>(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base,
<a name="l00490"></a>00490 <span class="keyword">const</span> <span class="keywordtype">char</span> **names,
<a name="l00491"></a>00491 <span class="keyword">const</span> float32 *weights)
<a name="l00492"></a>00492 {
<a name="l00493"></a>00493 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00494"></a>00494
<a name="l00495"></a>00495 <span class="comment">/* If we have a set of weights here, then set them. */</span>
<a name="l00496"></a>00496 <span class="keywordflow">if</span> (names && weights) {
<a name="l00497"></a>00497 int32 i, j;
<a name="l00498"></a>00498
<a name="l00499"></a>00499 <span class="comment">/* We hope there aren't many models. */</span>
<a name="l00500"></a>00500 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models; ++i) {
<a name="l00501"></a>00501 <span class="keywordflow">for</span> (j = 0; j < <span class="keyword">set</span>->n_models; ++j)
<a name="l00502"></a>00502 <span class="keywordflow">if</span> (0 == strcmp(names[i], set->names[j]))
<a name="l00503"></a>00503 <span class="keywordflow">break</span>;
<a name="l00504"></a>00504 <span class="keywordflow">if</span> (j == set->n_models) {
<a name="l00505"></a>00505 <a class="code" href="err_8h.html#a5f7b2f58f5a663a6bdd51f197ae21993" title="Print error message to standard error stream.">E_ERROR</a>(<span class="stringliteral">"Unknown LM name %s\n"</span>, names[i]);
<a name="l00506"></a>00506 <span class="keywordflow">return</span> NULL;
<a name="l00507"></a>00507 }
<a name="l00508"></a>00508 <span class="keyword">set</span>->lweights[j] = <a class="code" href="logmath_8h.html#aebb4711268322fa7aec31e5798fe7e90" title="Convert linear floating point number to integer log in base B.">logmath_log</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, weights[i]);
<a name="l00509"></a>00509 }
<a name="l00510"></a>00510 }
<a name="l00511"></a>00511 <span class="keywordflow">else</span> <span class="keywordflow">if</span> (weights) {
<a name="l00512"></a>00512 memcpy(set->lweights, weights, set->n_models * <span class="keyword">sizeof</span>(*set->lweights));
<a name="l00513"></a>00513 }
<a name="l00514"></a>00514 <span class="comment">/* Otherwise just enable existing weights. */</span>
<a name="l00515"></a>00515 <span class="keyword">set</span>->cur = -1;
<a name="l00516"></a>00516 <span class="keywordflow">return</span> base;
<a name="l00517"></a>00517 }
<a name="l00518"></a>00518
<a name="l00519"></a>00519 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *
<a name="l00520"></a><a class="code" href="ngram__model__set_8c.html#a53071575bca9465361740c809666349d">00520</a> <a class="code" href="ngram__model_8h.html#aab34758e97f75733330f3f4cff8a5311" title="Add a language model to a set.">ngram_model_set_add</a>(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base,
<a name="l00521"></a>00521 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *model,
<a name="l00522"></a>00522 <span class="keyword">const</span> <span class="keywordtype">char</span> *name,
<a name="l00523"></a>00523 float32 weight,
<a name="l00524"></a>00524 <span class="keywordtype">int</span> reuse_widmap)
<a name="l00525"></a>00525
<a name="l00526"></a>00526 {
<a name="l00527"></a>00527 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00528"></a>00528 float32 fprob;
<a name="l00529"></a>00529 int32 scale, i;
<a name="l00530"></a>00530
<a name="l00531"></a>00531 <span class="comment">/* Add it to the array of lms. */</span>
<a name="l00532"></a>00532 ++<span class="keyword">set</span>->n_models;
<a name="l00533"></a>00533 <span class="keyword">set</span>->lms = <a class="code" href="ckd__alloc_8h.html#afd496738b3e114bd494c5a0955f1bfb3" title="Macro for __ckd_realloc__.">ckd_realloc</a>(set->lms, set->n_models * <span class="keyword">sizeof</span>(*set->lms));
<a name="l00534"></a>00534 <span class="keyword">set</span>->lms[<span class="keyword">set</span>->n_models - 1] = model;
<a name="l00535"></a>00535 <span class="keyword">set</span>->names = <a class="code" href="ckd__alloc_8h.html#afd496738b3e114bd494c5a0955f1bfb3" title="Macro for __ckd_realloc__.">ckd_realloc</a>(set->names, set->n_models * <span class="keyword">sizeof</span>(*set->names));
<a name="l00536"></a>00536 <span class="keyword">set</span>->names[<span class="keyword">set</span>->n_models - 1] = <a class="code" href="ckd__alloc_8h.html#ad313f92478859f9e4ea99d0f6e78c393" title="Macro for __ckd_salloc__.">ckd_salloc</a>(name);
<a name="l00537"></a>00537 <span class="comment">/* Expand the history mapping table if necessary. */</span>
<a name="l00538"></a>00538 <span class="keywordflow">if</span> (model-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> > base-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a>) {
<a name="l00539"></a>00539 base-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> = model-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a>;
<a name="l00540"></a>00540 <span class="keyword">set</span>->maphist = <a class="code" href="ckd__alloc_8h.html#afd496738b3e114bd494c5a0955f1bfb3" title="Macro for __ckd_realloc__.">ckd_realloc</a>(set->maphist,
<a name="l00541"></a>00541 (model-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> - 1) * <span class="keyword">sizeof</span>(*set->maphist));
<a name="l00542"></a>00542 }
<a name="l00543"></a>00543
<a name="l00544"></a>00544 <span class="comment">/* Renormalize the interpolation weights. */</span>
<a name="l00545"></a>00545 fprob = weight * 1.0 / <span class="keyword">set</span>->n_models;
<a name="l00546"></a>00546 <span class="keyword">set</span>->lweights = <a class="code" href="ckd__alloc_8h.html#afd496738b3e114bd494c5a0955f1bfb3" title="Macro for __ckd_realloc__.">ckd_realloc</a>(set->lweights,
<a name="l00547"></a>00547 set->n_models * <span class="keyword">sizeof</span>(*set->lweights));
<a name="l00548"></a>00548 <span class="keyword">set</span>->lweights[<span class="keyword">set</span>->n_models - 1] = <a class="code" href="logmath_8h.html#aebb4711268322fa7aec31e5798fe7e90" title="Convert linear floating point number to integer log in base B.">logmath_log</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, fprob);
<a name="l00549"></a>00549 <span class="comment">/* Now normalize everything else to fit it in. This is</span>
<a name="l00550"></a>00550 <span class="comment"> * accomplished by simply scaling all the other probabilities</span>
<a name="l00551"></a>00551 <span class="comment"> * by (1-fprob). */</span>
<a name="l00552"></a>00552 scale = <a class="code" href="logmath_8h.html#aebb4711268322fa7aec31e5798fe7e90" title="Convert linear floating point number to integer log in base B.">logmath_log</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, 1.0 - fprob);
<a name="l00553"></a>00553 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models - 1; ++i)
<a name="l00554"></a>00554 set->lweights[i] += scale;
<a name="l00555"></a>00555
<a name="l00556"></a>00556 <span class="comment">/* Reuse the old word ID mapping if requested. */</span>
<a name="l00557"></a>00557 if (reuse_widmap) {
<a name="l00558"></a>00558 int32 **new_widmap;
<a name="l00559"></a>00559
<a name="l00560"></a>00560 <span class="comment">/* Tack another column onto the widmap array. */</span>
<a name="l00561"></a>00561 new_widmap = (int32 **)<a class="code" href="ckd__alloc_8h.html#a949e7b50fcd9697b1563fa57f50e9c4f" title="Macro for __ckd_calloc_2d__.">ckd_calloc_2d</a>(base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a>, set->n_models,
<a name="l00562"></a>00562 sizeof (**new_widmap));
<a name="l00563"></a>00563 <span class="keywordflow">for</span> (i = 0; i < base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a>; ++i) {
<a name="l00564"></a>00564 <span class="comment">/* Copy all the existing mappings. */</span>
<a name="l00565"></a>00565 memcpy(new_widmap[i], set->widmap[i],
<a name="l00566"></a>00566 (set->n_models - 1) * <span class="keyword">sizeof</span>(**new_widmap));
<a name="l00567"></a>00567 <span class="comment">/* Create the new mapping. */</span>
<a name="l00568"></a>00568 new_widmap[i][<span class="keyword">set</span>->n_models-1] = <a class="code" href="ngram__model_8h.html#ad03d4355d4ea659815dc25bce8d83880" title="Look up numerical word ID.">ngram_wid</a>(model, base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i]);
<a name="l00569"></a>00569 }
<a name="l00570"></a>00570 <a class="code" href="ckd__alloc_8h.html#a8246c071ac12e98b7e1df9fe6da15aef" title="Free a 2-D array (ptr) previously allocated by ckd_calloc_2d.">ckd_free_2d</a>((<span class="keywordtype">void</span> **)set->widmap);
<a name="l00571"></a>00571 <span class="keyword">set</span>->widmap = new_widmap;
<a name="l00572"></a>00572 }
<a name="l00573"></a>00573 <span class="keywordflow">else</span> {
<a name="l00574"></a>00574 build_widmap(base, base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, base-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a>);
<a name="l00575"></a>00575 }
<a name="l00576"></a>00576 <span class="keywordflow">return</span> model;
<a name="l00577"></a>00577 }
<a name="l00578"></a>00578
<a name="l00579"></a>00579 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *
<a name="l00580"></a><a class="code" href="ngram__model__set_8c.html#a1fadaf8cdc6799869b74a39a8652a7e0">00580</a> <a class="code" href="ngram__model_8h.html#af761011a07b87c209e6d174e07b34617" title="Remove a language model from a set.">ngram_model_set_remove</a>(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base,
<a name="l00581"></a>00581 <span class="keyword">const</span> <span class="keywordtype">char</span> *name,
<a name="l00582"></a>00582 <span class="keywordtype">int</span> reuse_widmap)
<a name="l00583"></a>00583 {
<a name="l00584"></a>00584 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00585"></a>00585 <a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *submodel;
<a name="l00586"></a>00586 int32 lmidx, scale, n, i;
<a name="l00587"></a>00587 float32 fprob;
<a name="l00588"></a>00588
<a name="l00589"></a>00589 <span class="keywordflow">for</span> (lmidx = 0; lmidx < <span class="keyword">set</span>->n_models; ++lmidx)
<a name="l00590"></a>00590 <span class="keywordflow">if</span> (0 == strcmp(name, set->names[lmidx]))
<a name="l00591"></a>00591 <span class="keywordflow">break</span>;
<a name="l00592"></a>00592 <span class="keywordflow">if</span> (lmidx == set->n_models)
<a name="l00593"></a>00593 <span class="keywordflow">return</span> NULL;
<a name="l00594"></a>00594 submodel = <span class="keyword">set</span>->lms[lmidx];
<a name="l00595"></a>00595
<a name="l00596"></a>00596 <span class="comment">/* Renormalize the interpolation weights by scaling them by</span>
<a name="l00597"></a>00597 <span class="comment"> * 1/(1-fprob) */</span>
<a name="l00598"></a>00598 fprob = <a class="code" href="logmath_8h.html#ae8b0a168e29e448c0d6de66dc46e099e" title="Convert integer log in base B to linear floating point.">logmath_exp</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, set->lweights[lmidx]);
<a name="l00599"></a>00599 scale = <a class="code" href="logmath_8h.html#aebb4711268322fa7aec31e5798fe7e90" title="Convert linear floating point number to integer log in base B.">logmath_log</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, 1.0 - fprob);
<a name="l00600"></a>00600
<a name="l00601"></a>00601 <span class="comment">/* Remove it from the array of lms, renormalize remaining weights,</span>
<a name="l00602"></a>00602 <span class="comment"> * and recalcluate n. */</span>
<a name="l00603"></a>00603 --<span class="keyword">set</span>->n_models;
<a name="l00604"></a>00604 n = 0;
<a name="l00605"></a>00605 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(set->names[lmidx]);
<a name="l00606"></a>00606 <span class="keyword">set</span>->names[lmidx] = NULL;
<a name="l00607"></a>00607 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models; ++i) {
<a name="l00608"></a>00608 <span class="keywordflow">if</span> (i >= lmidx) {
<a name="l00609"></a>00609 <span class="keyword">set</span>->lms[i] = <span class="keyword">set</span>->lms[i+1];
<a name="l00610"></a>00610 <span class="keyword">set</span>->names[i] = <span class="keyword">set</span>->names[i+1];
<a name="l00611"></a>00611 <span class="keyword">set</span>->lweights[i] = <span class="keyword">set</span>->lweights[i+1];
<a name="l00612"></a>00612 }
<a name="l00613"></a>00613 <span class="keyword">set</span>->lweights[i] -= scale;
<a name="l00614"></a>00614 <span class="keywordflow">if</span> (set->lms[i]->n > n)
<a name="l00615"></a>00615 n = <span class="keyword">set</span>->lms[i]->n;
<a name="l00616"></a>00616 }
<a name="l00617"></a>00617 <span class="comment">/* There's no need to shrink these arrays. */</span>
<a name="l00618"></a>00618 <span class="keyword">set</span>->lms[<span class="keyword">set</span>->n_models] = NULL;
<a name="l00619"></a>00619 <span class="keyword">set</span>->lweights[<span class="keyword">set</span>->n_models] = base-><a class="code" href="structngram__model__s.html#a65425a599c4bcc4dda809d81149b8fc0" title="Zero probability, cached here for quick lookup.">log_zero</a>;
<a name="l00620"></a>00620 <span class="comment">/* No need to shrink maphist either. */</span>
<a name="l00621"></a>00621
<a name="l00622"></a>00622 <span class="comment">/* Reuse the existing word ID mapping if requested. */</span>
<a name="l00623"></a>00623 <span class="keywordflow">if</span> (reuse_widmap) {
<a name="l00624"></a>00624 <span class="comment">/* Just go through and shrink each row. */</span>
<a name="l00625"></a>00625 <span class="keywordflow">for</span> (i = 0; i < base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a>; ++i) {
<a name="l00626"></a>00626 memmove(set->widmap[i] + lmidx, set->widmap[i] + lmidx + 1,
<a name="l00627"></a>00627 (set->n_models - lmidx) * <span class="keyword">sizeof</span>(**set->widmap));
<a name="l00628"></a>00628 }
<a name="l00629"></a>00629 }
<a name="l00630"></a>00630 <span class="keywordflow">else</span> {
<a name="l00631"></a>00631 build_widmap(base, base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, n);
<a name="l00632"></a>00632 }
<a name="l00633"></a>00633 <span class="keywordflow">return</span> submodel;
<a name="l00634"></a>00634 }
<a name="l00635"></a>00635
<a name="l00636"></a>00636 <span class="keywordtype">void</span>
<a name="l00637"></a><a class="code" href="ngram__model__set_8c.html#af3ac3bbec3eed0b9638c4e64c8b75949">00637</a> <a class="code" href="ngram__model_8h.html#a3e8abe87ae12cfbe1b79fd99213abe2d" title="Set the word-to-ID mapping for this model set.">ngram_model_set_map_words</a>(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base,
<a name="l00638"></a>00638 <span class="keyword">const</span> <span class="keywordtype">char</span> **words,
<a name="l00639"></a>00639 int32 n_words)
<a name="l00640"></a>00640 {
<a name="l00641"></a>00641 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00642"></a>00642 int32 i;
<a name="l00643"></a>00643
<a name="l00644"></a>00644 <span class="comment">/* Recreate the word mapping. */</span>
<a name="l00645"></a>00645 <span class="keywordflow">if</span> (base-><a class="code" href="structngram__model__s.html#a78a3253febced2cae4732044da466ee6" title="Are word strings writable?">writable</a>) {
<a name="l00646"></a>00646 <span class="keywordflow">for</span> (i = 0; i < base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a>; ++i) {
<a name="l00647"></a>00647 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i]);
<a name="l00648"></a>00648 }
<a name="l00649"></a>00649 }
<a name="l00650"></a>00650 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>);
<a name="l00651"></a>00651 <a class="code" href="ckd__alloc_8h.html#a8246c071ac12e98b7e1df9fe6da15aef" title="Free a 2-D array (ptr) previously allocated by ckd_calloc_2d.">ckd_free_2d</a>((<span class="keywordtype">void</span> **)set->widmap);
<a name="l00652"></a>00652 base-><a class="code" href="structngram__model__s.html#a78a3253febced2cae4732044da466ee6" title="Are word strings writable?">writable</a> = TRUE;
<a name="l00653"></a>00653 base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a> = base-><a class="code" href="structngram__model__s.html#a3e41109b30668bdfc077614c1ef49960" title="Number of allocated word strings (for new word addition)">n_1g_alloc</a> = n_words;
<a name="l00654"></a>00654 base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a> = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(n_words, <span class="keyword">sizeof</span>(*base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>));
<a name="l00655"></a>00655 <span class="keyword">set</span>->widmap = (int32 **)<a class="code" href="ckd__alloc_8h.html#a949e7b50fcd9697b1563fa57f50e9c4f" title="Macro for __ckd_calloc_2d__.">ckd_calloc_2d</a>(n_words, set->n_models, <span class="keyword">sizeof</span>(**set->widmap));
<a name="l00656"></a>00656 <a class="code" href="hash__table_8h.html#acab374d21e25009d397642e3465308c7" title="Delete all entries from a hash_table.">hash_table_empty</a>(base-><a class="code" href="structngram__model__s.html#a75567419a8002ef6e916c81f5d9ee9ed" title="Mapping of unigram names to word IDs.">wid</a>);
<a name="l00657"></a>00657 <span class="keywordflow">for</span> (i = 0; i < n_words; ++i) {
<a name="l00658"></a>00658 int32 j;
<a name="l00659"></a>00659 base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i] = <a class="code" href="ckd__alloc_8h.html#ad313f92478859f9e4ea99d0f6e78c393" title="Macro for __ckd_salloc__.">ckd_salloc</a>(words[i]);
<a name="l00660"></a>00660 (void)<a class="code" href="hash__table_8h.html#a393c56322e54607a48e6bc61169d92bf" title="Add a 32-bit integer value to a hash table.">hash_table_enter_int32</a>(base-><a class="code" href="structngram__model__s.html#a75567419a8002ef6e916c81f5d9ee9ed" title="Mapping of unigram names to word IDs.">wid</a>, base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i], i);
<a name="l00661"></a>00661 <span class="keywordflow">for</span> (j = 0; j < <span class="keyword">set</span>->n_models; ++j) {
<a name="l00662"></a>00662 <span class="keyword">set</span>->widmap[i][j] = <a class="code" href="ngram__model_8h.html#ad03d4355d4ea659815dc25bce8d83880" title="Look up numerical word ID.">ngram_wid</a>(set->lms[j], base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[i]);
<a name="l00663"></a>00663 }
<a name="l00664"></a>00664 }
<a name="l00665"></a>00665 }
<a name="l00666"></a>00666
<a name="l00667"></a>00667 <span class="keyword">static</span> <span class="keywordtype">int</span>
<a name="l00668"></a>00668 ngram_model_set_apply_weights(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base, float32 lw,
<a name="l00669"></a>00669 float32 wip, float32 uw)
<a name="l00670"></a>00670 {
<a name="l00671"></a>00671 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00672"></a>00672 int32 i;
<a name="l00673"></a>00673
<a name="l00674"></a>00674 <span class="comment">/* Apply weights to each sub-model. */</span>
<a name="l00675"></a>00675 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models; ++i)
<a name="l00676"></a>00676 <a class="code" href="ngram__model_8h.html#aa4b8d7c1f3d873b8458c0cfee13af4da" title="Apply a language weight, insertion penalty, and unigram weight to a language model.">ngram_model_apply_weights</a>(set->lms[i], lw, wip, uw);
<a name="l00677"></a>00677 <span class="keywordflow">return</span> 0;
<a name="l00678"></a>00678 }
<a name="l00679"></a>00679
<a name="l00680"></a>00680 <span class="keyword">static</span> int32
<a name="l00681"></a>00681 ngram_model_set_score(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base, int32 wid,
<a name="l00682"></a>00682 int32 *history, int32 n_hist,
<a name="l00683"></a>00683 int32 *n_used)
<a name="l00684"></a>00684 {
<a name="l00685"></a>00685 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00686"></a>00686 int32 mapwid;
<a name="l00687"></a>00687 int32 score;
<a name="l00688"></a>00688 int32 i;
<a name="l00689"></a>00689
<a name="l00690"></a>00690 <span class="comment">/* Truncate the history. */</span>
<a name="l00691"></a>00691 <span class="keywordflow">if</span> (n_hist > base-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> - 1)
<a name="l00692"></a>00692 n_hist = base-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> - 1;
<a name="l00693"></a>00693
<a name="l00694"></a>00694 <span class="comment">/* Interpolate if there is no current. */</span>
<a name="l00695"></a>00695 <span class="keywordflow">if</span> (set->cur == -1) {
<a name="l00696"></a>00696 score = base-><a class="code" href="structngram__model__s.html#a65425a599c4bcc4dda809d81149b8fc0" title="Zero probability, cached here for quick lookup.">log_zero</a>;
<a name="l00697"></a>00697 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models; ++i) {
<a name="l00698"></a>00698 int32 j;
<a name="l00699"></a>00699 <span class="comment">/* Map word and history IDs for each model. */</span>
<a name="l00700"></a>00700 mapwid = <span class="keyword">set</span>->widmap[wid][i];
<a name="l00701"></a>00701 <span class="keywordflow">for</span> (j = 0; j < n_hist; ++j) {
<a name="l00702"></a>00702 <span class="keywordflow">if</span> (history[j] == <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>)
<a name="l00703"></a>00703 <span class="keyword">set</span>->maphist[j] = <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>;
<a name="l00704"></a>00704 <span class="keywordflow">else</span>
<a name="l00705"></a>00705 <span class="keyword">set</span>->maphist[j] = <span class="keyword">set</span>->widmap[history[j]][i];
<a name="l00706"></a>00706 }
<a name="l00707"></a>00707 score = <a class="code" href="logmath_8h.html#a5eb70928578b0115c9c7ac2765396a06" title="Add two values in log space (i.e.">logmath_add</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, score,
<a name="l00708"></a>00708 set->lweights[i] +
<a name="l00709"></a>00709 <a class="code" href="ngram__model_8h.html#a6ac5799e78ea4ad82a11e2439016471e" title="Quick general N-Gram score lookup.">ngram_ng_score</a>(set->lms[i],
<a name="l00710"></a>00710 mapwid, set->maphist, n_hist, n_used));
<a name="l00711"></a>00711 }
<a name="l00712"></a>00712 }
<a name="l00713"></a>00713 <span class="keywordflow">else</span> {
<a name="l00714"></a>00714 int32 j;
<a name="l00715"></a>00715 <span class="comment">/* Map word and history IDs (FIXME: do this in a function?) */</span>
<a name="l00716"></a>00716 mapwid = <span class="keyword">set</span>->widmap[wid][<span class="keyword">set</span>->cur];
<a name="l00717"></a>00717 <span class="keywordflow">for</span> (j = 0; j < n_hist; ++j) {
<a name="l00718"></a>00718 <span class="keywordflow">if</span> (history[j] == <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>)
<a name="l00719"></a>00719 <span class="keyword">set</span>->maphist[j] = <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>;
<a name="l00720"></a>00720 <span class="keywordflow">else</span>
<a name="l00721"></a>00721 <span class="keyword">set</span>->maphist[j] = <span class="keyword">set</span>->widmap[history[j]][<span class="keyword">set</span>->cur];
<a name="l00722"></a>00722 }
<a name="l00723"></a>00723 score = <a class="code" href="ngram__model_8h.html#a6ac5799e78ea4ad82a11e2439016471e" title="Quick general N-Gram score lookup.">ngram_ng_score</a>(set->lms[set->cur],
<a name="l00724"></a>00724 mapwid, set->maphist, n_hist, n_used);
<a name="l00725"></a>00725 }
<a name="l00726"></a>00726
<a name="l00727"></a>00727 <span class="keywordflow">return</span> score;
<a name="l00728"></a>00728 }
<a name="l00729"></a>00729
<a name="l00730"></a>00730 <span class="keyword">static</span> int32
<a name="l00731"></a>00731 ngram_model_set_raw_score(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base, int32 wid,
<a name="l00732"></a>00732 int32 *history, int32 n_hist,
<a name="l00733"></a>00733 int32 *n_used)
<a name="l00734"></a>00734 {
<a name="l00735"></a>00735 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00736"></a>00736 int32 mapwid;
<a name="l00737"></a>00737 int32 score;
<a name="l00738"></a>00738 int32 i;
<a name="l00739"></a>00739
<a name="l00740"></a>00740 <span class="comment">/* Truncate the history. */</span>
<a name="l00741"></a>00741 <span class="keywordflow">if</span> (n_hist > base-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> - 1)
<a name="l00742"></a>00742 n_hist = base-><a class="code" href="structngram__model__s.html#a3c87bc1b678662a2c8930b3b8c33a80f" title="This is an n-gram model (1, 2, 3, ...).">n</a> - 1;
<a name="l00743"></a>00743
<a name="l00744"></a>00744 <span class="comment">/* Interpolate if there is no current. */</span>
<a name="l00745"></a>00745 <span class="keywordflow">if</span> (set->cur == -1) {
<a name="l00746"></a>00746 score = base-><a class="code" href="structngram__model__s.html#a65425a599c4bcc4dda809d81149b8fc0" title="Zero probability, cached here for quick lookup.">log_zero</a>;
<a name="l00747"></a>00747 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models; ++i) {
<a name="l00748"></a>00748 int32 j;
<a name="l00749"></a>00749 <span class="comment">/* Map word and history IDs for each model. */</span>
<a name="l00750"></a>00750 mapwid = <span class="keyword">set</span>->widmap[wid][i];
<a name="l00751"></a>00751 <span class="keywordflow">for</span> (j = 0; j < n_hist; ++j) {
<a name="l00752"></a>00752 <span class="keywordflow">if</span> (history[j] == <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>)
<a name="l00753"></a>00753 <span class="keyword">set</span>->maphist[j] = <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>;
<a name="l00754"></a>00754 <span class="keywordflow">else</span>
<a name="l00755"></a>00755 <span class="keyword">set</span>->maphist[j] = <span class="keyword">set</span>->widmap[history[j]][i];
<a name="l00756"></a>00756 }
<a name="l00757"></a>00757 score = <a class="code" href="logmath_8h.html#a5eb70928578b0115c9c7ac2765396a06" title="Add two values in log space (i.e.">logmath_add</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, score,
<a name="l00758"></a>00758 set->lweights[i] +
<a name="l00759"></a>00759 <a class="code" href="ngram__model_8h.html#a218d8d140b93d3d8008f8933f9e04ec6" title="Quick &quot;raw&quot; probability lookup for a general N-Gram.">ngram_ng_prob</a>(set->lms[i],
<a name="l00760"></a>00760 mapwid, set->maphist, n_hist, n_used));
<a name="l00761"></a>00761 }
<a name="l00762"></a>00762 }
<a name="l00763"></a>00763 <span class="keywordflow">else</span> {
<a name="l00764"></a>00764 int32 j;
<a name="l00765"></a>00765 <span class="comment">/* Map word and history IDs (FIXME: do this in a function?) */</span>
<a name="l00766"></a>00766 mapwid = <span class="keyword">set</span>->widmap[wid][<span class="keyword">set</span>->cur];
<a name="l00767"></a>00767 <span class="keywordflow">for</span> (j = 0; j < n_hist; ++j) {
<a name="l00768"></a>00768 <span class="keywordflow">if</span> (history[j] == <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>)
<a name="l00769"></a>00769 <span class="keyword">set</span>->maphist[j] = <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>;
<a name="l00770"></a>00770 <span class="keywordflow">else</span>
<a name="l00771"></a>00771 <span class="keyword">set</span>->maphist[j] = <span class="keyword">set</span>->widmap[history[j]][<span class="keyword">set</span>->cur];
<a name="l00772"></a>00772 }
<a name="l00773"></a>00773 score = <a class="code" href="ngram__model_8h.html#a218d8d140b93d3d8008f8933f9e04ec6" title="Quick &quot;raw&quot; probability lookup for a general N-Gram.">ngram_ng_prob</a>(set->lms[set->cur],
<a name="l00774"></a>00774 mapwid, set->maphist, n_hist, n_used);
<a name="l00775"></a>00775 }
<a name="l00776"></a>00776
<a name="l00777"></a>00777 <span class="keywordflow">return</span> score;
<a name="l00778"></a>00778 }
<a name="l00779"></a>00779
<a name="l00780"></a>00780 <span class="keyword">static</span> int32
<a name="l00781"></a>00781 ngram_model_set_add_ug(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base,
<a name="l00782"></a>00782 int32 wid, int32 lweight)
<a name="l00783"></a>00783 {
<a name="l00784"></a>00784 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00785"></a>00785 int32 *newwid;
<a name="l00786"></a>00786 int32 i, prob;
<a name="l00787"></a>00787
<a name="l00788"></a>00788 <span class="comment">/* At this point the word has already been added to the master</span>
<a name="l00789"></a>00789 <span class="comment"> model and we have a new word ID for it. Add it to active</span>
<a name="l00790"></a>00790 <span class="comment"> submodels and track the word IDs. */</span>
<a name="l00791"></a>00791 newwid = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e" title="Macros to simplify the use of above functions.">ckd_calloc</a>(set->n_models, <span class="keyword">sizeof</span>(*newwid));
<a name="l00792"></a>00792 prob = base-><a class="code" href="structngram__model__s.html#a65425a599c4bcc4dda809d81149b8fc0" title="Zero probability, cached here for quick lookup.">log_zero</a>;
<a name="l00793"></a>00793 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models; ++i) {
<a name="l00794"></a>00794 int32 wprob, n_hist;
<a name="l00795"></a>00795
<a name="l00796"></a>00796 <span class="comment">/* Only add to active models. */</span>
<a name="l00797"></a>00797 <span class="keywordflow">if</span> (set->cur == -1 || set->cur == i) {
<a name="l00798"></a>00798 <span class="comment">/* Did this word already exist? */</span>
<a name="l00799"></a>00799 newwid[i] = <a class="code" href="ngram__model_8h.html#ad03d4355d4ea659815dc25bce8d83880" title="Look up numerical word ID.">ngram_wid</a>(set->lms[i], base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wid]);
<a name="l00800"></a>00800 <span class="keywordflow">if</span> (newwid[i] == <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>) {
<a name="l00801"></a>00801 <span class="comment">/* Add it to the submodel. */</span>
<a name="l00802"></a>00802 newwid[i] = <a class="code" href="ngram__model_8h.html#a38e034d58ee0d6a4c0dd6f852cb71c3c" title="Add a word (unigram) to the language model.">ngram_model_add_word</a>(set->lms[i], base-><a class="code" href="structngram__model__s.html#ae625e779e340845f03fb3da164e93039" title="Unigram names.">word_str</a>[wid],
<a name="l00803"></a>00803 <a class="code" href="logmath_8h.html#ae8b0a168e29e448c0d6de66dc46e099e" title="Convert integer log in base B to linear floating point.">logmath_exp</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, lweight));
<a name="l00804"></a>00804 <span class="keywordflow">if</span> (newwid[i] == <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>) {
<a name="l00805"></a>00805 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(newwid);
<a name="l00806"></a>00806 <span class="keywordflow">return</span> base-><a class="code" href="structngram__model__s.html#a65425a599c4bcc4dda809d81149b8fc0" title="Zero probability, cached here for quick lookup.">log_zero</a>;
<a name="l00807"></a>00807 }
<a name="l00808"></a>00808 }
<a name="l00809"></a>00809 <span class="comment">/* Now get the unigram probability for the new word and either</span>
<a name="l00810"></a>00810 <span class="comment"> * interpolate it or use it (if this is the current model). */</span>
<a name="l00811"></a>00811 wprob = <a class="code" href="ngram__model_8h.html#a218d8d140b93d3d8008f8933f9e04ec6" title="Quick &quot;raw&quot; probability lookup for a general N-Gram.">ngram_ng_prob</a>(set->lms[i], newwid[i], NULL, 0, &n_hist);
<a name="l00812"></a>00812 <span class="keywordflow">if</span> (set->cur == i)
<a name="l00813"></a>00813 prob = wprob;
<a name="l00814"></a>00814 <span class="keywordflow">else</span> <span class="keywordflow">if</span> (set->cur == -1)
<a name="l00815"></a>00815 prob = <a class="code" href="logmath_8h.html#a5eb70928578b0115c9c7ac2765396a06" title="Add two values in log space (i.e.">logmath_add</a>(base-><a class="code" href="structngram__model__s.html#a2ca373109c651ac998b33153eb38fd95" title="Log-math object.">lmath</a>, prob, set->lweights[i] + wprob);
<a name="l00816"></a>00816 }
<a name="l00817"></a>00817 <span class="keywordflow">else</span> {
<a name="l00818"></a>00818 newwid[i] = <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab" title="Impossible word ID.">NGRAM_INVALID_WID</a>;
<a name="l00819"></a>00819 }
<a name="l00820"></a>00820 }
<a name="l00821"></a>00821 <span class="comment">/* Okay we have the word IDs for this in all the submodels. Now</span>
<a name="l00822"></a>00822 <span class="comment"> do some complicated memory mangling to add this to the</span>
<a name="l00823"></a>00823 <span class="comment"> widmap. */</span>
<a name="l00824"></a>00824 <span class="keyword">set</span>->widmap = <a class="code" href="ckd__alloc_8h.html#afd496738b3e114bd494c5a0955f1bfb3" title="Macro for __ckd_realloc__.">ckd_realloc</a>(set->widmap, base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a> * <span class="keyword">sizeof</span>(*set->widmap));
<a name="l00825"></a>00825 <span class="keyword">set</span>->widmap[0] = <a class="code" href="ckd__alloc_8h.html#afd496738b3e114bd494c5a0955f1bfb3" title="Macro for __ckd_realloc__.">ckd_realloc</a>(set->widmap[0],
<a name="l00826"></a>00826 base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a>
<a name="l00827"></a>00827 * set->n_models
<a name="l00828"></a>00828 * <span class="keyword">sizeof</span>(**set->widmap));
<a name="l00829"></a>00829 <span class="keywordflow">for</span> (i = 0; i < base-><a class="code" href="structngram__model__s.html#a74f85927ef0d5513a1e6c02d13864be3" title="Number of actual word strings (NOT the same as the number of unigrams, due to class words)...">n_words</a>; ++i)
<a name="l00830"></a>00830 set->widmap[i] = set->widmap[0] + i * set->n_models;
<a name="l00831"></a>00831 memcpy(set->widmap[wid], newwid, set->n_models * <span class="keyword">sizeof</span>(*newwid));
<a name="l00832"></a>00832 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(newwid);
<a name="l00833"></a>00833 <span class="keywordflow">return</span> prob;
<a name="l00834"></a>00834 }
<a name="l00835"></a>00835
<a name="l00836"></a>00836 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00837"></a>00837 ngram_model_set_free(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base)
<a name="l00838"></a>00838 {
<a name="l00839"></a>00839 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00840"></a>00840 int32 i;
<a name="l00841"></a>00841
<a name="l00842"></a>00842 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models; ++i)
<a name="l00843"></a>00843 <a class="code" href="ngram__model_8h.html#aec73d28e7285e539a0b44a7ac0cbe489" title="Release memory associated with an N-Gram model.">ngram_model_free</a>(set->lms[i]);
<a name="l00844"></a>00844 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(set->lms);
<a name="l00845"></a>00845 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models; ++i)
<a name="l00846"></a>00846 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(set->names[i]);
<a name="l00847"></a>00847 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(set->names);
<a name="l00848"></a>00848 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(set->lweights);
<a name="l00849"></a>00849 <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf" title="Test and free a 1-D array.">ckd_free</a>(set->maphist);
<a name="l00850"></a>00850 <a class="code" href="ckd__alloc_8h.html#a8246c071ac12e98b7e1df9fe6da15aef" title="Free a 2-D array (ptr) previously allocated by ckd_calloc_2d.">ckd_free_2d</a>((<span class="keywordtype">void</span> **)set->widmap);
<a name="l00851"></a>00851 }
<a name="l00852"></a>00852
<a name="l00853"></a>00853 <span class="keyword">static</span> <span class="keywordtype">void</span>
<a name="l00854"></a>00854 ngram_model_set_flush(<a class="code" href="structngram__model__s.html" title="Common implementation of ngram_model_t.">ngram_model_t</a> *base)
<a name="l00855"></a>00855 {
<a name="l00856"></a>00856 <a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *<span class="keyword">set</span> = (<a class="code" href="structngram__model__set__s.html" title="Subclass of ngram_model for grouping language models.">ngram_model_set_t</a> *)base;
<a name="l00857"></a>00857 int32 i;
<a name="l00858"></a>00858
<a name="l00859"></a>00859 <span class="keywordflow">for</span> (i = 0; i < <span class="keyword">set</span>->n_models; ++i)
<a name="l00860"></a>00860 <a class="code" href="ngram__model_8h.html#a8304f56d82278824b0a94c254d7235aa" title="Flush any cached N-Gram information.">ngram_model_flush</a>(set->lms[i]);
<a name="l00861"></a>00861 }
<a name="l00862"></a>00862
<a name="l00863"></a>00863 <span class="keyword">static</span> <a class="code" href="structngram__funcs__s.html" title="Implementation-specific functions for operating on ngram_model_t objects.">ngram_funcs_t</a> ngram_model_set_funcs = {
<a name="l00864"></a>00864 ngram_model_set_free, <span class="comment">/* free */</span>
<a name="l00865"></a>00865 ngram_model_set_apply_weights, <span class="comment">/* apply_weights */</span>
<a name="l00866"></a>00866 ngram_model_set_score, <span class="comment">/* score */</span>
<a name="l00867"></a>00867 ngram_model_set_raw_score, <span class="comment">/* raw_score */</span>
<a name="l00868"></a>00868 ngram_model_set_add_ug, <span class="comment">/* add_ug */</span>
<a name="l00869"></a>00869 ngram_model_set_flush <span class="comment">/* flush */</span>
<a name="l00870"></a>00870 };
</pre></div></div>
</div>
<div id="nav-path" class="navpath">
<ul>
<li class="navelem"><a class="el" href="ngram__model__set_8c.html">ngram_model_set.c</a> </li>
<li class="footer">Generated on Tue Apr 19 2011 for SphinxBase by 
<a href="http://www.doxygen.org/index.html">
<img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.7.3 </li>
</ul>
</div>
</body>
</html>
distrib
>
Fedora
>
14
>
x86_64
>
media
>
updates
>
by-pkgid
>
0b420d0fce195cf4115dc6a3be5c2da2
>
files
>
303
