Commit f85a5cc9 authored by Dan Povey's avatar Dan Povey
Browse files

trunk: various small fixes to scripts; minor cosmetic changes to code and docs.

git-svn-id: https://svn.code.sf.net/p/kaldi/code/trunk@4755 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8
parent a6cf18ea
......@@ -5,12 +5,12 @@
# iterations.)
. cmd.sh
## SGMM on top of LDA+MLLT+SAT features.
if [ ! -f exp/ubm4a/final.mdl ]; then
if [ ! -f exp/ubm4a/final.ubm ] && [ ! data/train/feats.scp -nt exp/ubm4a/final.ubm ]; then
steps/train_ubm.sh --silence-weight 0.5 --cmd "$train_cmd" 400 data/train data/lang exp/tri3b_ali exp/ubm4a || exit 1;
fi
steps/train_sgmm2.sh --cmd "$train_cmd" 5000 7000 data/train data/lang exp/tri3b_ali exp/ubm4a/final.ubm exp/sgmm2_4a || exit 1;
steps/train_sgmm2.sh --cmd "$train_cmd" 5000 7000 data/train data/lang exp/tri3b_ali exp/ubm4a/final.ubm exp/sgmm2_4a || exit 1;
utils/mkgraph.sh data/lang exp/sgmm2_4a exp/sgmm2_4a/graph || exit 1;
......
......@@ -7,8 +7,8 @@ About the Switchboard corpus
we download separately from
http://www.isip.piconepress.com/projects/switchboard/releases/switchboard_word_alignments.tar.gz
We are using the eval2000 evaluation data. The acoustics are LDC2002S09 and
the text is LDC2002T43.
We are using the eval2000 a.k.a. hub5'00 evaluation data. The acoustics are
LDC2002S09 and the text is LDC2002T43.
About the Fisher corpus for language modeling
......
......@@ -10,8 +10,13 @@
#export decode_cmd=run.pl
#export cuda_cmd=run.pl
#JHU cluster:
export train_cmd="queue.pl"
export decode_cmd="queue.pl --mem 3G"
export cuda_cmd="queue.pl --gpu 1"
# BUT cluster:
export train_cmd="queue.pl -q all.q@blade[01][0126789][123456789] -l ram_free=2500M,mem_free=2500M,matylda5=0.5"
export decode_cmd="queue.pl -q all.q@blade[01][0126789][123456789] -l ram_free=3000M,mem_free=3000M,matylda5=0.1"
export cuda_cmd="queue.pl -q long.q@pcspeech-gpu -l gpu=1"
#export train_cmd="queue.pl -q all.q@blade[01][0126789][123456789] -l ram_free=2500M,mem_free=2500M,matylda5=0.5"
#export decode_cmd="queue.pl -q all.q@blade[01][0126789][123456789] -l ram_free=3000M,mem_free=3000M,matylda5=0.1"
#export cuda_cmd="queue.pl -q long.q@pcspeech-gpu -l gpu=1"
......@@ -32,28 +32,6 @@ gunzip -c "$arpa_lm" | \
echo "Checking how stochastic G is (the first of these numbers should be small):"
fstisstochastic data/lang_test/G.fst
## Check lexicon.
## just have a look and make sure it seems sane.
echo "First few lines of lexicon FST:"
fstprint --isymbols=data/lang/phones.txt --osymbols=data/lang/words.txt data/lang/L.fst | head
echo Performing further checks
# Checking that G.fst is determinizable.
fstdeterminize data/lang_test/G.fst /dev/null || echo Error determinizing G.
# Checking that L_disambig.fst is determinizable.
fstdeterminize data/lang_test/L_disambig.fst /dev/null || echo Error determinizing L.
# Checking that disambiguated lexicon times G is determinizable
# Note: we do this with fstdeterminizestar not fstdeterminize, as
# fstdeterminize was taking forever (presumbaly relates to a bug
# in this version of OpenFst that makes determinization slow for
# some case).
fsttablecompose data/lang_test/L_disambig.fst data/lang_test/G.fst | \
fstdeterminizestar >/dev/null || { echo Error determinizing LG; exit 1; }
# Checking that LG is stochastic:
fsttablecompose data/lang/L_disambig.fst data/lang_test/G.fst | \
fstisstochastic || echo LG is not stochastic
utils/validate_lang.pl data/lang_test || exit 1;
exit 0;
......@@ -36,6 +36,7 @@ if [ $stage -le 0 ]; then
utils/prepare_lang.sh data/local/dict "<UNK>" data/local/lang data/lang || exit 1
local/prepare_lm.sh || exit 1
fi
# Feature extraction
......
......@@ -35,7 +35,7 @@ if [ ! -f data_bnf/train_bnf/.done ]; then
mkdir -p data_bnf
# put the archives in param_bnf/.
steps/nnet2/dump_bottleneck_features.sh --cmd "$train_cmd" \
--transform-dir exp/tri4a data/train_si284 data_bnf/train_bnf exp_bnf/tri6_bnf param_bnf exp_bnf/dump_bnf
--transform-dir exp/tri4b_ali_si284 data/train_si284 data_bnf/train_bnf exp_bnf/tri6_bnf param_bnf exp_bnf/dump_bnf
touch data_bnf/train_bnf/.done
fi
......
......@@ -21,6 +21,7 @@ use_graphs=false
scale_opts="--transition-scale=1.0 --acoustic-scale=0.1 --self-loop-scale=0.1"
beam=10
retry_beam=40
careful=false
boost_silence=1.0 # factor by which to boost silence during alignment.
fmllr_update_type=full
# End configuration options.
......@@ -110,7 +111,7 @@ fi
if [ $stage -le 1 ]; then
echo "$0: aligning data in $data using $alimdl and speaker-independent features."
$cmd JOB=1:$nj $dir/log/align_pass1.JOB.log \
gmm-align-compiled $scale_opts --beam=$beam --retry-beam=$retry_beam "$alimdl_cmd" \
gmm-align-compiled $scale_opts --beam=$beam --retry-beam=$retry_beam --careful=$careful "$alimdl_cmd" \
"ark:gunzip -c $graphdir/fsts.JOB.gz|" "$sifeats" "ark:|gzip -c >$dir/pre_ali.JOB.gz" || exit 1;
fi
......@@ -139,7 +140,7 @@ feats="$sifeats transform-feats --utt2spk=ark:$sdata/JOB/utt2spk ark:$dir/trans.
if [ $stage -le 3 ]; then
echo "$0: doing final alignment."
$cmd JOB=1:$nj $dir/log/align_pass2.JOB.log \
gmm-align-compiled $scale_opts --beam=$beam --retry-beam=$retry_beam "$mdl_cmd" \
gmm-align-compiled $scale_opts --beam=$beam --retry-beam=$retry_beam --careful=$careful "$mdl_cmd" \
"ark:gunzip -c $graphdir/fsts.JOB.gz|" "$feats" "ark:|gzip -c >$dir/ali.JOB.gz" || exit 1;
fi
......
......@@ -9,6 +9,7 @@
cmd=run.pl
num_epochs=4 # Number of epochs of training
learning_rate=0.00002
effective_lrate= # If supplied, overrides the learning rate, which gets set to effective_lrate * num_jobs_nnet.
acoustic_scale=0.1 # acoustic scale for MMI/MPFE/SMBR training.
criterion=smbr
boost=0.0 # option relevant for MMI
......@@ -60,8 +61,9 @@ if [ $# != 6 ]; then
echo " --config <config-file> # config file containing options"
echo " --cmd (utils/run.pl|utils/queue.pl <queue opts>) # how to run jobs."
echo " --num-epochs <#epochs|4> # Number of epochs of training"
echo " --initial-learning-rate <initial-learning-rate|0.0002> # Learning rate at start of training"
echo " --final-learning-rate <final-learning-rate|0.0004> # Learning rate at end of training"
echo " --learning-rate <learning-rate|0.0002> # Learning rate to use"
echo " --effective-lrate <effective-learning-rate> # If supplied, learning rate will be set to"
echo " # this value times num-jobs-nnet."
echo " --num-jobs-nnet <num-jobs|8> # Number of parallel jobs to use for main neural net"
echo " # training (will affect results as well as speed; try 8, 16)"
echo " # Note: if you increase this, you may want to also increase"
......@@ -237,6 +239,8 @@ if [ -z "$degs_dir" ] && [ -d $dir/degs/storage ]; then
done
fi
if [ $stage -le -7 ]; then
echo "$0: Copying initial model and modifying preconditioning setup"
......@@ -247,6 +251,10 @@ if [ $stage -le -7 ]; then
# little, later on, although I doubt it matters once the --num-samples-history
# is large enough.
if [ ! -z "$effective_lrate" ]; then
learning_rate=$(perl -e "print ($num_jobs_nnet*$effective_lrate);")
echo "$0: setting learning rate to $learning_rate = --num-jobs-nnet * --effective-lrate."
fi
$cmd $dir/log/convert.log \
nnet-am-copy --learning-rate=$learning_rate "$src_model" - \| \
nnet-am-switch-preconditioning --num-samples-history=50000 - $dir/0.mdl || exit 1;
......
......@@ -10,6 +10,7 @@
cmd=run.pl
num_epochs=4 # Number of epochs of training
learning_rate=0.00002
effective_lrate= # If supplied, overrides the learning rate, which gets set to effective_lrate * num_jobs_nnet.
acoustic_scale=0.1 # acoustic scale for MMI/MPFE/SMBR training.
boost=0.0 # option relevant for MMI
......@@ -61,8 +62,9 @@ if [ $# != 2 ]; then
echo " --config <config-file> # config file containing options"
echo " --cmd (utils/run.pl|utils/queue.pl <queue opts>) # how to run jobs."
echo " --num-epochs <#epochs|4> # Number of epochs of training"
echo " --initial-learning-rate <initial-learning-rate|0.0002> # Learning rate at start of training"
echo " --final-learning-rate <final-learning-rate|0.0004> # Learning rate at end of training"
echo " --learning-rate <learning-rate|0.0002> # Learning rate to use"
echo " --effective-lrate <effective-learning-rate> # If supplied, learning rate will be set to"
echo " # this value times num-jobs-nnet."
echo " --num-jobs-nnet <num-jobs|8> # Number of parallel jobs to use for main neural net"
echo " # training (will affect results as well as speed; try 8, 16)"
echo " # Note: if you increase this, you may want to also increase"
......@@ -130,6 +132,11 @@ if [ $stage -le -1 ]; then
# little, later on, although I doubt it matters once the --num-samples-history
# is large enough.
if [ ! -z "$effective_lrate" ]; then
learning_rate=$(perl -e "print ($num_jobs_nnet*$effective_lrate);")
echo "$0: setting learning rate to $learning_rate = --num-jobs-nnet * --effective-lrate."
fi
$cmd $dir/log/convert.log \
nnet-am-copy --learning-rate=$learning_rate "$src_model" - \| \
nnet-am-switch-preconditioning --num-samples-history=50000 - $dir/0.mdl || exit 1;
......
......@@ -23,8 +23,7 @@ doc/make_tools.sh
doxygen
cp doc/*.pptx html/;
if [[ $(hostname) == *.clsp.jhu.edu ]]; then
if [[ $(hostname -f) == *.clsp.jhu.edu ]]; then
tar -czf html.tar.gz html
scp html.tar.gz danielpovey@web.sourceforge.net:/home/project-web/kaldi/htdocs/
......
......@@ -702,16 +702,16 @@ One of the major causes of crashes in compute clusters is memory exhaustion.
The default OOM-killer in Linux is not very good, so if you exhaust memory, it
may end up killing an important system process, which tends to cause
hard-to-diagnose instability. Even if nothing is killed, <tt>malloc()</tt> may
start failing when called from processes on the system, and very few programs
start failing when called from processes on the system; and very few programs
deal with this gracefully. In the CLSP grid we wrote our own version of an OOM
killer, which we run as root, and we wrote the corresponding init scripts. When
our OOM killer detects memory overload, it kills the largest process of whichver
our OOM killer detects memory overload, it kills the largest process of whichever
non-system user is using the most memory. This is usually the right thing to
do. These scripts have been made public as part of the <tt>kluster</tt>
project, and you can get them as shown below if you want to add them to your
system. The following commands will only work as-is if you have LSB-style init
scripts, which is the case in Debian
<tt>wheezy</tt>. The next Debian elease, <tt>jessie</tt>, won't have init scripts at all and
<tt>wheezy</tt>. The next Debian release, <tt>jessie</tt>, won't have init scripts at all and
will use <tt>systemd</tt> instead (the so-called "systemd controversy"). If someone can figure out how to do the
following in <tt>systemd</tt>, please let us know.
Type
......@@ -780,9 +780,10 @@ In this section we have some general observations about how to configure
and manage a compute grid.
In CLSP we use a lot of NFS hosts, not just one or two; in fact, most of our
nodes also export data via NFS. (If you do this you should use
nodes also export data via NFS. If you do this you should use
our <tt>mem-killer.pl</tt> or a similar script, or you will get instability due
to memory exhaustion when users make mistakes). Having a large number of file
to memory exhaustion when users make mistakes.
Having a large number of file
servers is a particularly good idea for queues that are shared by many people,
because it's inevitable that people will overload file servers, and if there are
only one or two file servers, the queue will end up being in a degraded state
......
......@@ -35,7 +35,12 @@ int main(int argc, char *argv[]) {
"phones. The output symbols are still words, unless you specify --replace-output-symbols=true\n"
"Usage: lattice-align-phones [options] <model> <lattice-rspecifier> <lattice-wspecifier>\n"
" e.g.: lattice-align-phones final.mdl ark:1.lats ark:phone_aligned.lats\n"
"See also: lattice-to-phone-lattice, lattice-align-words, lattice-align-words-lexicon\n";
"See also: lattice-to-phone-lattice, lattice-align-words, lattice-align-words-lexicon\n"
"Note: if you just want the phone alignment from a lattice, the easiest path is\n"
" lattice-1best | nbest-to-linear [keeping only alignment] | ali-to-phones\n"
"If you want the words and phones jointly (i.e. pronunciations of words, with word\n"
"alignment), try\n"
" lattice-1best | nbest-to-prons\n";
ParseOptions po(usage);
bool output_if_error = true;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment