Commit 48c72043 authored by Dan Povey's avatar Dan Povey
Browse files

Merging recent trunk/ changes back into discrim/


git-svn-id: https://svn.code.sf.net/p/kaldi/code/sandbox/discrim@464 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8
parents 7565fb1a 5b5ba4aa
...@@ -86,32 +86,30 @@ exp/decode_tri2m_vtln_diag/wer:Average WER is 3.087848 (387 / 12533) # + diagon ...@@ -86,32 +86,30 @@ exp/decode_tri2m_vtln_diag/wer:Average WER is 3.087848 (387 / 12533) # + diagon
exp/decode_tri2m_vtln_diag_utt/wer:Average WER is 4.340541 (544 / 12533) # [per-utterance] exp/decode_tri2m_vtln_diag_utt/wer:Average WER is 4.340541 (544 / 12533) # [per-utterance]
exp/decode_tri2m_vtln_nofmllr/wer:Average WER is 5.784728 (725 / 12533) # feature-space VTLN, with no fMLLR exp/decode_tri2m_vtln_nofmllr/wer:Average WER is 5.784728 (725 / 12533) # feature-space VTLN, with no fMLLR
# sgmma is SGMM without speaker vectors. # sgmma is SGMM without speaker vectors.
exp/decode_sgmma/wer:Average WER is 3.319237 (416 / 12533) exp/decode_sgmma/wer:Average WER is 3.319237 (416 / 12533)
exp/decode_sgmma_fmllr/wer:Average WER is 2.934308 (289 / 9849) exp/decode_sgmma_fmllr/wer:Average WER is 2.928269 (367 / 12533)
exp/decode_sgmma_fmllr_utt/wer:Average WER is 3.303279 (414 / 12533) exp/decode_sgmma_fmllr_utt/wer:Average WER is 3.303279 (414 / 12533)
exp/decode_sgmma_fmllrbasis_utt/wer:Average WER is 3.191574 (400 / 12533) exp/decode_sgmma_fmllrbasis_utt/wer:Average WER is 3.191574 (400 / 12533)
# sgmmb is SGMM with speaker vectors. # sgmmb is SGMM with speaker vectors.
exp/decode_sgmmb/wer:Average WER is 2.760712 (346 / 12533) exp/decode_sgmmb/wer:Average WER is 2.521344 (316 / 12533)
exp/decode_sgmmb_fmllr/wer:Average WER is 2.585175 (324 / 12533) exp/decode_sgmmb_fmllr/wer:Average WER is 2.377723 (298 / 12533)
exp/decode_sgmmb_utt/wer:Average WER is 2.808585 (352 / 12533) exp/decode_sgmmb_utt/wer:Average WER is 2.728796 (342 / 12533)
# sgmmc is like sgmmb but with gender dependency # sgmmc is like sgmmb but with gender dependency
exp/decode_sgmmc/wer:Average WER is 2.696880 (338 / 12533) exp/decode_sgmmc/wer:Average WER is 2.720817 (341 / 12533)
exp/decode_sgmmc_fmllr/wer:Average WER is 2.457512 (308 / 12533) exp/decode_sgmmc_fmllr/wer:Average WER is 2.489428 (312 / 12533)
# "norm" is normalizing weights per gender..
exp/decode_sgmmc_norm/wer:Average WER is 2.696880 (338 / 12533)
exp/decode_sgmmc_fmllr_norm/wer:Average WER is 2.425596 (304 / 12533)
# sgmmd is like sgmmb but with LDA+MLLT features. # sgmmd is like sgmmb but with LDA+MLLT features.
exp/decode_sgmmd/wer:Average WER is 2.449533 (307 / 12533) exp/decode_sgmmd/wer:Average WER is 2.656986 (333 / 12533)
exp/decode_sgmmd_fmllr/wer:Average WER is 2.305912 (289 / 12533) exp/decode_sgmmd_fmllr/wer:Average WER is 2.409639 (302 / 12533)
# sgmme is like sgmmb but with LDA+ET features. # sgmme is like sgmmb but with LDA+ET features.
exp/decode_sgmme/wer:Average WER is 2.321870 (291 / 12533) exp/decode_sgmme/wer:Average WER is 2.337828 (293 / 12533)
exp/decode_sgmme_fmllr/wer:Average WER is 2.154313 (270 / 12533) exp/decode_sgmme_fmllr/wer:Average WER is 2.266018 (284 / 12533)
#### Note: stuff below this line may be out of date / not computed #### Note: stuff below this line may be out of date / not computed
...@@ -167,3 +165,4 @@ exp/decode_sgmmc_fmllr/wer:Average WER is 2.688901 (337 / 12533) ...@@ -167,3 +165,4 @@ exp/decode_sgmmc_fmllr/wer:Average WER is 2.688901 (337 / 12533)
# 64-bit+ATLAS was 0.171s # 64-bit+ATLAS was 0.171s
# 32-bit+ATLAS was 0.205s # 32-bit+ATLAS was 0.205s
# 64-bit+MKL was 0.291s # 64-bit+MKL was 0.291s
...@@ -112,8 +112,8 @@ while [ $iter -lt $numiters ]; do ...@@ -112,8 +112,8 @@ while [ $iter -lt $numiters ]; do
if echo $realign_iters | grep -w $iter >/dev/null; then if echo $realign_iters | grep -w $iter >/dev/null; then
echo "Aligning data" echo "Aligning data"
sgmm-align-compiled $spkvecs_opt $utt2spk_opt $scale_opts "$gselect_opt" \ sgmm-align-compiled $spkvecs_opt $utt2spk_opt $scale_opts "$gselect_opt" \
--retry-beam=40 $dir/$iter.mdl "ark:gunzip -c $dir/graphs.fsts.gz|" "$feats" \ --beam=8 --retry-beam=40 $dir/$iter.mdl "ark:gunzip -c $dir/graphs.fsts.gz|" \
ark:$dir/cur.ali 2> $dir/align.$iter.log || exit 1; "$feats" ark:$dir/cur.ali 2> $dir/align.$iter.log || exit 1;
fi fi
if echo $spkvec_iters | grep -w $iter >/dev/null; then if echo $spkvec_iters | grep -w $iter >/dev/null; then
( ali-to-post ark:$dir/cur.ali ark:- | \ ( ali-to-post ark:$dir/cur.ali ark:- | \
......
...@@ -118,8 +118,8 @@ while [ $iter -lt $numiters ]; do ...@@ -118,8 +118,8 @@ while [ $iter -lt $numiters ]; do
if echo $realign_iters | grep -w $iter >/dev/null; then if echo $realign_iters | grep -w $iter >/dev/null; then
echo "Aligning data" echo "Aligning data"
sgmm-align-compiled $spkvecs_opt $utt2spk_opt $scale_opts "$gselect_opt" \ sgmm-align-compiled $spkvecs_opt $utt2spk_opt $scale_opts "$gselect_opt" \
--retry-beam=40 $dir/$iter.mdl "ark:gunzip -c $dir/graphs.fsts.gz|" "$feats" \ --beam=8 --retry-beam=40 $dir/$iter.mdl "ark:gunzip -c $dir/graphs.fsts.gz|" \
ark:$dir/cur.ali 2> $dir/align.$iter.log || exit 1; "$feats" ark:$dir/cur.ali 2> $dir/align.$iter.log || exit 1;
fi fi
if echo $spkvec_iters | grep -w $iter >/dev/null; then if echo $spkvec_iters | grep -w $iter >/dev/null; then
( ali-to-post ark:$dir/cur.ali ark:- | \ ( ali-to-post ark:$dir/cur.ali ark:- | \
......
...@@ -114,7 +114,7 @@ while [ $iter -lt $numiters ]; do ...@@ -114,7 +114,7 @@ while [ $iter -lt $numiters ]; do
if echo $realign_iters | grep -w $iter >/dev/null; then if echo $realign_iters | grep -w $iter >/dev/null; then
echo "Aligning data" echo "Aligning data"
sgmm-align-compiled $spkvecs_opt $utt2spk_opt $scale_opts "$gselect_opt" \ sgmm-align-compiled $spkvecs_opt $utt2spk_opt $scale_opts "$gselect_opt" \
--retry-beam=40 $dir/$iter.mdl "ark:gunzip -c $dir/graphs.fsts.gz|" "$feats" \ --beam=8 --retry-beam=40 $dir/$iter.mdl "ark:gunzip -c $dir/graphs.fsts.gz|" "$feats" \
ark:$dir/cur.ali 2> $dir/align.$iter.log || exit 1; ark:$dir/cur.ali 2> $dir/align.$iter.log || exit 1;
fi fi
if echo $spkvec_iters | grep -w $iter >/dev/null; then if echo $spkvec_iters | grep -w $iter >/dev/null; then
......
...@@ -115,8 +115,8 @@ while [ $iter -lt $numiters ]; do ...@@ -115,8 +115,8 @@ while [ $iter -lt $numiters ]; do
if echo $realign_iters | grep -w $iter >/dev/null; then if echo $realign_iters | grep -w $iter >/dev/null; then
echo "Aligning data" echo "Aligning data"
sgmm-align-compiled $spkvecs_opt $utt2spk_opt $scale_opts "$gselect_opt" \ sgmm-align-compiled $spkvecs_opt $utt2spk_opt $scale_opts "$gselect_opt" \
--retry-beam=40 $dir/$iter.mdl "ark:gunzip -c $dir/graphs.fsts.gz|" "$feats" \ --beam=8 --retry-beam=40 $dir/$iter.mdl "ark:gunzip -c $dir/graphs.fsts.gz|" \
ark:$dir/cur.ali 2> $dir/align.$iter.log || exit 1; "$feats" ark:$dir/cur.ali 2> $dir/align.$iter.log || exit 1;
fi fi
if echo $spkvec_iters | grep -w $iter >/dev/null; then if echo $spkvec_iters | grep -w $iter >/dev/null; then
( ali-to-post ark:$dir/cur.ali ark:- | \ ( ali-to-post ark:$dir/cur.ali ark:- | \
......
#!/bin/bash #!/bin/bash
# This script basically calls the supplied decoding script
# once for each test set (in parallel on the same machine),
# and then averages the resulting WERs.
# The interpretation of the decode-dir-1, etc., as inputs,
# outputs and so on, depends on the decoding script you call.
# It assumes the model directory is one level of from decode-dir-1.
mono_opt=
if [ "$1" == "--mono" ]; then
mono_opt=$1;
shift;
fi
script=$1 script=$1
dir=$2 decode_dir_1=$2 # e.g. exp/sgmm3b/decode
decode_dir_2=$3
decode_dir_3=$4
dir=`dirname $decode_dir_1` # e.g. exp/sgmm3b
if [ $# -lt 2 -o $# -gt 4 ]; then
echo "Usage: scripts/decode.sh <decode-script> <decode-dir-1> [<decode-dir-2> [<decode-dir-3>] ]"
exit 1;
fi
if [ ! -x $script -o ! -d $dir ]; then
echo "scripts/decode.sh: Either no such script $script or not executable, or no such dir $dir"
exit 1;
fi
scripts/mkgraph.sh $mono_opt data/lang_test $dir $dir/graph
if [ $# -ne 2 -o ! -x $script -o ! -d $dir ]; then if [ $# -eq 2 ]; then # normal case: 2 args.
echo "Usage: scripts/decode.sh <decode-script> <decode-dir>" for test in mar87 oct87 feb89 oct89 feb91 sep92; do
echo "[check your command line arguments]" $script $dir data/test_$test data/lang $decode_dir_1/$test &
done
elif [ $# -eq 3 ]; then
for test in mar87 oct87 feb89 oct89 feb91 sep92; do
$script $dir data/test_$test data/lang $decode_dir_1/$test $decode_dir_2/$test &
done
else
for test in mar87 oct87 feb89 oct89 feb91 sep92; do
$script $dir data/test_$test data/lang $decode_dir_1/$test $decode_dir_2/$test $decode_dir_3/$test &
done
fi fi
wait
scripts/mkgraph.sh data/lang_test $dir $dir/graph
for test in mar87 oct87 feb89 oct89 feb91 sep92; do # Average the WERs... there may be various wer files named e.g. wer, wer_10, etc.,
$script $dir data/test_$test data/lang $dir/decode_$test & # so do this for each one.
for w in $decode_dir_1/mar87/wer*; do
wername=`basename $w`
scripts/average_wer.sh $decode_dir_1/?????/$wername > $decode_dir_1/$wername
done done
wait grep WER $decode_dir_1/wer* || echo "Error decoding $decode_dir: no WER results found."
scripts/average_wer.sh $dir/decode_?????/wer > $dir/wer
cat $dir/wer
#!/bin/bash
# This script just calls the supplied decoding script
# (typically steps/decode_combine.sh, which is not the
# same as this script) once for each test set,
# and then averages the resulting WERs.
script=$1
decode_dir_in1=$2
decode_dir_in2=$3
decode_dir_out=$4
if [ $# -ne 4 ]; then
echo "Usage: scripts/decode_combine.sh <decode-script> <decode-dir-in1> <decode-dir-in2> <decode-dir-out>"
exit 1;
fi
if [ ! -x $script -o ! -d $decode_dir_in1 -o ! -d $decode_dir_in2 ]; then
echo "scripts/decode.sh: Either no such script $script or not executable, or no such dir $decode_dir_in1 or $decode_dir_in2"
exit 1;
fi
mkdir -p $decode_dir_out
for test in mar87 oct87 feb89 oct89 feb91 sep92; do
$script data/test_$test data/lang $decode_dir_in1/$test $decode_dir_in2/$test $decode_dir_out/$test
done
wait
# Average the WERs... there may be various wer files named e.g. wer, wer_10, etc.,
# so do this for each one.
for w in $decode_dir_out/mar87/wer*; do
wername=`basename $w`
scripts/average_wer.sh $decode_dir_out/?????/$wername > $decode_dir_out/$wername
done
grep WER $decode_dir_out/wer* || echo "Error decoding $decode_dir_out: no WER results found."
...@@ -41,15 +41,9 @@ scripts/subset_data_dir.sh data/train 1000 data/train.1k ...@@ -41,15 +41,9 @@ scripts/subset_data_dir.sh data/train 1000 data/train.1k
# train monophone system. # train monophone system.
steps/train_mono.sh data/train.1k data/lang exp/mono steps/train_mono.sh data/train.1k data/lang exp/mono
# decode mono [do this "manually" in the next few lines of
# script; generally this stuff gets called by "local/decode.sh" local/decode.sh --mono steps/decode_deltas.sh exp/mono/decode
# but here we want to pass the --mono option to mkgraph.sh.
scripts/mkgraph.sh --mono data/lang_test exp/mono exp/mono/graph
for test in mar87 oct87 feb89 oct89 feb91 sep92; do
steps/decode_deltas.sh exp/mono data/test_$test data/lang exp/mono/decode_$test &
done
wait
scripts/average_wer.sh exp/mono/decode_?????/wer > exp/mono/wer
# Get alignments from monophone system. # Get alignments from monophone system.
steps/align_deltas.sh data/train data/lang exp/mono exp/mono_ali steps/align_deltas.sh data/train data/lang exp/mono exp/mono_ali
...@@ -57,7 +51,7 @@ steps/align_deltas.sh data/train data/lang exp/mono exp/mono_ali ...@@ -57,7 +51,7 @@ steps/align_deltas.sh data/train data/lang exp/mono exp/mono_ali
# train tri1 [first triphone pass] # train tri1 [first triphone pass]
steps/train_deltas.sh data/train data/lang exp/mono_ali exp/tri1 steps/train_deltas.sh data/train data/lang exp/mono_ali exp/tri1
# decode tri1 # decode tri1
local/decode.sh steps/decode_deltas.sh exp/tri1 local/decode.sh steps/decode_deltas.sh exp/tri1/decode
# align tri1 # align tri1
steps/align_deltas.sh --graphs "ark,s,cs:gunzip -c exp/tri1/graphs.fsts.gz|" \ steps/align_deltas.sh --graphs "ark,s,cs:gunzip -c exp/tri1/graphs.fsts.gz|" \
...@@ -66,24 +60,24 @@ steps/align_deltas.sh --graphs "ark,s,cs:gunzip -c exp/tri1/graphs.fsts.gz|" \ ...@@ -66,24 +60,24 @@ steps/align_deltas.sh --graphs "ark,s,cs:gunzip -c exp/tri1/graphs.fsts.gz|" \
# train tri2a [delta+delta-deltas] # train tri2a [delta+delta-deltas]
steps/train_deltas.sh data/train data/lang exp/tri1_ali exp/tri2a steps/train_deltas.sh data/train data/lang exp/tri1_ali exp/tri2a
# decode tri2a # decode tri2a
local/decode.sh steps/decode_deltas.sh exp/tri2a local/decode.sh steps/decode_deltas.sh exp/tri2a/decode
# train tri2b [LDA+MLLT] # train tri2b [LDA+MLLT]
steps/train_lda_mllt.sh data/train data/lang exp/tri1_ali exp/tri2b steps/train_lda_mllt.sh data/train data/lang exp/tri1_ali exp/tri2b
# decode tri2b # decode tri2b
local/decode.sh steps/decode_lda_mllt.sh exp/tri2b local/decode.sh steps/decode_lda_mllt.sh exp/tri2b/decode
# Train and test ET. # Train and test ET.
steps/train_lda_et.sh data/train data/lang exp/tri1_ali exp/tri2c steps/train_lda_et.sh data/train data/lang exp/tri1_ali exp/tri2c
scripts/mkgraph.sh data/lang_test exp/tri2c exp/tri2c/graph scripts/mkgraph.sh data/lang_test exp/tri2c exp/tri2c/graph
local/decode.sh steps/decode_lda_et.sh exp/tri2c local/decode.sh steps/decode_lda_et.sh exp/tri2c/decode
# Align all data with LDA+MLLT system (tri2b) and do LDA+MLLT+SAT # Align all data with LDA+MLLT system (tri2b) and do LDA+MLLT+SAT
steps/align_lda_mllt.sh --graphs "ark,s,cs:gunzip -c exp/tri2b/graphs.fsts.gz|" \ steps/align_lda_mllt.sh --graphs "ark,s,cs:gunzip -c exp/tri2b/graphs.fsts.gz|" \
data/train data/lang exp/tri2b exp/tri2b_ali data/train data/lang exp/tri2b exp/tri2b_ali
steps/train_lda_mllt_sat.sh data/train data/lang exp/tri2b_ali exp/tri3d steps/train_lda_mllt_sat.sh data/train data/lang exp/tri2b_ali exp/tri3d
scripts/mkgraph.sh data/lang_test exp/tri3d exp/tri3d/graph scripts/mkgraph.sh data/lang_test exp/tri3d exp/tri3d/graph
local/decode.sh steps/decode_lda_mllt_sat.sh exp/tri3d local/decode.sh steps/decode_lda_mllt_sat.sh exp/tri3d/decode
# Align all data with LDA+MLLT+SAT system (tri3d) # Align all data with LDA+MLLT+SAT system (tri3d)
steps/align_lda_mllt_sat.sh --graphs "ark,s,cs:gunzip -c exp/tri3d/graphs.fsts.gz|" \ steps/align_lda_mllt_sat.sh --graphs "ark,s,cs:gunzip -c exp/tri3d/graphs.fsts.gz|" \
...@@ -92,8 +86,58 @@ steps/align_lda_mllt_sat.sh --graphs "ark,s,cs:gunzip -c exp/tri3d/graphs.fsts.g ...@@ -92,8 +86,58 @@ steps/align_lda_mllt_sat.sh --graphs "ark,s,cs:gunzip -c exp/tri3d/graphs.fsts.g
# Try another pass on top of that. # Try another pass on top of that.
steps/train_lda_mllt_sat.sh data/train data/lang exp/tri3d_ali exp/tri4d steps/train_lda_mllt_sat.sh data/train data/lang exp/tri3d_ali exp/tri4d
scripts/mkgraph.sh data/lang_test exp/tri4d exp/tri4d/graph scripts/mkgraph.sh data/lang_test exp/tri4d exp/tri4d/graph
local/decode.sh steps/decode_lda_mllt_sat.sh exp/tri4d local/decode.sh steps/decode_lda_mllt_sat.sh exp/tri4d/decode
# Next, SGMM system-- train SGMM system with speaker vectors, on top
# of LDA+MLLT features.
steps/train_ubm_lda_etc.sh data/train data/lang exp/tri2b_ali exp/ubm3d
steps/train_sgmm_lda_etc.sh data/train data/lang exp/tri2b_ali exp/ubm3d/final.ubm exp/sgmm3d
scripts/mkgraph.sh data/lang_test exp/sgmm3d exp/sgmm3d/graph
local/decode.sh steps/decode_sgmm_lda_etc.sh exp/sgmm3d/decode
# Align LDA+ET system prior to training corresponding SGMM system.
steps/align_lda_et.sh --graphs "ark,s,cs:gunzip -c exp/tri2c/graphs.fsts.gz|" \
data/train data/lang exp/tri2c exp/tri2c_ali
# Train SGMM system on top of LDA+ET.
steps/train_ubm_lda_etc.sh data/train data/lang exp/tri2c_ali exp/ubm3e
steps/train_sgmm_lda_etc.sh data/train data/lang exp/tri2c_ali exp/ubm3e/final.ubm exp/sgmm3e
local/decode.sh steps/decode_sgmm_lda_etc.sh exp/sgmm3e/decode exp/tri2c/decode
# Now train SGMM system on top of LDA+MLLT+SAT
steps/train_ubm_lda_etc.sh data/train data/lang exp/tri3d_ali exp/ubm4f
steps/train_sgmm_lda_etc.sh data/train data/lang exp/tri3d_ali exp/ubm4f/final.ubm exp/sgmm4f
local/decode.sh steps/decode_sgmm_lda_etc.sh exp/sgmm4f/decode exp/tri3d/decode
# Decode with fMLLR
sgmm-comp-prexform exp/sgmm4f/final.{mdl,occs,fmllr_mdl}
local/decode.sh steps/decode_sgmm_lda_etc_fmllr.sh exp/sgmm4f/decode_fmllr exp/sgmm4f/decode exp/tri3d/decode
# Some system combination experiments (just compose lattices).
local/decode_combine.sh steps/decode_combine.sh exp/tri1/decode exp/tri2a/decode exp/combine_1_2a/decode
local/decode_combine.sh steps/decode_combine.sh exp/sgmm4f/decode/ exp/tri3d/decode exp/combine_sgmm4f_tri3d/decode
for x in exp/*/decode*; do grep WER $x/wer_* | scripts/best_wer.sh; done
exp/combine_1_2a/decode/wer_7:%WER 3.399027 [ 426 / 12533, 55 ins, 94 del, 277 sub ]
exp/combine_sgmm4f_tri3d/decode/wer_5:%WER 1.731429 [ 217 / 12533, 30 ins, 43 del, 144 sub ]
exp/mono/decode/wer_6:%WER 10.340701 [ 1296 / 12533, 95 ins, 391 del, 810 sub ]
exp/sgmm3d/decode/wer_5:%WER 2.267284 [ 284 / 12526, 38 ins, 51 del, 195 sub ]
exp/sgmm3e/decode/wer_6:%WER 2.122397 [ 266 / 12533, 37 ins, 51 del, 178 sub ]
exp/sgmm4f/decode/wer_4:%WER 1.795261 [ 225 / 12533, 45 ins, 37 del, 143 sub ]
exp/tri1/decode/wer_6:%WER 3.566584 [ 447 / 12533, 74 ins, 88 del, 285 sub ]
exp/tri2a/decode/wer_7:%WER 3.518711 [ 441 / 12533, 57 ins, 91 del, 293 sub ]
exp/tri2b/decode/wer_9:%WER 3.614458 [ 453 / 12533, 59 ins, 111 del, 283 sub ]
exp/tri2c/decode/wer_6:%WER 2.833653 [ 355 / 12528, 54 ins, 71 del, 230 sub ]
exp/tri3d/decode/wer_7:%WER 2.489428 [ 312 / 12533, 43 ins, 63 del, 206 sub ]
exp/tri4d/decode/wer_7:%WER 2.649007 [ 332 / 12533, 53 ins, 67 del, 212 sub ]
local/decode_combine.sh steps/decode_combine.sh exp/tri1/decode exp/tri2a/decode exp/combine_tri3d_sgmm4f
##### Below here is trash. ###### ##### Below here is trash. ######
......
...@@ -18,4 +18,4 @@ ...@@ -18,4 +18,4 @@
# To be run from one directory above this script. # To be run from one directory above this script.
grep WER $* | \ grep WER $* | \
awk '{n=n+$4; d=d+$6} END{ printf("Average WER is %f (%d / %d) \n", (100.0*n)/d, n, d); }' awk '{e=e+$4; n=n+$6; i=i+$7; d=d+$9; s=s+$11;} END{ printf("%%WER %f [ %d / %d, %d ins, %d del, %d sub ]\n", (100.0*e)/n, e, n, i, d, s); }'
#!/usr/bin/perl -w
# Copyright 2010-2011 Microsoft Corporation
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
# MERCHANTABLITY OR NON-INFRINGEMENT.
# See the Apache 2 License for the specific language governing permissions and
# limitations under the License.
# This script takes as input any two files in an "scp-like" format that
# has e.g. utterance id's as the first token on each line; and it outputs
# something that will normally be identical to the first file, except that
# if any utterance-id was present in the first input but not the second, it
# will choose the second. Basically this implements a form of backoff.
if(@ARGV != 2) {
die "Usage: backoff_scp.pl in1.scp in2.scp > out.scp ";
}
($f1, $f2) = @ARGV;
open(O, "|sort"); # Sort and put into standard out.
open(F1, "<$f1") || die "Could not open input $f1";
while(<F1>) {
@A = split;
@A>=1 || die "Invalid id-list file line $_";
$seen{$A[0]} = 1;
print O;
}
open(F2, "<$f2") || die "Could not open input $f2";
while(<F2>) {
@A = split;
@A > 0 || die "Invalid scp file line $_";
if(! $seen{$A[0]}) {
print O;
}
}
#!/bin/bash
#
# Copyright 2010-2011 Microsoft Corporation
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
# MERCHANTABLITY OR NON-INFRINGEMENT.
# See the Apache 2 License for the specific language governing permissions and
# limitations under the License.
# To be run from one directory above this script.
perl -e 'while(<>){
if (m/WER (\S+)/ && !defined $bestwer || $bestwer > $1){ $bestwer = $1; $bestline=$_; }}
if (defined $bestline){ print $bestline; } '
...@@ -50,7 +50,7 @@ model=$srcdir/final.mdl ...@@ -50,7 +50,7 @@ model=$srcdir/final.mdl
mkdir -p $dir mkdir -p $dir
cp $srcdir/{tree,final.mdl} $dir || exit 1; # Create copy of the tree and model... cp $srcdir/{tree,final.mdl,final.occs} $dir || exit 1; # Create copy of the tree and model and occs...
scale_opts="--transition-scale=1.0 --acoustic-scale=0.1 --self-loop-scale=0.1" scale_opts="--transition-scale=1.0 --acoustic-scale=0.1 --self-loop-scale=0.1"
......
#!/bin/bash
# Copyright 2010-2011 Microsoft Corporation Arnab Ghoshal
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
# MERCHANTABLITY OR NON-INFRINGEMENT.
# See the Apache 2 License for the specific language governing permissions and
# limitations under the License.
# To be run from ..
# This script does training-data alignment given a model built using CMN +
# splice-9-frames + LDA + ET features. Its output, all in
# its own experimental directory, is cmvn.ark, trans.ark, ali, tree, final.mdl,
# final.alimdl, final.occs, final.mat and final.et (the last six are just copied
# from the source directory).
# Option to use precompiled graphs from last phase, if these
# are available (i.e. if they were built with the same data).
graphs=
if [ "$1" == --graphs ]; then
shift;
graphs=$1
shift
fi
if [ $# != 4 ]; then
echo "Usage: steps/align_lda_et.sh <data-dir> <lang-dir> <src-dir> <exp-dir>"
echo " e.g.: steps/align_lda_et.sh data/train data/lang exp/tri2c exp/tri2c_ali"
exit 1;
fi
if [ -f path.sh ]; then . path.sh; fi
data=$1
lang=$2
srcdir=$3
dir=$4
requirements="$srcdir/final.mdl $srcdir/final.alimdl $srcdir/final.mat $srcdir/final.et $srcdir/tree"
for f in $requirements; do
if [ ! -f $f ]; then
echo "align_lda_mllt.sh: no such file $f"
exit 1;
fi
done
mkdir -p $dir
cp $srcdir/{final.mdl,final.occs,final.alimdl,tree,final.mat,final.et} $dir || exit 1; # Create copies in $dir
scale_opts="--transition-scale=1.0 --acoustic-scale=0.1 --self-loop-scale=0.1"
echo "Computing cepstral mean and variance statistics"
compute-cmvn-stats --spk2utt=ark:$data/spk2utt scp:$data/feats.scp \
ark:$dir/cmvn.ark 2>$dir/cmvn.log || exit 1;
sifeats="ark:apply-cmvn --norm-vars=false --utt2spk=ark:$data/utt2spk ark:$dir/cmvn.ark scp:$data/feats.scp ark:- | splice-feats ark:- ark:- | transform-feats $dir/final.mat ark:- ark:- |"
# Align all training data using the supplied alignment model.
echo "Aligning all training data [with alignment model]"
if [ -z "$graphs" ]; then # --graphs option not supplied [-z means empty string]
# compute integer form of transcripts.
scripts/sym2int.pl --ignore-first-field $lang/words.txt < $data/text > $dir/train.tra \
|| exit 1;
gmm-align $scale_opts --beam=8 --retry-beam=40 $dir/tree $dir/final.alimdl $lang/L.fst \
"$sifeats" ark:$dir/train.tra ark:$dir/pre.ali 2> $dir/align_pass1.log || exit 1;
rm $dir/train.tra
else
gmm-align-compiled $scale_opts --beam=8 --retry-beam=40 $dir/final.alimdl \
"$graphs" "$sifeats" ark:$dir/pre.ali 2> $dir/align_pass1.log || exit 1;
fi
echo "Computing exponential transforms"
silphonelist=`cat $lang/silphones.csl`
[ -z $silphonelist ] && exit 1;
( ali-to-post ark:$dir/pre.ali ark:- | \
weight-silence-post 0.0 $silphonelist $dir/final.alimdl ark:- ark:- | \
gmm-post-to-gpost $dir/final.alimdl "$sifeats" ark:- ark:- | \
gmm-est-et --spk2utt=ark:$data/spk2utt $dir/final.mdl $dir/final.et \
"$sifeats" ark,s,cs:- ark:$dir/trans.ark ark,t:$dir/warp ) \
2>$dir/trans.log || exit 1;
feats="$sifeats transform-feats --utt2spk=ark:$data/utt2spk ark:$dir/trans.ark ark:- ark:- |"
echo "Aligning all training data [with final model and features]"
if [ -z "$graphs" ]; then # --graphs option not supplied [-z means empty string]
# compute integer form of transcripts.
scripts/sym2int.pl --ignore-first-field $lang/words.txt < $data/text > $dir/train.tra \
|| exit 1;
gmm-align $scale_opts --beam=8 --retry-beam=40 $dir/tree $dir/final.mdl $lang/L.fst \
"$feats" ark:$dir/train.tra ark:$dir/ali 2> $dir/align_pass2.log || exit 1;
rm $dir/train.tra
else
gmm-align-compiled $scale_opts --beam=8 --retry-beam=40 $dir/final.mdl \
"$graphs" "$feats" ark:$dir/ali 2> $dir/align_pass2.log || exit 1;
fi
rm $dir/pre.ali
echo "Done."
...@@ -54,7 +54,7 @@ for f in $requirements; do ...@@ -54,7 +54,7 @@ for f in $requirements; do
done done
mkdir -p $dir mkdir -p $dir
cp $srcdir/{final.mdl,tree,final.mat} $dir || exit 1; # Create copies in $dir cp $srcdir/{final.mdl,final.occs,tree,final.mat} $dir || exit 1; # Create copies in $dir
scale_opts="--transition-scale=1.0 --acoustic-scale=0.1 --self-loop-scale=0.1" scale_opts="--transition-scale=1.0 --acoustic-scale=0.1 --self-loop-scale=0.1"
......
...@@ -54,7 +54,7 @@ for f in $requirements; do ...@@ -54,7 +54,7 @@ for f in $requirements; do
done done
mkdir -p $dir mkdir -p $dir
cp $srcdir/{final.mdl,final.alimdl,tree,final.mat} $dir || exit 1; # Create copies in $dir cp $srcdir/{final.mdl,final.occs,final.alimdl,tree,final.mat} $dir || exit 1; # Create copies in $dir
scale_opts="--transition-scale=1.0 --acoustic-scale=0.1 --self-loop-scale=0.1"