Commit 8888620e authored by Dan Povey's avatar Dan Povey
Browse files

Removing some now-unused code RE CMVN; adding 's5b' version of Switchboard...

Removing some now-unused code RE CMVN; adding 's5b' version of Switchboard script based on Arnab's Edinburgh recipe; some updates to READMEs and minor fixes/improvements.

git-svn-id: https://svn.code.sf.net/p/kaldi/code/trunk@2290 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8
parent 083c90ef
#!/bin/bash
. conf/common_vars.sh
. conf/languages/106-tagalog-limitedLP.official.conf
if false; then
#Preparing dev and train directories
if [[ ! -z "$train_data_list" ]] ; then
echo ---------------------------------------------------------------------
echo "Subsetting the TRAIN set"
echo ---------------------------------------------------------------------
local/make_corpus_subset.sh $train_data_dir $train_data_list ./data/raw_train_data || exit 1
train_data_dir=`readlink -f ./data/raw_train_data`
nj_max=`cat $train_data_list | wc -l`
if [[ "$nj_max" -lt "$train_nj" ]] ; then
echo "The maximum reasonable number of jobs is $nj_max (you have $train_nj)! (The training and decoding process has file-granularity)"
train_nj=$nj_max
fi
fi
if [[ ! -z $dev_data_list ]] ; then
echo ---------------------------------------------------------------------
echo "Subsetting the DEV set"
echo ---------------------------------------------------------------------
local/make_corpus_subset.sh $dev_data_dir $dev_data_list ./data/raw_dev_data || exit 1
dev_data_dir=`readlink -f ./data/raw_dev_data`
nj_max=`cat $dev_data_list | wc -l`
if [[ "$nj_max" -lt "$decode_nj" ]] ; then
echo "The maximum reasonable number of jobs is $nj_max -- you have $decode_nj! (The training and decoding process has file-granularity)"
decode_nj=$nj_max
fi
fi
if [[ $filter_lexicon ]]; then
echo ---------------------------------------------------------------------
echo "Subsetting the LEXICON"
echo ---------------------------------------------------------------------
lexicon_dir=./data/raw_lex_data
mkdir -p $lexicon_dir
local/make_lexicon_subset.sh $train_data_dir/transcription \
$lexicon_file $lexicon_dir/lexicon.txt || exit 1
lexicon_file=$lexicon_dir/lexicon.txt
fi
echo ---------------------------------------------------------------------
echo "Preparing lexicon in data/local on" `date`
echo ---------------------------------------------------------------------
mkdir -p data/local
local/prepare_lexicon.pl \
$lexiconFlags $lexicon_file data/local || exit 1
echo ---------------------------------------------------------------------
echo "Creating L.fst etc in data/lang on" `date`
echo ---------------------------------------------------------------------
mkdir -p data/lang
utils/prepare_lang.sh \
--share-silence-phones true \
data/local $oovSymbol data/local/tmp.lang data/lang || exit 1
echo ---------------------------------------------------------------------
echo "Preparing acoustic training lists in data/train on" `date`
echo ---------------------------------------------------------------------
mkdir -p data/train
local/prepare_acoustic_training_data.pl \
--vocab data/local/lexicon.txt --fragmentMarkers \-\*\~ \
$train_data_dir data/train > data/train/skipped_utts.log || exit 1
echo ---------------------------------------------------------------------
echo "Preparing dev data lists in data/dev on" `date`
echo ---------------------------------------------------------------------
mkdir -p data/dev
local/prepare_acoustic_training_data.pl \
--fragmentMarkers \-\*\~ \
$dev_data_dir data/dev > data/dev/skipped_utts.log || exit 1
echo ---------------------------------------------------------------------
echo "Preparing dev stm files in data/dev on" `date`
echo ---------------------------------------------------------------------
local/prepare_stm.pl --fragmentMarkers \-\*\~ data/dev || exit 1
cp $glmFile data/dev/glm
echo ---------------------------------------------------------------------
echo "Creating a basic G.fst in data/lang on" `date`
echo ---------------------------------------------------------------------
# We will simply override the default G.fst by the G.fst generated using SRILM
local/train_lms_srilm.sh data data/srilm || exit 1;
local/arpa2G.sh data/srilm/lm.gz data/lang data/lang || exit 1;
if [[ $subset_ecf ]] ; then
local/kws_setup.sh --case-insensitive $case_insensitive --subset-ecf $dev_data_list $ecf_file $kwlist_file $rttm_file data/lang data/dev || exit 1
else
local/kws_setup.sh --case-insensitive $case_insensitive $ecf_file $kwlist_file $rttm_file data/lang data/dev || exit 1
fi
fi ##TEMP
echo ---------------------------------------------------------------------
echo "Starting plp feature extraction in plp on" `date`
echo ---------------------------------------------------------------------
steps/make_plp.sh \
--cmd "$train_cmd" --nj $train_nj \
data/train exp/make_plp/train plp || exit 1
steps/compute_cmvn_stats.sh \
data/train exp/make_plp/train plp || exit 1
# In case plp extraction failed on some utterance, delist them
utils/fix_data_dir.sh data/train
steps/make_plp.sh \
--cmd "$train_cmd" --nj $decode_nj \
data/dev exp/make_plp/dev plp || exit 1
steps/compute_cmvn_stats.sh \
data/dev exp/make_plp/dev plp || exit 1
# In case plp extraction failed on some utterance, delist them
utils/fix_data_dir.sh data/dev
echo ---------------------------------------------------------------------
echo "Subsetting monophone training data in data/train_sub1 on" `date`
echo ---------------------------------------------------------------------
utils/subset_data_dir.sh data/train 5000 data/train_sub1 || exit 1
echo ---------------------------------------------------------------------
echo "Starting (small) monophone training in exp/mono on" `date`
echo ---------------------------------------------------------------------
steps/train_mono.sh \
--boost-silence 1.5 --nj 12 --cmd "$train_cmd" \
data/train_sub1 data/lang exp/mono || exit 1
echo ---------------------------------------------------------------------
echo "Starting (first) triphone training in exp/tri1 on" `date`
echo ---------------------------------------------------------------------
steps/align_si.sh \
--boost-silence 1.5 --nj $(( $train_nj/2 )) --cmd "$train_cmd" \
data/train data/lang exp/mono exp/mono_ali || exit 1
steps/train_deltas.sh \
--boost-silence 1.5 --cmd "$train_cmd" \
$numLeavesTri1 $numGaussTri1 data/train data/lang exp/mono_ali exp/tri1 || exit 1
echo ---------------------------------------------------------------------
echo "Spawning decoding with first triphone models in exp/tri1 on" `date`
echo ---------------------------------------------------------------------
(
mkdir -p exp/tri1/graph
utils/mkgraph.sh data/lang exp/tri1 exp/tri1/graph &> exp/tri1/mkgraph.log
mkdir -p exp/tri1/decode
steps/decode.sh --nj $decode_nj --cmd "$decode_cmd" \
exp/tri1/graph data/dev exp/tri1/decode &> exp/tri1/decode.log
local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
data/lang data/dev exp/tri1/decode
) &
tri1decode=$!; # Grab the PID of the subshell
sleep 5; # Let any "start-up error" messages from the subshell get logged
echo "See exp/tri1/mkgraph.log and exp/tri1/decode.log for decoding outcomes"
echo ---------------------------------------------------------------------
echo "Starting second triphone training in exp/tri2 on" `date`
echo ---------------------------------------------------------------------
steps/align_si.sh \
--boost-silence 1.5 --nj $train_nj --cmd "$train_cmd" \
data/train data/lang exp/tri1 exp/tri1_ali || exit 1
steps/train_deltas.sh \
--boost-silence 1.5 --cmd "$train_cmd" \
$numLeavesTri2 $numGaussTri2 data/train data/lang exp/tri1_ali exp/tri2 || exit 1
echo ---------------------------------------------------------------------
echo "Spawning decoding with triphone models in exp/tri2 on" `date`
echo ---------------------------------------------------------------------
(
mkdir -p exp/tri2/graph
utils/mkgraph.sh data/lang exp/tri2 exp/tri2/graph &> exp/tri2/mkgraph.log
mkdir -p exp/tri2/decode
steps/decode.sh --nj $decode_nj --cmd "$decode_cmd" \
exp/tri2/graph data/dev exp/tri2/decode &> exp/tri2/decode.log
local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
data/lang data/dev exp/tri2/decode
) &
tri2decode=$!; # Grab the PID of the subshell
sleep 5; # Let any "start-up error" messages from the subshell get logged
echo "See exp/tri2/mkgraph.log and exp/tri2/decode.log for decoding outcomes"
echo ---------------------------------------------------------------------
echo "Starting (lda_mllt) triphone training in exp/tri4 on" `date`
echo ---------------------------------------------------------------------
steps/align_si.sh \
--boost-silence 1.5 --nj $train_nj --cmd "$train_cmd" \
data/train data/lang exp/tri2 exp/tri2_ali || exit 1
steps/train_lda_mllt.sh \
--boost-silence 1.5 --cmd "$train_cmd" \
$numLeavesMLLT $numGaussMLLT data/train data/lang exp/tri2_ali exp/tri3 || exit 1
echo ---------------------------------------------------------------------
echo "Spawning decoding with lda_mllt models in exp/tri3 on" `date`
echo ---------------------------------------------------------------------
(
mkdir -p exp/tri3/graph
utils/mkgraph.sh \
data/lang exp/tri3 exp/tri3/graph &> exp/tri3/mkgraph.log
mkdir -p exp/tri3/decode
steps/decode.sh --nj $decode_nj --cmd "$decode_cmd" \
exp/tri3/graph data/dev exp/tri3/decode &> exp/tri3/decode.log
local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
data/lang data/dev exp/tri3/decode
) &
tri3decode=$!; # Grab the PID of the subshell
sleep 5; # Let any "start-up error" messages from the subshell get logged
echo "See exp/tri3/mkgraph.log and exp/tri3/decode.log for decoding outcomes"
echo ---------------------------------------------------------------------
echo "Starting (SAT) triphone training in exp/tri4 on" `date`
echo ---------------------------------------------------------------------
steps/align_si.sh \
--boost-silence 1.5 --nj $train_nj --cmd "$train_cmd" \
data/train data/lang exp/tri3 exp/tri3_ali || exit 1
steps/train_sat.sh \
--boost-silence 1.5 --cmd "$train_cmd" \
$numLeavesSAT $numGaussSAT data/train data/lang exp/tri3_ali exp/tri4 || exit 1
echo ---------------------------------------------------------------------
echo "Spawning decoding with SAT models on" `date`
echo ---------------------------------------------------------------------
(
mkdir -p exp/tri4/graph
utils/mkgraph.sh \
data/lang exp/tri4 exp/tri4/graph &> exp/tri4/mkgraph.log
mkdir -p exp/tri4/decode
touch exp/tri4/decode.started # A signal to the SGMM2 decoding step
steps/decode_fmllr.sh --nj $decode_nj --cmd "$decode_cmd" \
exp/tri4/graph data/dev exp/tri4/decode &> exp/tri4/decode.log \
&& touch exp/tri4/decode.finished # so SGMM2 decoding may proceed
local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
data/lang data/dev exp/tri4/decode.si
local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
data/lang data/dev exp/tri4/decode
) &
tri4decode=$!; # Grab the PID of the subshell; needed for SGMM2 decoding
sleep 5; # Let any "start-up error" messages from the subshell get logged
echo "See exp/tri4/mkgraph.log and exp/tri4/decode.log for decoding outcomes"
################################################################################
# Ready to start SGMM training
################################################################################
echo ---------------------------------------------------------------------
echo "Starting exp/ubm5 on" `date`
echo ---------------------------------------------------------------------
steps/align_fmllr.sh --boost-silence 1.5 --nj $train_nj --cmd "$train_cmd" \
data/train data/lang exp/tri4 exp/tri4_ali || exit 1
steps/train_ubm.sh --cmd "$train_cmd" \
$numGaussUBM data/train data/lang exp/tri4_ali exp/ubm5 || exit 1
echo ---------------------------------------------------------------------
echo "Starting exp/sgmm5 on" `date`
echo ---------------------------------------------------------------------
steps/train_sgmm2.sh --cmd "$train_cmd" \
$numLeavesSGMM $numGaussSGMM data/train data/lang exp/tri4_ali exp/ubm5/final.ubm exp/sgmm5 || exit 1
################################################################################
# Ready to decode with SGMM2 models
################################################################################
echo ---------------------------------------------------------------------
echo "Spawning exp/sgmm5/decode[_fmllr] on" `date`
echo ---------------------------------------------------------------------
echo "exp/sgmm5/decode will wait on PID $tri4decode if necessary"
wait $tri4decode; # Need lattices from the corresponding SGMM decoding passes
(
sleep 5; # Let the status message after the subshell get logged
## The next (now commented) block should ensure we starting decoding of sgmm5 only after
## the tri4 decoding finishes. The same can be achieved by "wait"int for tri4decode pid
# while [ ! -f exp/tri4/decode.started -o ! -f exp/tri4/decode.finished ]; do
# echo "exp/sgmm5/decode is waiting on SAT decoding ..." `date`
# sleep 5
# done
# while [ exp/tri4/decode.finished -ot exp/tri4/decode.started ]; do
# echo "exp/tri4/decode.finished is older than exp/tri4/decode.started"; \
# ls -lt exp/tri4/decode.finished exp/tri4/decode.started; \
# echo "Perhaps SAT decoding was restarted and is still running?"; \
# echo "exp/sgmm5/decode is still waiting on SAT decoding ..." `date`
# sleep 5
# done
# rm exp/tri4/decode.started exp/tri4/decode.finished
mkdir -p exp/sgmm5/graph
utils/mkgraph.sh \
data/lang exp/sgmm5 exp/sgmm5/graph &> exp/sgmm5/mkgraph.log
mkdir -p exp/sgmm5/decode
steps/decode_sgmm2.sh \
--nj $decode_nj --cmd "$decode_cmd" --transform-dir exp/tri4/decode \
exp/sgmm5/graph data/dev/ exp/sgmm5/decode &> exp/sgmm5/decode.log
steps/decode_sgmm2.sh --use-fmllr true --nj $decode_nj --cmd "$decode_cmd" \
--transform-dir exp/tri4/decode \
exp/sgmm5/graph data/dev/ exp/sgmm5/decode_fmllr &> exp/sgmm5/decode_fmllr.log
local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
data/lang data/dev exp/sgmm5/decode
local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
data/lang data/dev exp/sgmm5/decode_fmllr
) &
sgmm5decode=$!; # Grab the PID of the subshell; needed for MMI rescoring
sleep 5; # Let any "start-up error" messages from the subshell get logged
echo "See exp/sgmm5/mkgraph.log, exp/sgmm5/decode.log and exp/sgmm5/decode_fmllr.log for decoding outcomes"
################################################################################
# Ready to start discriminative SGMM training
################################################################################
echo ---------------------------------------------------------------------
echo "Starting exp/sgmm5_ali on" `date`
echo ---------------------------------------------------------------------
steps/align_sgmm2.sh \
--nj $train_nj --cmd "$train_cmd" --transform-dir exp/tri4_ali --use-graphs true --use-gselect true \
data/train data/lang exp/sgmm5 exp/sgmm5_ali || exit 1
echo ---------------------------------------------------------------------
echo "Starting exp/sgmm5_denlats on" `date`
echo ---------------------------------------------------------------------
steps/make_denlats_sgmm2.sh \
--nj $train_nj --sub-split $train_nj \
--beam 10.0 --lattice-beam 6 --cmd "$decode_cmd" --transform-dir exp/tri4_ali \
data/train data/lang exp/sgmm5_ali exp/sgmm5_denlats || exit 1
echo ---------------------------------------------------------------------
echo "Starting exp/sgmm5_mmi_b0.1 on" `date`
echo ---------------------------------------------------------------------
steps/train_mmi_sgmm2.sh \
--cmd "$decode_cmd" --zero-if-disjoint true --transform-dir exp/tri4_ali --boost 0.1 \
data/train data/lang exp/sgmm5_ali exp/sgmm5_denlats \
exp/sgmm5_mmi_b0.1 || exit 1
################################################################################
# Ready to decode with discriminative SGMM2 models
################################################################################
echo "exp/sgmm5_mmi_b0.1/decode will wait on PID $sgmm5decode if necessary"
wait $sgmm5decode; # Need lattices from the corresponding SGMM decoding passes
echo ---------------------------------------------------------------------
echo "Starting exp/sgmm5_mmi_b0.1/decode[_fmllr] on" `date`
echo ---------------------------------------------------------------------
for iter in 1 2 3 4; do
steps/decode_sgmm2_rescore.sh \
--cmd "$decode_cmd" --iter $iter --transform-dir exp/tri4/decode \
data/lang data/dev exp/sgmm5/decode exp/sgmm5_mmi_b0.1/decode_it$iter
steps/decode_sgmm2_rescore.sh \
--cmd "$decode_cmd" --iter $iter --transform-dir exp/tri4/decode \
data/lang data/dev exp/sgmm5/decode_fmllr exp/sgmm5_mmi_b0.1/decode_fmllr_it$iter
local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
data/lang data/dev exp/sgmm5_mmi_b0.1/decode_it$iter
local/kws_search.sh --cmd "$decode_cmd" --duptime $duptime \
data/lang data/dev exp/sgmm5_mmi_b0.1/decode_fmllr_it$iter
done
wait
# No need to wait on $tri4decode ---> $sgmm5decode ---> sgmm5_mmi_b0.1decode
echo ---------------------------------------------------------------------
echo "Finished successfully on" `date`
echo ---------------------------------------------------------------------
exit 0
......@@ -28,4 +28,3 @@ scripts at the moment.
unlike previous recipes, this now uses the same underlying
scripts as the WSJ recipe.
s6: this is not finished.
......@@ -32,7 +32,15 @@ steps/decode_sgmm2.sh --use-fmllr true --config conf/decode.config --nj 20 --cmd
steps/decode_sgmm2_rescore.sh --cmd "$decode_cmd" --iter $iter \
--transform-dir exp/tri3b/decode data/lang data/test exp/sgmm2_4a/decode exp/sgmm2_4a_mmi_b0.2/decode_it$iter &
done
(
steps/train_mmi_sgmm2.sh --cmd "$decode_cmd" --transform-dir exp/tri3b --boost 0.2 --zero-if-disjoint true \
data/train data/lang exp/sgmm2_4a_ali exp/sgmm2_4a_denlats exp/sgmm2_4a_mmi_b0.2_x
for iter in 1 2 3 4; do
steps/decode_sgmm2_rescore.sh --cmd "$decode_cmd" --iter $iter \
--transform-dir exp/tri3b/decode data/lang data/test exp/sgmm2_4a/decode exp/sgmm2_4a_mmi_b0.2_x/decode_it$iter &
done
)
wait
steps/decode_combine.sh data/test data/lang exp/tri1/decode exp/tri2a/decode exp/combine_1_2a/decode || exit 1;
steps/decode_combine.sh data/test data/lang exp/sgmm2_4a/decode exp/tri3b_mmi/decode exp/combine_sgmm2_4a_3b/decode || exit 1;
......
......@@ -20,4 +20,5 @@ scripts for a sequence of experiments.
s5: This is the "new-new-style" recipe.
All further work will be on top of this style of recipe.
s6: This is still under construction.
s5b: This is a cleaned-up version of s5, based on Arnab's
"edinburgh" recipe, but still somewhat under construction.
......@@ -39,4 +39,30 @@ for iter in 1 2 3 4; do
--transform-dir exp/tri4a/decode_eval2000 data/lang_test data/eval2000 exp/sgmm2_5a/decode_eval2000 \
exp/sgmm2_5a_mmi_b0.1/decode_eval2000_it$iter &
done
( # testing zero-if-disjoint.
steps/train_mmi_sgmm2.sh --cmd "$decode_cmd" --transform-dir exp/tri4a_ali_100k_nodup --boost 0.1 --zero-if-disjoint true \
data/train_100k_nodup data/lang exp/sgmm2_5a_ali_100k_nodup exp/sgmm2_5a_denlats_100k_nodup exp/sgmm2_5a_mmi_b0.1_z
for iter in 1 2 3 4; do
steps/decode_sgmm2_rescore.sh --cmd "$decode_cmd" --iter $iter \
--transform-dir exp/tri4a/decode_eval2000 data/lang_test data/eval2000 exp/sgmm2_5a/decode_eval2000 \
exp/sgmm2_5a_mmi_b0.1_z/decode_eval2000_it$iter &
done
wait
)
( # testing zero-if-disjoint.
# The same after a code speedup.
steps/train_mmi_sgmm2.sh --cmd "$decode_cmd" --transform-dir exp/tri4a_ali_100k_nodup --boost 0.1 --zero-if-disjoint true \
data/train_100k_nodup data/lang exp/sgmm2_5a_ali_100k_nodup exp/sgmm2_5a_denlats_100k_nodup exp/sgmm2_5a_mmi_b0.1_z2
for iter in 1 2 3 4; do
steps/decode_sgmm2_rescore.sh --cmd "$decode_cmd" --iter $iter \
--transform-dir exp/tri4a/decode_eval2000 data/lang_test data/eval2000 exp/sgmm2_5a/decode_eval2000 \
exp/sgmm2_5a_mmi_b0.1_z2/decode_eval2000_it$iter &
done
wait
)
......@@ -163,8 +163,8 @@ steps/align_fmllr.sh --nj 30 --cmd "$train_cmd" \
#local/run_nnet_cpu.sh &
local/run_sgmm.sh
#local/run_sgmm2.sh
#local/run_sgmm.sh
local/run_sgmm2.sh
# Building a larger SAT system.
......
# Note: these numbers are on eval2000 but don't compare them with the
# numbers Microsoft quotes in their DNN papers; those are just on the
# Switchboard portion of eval2000, excluding CallHome, which is
# substantially easier.
# These results are slightly out of date: since then I changed
# the LDA+MLLT to use 7, not 9 frames of context, and also increased
# the learning rate for the "indirect" fMMI.
for x in exp/{mono,tri,sgmm,nnet}*/decode*; do [ -d $x ] && grep Sum $x/score_*/*.sys | utils/best_wer.sh; done 2>/dev/null
for x in exp/{mono,tri,sgmm,nnet}*/decode*; do [ -d $x ] && grep WER $x/wer_* | utils/best_wer.sh; done 2>/dev/null
# These results are still partial.
exp/tri1/decode_eval2000_sw1_fsh_tgpr/score_13/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 61.5 26.8 11.7 3.2 41.7 70.2 |
exp/tri1/decode_eval2000_sw1_tg/score_13/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 61.1 27.3 11.6 3.5 42.3 70.3 |
exp/tri2/decode_eval2000_sw1_fsh_tgpr/score_13/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 62.8 26.1 11.0 3.2 40.3 70.1 |
exp/tri2/decode_eval2000_sw1_tg/score_12/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 62.7 26.9 10.3 3.7 40.9 70.5 |
exp/tri3a/decode_eval2000_sw1_fsh_tgpr/score_12/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 69.7 21.6 8.6 3.2 33.5 68.0 |
exp/tri3a/decode_eval2000_sw1_tg/score_12/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 69.3 22.0 8.7 3.4 34.1 67.3 |
exp/tri3b/decode_eval2000_sw1_fsh_tgpr/score_13/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 74.3 18.0 7.6 2.9 28.6 65.6 |
exp/tri3b/decode_eval2000_sw1_fsh_tgpr_newcode/score_13/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 74.3 18.0 7.6 2.9 28.6 65.6 |
exp/tri3b/decode_eval2000_sw1_tg/score_13/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 74.0 18.7 7.3 3.0 29.0 66.5 |
exp/tri3b/decode_eval2000_sw1_tg_newcode/score_13/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 74.0 18.7 7.3 3.0 29.0 66.5 |
exp/tri4a/decode_eval2000_sw1_fsh_tgpr/score_13/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 78.4 15.2 6.3 2.6 24.1 61.4 |
exp/tri4a/decode_eval2000_sw1_fsh_tgpr.si/score_11/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 71.8 20.7 7.5 3.6 31.8 67.4 |
exp/tri4a/decode_eval2000_sw1_tg/score_13/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 78.1 15.6 6.3 2.7 24.6 61.7 |
exp/tri4a/decode_eval2000_sw1_tg.si/score_11/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 71.3 21.2 7.5 3.8 32.5 67.7 |
exp/tri4b/decode_eval2000_sw1_fsh_tgpr/score_16/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 80.4 13.3 6.2 2.1 21.7 60.0 |
exp/tri4b/decode_eval2000_sw1_fsh_tgpr.si/score_13/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 73.7 18.9 7.4 3.0 29.3 65.9 |
exp/tri4b/decode_eval2000_sw1_tg/score_14/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 80.1 14.1 5.8 2.5 22.4 60.8 |
exp/tri4b/decode_eval2000_sw1_tg.si/score_12/eval2000.ctm.swbd.filt.sys: | Sum/Avg | 1831 21395 | 73.6 19.3 7.1 3.6 30.0 66.2 |
# "queue.pl" uses qsub. The options to it are
# options to qsub. If you have GridEngine installed,
# change this to a queue you have access to.
# Otherwise, use "run.pl", which will run jobs locally
# (make sure your --num-jobs options are no more than
# the number of cpus on your machine.
#a) JHU cluster options
export train_cmd="queue.pl -l arch=*64*"
export decode_cmd="queue.pl -l arch=*64* -l ram_free=4G,mem_free=4G"
#export cuda_cmd="..."
export mkgraph_cmd="queue.pl -l arch=*64* ram_free=4G,mem_free=4G"
#b) BUT cluster options
#export train_cmd="queue.pl -q all.q@@blade -l ram_free=1200M,mem_free=1200M"
#export decode_cmd="queue.pl -q all.q@@blade -l ram_free=1700M,mem_free=1700M"
#export decodebig_cmd="queue.pl -q all.q@@blade -l ram_free=4G,mem_free=4G"
#export cuda_cmd="queue.pl -q long.q@@pco203 -l gpu=1"
#export cuda_cmd="queue.pl -q long.q@pcspeech-gpu"
#export mkgraph_cmd="queue.pl -q all.q@@servers -l ram_free=4G,mem_free=4G"
#c) run it locally...
#export train_cmd=run.pl
#export decode_cmd=run.pl
export cuda_cmd=run.pl
#export mkgraph_cmd=run.pl
beam=11.0 # beam for decoding. Was 13.0 in the scripts.
first_beam=8.0 # beam for 1st-pass decoding in SAT.
--window-type=hamming # disable Dans window, use the standard
--use-energy=false # only fbank outputs
--sample-frequency=8000 # Cantonese is sampled at 8kHz
--low-freq=64 # typical setup from Frantisek Grezl
--high-freq=3800
--dither=1
--num-mel-bins=15 # 8kHz so we use 15 bins
--htk-compat=true # try to make it compatible with HTK
--use-energy=false # only non-default option.
--sample-frequency=8000 # Switchboard is sampled at 8kHz
<Topology>
<TopologyEntry>
<ForPhones>
NONSILENCEPHONES
</ForPhones>
<State> 0 <PdfClass> 0 <Transition> 0 0.75 <Transition> 1 0.25 </State>
<State> 1 <PdfClass> 1 <Transition> 1 0.75 <Transition> 2 0.25 </State>
<State> 2 <PdfClass> 2 <Transition> 2 0.75 <Transition> 3 0.25 </State>
<State> 3 </State>
</TopologyEntry>
<TopologyEntry>
<ForPhones>
SILENCEPHONES
</ForPhones>
<State> 0 <PdfClass> 0 <Transition> 0 0.25 <Transition> 1 0.25 <Transition> 2 0.25 <Transition> 3 0.25 </State>
<State> 1 <PdfClass> 1 <Transition> 1 0.25 <Transition> 2 0.25 <Transition> 3 0.25 <Transition> 4 0.25 </State>
<State> 2 <PdfClass> 2 <Transition> 1 0.25 <Transition> 2 0.25 <Transition> 3 0.25 <Transition> 4 0.25 </State>
<State> 3 <PdfClass> 3 <Transition> 1 0.25 <Transition> 2 0.25 <Transition> 3 0.25 <Transition> 4 0.25 </State>
<State> 4 <PdfClass> 4 <Transition> 4 0.25 <Transition> 5 0.75 </State>
<State> 5 </State>
</TopologyEntry>
</Topology>
#!/bin/bash
# Hub-5 Eval 1997 data preparation
# Author: Arnab Ghoshal (Jan 2013)
# To be run from one directory above this script.
# The input is a directory name containing the 1997 Hub5 english evaluation
# test set and transcripts, which is LDC2002S10
# e.g. see
# http://www.ldc.upenn.edu/Catalog/CatalogEntry.jsp?catalogId=LDC2002S10
#
# It is assumed that the transcripts are in a subdirectory called transcr
# However, we download the STM from NIST site:
# ftp://jaguar.ncsl.nist.gov/lvcsr/mar97/eval/hub5e97.english.980618.stm
if [ $# -ne 1 ]; then
echo "Usage: "`basename $0`" <speech-dir>"
echo "See comments in the script for more details"
exit 1
fi
sdir=$1
[ ! -d $sdir/speech ] \
&& echo Expecting directory $sdir/speech to be present && exit 1;
[ ! -d $sdir/transcr ] \
&& echo Expecting directory $sdir/transcr to be present && exit 1;
. path.sh
dir=data/local/eval1997
mkdir -p $dir
find $sdir/speech -iname '*.sph' | sort > $dir/sph.flist
sed -e 's?.*/??' -e 's?.sph??' $dir/sph.flist | paste - $dir/sph.flist \
> $dir/sph.scp
sph2pipe=$KALDI_ROOT/tools/sph2pipe_v2.5/sph2pipe
[ ! -x $sph2pipe ] \
&& echo "Could not execute the sph2pipe program at $sph2pipe" && exit 1;
awk -v sph2pipe=$sph2pipe '{
printf("%s-A %s -f wav -p -c 1 %s |\n", $1, sph2pipe, $2);
printf("%s-B %s -f wav -p -c 2 %s |\n", $1, sph2pipe, $2);
}' < $dir/sph.scp | sort > $dir/wav.scp || exit 1;
#side A - channel 1, side B - channel 2
# Get segments file...
# segments file format is: utt-id side-id start-time end-time, e.g.:
# sw02001-A_000098-001156 sw02001-A 0.98 11.56
pem=$sdir/speech/97_hub5e.pem
[ ! -f $pem ] && echo "No such file $pem" && exit 1;
# pem file has lines like:
# en_4156 A unknown_speaker 301.85 302.48
# There is one line in the 97_hub5e.pem with an extra : on the channel