trunk,ami : refactoring the ami recipe which will serve as a baseline for JSALT workshop

git-svn-id: https://svn.code.sf.net/p/kaldi/code/trunk@5204 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8

egs/ami/s5/README

+= README =
+- In this recipe you can build systems for 3 different conditions:
+  - IHM : independent headset microphone, run_ihm.sh
+  - MDM : multiple distant microphone, run_mdm.sh
+  - SDM : single distant microphone, run_sdm.sh
+
+- Please run 'run_prepare_shared.sh' before buildin any system (prepares annotation and LM)
+
+- To show the results run './RESULTS_(ihm,mdm,sdm)'
+
+- For information about the database see : http://groups.inf.ed.ac.uk/ami/corpus/overview.shtml
+
+= DONE =
+- Refactored the scripts,
+  - separate preparation of annotations + LM, which is independent on condition,
+  - put 'stages' to the recipes,
+  - fix the DNN recipe for LDA-MLLT features (we have no fMLLR),
+  - use sclite scoring also for IHM => better logs,
+  - fixed check on #wavs after downloading, 
+  - checked the IHM accuracy is same as in RESULTS_*!
+  - defined 10h subset (now unused, 0.2-0.3 degradation at 'tri4a' if subset used for mono + tri1)
+
+= TODO =
+- update the RESULTS* files
+- MDM download fails for IS1003b, IS1007d. Were those badly and should be excluded?
+- wav download is slow, propose mirror for wavs? compression for wget?
+
+= HOW TO IMPROVE RESULTS =
+- Try adding pitch features,
+- Try Bottlenck-DNN system like in ../../tedlium/s5/local/nnet/run_dnn_bn.sh,
+- Consider use of diarization in MDM recipe, plus adding SAT training for MDM condition,
+
+

egs/ami/s5/RESULTS_ihm

-
+#!/bin/bash
+# kaldi:
+for x in exp/ihm/{mono,tri,sgmm,nnet,dnn,lstm}*/decode*; do [ -d $x ] && [[ $x =~ "$1" ]] && grep WER $x/wer_* | utils/best_wer.sh; done 2>/dev/null
+# sclite / asclite:
+for x in exp/ihm/{mono,tri,sgmm,nnet,dnn,lstm}*/decode*; do [ -d $x ] && [[ $x =~ "$1" ]] && grep Sum $x/ascore_*/*.sys | utils/best_wer.sh; done 2>/dev/null
+exit 0
 
 dev
 exp/ihm/tri2a/decode_dev_ami_fsh.o3g.kn.pr1-7/ascore_13/dev.ctm.filt.dtl:Percent Total Error       =   38.0%   (35925)

egs/ami/s5/RESULTS_mdm

+#!/bin/bash
+# kaldi:
+for x in exp/mdm*/{mono,tri,sgmm,nnet,dnn,lstm}*/decode*; do [ -d $x ] && [[ $x =~ "$1" ]] && grep WER $x/wer_* | utils/best_wer.sh; done 2>/dev/null
+# sclite / asclite:
+for x in exp/mdm*/{mono,tri,sgmm,nnet,dnn,lstm}*/decode*; do [ -d $x ] && [[ $x =~ "$1" ]] && grep Sum $x/ascore_*/*.sys | utils/best_wer.sh; done 2>/dev/null
+exit 0
 
 
 #Beamforming of 8 microphones, WER scores with up to 4 overlapping speakers

egs/ami/s5/RESULTS_sdm

-
+#!/bin/bash
+# kaldi:
+for x in exp/sdm*/{mono,tri,sgmm,nnet,dnn,lstm}*/decode*; do [ -d $x ] && [[ $x =~ "$1" ]] && grep WER $x/wer_* | utils/best_wer.sh; done 2>/dev/null
+# sclite / asclite:
+for x in exp/sdm*/{mono,tri,sgmm,nnet,dnn,lstm}*/decode*; do [ -d $x ] && [[ $x =~ "$1" ]] && grep Sum $x/ascore_*/*.sys | utils/best_wer.sh; done 2>/dev/null
+exit 0
 
 #the below are WER scores with up to 4 overlapping speakers
 

egs/ami/s5/cmd.sh

@@ -15,4 +15,16 @@
 export train_cmd=run.pl
 export decode_cmd=run.pl
 export highmem_cmd=run.pl
-export cuda_cmd=run.pl
\ No newline at end of file
+export cuda_cmd=run.pl
+
+host=$(hostname -f)
+if [ ${host#*.} == "fit.vutbr.cz" ]; then
+  # BUT cluster:
+  queue="all.q@@blade,all.q@@speech"
+  gpu_queue="long.q@supergpu*,long.q@dellgpu*,long.q@pcspeech-gpu,long.q@pcgpu*"
+  storage="matylda5"
+  export train_cmd="queue.pl -q $queue -l ram_free=1500M,mem_free=1500M,${storage}=1"
+  export decode_cmd="queue.pl -q $queue -l ram_free=2500M,mem_free=2500M,${storage}=0.5"
+  export cuda_cmd="queue.pl -q $gpu_queue -l gpu=1"
+fi 
+

egs/ami/s5/conf/decode_dnn.conf

+beam=13.0 # beam for decoding.  Was 13.0 in the scripts.
+lattice_beam=8.0 # this has most effect on size of the lattices.

egs/ami/s5/conf/decode_tandem.conf

+beam=16.0
+lattice_beam=8.0
+scoring_opts="--min-lmwt 20 --max-lmwt 30"

egs/ami/s5/conf/pitch.conf

+--sample-frequency=16000

egs/ami/s5/local/ami_beamform.sh

 #!/bin/bash
 
-#Copyright 2014, University of Edinburgh (Author: Pawel Swietojanski)
-#Apache 2.0
+# Copyright 2014, University of Edinburgh (Author: Pawel Swietojanski)
+# Apache 2.0
 
 wiener_filtering=false
 nj=4
@@ -28,12 +28,17 @@ sdir=$2
 odir=$3
 wdir=data/local/beamforming
 
+set -e 
+set -u
+
 mkdir -p $odir
 mkdir -p $wdir/log
 
 meetings=$wdir/meetings.list
 
 cat local/split_train.orig local/split_dev.orig local/split_eval.orig | sort > $meetings
+# Removing ``lost'' MDM session-ids : http://groups.inf.ed.ac.uk/ami/corpus/dataproblems.shtml 
+mv $meetings{,.orig}; grep -v "IS1003b\|IS1007d" $meetings.orig >$meetings
 
 ch_inc=$((8/$numch))
 bmf=
@@ -43,7 +48,7 @@ done
 
 echo "Will use the following channels: $bmf"
 
-#make the channel file
+# make the channel file,
 if [ -f $wdir/channels_$numch ]; then
   rm $wdir/channels_$numch
 fi
@@ -58,17 +63,14 @@ do
   echo $channels >> $wdir/channels_$numch
 done < $meetings
 
-#do noise cancellation
-
+# do noise cancellation,
 if [ $wiener_filtering == "true" ]; then
   echo "Wiener filtering not yet implemented."
   exit 1;
 fi
 
-#do beamforming
-
+# do beamforming,
 echo -e "Beamforming\n"
-
 $cmd JOB=1:$nj $wdir/log/beamform.JOB.log \
      local/beamformit.sh $nj JOB $numch $meetings $sdir $odir
 

egs/ami/s5/local/ami_download.sh

 #!/bin/bash
 
 # Copyright 2014, University of Edinburgh (Author: Pawel Swietojanski, Jonathan Kilgour)
+# Copyright 2015, Brno University of Technology (Author: Karel Vesely)
+
+
+mics=
+. utils/parse_options.sh
 
 if [ $# -ne 2 ]; then
   echo "Usage: $0 <mic> <ami-dir>"
   echo " where <mic> is either ihm, sdm or mdm and <ami-dir> is download space."
   exit 1;
 fi
-
 mic=$1
 adir=$2
+
 amiurl=http://groups.inf.ed.ac.uk/ami
-annotver=ami_public_manual_1.6.1
+#annotver=ami_public_manual_1.6.1
 wdir=data/local/downloads
 
-if [[ ! "$mic" =~ ^(ihm|sdm|mdm)$ ]]; then
-  echo "$0. Wrong <mic> option." 
-  exit 1;
-fi
-
-mics="1 2 3 4 5 6 7 8"
-if [ "$mic" == "sdm" ]; then
-  mics=1
-fi
+case $mic in
+  ihm) echo "Ignoring mics '$mics'"
+  ;;
+  mdm) mics="1 2 3 4 5 6 7 8"
+    echo "mics set to '$mics'"
+  ;;
+  sdm) [ -z "$mics" -o ! -z "${mics#[1-8]}" ] && echo 'Select microphone by --mics "N", (N=1..8)' && exit 1
+    echo "Selected microphone '$mics'"
+  ;;
+  *) echo "Wrong 'mic' option $mic" && exit 1 
+  ;;
+esac
 
 mkdir -p $adir
 mkdir -p $wdir/log
 
 #download annotations
-
-annot="$adir/$annotver"
-if [[ ! -d $adir/annotations || ! -f "$annot" ]]; then
-  echo "Downloading annotiations..."
-  wget -nv -O $annot.zip $amiurl/AMICorpusAnnotations/$annotver.zip &> $wdir/log/download_ami_annot.log
-  mkdir -p $adir/annotations
-  unzip -o -d $adir/annotations $annot.zip &> /dev/null
-fi
-[ ! -f "$adir/annotations/AMI-metadata.xml" ] && echo "$0: File AMI-Metadata.xml not found under $adir/annotations." && exit 1;
+#
+#annot="$adir/$annotver"
+#if [[ ! -d $adir/annotations || ! -f "$annot" ]]; then
+#  echo "Downloading annotiations..."
+#  wget -nv -O $annot.zip $amiurl/AMICorpusAnnotations/$annotver.zip &> $wdir/log/download_ami_annot.log
+#  mkdir -p $adir/annotations
+#  unzip -o -d $adir/annotations $annot.zip &> /dev/null
+#fi
+#[ ! -f "$adir/annotations/AMI-metadata.xml" ] && echo "$0: File AMI-Metadata.xml not found under $adir/annotations." && exit 1;
 
 #download waves
 
 cat local/split_train.orig local/split_eval.orig local/split_dev.orig > $wdir/ami_meet_ids.flist
 
 wgetfile=$wdir/wget_$mic.sh
+
+# TODO fix this with Pawel, files don't exist anymore,
 manifest="wget -O $adir/MANIFEST.TXT http://groups.inf.ed.ac.uk/ami/download/temp/amiBuild-04237-Sun-Jun-15-2014.manifest.txt"
 license="wget -O $adir/LICENCE.TXT http://groups.inf.ed.ac.uk/ami/download/temp/Creative-Commons-Attribution-NonCommercial-ShareAlike-2.5.txt"
 
@@ -56,11 +66,13 @@ while read line; do
        [ "$mtg" == "$line" ] && extra_headset=4;
      done
      for m in 0 1 2 3 $extra_headset; do
-       echo "wget -nv -c -P $adir/$line/audio $amiurl/AMICorpusMirror/amicorpus/$line/audio/$line.Headset-$m.wav" >> $wgetfile
+       # Hint: avoiding re-download by '--continue',
+       echo "wget -nv --continue -P $adir/$line/audio $amiurl/AMICorpusMirror/amicorpus/$line/audio/$line.Headset-$m.wav" >> $wgetfile
      done
    else
      for m in $mics; do
-       echo "wget -nv -c -P $adir/$line/audio $amiurl/AMICorpusMirror/amicorpus/$line/audio/$line.Array1-0$m.wav" >> $wgetfile
+       # Hint: avoiding re-download by '--continue',
+       echo "wget -nv --continue -P $adir/$line/audio $amiurl/AMICorpusMirror/amicorpus/$line/audio/$line.Array1-0$m.wav" >> $wgetfile
      done
    fi
 done < $wdir/ami_meet_ids.flist
@@ -68,17 +80,17 @@ done < $wdir/ami_meet_ids.flist
 chmod +x $wgetfile
 echo "Downloading audio files for $mic scenario."
 echo "Look at $wdir/log/download_ami_$mic.log for progress"
-$wgetfile &> $wdir/log/download_ami_$mic.log
+$wgetfile &> $wdir/log/download_ami_$mic.log 
 
-#do rough check if #wavs is as expected, it will fail anyway in data prep stage if it isn't
+# Do rough check if #wavs is as expected, it will fail anyway in data prep stage if it isn't,
 if [ "$mic" == "ihm" ]; then
-  num_files=`find $adir -iname *Headset*`
+  num_files=$(find $adir -iname *Headset* | wc -l)
   if [ $num_files -ne 687 ]; then
     echo "Warning: Found $num_files headset wavs but expected 687. Check $wdir/log/download_ami_$mic.log for details."
     exit 1;
   fi
 else
-  num_files=`find $adir -iname *Array1*`
+  num_files=$(find $adir -iname *Array1* | wc -l)
   if [[ $num_files -lt 1352 && "$mic" == "mdm" ]]; then
     echo "Warning: Found $num_files distant Array1 waves but expected 1352 for mdm. Check $wdir/log/download_ami_$mic.log for details."
     exit 1;

egs/ami/s5/local/ami_text_prep.sh

 #!/bin/bash
 
+# Copyright 2015, Brno University of Technology (Author: Karel Vesely)
 # Copyright 2014, University of Edinburgh (Author: Pawel Swietojanski), 2014, Apache 2.0
 
 if [ $# -ne 1 ]; then
   echo "Usage: $0 <ami-dir>"
+  echo " <ami-dir> is download space."
   exit 1;
 fi
 
+set -e
+set -u
+
 amidir=$1
-wdir=data/local/annotations
+mkdir -p $amidir
+
+echo "Downloading annotiations..."
+
+amiurl=http://groups.inf.ed.ac.uk/ami
+annotver=ami_public_manual_1.6.1
+annot="$amidir/$annotver"
+
+logdir=data/local/downloads; mkdir -p $logdir/log
+[ ! -f $annot.zip ] && wget -nv -O $annot.zip $amiurl/AMICorpusAnnotations/$annotver.zip &> $logdir/log/download_ami_annot.log
+
+mkdir -p $amidir/annotations
+unzip -o -d $amidir/annotations $annot.zip &> /dev/null
+
+[ ! -f "$amidir/annotations/AMI-metadata.xml" ] && echo "$0: File AMI-Metadata.xml not found under $amidir/annotations." && exit 1;
 
-#extract text from AMI XML annotations
+
+# extract text from AMI XML annotations,
 local/ami_xml2text.sh $amidir
 
+wdir=data/local/annotations
 [ ! -f $wdir/transcripts1 ] && echo "$0: File $wdir/transcripts1 not found." && exit 1;
 
 echo "Preprocessing transcripts..."
@@ -25,8 +46,3 @@ for dset in train eval dev; do
 done
 
 
-
-
-
-
-

egs/ami/s5/local/ami_train_lms.sh

@@ -59,8 +59,8 @@ set -o errexit
 mkdir -p $dir
 export LC_ALL=C 
 
-cut -d' ' -f2- $train | gzip -c > $dir/train.gz
-cut -d' ' -f2- $dev | gzip -c > $dir/dev.gz
+cut -d' ' -f6- $train | gzip -c > $dir/train.gz
+cut -d' ' -f6- $dev | gzip -c > $dir/dev.gz
 
 awk '{print $1}' $lexicon | sort -u > $dir/wordlist.lex
 gunzip -c $dir/train.gz | tr ' ' '\n' | grep -v ^$ | sort -u > $dir/wordlist.train
@@ -98,14 +98,14 @@ if [ ! -z "$swbd" ]; then
 fi
 
 if [ ! -z "$fisher" ]; then
-  [ ! -d "$fisher/part1/data/trans" ] \
+  [ ! -d "$fisher/data/trans" ] \
     && echo "Cannot find transcripts in Fisher directory: '$fisher'" \
     && exit 1;
   mkdir -p $dir/fisher
 
   find $fisher -follow -path '*/trans/*fe*.txt' -exec cat {} \; | grep -v ^# | grep -v ^$ \
     | cut -d' ' -f4- | gzip -c > $dir/fisher/text0.gz
-  gunzip -c $dir/fisher/text0.gz | fisher_map_words.pl \
+  gunzip -c $dir/fisher/text0.gz | local/fisher_map_words.pl \
     | gzip -c > $dir/fisher/text1.gz
   ngram-count -debug 0 -text $dir/fisher/text1.gz -order $order -limit-vocab \
     -vocab $dir/wordlist -unk -map-unk "<unk>" -kndiscount -interpolate \

egs/ami/s5/local/beamformit.sh

 #!/bin/bash
 
-# Copyright 2014, University of Edibnurgh (Author: Pawel Swietojanski)
+# Copyright 2014, University of Edinburgh (Author: Pawel Swietojanski)
 
 . ./path.sh
 
@@ -12,22 +12,25 @@ sdir=$5
 odir=$6
 wdir=data/local/beamforming
 
+set -e
+set -u
+
 utils/split_scp.pl -j $nj $((job-1)) $meetings $meetings.$job
 
 while read line; do
 
   mkdir -p $odir/$line
   BeamformIt -s $line -c $wdir/channels_$numch \
-                        --config_file `pwd`/conf/ami.cfg \
+                        --config_file `pwd`/conf/ami_beamformit.cfg \
                         --source_dir $sdir \
                         --result_dir $odir/$line
   mkdir -p $odir/$line
   mv $odir/$line/${line}.del  $odir/$line/${line}_MDM$numch.del
   mv $odir/$line/${line}.del2 $odir/$line/${line}_MDM$numch.del2
   mv $odir/$line/${line}.info $odir/$line/${line}_MDM$numch.info
-  mv $odir/$line/${line}.ovl  $odir/$line/${line}_MDM$numch.ovl
   mv $odir/$line/${line}.weat $odir/$line/${line}_MDM$numch.weat
   mv $odir/$line/${line}.wav  $odir/$line/${line}_MDM$numch.wav
+  #mv $odir/$line/${line}.ovl  $odir/$line/${line}_MDM$numch.ovl # Was not created!
 
 done < $meetings.$job
 

egs/ami/s5/local/run_dnn.sh → egs/ami/s5/local/nnet/run_dnn.sh

@@ -4,139 +4,114 @@
 . ./path.sh
 
 # DNN training. This script is based on egs/swbd/s5b/local/run_dnn.sh
-# Shinji Watanabe
+# Shinji Watanabe, Karel Vesely,
+
+# Config:
+nj=80
+nj_decode=30
+stage=0 # resume training with --stage=N
+. utils/parse_options.sh || exit 1;
+#
 
 if [ $# -ne 1 ]; then
   printf "\nUSAGE: %s <mic condition(ihm|sdm|mdm)>\n\n" `basename $0`
   exit 1;
 fi
-
 mic=$1
-final_lm=`cat data/local/lm/final_lm`
-LM=$final_lm.pr1-7
 
-for lm_suffix in $LM; do
-# Config:
 gmmdir=exp/$mic/tri4a
-graph_dir=exp/$mic/tri4a/graph_${lm_suffix}
 data_fmllr=data-fmllr-tri4
-stage=0 # resume training with --stage=N
-# End of config.
-. utils/parse_options.sh || exit 1;
-#
 
+final_lm=`cat data/local/lm/final_lm`
+LM=$final_lm.pr1-7
+graph_dir=$gmmdir/graph_${LM}
+
+# Set bash to 'debug' mode, it will exit on : 
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+set -x
+
+# Store fMLLR features, so we can train on them easily,
 if [ $stage -le 0 ]; then
-  # Store fMLLR features, so we can train on them easily,
-  # test
+  # eval
   dir=$data_fmllr/$mic/eval
-  steps/nnet/make_fmllr_feats.sh --nj 1 --cmd "$train_cmd" \
-     --transform-dir $gmmdir/decode_eval_${lm_suffix} \
-     $dir data/$mic/eval $gmmdir $dir/log $dir/data || exit 1
+  steps/nnet/make_fmllr_feats.sh --nj 15 --cmd "$train_cmd" \
+     --transform-dir $gmmdir/decode_eval_${LM} \
+     $dir data/$mic/eval $gmmdir $dir/log $dir/data
   # dev
   dir=$data_fmllr/$mic/dev
-  steps/nnet/make_fmllr_feats.sh --nj 1 --cmd "$train_cmd" \
-     --transform-dir $gmmdir/decode_dev_${lm_suffix} \
-     $dir data/$mic/dev $gmmdir $dir/log $dir/data || exit 1
+  steps/nnet/make_fmllr_feats.sh --nj 15 --cmd "$train_cmd" \
+     --transform-dir $gmmdir/decode_dev_${LM} \
+     $dir data/$mic/dev $gmmdir $dir/log $dir/data
   # train
   dir=$data_fmllr/$mic/train
-  steps/nnet/make_fmllr_feats.sh --nj 1 --cmd "$train_cmd" \
+  steps/nnet/make_fmllr_feats.sh --nj 15 --cmd "$train_cmd" \
      --transform-dir ${gmmdir}_ali \
-     $dir data/$mic/train $gmmdir $dir/log $dir/data || exit 1
+     $dir data/$mic/train $gmmdir $dir/log $dir/data
   # split the data : 90% train 10% cross-validation (held-out)
-  utils/subset_data_dir_tr_cv.sh $dir ${dir}_tr90 ${dir}_cv10 || exit 1
+  utils/subset_data_dir_tr_cv.sh $dir ${dir}_tr90 ${dir}_cv10
 fi
 
+# Pre-train DBN, i.e. a stack of RBMs,
 if [ $stage -le 1 ]; then
-  # Pre-train DBN, i.e. a stack of RBMs
   dir=exp/$mic/dnn4_pretrain-dbn
-  (tail --pid=$$ -F $dir/log/pretrain_dbn.log 2>/dev/null)& # forward log
   $cuda_cmd $dir/log/pretrain_dbn.log \
-    steps/nnet/pretrain_dbn.sh --rbm-iter 1 $data_fmllr/$mic/train $dir || exit 1;
+    steps/nnet/pretrain_dbn.sh --rbm-iter 1 $data_fmllr/$mic/train $dir
 fi
 
+# Train the DNN optimizing per-frame cross-entropy,
 if [ $stage -le 2 ]; then
-  # Train the DNN optimizing per-frame cross-entropy.
   dir=exp/$mic/dnn4_pretrain-dbn_dnn
   ali=${gmmdir}_ali
   feature_transform=exp/$mic/dnn4_pretrain-dbn/final.feature_transform
   dbn=exp/$mic/dnn4_pretrain-dbn/6.dbn
-  (tail --pid=$$ -F $dir/log/train_nnet.log 2>/dev/null)& # forward log
   # Train
   $cuda_cmd $dir/log/train_nnet.log \
     steps/nnet/train.sh --feature-transform $feature_transform --dbn $dbn --hid-layers 0 --learn-rate 0.008 \
-    $data_fmllr/$mic/train_tr90 $data_fmllr/$mic/train_cv10 data/lang $ali $ali $dir || exit 1;
+    $data_fmllr/$mic/train_tr90 $data_fmllr/$mic/train_cv10 data/lang $ali $ali $dir
   # Decode (reuse HCLG graph)
-  steps/nnet/decode.sh --nj 6 --cmd "$decode_cmd" --config conf/decode_dnn.conf --acwt 0.1 \
+  steps/nnet/decode.sh --nj $nj_decode --cmd "$decode_cmd" --config conf/decode_dnn.conf --acwt 0.1 \
     --num-threads 3 \
-    $graph_dir $data_fmllr/$mic/dev $dir/decode_dev_${lm_suffix} || exit 1;
-  steps/nnet/decode.sh --nj 6 --cmd "$decode_cmd" --config conf/decode_dnn.conf --acwt 0.1 \
+    $graph_dir $data_fmllr/$mic/dev $dir/decode_dev_${LM}
+  steps/nnet/decode.sh --nj $nj_decode --cmd "$decode_cmd" --config conf/decode_dnn.conf --acwt 0.1 \
     --num-threads 3 \
-    $graph_dir $data_fmllr/$mic/eval $dir/decode_eval_${lm_suffix} || exit 1;
+    $graph_dir $data_fmllr/$mic/eval $dir/decode_eval_${LM}
 fi
 
-# Sequence training using sMBR criterion, we do Stochastic-GD 
-# with per-utterance updates. We use usually good acwt 0.1
-# Lattices are re-generated after 1st epoch, to get faster convergence.
+
+# Sequence training using sMBR criterion, we do Stochastic-GD with 
+# per-utterance updates. We use usually good acwt 0.1.
+# Lattices are not regenerated (it is faster).
+
 dir=exp/$mic/dnn4_pretrain-dbn_dnn_smbr
 srcdir=exp/$mic/dnn4_pretrain-dbn_dnn
 acwt=0.1
 
+# Generate lattices and alignments,
 if [ $stage -le 3 ]; then
-  # First we generate lattices and alignments:
-  steps/nnet/align.sh --nj 6 --cmd "$train_cmd" \
-    $data_fmllr/$mic/train data/lang $srcdir ${srcdir}_ali || exit 1;
-  steps/nnet/make_denlats.sh --nj 6 --cmd "$decode_cmd" --config conf/decode_dnn.conf \
-    --acwt $acwt $data_fmllr/$mic/train data/lang $srcdir ${srcdir}_denlats || exit 1;
+  steps/nnet/align.sh --nj $nj --cmd "$train_cmd" \
+    $data_fmllr/$mic/train data/lang $srcdir ${srcdir}_ali
+  steps/nnet/make_denlats.sh --nj $nj --cmd "$decode_cmd" --config conf/decode_dnn.conf \
+    --acwt $acwt $data_fmllr/$mic/train data/lang $srcdir ${srcdir}_denlats
 fi
 
+# Re-train the DNN by 4 epochs of sMBR,
 if [ $stage -le 4 ]; then
-  # Re-train the DNN by 1 iteration of sMBR 
-  steps/nnet/train_mpe.sh --cmd "$cuda_cmd" --num-iters 1 --acwt $acwt --do-smbr true \
-    $data_fmllr/$mic/train data/lang $srcdir ${srcdir}_ali ${srcdir}_denlats $dir || exit 1
-  # Decode (reuse HCLG graph)
-  for ITER in 1; do
-    steps/nnet/decode.sh --nj 6 --cmd "$decode_cmd" --config conf/decode_dnn.conf \
-      --num-threads 3 \
-      --nnet $dir/${ITER}.nnet --acwt $acwt \
-      $graph_dir $data_fmllr/$mic/dev $dir/decode_dev_${lm_suffix} || exit 1;
-    steps/nnet/decode.sh --nj 6 --cmd "$decode_cmd" --config conf/decode_dnn.conf \
-      --num-threads 3 \
-      --nnet $dir/${ITER}.nnet --acwt $acwt \
-      $graph_dir $data_fmllr/$mic/eval $dir/decode_eval_${lm_suffix} || exit 1;
-  done 
-fi
-
-# Re-generate lattices, run 4 more sMBR iterations
-dir=exp/$mic/dnn4_pretrain-dbn_dnn_smbr_i1lats
-srcdir=exp/$mic/dnn4_pretrain-dbn_dnn_smbr
-acwt=0.1
-
-if [ $stage -le 5 ]; then
-  # First we generate lattices and alignments:
-  steps/nnet/align.sh --nj 6 --cmd "$train_cmd" \
-    $data_fmllr/$mic/train data/lang $srcdir ${srcdir}_ali || exit 1;
-  steps/nnet/make_denlats.sh --nj 6 --cmd "$decode_cmd" --config conf/decode_dnn.conf \
-    --acwt $acwt $data_fmllr/$mic/train data/lang $srcdir ${srcdir}_denlats || exit 1;
-fi
-
-if [ $stage -le 6 ]; then
-  # Re-train the DNN by 1 iteration of sMBR 
   steps/nnet/train_mpe.sh --cmd "$cuda_cmd" --num-iters 4 --acwt $acwt --do-smbr true \
-    $data_fmllr/$mic/train data/lang $srcdir ${srcdir}_ali ${srcdir}_denlats $dir || exit 1
+    $data_fmllr/$mic/train data/lang $srcdir ${srcdir}_ali ${srcdir}_denlats $dir
   # Decode (reuse HCLG graph)
-  for ITER in 1 2 3 4; do
-    steps/nnet/decode.sh --nj 6 --cmd "$decode_cmd" --config conf/decode_dnn.conf \
-      --num-threads 3 \
-      --nnet $dir/${ITER}.nnet --acwt $acwt \
-      $graph_dir $data_fmllr/$mic/dev $dir/decode_dev_${lm_suffix}_$ITER || exit 1;
-    steps/nnet/decode.sh --nj 6 --cmd "$decode_cmd" --config conf/decode_dnn.conf \
-      --num-threads 3 \
-      --nnet $dir/${ITER}.nnet --acwt $acwt \
-      $graph_dir $data_fmllr/$mic/eval $dir/decode_eval_${lm_suffix}_$ITER || exit 1;
-  done 
+  for ITER in 4 3 2 1; do
+    steps/nnet/decode.sh --nj $nj_decode --cmd "$decode_cmd" --config conf/decode_dnn.conf \
+      --num-threads 3 --nnet $dir/${ITER}.nnet --acwt $acwt \
+      $graph_dir $data_fmllr/$mic/dev $dir/decode_dev_${LM}
+    steps/nnet/decode.sh --nj $nj_decode --cmd "$decode_cmd" --config conf/decode_dnn.conf \
+      --num-threads 3 --nnet $dir/${ITER}.nnet --acwt $acwt \
+      $graph_dir $data_fmllr/$mic/eval $dir/decode_eval_${LM}
+  done
 fi
 
-done
 # Getting results [see RESULTS file]
 # for x in exp/$mic/*/decode*; do [ -d $x ] && grep WER $x/wer_* | utils/best_wer.sh; done
 

egs/ami/s5/local/nnet/run_dnn_lda_mllt.sh

+#!/bin/bash -u
+
+. ./cmd.sh
+. ./path.sh
+
+# DNN training. This script is based on egs/swbd/s5b/local/run_dnn.sh
+# Shinji Watanabe, Karel Vesely,
+
+# Config:
+nj=80
+
+stage=0 # resume training with --stage=N
+. utils/parse_options.sh || exit 1;
+#
+
+if [ $# -ne 1 ]; then
+  printf "\nUSAGE: %s <mic condition(ihm|sdm|mdm)>\n\n" `basename $0`
+  exit 1;
+fi
+mic=$1
+
+gmmdir=exp/$mic/tri3a
+
+final_lm=`cat data/local/lm/final_lm`
+LM=$final_lm.pr1-7
+graph_dir=$gmmdir/graph_${LM}