nnet1,wsj: updating the recipe and results

- no lattice re-generation after 1st epoch of sMBR
- the results are now better
- adding auto-selection of queue options for BUT

egs/wsj/s5/RESULTS

@@ -2,7 +2,7 @@
 
 # this RESULTS file was obtained by Haihua Xu in July 2013.
 
-for x in exp/*/decode*; do [ -d $x ] && grep WER $x/wer_* | utils/best_wer.sh; done
+for x in exp/*/decode*; do [ -d $x ] && [[ $x =~ "$1" ]] && grep WER $x/wer_* | utils/best_wer.sh; done
 exit 0
 
 # Use caution when comparing these results with other published results.
@@ -182,16 +182,13 @@ exit 0
 # not updated
 
 
-# DNN on fMLLR features (Karel's setup, updated recipe [5.3.2014]). 
+# DNN on fMLLR features (Karel's setup, [7.8.2015]). 
 # frame cross-entropy training
-%WER 6.74 [ 555 / 8234, 67 ins, 73 del, 415 sub ] exp/dnn5b_pretrain-dbn_dnn/decode_bd_tgpr_dev93/wer_10
-%WER 4.09 [ 231 / 5643, 33 ins, 15 del, 183 sub ] exp/dnn5b_pretrain-dbn_dnn/decode_bd_tgpr_eval92/wer_12
-# sMBR training (1 iteration)
-%WER 6.39 [ 526 / 8234, 56 ins, 77 del, 393 sub ] exp/dnn5b_pretrain-dbn_dnn_smbr/decode_bd_tgpr_dev93_iter1/wer_11
-%WER 3.85 [ 217 / 5643, 23 ins, 16 del, 178 sub ] exp/dnn5b_pretrain-dbn_dnn_smbr/decode_bd_tgpr_eval92_iter1/wer_14
-# sMBR training (1+4 iterations, lattices+alignment updated after 1st iteration)
-%WER 6.15 [ 506 / 8234, 55 ins, 70 del, 381 sub ] exp/dnn5b_pretrain-dbn_dnn_smbr_i1lats/decode_bd_tgpr_dev93_iter4/wer_11
-%WER 3.56 [ 201 / 5643, 24 ins, 9 del, 168 sub ] exp/dnn5b_pretrain-dbn_dnn_smbr_i1lats/decode_bd_tgpr_eval92_iter4/wer_13
+%WER 6.05 [ 498 / 8234, 59 ins, 67 del, 372 sub ] exp/dnn5b_pretrain-dbn_dnn/decode_bd_tgpr_dev93/wer_11_0.0
+%WER 3.69 [ 208 / 5643, 19 ins, 19 del, 170 sub ] exp/dnn5b_pretrain-dbn_dnn/decode_bd_tgpr_eval92/wer_11_1.0
+# sMBR training (5 iteration, no lattice-regeneration),
+%WER 5.60 [ 461 / 8234, 58 ins, 62 del, 341 sub ] exp/dnn5b_pretrain-dbn_dnn_smbr/decode_bd_tgpr_dev93_it4/wer_11_0.0
+%WER 3.35 [ 189 / 5643, 18 ins, 14 del, 157 sub ] exp/dnn5b_pretrain-dbn_dnn_smbr/decode_bd_tgpr_eval92_it5/wer_13_1.0
 
 
 #DNN results  with cpu based setup
@@ -280,3 +277,4 @@ for x in exp/nnet2_online/nnet_ms_a_online/decode_*; do grep WER $x/wer_* | util
 %WER 6.68 [ 377 / 5643, 102 ins, 13 del, 262 sub ] exp/nnet2_online/nnet_ms_a_online/decode_tgpr_eval92_utt/wer_10
 %WER 6.56 [ 370 / 5643, 100 ins, 12 del, 258 sub ] exp/nnet2_online/nnet_ms_a_online/decode_tgpr_eval92_utt_offline/wer_10
 
+

egs/wsj/s5/cmd.sh

@@ -12,19 +12,19 @@ export mkgraph_cmd="queue.pl -l arch=*64 --mem 4G"
 export big_memory_cmd="queue.pl -l arch=*64 --mem 8G"
 export cuda_cmd="queue.pl -l gpu=1"
 
-
-
-#b) BUT cluster options
-#export train_cmd="queue.pl -q all.q@@blade -l ram_free=1200M,mem_free=1200M"
-#export decode_cmd="queue.pl -q all.q@@blade -l ram_free=1700M,mem_free=1700M"
-#export decodebig_cmd="queue.pl -q all.q@@blade -l ram_free=4G,mem_free=4G"
-
-#export cuda_cmd="queue.pl -q long.q@@pco203 -l gpu=1"
-#export cuda_cmd="queue.pl -q long.q@pcspeech-gpu"
-#export mkgraph_cmd="queue.pl -q all.q@@servers -l ram_free=4G,mem_free=4G"
-
-#c) run it locally...
+#b) run it locally...
 #export train_cmd=run.pl
 #export decode_cmd=run.pl
 #export cuda_cmd=run.pl
 #export mkgraph_cmd=run.pl
+
+#c) BUT cluster:
+if [ "$(hostname -d)" == "fit.vutbr.cz" ]; then
+  queue="all.q@@blade,all.q@@speech"
+  gpu_queue="long.q@supergpu*,long.q@dellgpu*,long.q@pcspeech-gpu,long.q@pcgpu*"
+  storage="matylda5"
+  export train_cmd="queue.pl -q $queue -l ram_free=1500M,mem_free=1500M,${storage}=1"
+  export decode_cmd="queue.pl -q $queue -l ram_free=2500M,mem_free=2500M,${storage}=0.5"
+  export cuda_cmd="queue.pl -q $gpu_queue -l gpu=1" 
+fi
+

egs/wsj/s5/local/nnet/run_dnn.sh

@@ -93,10 +93,10 @@ fi
 
 if [ $stage -le 4 ]; then
   # Re-train the DNN by 1 iteration of sMBR 
-  steps/nnet/train_mpe.sh --cmd "$cuda_cmd" --num-iters 1 --acwt $acwt --do-smbr true \
+  steps/nnet/train_mpe.sh --cmd "$cuda_cmd" --num-iters 5 --acwt $acwt --do-smbr true \
     $data_fmllr/train_si284 data/lang $srcdir ${srcdir}_ali ${srcdir}_denlats $dir || exit 1
   # Decode (reuse HCLG graph)
-  for ITER in 1; do
+  for ITER in 5 4 3 1; do
     steps/nnet/decode.sh --nj 10 --cmd "$decode_cmd" --config conf/decode_dnn.config \
       --nnet $dir/${ITER}.nnet --acwt $acwt \
       $gmmdir/graph_bd_tgpr $data_fmllr/test_dev93 $dir/decode_bd_tgpr_dev93_it${ITER} || exit 1;
@@ -106,33 +106,5 @@ if [ $stage -le 4 ]; then
   done 
 fi
 
-# Re-generate lattices, run 4 more sMBR iterations
-dir=exp/dnn5b_pretrain-dbn_dnn_smbr_i1lats
-srcdir=exp/dnn5b_pretrain-dbn_dnn_smbr
-acwt=0.1
-
-if [ $stage -le 5 ]; then
-  # Generate lattices and alignments:
-  steps/nnet/align.sh --nj 100 --cmd "$train_cmd" \
-    $data_fmllr/train_si284 data/lang $srcdir ${srcdir}_ali || exit 1;
-  steps/nnet/make_denlats.sh --nj 100 --cmd "$decode_cmd" --config conf/decode_dnn.config --acwt $acwt \
-    $data_fmllr/train_si284 data/lang $srcdir ${srcdir}_denlats || exit 1;
-fi
-
-if [ $stage -le 6 ]; then
-  # Re-train the DNN by 1 iteration of sMBR 
-  steps/nnet/train_mpe.sh --cmd "$cuda_cmd" --num-iters 4 --acwt $acwt --do-smbr true \
-    $data_fmllr/train_si284 data/lang $srcdir ${srcdir}_ali ${srcdir}_denlats $dir || exit 1
-  # Decode (reuse HCLG graph)
-  for ITER in 1 2 3 4; do
-    steps/nnet/decode.sh --nj 10 --cmd "$decode_cmd" --config conf/decode_dnn.config \
-      --nnet $dir/${ITER}.nnet --acwt $acwt \
-      $gmmdir/graph_bd_tgpr $data_fmllr/test_dev93 $dir/decode_bd_tgpr_dev93_iter${ITER} || exit 1;
-    steps/nnet/decode.sh --nj 8 --cmd "$decode_cmd" --config conf/decode_dnn.config \
-      --nnet $dir/${ITER}.nnet --acwt $acwt \
-      $gmmdir/graph_bd_tgpr $data_fmllr/test_eval92 $dir/decode_bd_tgpr_eval92_iter${ITER} || exit 1;
-  done 
-fi
-
 # Getting results [see RESULTS file]
 # for x in exp/*/decode*; do [ -d $x ] && grep WER $x/wer_* | utils/best_wer.sh; done