sandbox/akirkedal: merging changes from trunk.

git-svn-id: https://svn.code.sf.net/p/kaldi/code/sandbox/akirkedal@4260 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8

egs/babel/s5b/conf/common.fullLP

@@ -17,6 +17,7 @@ bnf_num_gauss_sgmm=50000 # use fewer SGMM sub-states than the
                          # non-bottleneck system (which has 80000).
 bnf_decode_acwt=0.066666
 
+
 # DNN hybrid system training parameters
 dnn_num_hidden_layers=4
 dnn_input_dim=4000
@@ -48,9 +49,16 @@ if [[ `hostname` == *.tacc.utexas.edu ]] ; then
   sgmm_train_extra_opts=( )
   sgmm_group_extra_opts=( --num_iters 25 ) 
   sgmm_denlats_extra_opts=( --num-threads 2 )
+  sgmm_mmi_extra_opts=(--cmd "local/lonestar.py -pe smp 2")
   dnn_denlats_extra_opts=( --num-threads 2 )
-  dnn_parallel_opts="-l gpu=1"
+
+  dnn_cpu_parallel_opts=(--minibatch-size 128 --max-change 10 --num-jobs-nnet 8 --num-threads 16 \
+                         --parallel-opts "-pe smp 16" )
+  dnn_gpu_parallel_opts=(--minibatch-size 512 --max-change 40 --num-jobs-nnet 8 --num-threads 1)
+
+  dnn_gpu_mpe_parallel_opts=(--num-jobs-nnet 8 --num-threads 1)
   dnn_gpu_mpe_parallel_opts=(--num-jobs-nnet 8 --num-threads 1)
+  dnn_parallel_opts="-l gpu=1"
 else
   decode_extra_opts=(--num-threads 6 --parallel-opts "-pe smp 6 -l mem_free=4G,ram_free=0.7G")
   sgmm_train_extra_opts=( --num-iters 25 )

egs/babel/s5b/conf/common.limitedLP

@@ -49,14 +49,15 @@ dnn_update_egs_opts=(--weight-threshold 0.7 --splice-width 4 --samples-per-iter
 
 if [[ `hostname` == *.tacc.utexas.edu ]] ; then
   decode_extra_opts=( --num-threads 4 --parallel-opts "-pe smp 4" )
-  sgmm_train_extra_opts=( )
+  sgmm_train_extra_opts=( --num-iters 25 )
   sgmm_group_extra_opts=( ) 
   sgmm_denlats_extra_opts=( --num-threads 1 )
   dnn_denlats_extra_opts=( --num-threads 1 )
 
   dnn_cpu_parallel_opts=(--minibatch-size 128 --num-jobs-nnet 8 --num-threads 16 \
                          --parallel-opts "-pe smp 16" )
-  dnn_gpu_parallel_opts=(--minibatch-size 512 --max-change 40 --num-jobs-nnet 4 --num-threads 1)
+  dnn_gpu_parallel_opts=(--minibatch-size 512 --max-change 40 --num-jobs-nnet 4 --num-threads 1
+                         --parallel-opts "-pe smp 16" )
 
   dnn_gpu_mpe_parallel_opts=(--num-jobs-nnet 4 --num-threads 1)
 

egs/babel/s5b/conf/lang/102-assamese-fullLP.official.conf

@@ -15,7 +15,8 @@ dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev
 dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev/IARPA-babel102b-v0.5a_conv-dev.mitllfa3.rttm
 dev2h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist.xml
 dev2h_more_kwlists=(
-                      [limitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-eval.kwlist4.xml
                     )
 dev2h_subset_ecf=true
 dev2h_nj=24
@@ -29,10 +30,12 @@ dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-de
 dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev/IARPA-babel102b-v0.5a_conv-dev.mitllfa3.rttm
 dev10h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist.xml
 dev10h_more_kwlists=(
-                      [limitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-eval.kwlist4.xml
                     )
 dev10h_nj=32
 
+
 #Official EVAL period evaluation data files 
 eval_data_dir=/export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_102/conversational/eval/
 eval_data_list=/export/babel/data/splits/Assamese_Babel102/eval.list
@@ -46,19 +49,20 @@ shadow_data_dir=(
                 /export/babel/data/102-assamese/release-current/conversational/dev
                 /export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_102/conversational/eval/
               )
+shadow_data_cmudb=/export/babel/data/splits/Assamese_Babel102/uem/102-shadow-v0-cleaned-utt.dat
 shadow_data_list=(
                 /export/babel/data/splits/Assamese_Babel102/dev.list
                 /export/babel/data/splits/Assamese_Babel102/eval.list
               )
-shadow_data_cmudb=/export/babel/data/splits/Assamese_Babel102/uem/102-shadow-v0-utt.dat
 shadow_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.ecf.xml
-shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-eval.kwlist4.xml
+shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist.xml
 shadow_more_kwlists=(
-                      [FullLPdev]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-eval.kwlist4.xml
+
                     )
 shadow_nj=32
 
-
 # Acoustic model parameters
 numLeavesTri1=1000
 numGaussTri1=10000
@@ -74,7 +78,6 @@ numGaussUBM=800
 numLeavesSGMM=10000
 numGaussSGMM=80000
 
-
 # Lexicon and Language Model parameters
 oovSymbol="<unk>"
 lexiconFlags="--romanized --oov <unk>"

egs/babel/s5b/conf/lang/102-assamese-limitedLP.official.conf

@@ -15,7 +15,8 @@ dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev
 dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev/IARPA-babel102b-v0.5a_conv-dev.mitllfa3.rttm
 dev2h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist.xml
 dev2h_more_kwlists=(
-                      [limitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-eval.kwlist4.xml
                     )
 dev2h_subset_ecf=true
 dev2h_nj=24
@@ -29,10 +30,12 @@ dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-de
 dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev/IARPA-babel102b-v0.5a_conv-dev.mitllfa3.rttm
 dev10h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist.xml
 dev10h_more_kwlists=(
-                      [limitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-eval.kwlist4.xml
                     )
 dev10h_nj=32
 
+
 #Official EVAL period evaluation data files 
 eval_data_dir=/export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_102/conversational/eval/
 eval_data_list=/export/babel/data/splits/Assamese_Babel102/eval.list
@@ -46,19 +49,20 @@ shadow_data_dir=(
                 /export/babel/data/102-assamese/release-current/conversational/dev
                 /export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_102/conversational/eval/
               )
+shadow_data_cmudb=/export/babel/data/splits/Assamese_Babel102/uem/102-shadow-v0-cleaned-utt.dat
 shadow_data_list=(
                 /export/babel/data/splits/Assamese_Babel102/dev.list
                 /export/babel/data/splits/Assamese_Babel102/eval.list
               )
-shadow_data_cmudb=/export/babel/data/splits/Assamese_Babel102/uem/102-shadow-v0-utt.dat
 shadow_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.ecf.xml
-shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-eval.kwlist4.xml
+shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist.xml
 shadow_more_kwlists=(
-                      [FullLPdev]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel102b-v0.5a_conv-eval.kwlist4.xml
+
                     )
 shadow_nj=32
 
-
 unsup_data_dir=(/export/babel/data/102-assamese//release-current/conversational/training/
                 /export/babel/data/102-assamese//release-current/conversational/untranscribed-training/
                   )  
@@ -83,7 +87,6 @@ numGaussUBM=750
 numLeavesSGMM=5000
 numGaussSGMM=18000
 
-
 # Lexicon and Language Model parameters
 oovSymbol="<unk>"
 lexiconFlags="--romanized --oov <unk>"

egs/babel/s5b/conf/lang/103-bengali-fullLP.official.conf

@@ -15,7 +15,8 @@ dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev
 dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev/IARPA-babel103b-v0.4b_conv-dev.mitllfa3.rttm
 dev2h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist.xml
 dev2h_more_kwlists=(
-                      [limitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-eval.kwlist4.xml
                     )
 dev2h_subset_ecf=true
 dev2h_nj=12
@@ -29,10 +30,12 @@ dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-de
 dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev/IARPA-babel103b-v0.4b_conv-dev.mitllfa3.rttm
 dev10h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist.xml
 dev10h_more_kwlists=(
-                      [limitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-eval.kwlist4.xml
                     )
 dev10h_nj=32
 
+
 #Official EVAL period evaluation data files 
 eval_data_dir=/export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_103/conversational/eval
 eval_data_list=/export/babel/data/splits/Bengali_Babel103//eval.list
@@ -46,15 +49,17 @@ shadow_data_dir=(
                 /export/babel/data/103-bengali/release-current/conversational/dev
                 /export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_103/conversational/eval/
               )
+shadow_data_cmudb=/export/babel/data/splits/Bengali_Babel103/uem/103-shadow-v0-cleaned-utt.dat
 shadow_data_list=(
                 /export/babel/data/splits/Bengali_Babel103/dev.list
                 /export/babel/data/splits/Bengali_Babel103/eval.list
               )
-shadow_data_cmudb=/export/babel/data/splits/Bengali_Babel103/uem/103-shadow-v0-utt.dat
 shadow_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-eval.ecf.xml
-shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-eval.kwlist4.xml
+shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist.xml
 shadow_more_kwlists=(
-                      [FullLPdev]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-eval.kwlist4.xml
+
                     )
 shadow_nj=32
 
@@ -73,7 +78,6 @@ numGaussUBM=800
 numLeavesSGMM=10000
 numGaussSGMM=80000
 
-
 # Lexicon and Language Model parameters
 oovSymbol="<unk>"
 lexiconFlags="--romanized --oov <unk>"

egs/babel/s5b/conf/lang/103-bengali-limitedLP.official.conf

@@ -15,7 +15,8 @@ dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev
 dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev/IARPA-babel103b-v0.4b_conv-dev.mitllfa3.rttm
 dev2h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist.xml
 dev2h_more_kwlists=(
-                      [limitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-eval.kwlist4.xml
                     )
 dev2h_subset_ecf=true
 dev2h_nj=12
@@ -29,10 +30,12 @@ dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-de
 dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev/IARPA-babel103b-v0.4b_conv-dev.mitllfa3.rttm
 dev10h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist.xml
 dev10h_more_kwlists=(
-                      [limitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-eval.kwlist4.xml
                     )
 dev10h_nj=32
 
+
 #Official EVAL period evaluation data files 
 eval_data_dir=/export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_103/conversational/eval
 eval_data_list=/export/babel/data/splits/Bengali_Babel103//eval.list
@@ -46,15 +49,17 @@ shadow_data_dir=(
                 /export/babel/data/103-bengali/release-current/conversational/dev
                 /export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_103/conversational/eval/
               )
+shadow_data_cmudb=/export/babel/data/splits/Bengali_Babel103/uem/103-shadow-v0-cleaned-utt.dat
 shadow_data_list=(
                 /export/babel/data/splits/Bengali_Babel103/dev.list
                 /export/babel/data/splits/Bengali_Babel103/eval.list
               )
-shadow_data_cmudb=/export/babel/data/splits/Bengali_Babel103/uem/103-shadow-v0-utt.dat
 shadow_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-eval.ecf.xml
-shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-eval.kwlist4.xml
+shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist.xml
 shadow_more_kwlists=(
-                      [FullLPdev]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel103b-v0.4b_conv-eval.kwlist4.xml
+
                     )
 shadow_nj=32
 
@@ -82,7 +87,6 @@ numGaussUBM=750
 numLeavesSGMM=5000
 numGaussSGMM=18000
 
-
 # Lexicon and Language Model parameters
 oovSymbol="<unk>"
 lexiconFlags="--romanized --oov <unk>"

egs/babel/s5b/conf/lang/201-haitian-fullLP.official.conf

@@ -15,7 +15,8 @@ dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev
 dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev/IARPA-babel201b-v0.2b_conv-dev.mitllfa3.rttm
 dev2h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist.xml
 dev2h_more_kwlists=(
-                      [LimitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-eval.kwlist4.xml
                     )
 dev2h_subset_ecf=true
 dev2h_nj=20
@@ -29,10 +30,12 @@ dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-de
 dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev/IARPA-babel201b-v0.2b_conv-dev.mitllfa3.rttm
 dev10h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist.xml
 dev10h_more_kwlists=(
-                      [LimitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-eval.kwlist4.xml
                     )
 dev10h_nj=32
 
+
 #Official EVAL period evaluation data files 
 eval_data_dir=/export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_201/conversational/eval
 eval_data_list=/export/babel/data/splits/Haitian_Babel201//eval.list
@@ -46,15 +49,17 @@ shadow_data_dir=(
                 /export/babel/data/201-haitian/release-current/conversational/dev
                 /export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_201/conversational/eval
               )
+shadow_data_cmudb=/export/babel/data/splits/Haitian_Babel201/uem/201-shadow-v0-cleaned-utt.dat
 shadow_data_list=(
                 /export/babel/data/splits/Haitian_Babel201/dev.list
                 /export/babel/data/splits/Haitian_Babel201/eval.list
               )
-shadow_data_cmudb=/export/babel/data/splits/Haitian_Babel201/uem/201-shadow-v0-utt.dat
 shadow_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-eval.ecf.xml
-shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-eval.kwlist4.xml
+shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist.xml
 shadow_more_kwlists=(
-                      [FullLPdev]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-eval.kwlist4.xml
+
                     )
 shadow_nj=32
 

egs/babel/s5b/conf/lang/201-haitian-limitedLP.official.conf

@@ -15,7 +15,8 @@ dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev
 dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev/IARPA-babel201b-v0.2b_conv-dev.mitllfa3.rttm
 dev2h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist.xml
 dev2h_more_kwlists=(
-                      [LimitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-eval.kwlist4.xml
                     )
 dev2h_subset_ecf=true
 dev2h_nj=20
@@ -29,10 +30,12 @@ dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-de
 dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev/IARPA-babel201b-v0.2b_conv-dev.mitllfa3.rttm
 dev10h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist.xml
 dev10h_more_kwlists=(
-                      [LimitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-eval.kwlist4.xml
                     )
 dev10h_nj=32
 
+
 #Official EVAL period evaluation data files 
 eval_data_dir=/export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_201/conversational/eval
 eval_data_list=/export/babel/data/splits/Haitian_Babel201//eval.list
@@ -46,15 +49,17 @@ shadow_data_dir=(
                 /export/babel/data/201-haitian/release-current/conversational/dev
                 /export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_201/conversational/eval
               )
+shadow_data_cmudb=/export/babel/data/splits/Haitian_Babel201/uem/201-shadow-v0-cleaned-utt.dat
 shadow_data_list=(
                 /export/babel/data/splits/Haitian_Babel201/dev.list
                 /export/babel/data/splits/Haitian_Babel201/eval.list
               )
-shadow_data_cmudb=/export/babel/data/splits/Haitian_Babel201/uem/201-shadow-v0-utt.dat
 shadow_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-eval.ecf.xml
-shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-eval.kwlist4.xml
+shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist.xml
 shadow_more_kwlists=(
-                      [FullLPdev]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel201b-v0.2b_conv-eval.kwlist4.xml
+
                     )
 shadow_nj=32
 

egs/babel/s5b/conf/lang/203-lao-fullLP.official.conf

@@ -15,7 +15,8 @@ dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev
 dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev/IARPA-babel203b-v3.1a_conv-dev.mitllfa3.rttm
 dev2h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist.xml
 dev2h_more_kwlists=(
-                      [limitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-eval.kwlist4.xml
                     )
 dev2h_subset_ecf=true
 dev2h_nj=18
@@ -29,10 +30,12 @@ dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-de
 dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev/IARPA-babel203b-v3.1a_conv-dev.mitllfa3.rttm
 dev10h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist.xml
 dev10h_more_kwlists=(
-                      [limitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-eval.kwlist4.xml
                     )
 dev10h_nj=32
 
+
 #Official EVAL period evaluation data files 
 eval_data_dir=/export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_203/conversational/eval
 eval_data_list=/export/babel/data/splits/Lao_Babel203//eval.list
@@ -46,27 +49,20 @@ shadow_data_dir=(
                 /export/babel/data/203-lao/release-current/conversational/dev
                 /export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_203/conversational/eval/
               )
+shadow_data_cmudb=/export/babel/data/splits/Lao_Babel203/uem/203-shadow-v0-cleaned-utt.dat
 shadow_data_list=(
                 /export/babel/data/splits/Lao_Babel203/dev.list
                 /export/babel/data/splits/Lao_Babel203/eval.list
               )
-shadow_data_cmudb=/export/babel/data/splits/Lao_Babel203/uem/203-shadow-v0-utt.dat
 shadow_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-eval.ecf.xml
-shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-eval.kwlist4.xml
+shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist.xml
 shadow_more_kwlists=(
-                      [FullLPdev]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-eval.kwlist4.xml
+
                     )
 shadow_nj=32
 
-unsup_data_dir=(/export/babel/data/203-lao/release-current/conversational/training/
-                /export/babel/data/203-lao/release-current/conversational/untranscribed-training/
-                  )  
-unsup_data_list=(
-                    /export/babel/data/splits/Lao_Babel203/train.LimitedLP.untranscribed.list
-                    /export/babel/data/splits/Lao_Babel203/train.untranscribed.list
-                    )
-unsup_nj=64
- 
 # Acoustic model parameters
 numLeavesTri1=1000
 numGaussTri1=10000

egs/babel/s5b/conf/lang/203-lao-limitedLP.official.conf

@@ -15,7 +15,8 @@ dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev
 dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev/IARPA-babel203b-v3.1a_conv-dev.mitllfa3.rttm
 dev2h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist.xml
 dev2h_more_kwlists=(
-                      [limitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-eval.kwlist4.xml
                     )
 dev2h_subset_ecf=true
 dev2h_nj=18
@@ -29,10 +30,12 @@ dev10h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-de
 dev10h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev/IARPA-babel203b-v3.1a_conv-dev.mitllfa3.rttm
 dev10h_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist.xml
 dev10h_more_kwlists=(
-                      [limitedLP]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist2.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-eval.kwlist4.xml
                     )
 dev10h_nj=32
 
+
 #Official EVAL period evaluation data files 
 eval_data_dir=/export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_203/conversational/eval
 eval_data_list=/export/babel/data/splits/Lao_Babel203//eval.list
@@ -46,15 +49,17 @@ shadow_data_dir=(
                 /export/babel/data/203-lao/release-current/conversational/dev
                 /export/babel/data/IARPA-BABEL_OP1_dev_eval/BABEL_OP1_203/conversational/eval/
               )
+shadow_data_cmudb=/export/babel/data/splits/Lao_Babel203/uem/203-shadow-v0-cleaned-utt.dat
 shadow_data_list=(
                 /export/babel/data/splits/Lao_Babel203/dev.list
                 /export/babel/data/splits/Lao_Babel203/eval.list
               )
-shadow_data_cmudb=/export/babel/data/splits/Lao_Babel203/uem/203-shadow-v0-utt.dat
 shadow_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-eval.ecf.xml
-shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-eval.kwlist4.xml
+shadow_kwlist_file=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist.xml
 shadow_more_kwlists=(
-                      [FullLPdev]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist.xml
+                      [llp]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-dev.kwlist2.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel203b-v3.1a_conv-eval.kwlist4.xml
+
                     )
 shadow_nj=32
 

egs/babel/s5b/conf/lang/204-tamil-fullLP.official.conf

@@ -14,9 +14,12 @@ dev2h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev
 dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev/IARPA-babel204b-v1.1b_conv-dev.scoring.ecf.xml
 dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev/IARPA-babel204b-v1.1b_conv-dev.mitllfa3.rttm
 dev2h_kwlist_file=/export/babel/data/splits/Tamil_Babel204/IARPA-babel204b-v1.1b_conv-dev.radical.kwlist.xml
-#dev2h_more_kwlists=(
-#                      [limitedLP]=
-#                    )
+dev2h_more_kwlists=(
+                      [bbn1]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist.xml
+                      [bbn2]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist2.xml
+                      [ibm1]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist3.xml
+                      [ibm2]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist4.xml
+                    )
 dev2h_subset_ecf=true
 dev2h_nj=18
 
@@ -33,19 +36,41 @@ dev10h_more_kwlists=(
                       [bbn2]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist2.xml
                       [ibm1]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist3.xml
                       [ibm2]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist4.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist5.xml
                     )
 dev10h_nj=32
 
 
-unsup_data_dir=(/export/babel/data/204-tamil/release-current/conversational/training/
-                /export/babel/data/204-tamil/release-current/conversational/untranscribed-training/
-                  )  
-unsup_data_cmudb=/export/babel/data/splits/Tamil_Babel204/uem/db-unsup-jhuseg-v8-utt.dat
-unsup_data_list=(
-                    /export/babel/data/splits/Tamil_Babel204/train.LimitedLP.untranscribed.list
-                    /export/babel/data/splits/Tamil_Babel204/train.untranscribed.list
+#Official EVAL period evaluation data files 
+eval_data_dir=/export/babel/data/204-tamil/release-current/conversational/eval/
+eval_data_list=/export/babel/data/splits/Tamil_Babel204/eval.list
+eval_data_cmudb=/export/babel/data/splits/Tamil_Babel204/uem/db-shadow-jhuseg-v8-utt.dat
+eval_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev/IARPA-babel204b-v1.1b_conv-dev.scoring.ecf.xml
+eval_kwlist_file=/export/babel/data/splits/Tamil_Babel204/IARPA-babel204b-v1.1b_conv-dev.radical.kwlist.xml
+eval_nj=64
+
+#Shadow data files 
+shadow_data_dir=(
+                /export/babel/data/204-tamil/release-current/conversational/dev/
+                /export/babel/data/204-tamil/release-current/conversational/eval/
+              )
+shadow_data_cmudb=/export/babel/data/splits/Tamil_Babel204/uem/204-shadow-v0-utt.dat
+shadow_data_list=(
+                /export/babel/data/splits/Tamil_Babel204/dev.list
+                /export/babel/data/splits/Tamil_Babel204/eval.list
+              )
+shadow_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev/IARPA-babel204b-v1.1b_conv-dev.scoring.ecf.xml
+shadow_kwlist_file=/export/babel/data/splits/Tamil_Babel204/IARPA-babel204b-v1.1b_conv-dev.radical.kwlist.xml
+shadow_more_kwlists=(
+                      [bbn1]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist.xml
+                      [bbn2]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist2.xml
+                      [ibm1]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist3.xml
+                      [ibm2]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist4.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist5.xml
                     )
-unsup_nj=64
+shadow_nj=64
+
+
  
 # Acoustic model parameters
 numLeavesTri1=1000

egs/babel/s5b/conf/lang/204-tamil-limitedLP.official.conf

@@ -14,9 +14,12 @@ dev2h_stm_file=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev
 dev2h_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev/IARPA-babel204b-v1.1b_conv-dev.scoring.ecf.xml
 dev2h_rttm_file=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev/IARPA-babel204b-v1.1b_conv-dev.mitllfa3.rttm
 dev2h_kwlist_file=/export/babel/data/splits/Tamil_Babel204/IARPA-babel204b-v1.1b_conv-dev.radical.kwlist.xml
-#dev2h_more_kwlists=(
-#                      [limitedLP]=
-#                    )
+dev2h_more_kwlists=(
+                      [bbn1]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist.xml
+                      [bbn2]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist2.xml
+                      [ibm1]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist3.xml
+                      [ibm2]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist4.xml
+                    )
 dev2h_subset_ecf=true
 dev2h_nj=18
 
@@ -33,10 +36,41 @@ dev10h_more_kwlists=(
                       [bbn2]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist2.xml
                       [ibm1]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist3.xml
                       [ibm2]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist4.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist5.xml
                     )
 dev10h_nj=32
 
 
+#Official EVAL period evaluation data files 
+eval_data_dir=/export/babel/data/204-tamil/release-current/conversational/eval/
+eval_data_list=/export/babel/data/splits/Tamil_Babel204/eval.list
+eval_data_cmudb=/export/babel/data/splits/Tamil_Babel204/uem/db-shadow-jhuseg-v8-utt.dat
+eval_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev/IARPA-babel204b-v1.1b_conv-dev.scoring.ecf.xml
+eval_kwlist_file=/export/babel/data/splits/Tamil_Babel204/IARPA-babel204b-v1.1b_conv-dev.radical.kwlist.xml
+eval_nj=64
+
+#Shadow data files 
+shadow_data_dir=(
+                /export/babel/data/204-tamil/release-current/conversational/dev/
+                /export/babel/data/204-tamil/release-current/conversational/eval/
+              )
+shadow_data_cmudb=/export/babel/data/splits/Tamil_Babel204/uem/204-shadow-v0-utt.dat
+shadow_data_list=(
+                /export/babel/data/splits/Tamil_Babel204/dev.list
+                /export/babel/data/splits/Tamil_Babel204/eval.list
+              )
+shadow_ecf_file=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev/IARPA-babel204b-v1.1b_conv-dev.scoring.ecf.xml
+shadow_kwlist_file=/export/babel/data/splits/Tamil_Babel204/IARPA-babel204b-v1.1b_conv-dev.radical.kwlist.xml
+shadow_more_kwlists=(
+                      [bbn1]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist.xml
+                      [bbn2]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist2.xml
+                      [ibm1]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist3.xml
+                      [ibm2]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist4.xml
+                      [eval]=/export/babel/data/scoring/IndusDB/IARPA-babel204b-v1.1b_conv-dev.kwlist5.xml
+                    )
+