Commit 8f47060b authored by Ho Yin Chan's avatar Ho Yin Chan
Browse files

trunk:egs/hkust update on chinese character error rate (CER) scoring script....

trunk:egs/hkust update on chinese character error rate (CER) scoring script. Add --useword2charmap option and comments.

git-svn-id: https://svn.code.sf.net/p/kaldi/code/trunk@2872 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8
parent 512cead2
#!/bin/bash #!/bin/bash
# Copyright Johns Hopkins University (Author: Daniel Povey) 2012-2013. # Copyright Johns Hopkins University (Author: Daniel Povey) 2012-2013.
# Hong Kong University of Science and Technology (Ricky Chan). Apache 2.0. # Hong Kong University of Science and Technology (Author: Ricky Chan Ho Yin). Apache 2.0.
# #
# An alternative chinese character scoring script; If the wordlist (words in the lexicon) used by the decoder is a subset of the words in 195k_chinese_word2char_map (1st column of 195k_chinese_word2char_map contains chinese words, 2nd to end column fields are corresponding seperated chinese characters), scoring with score_basic_ext.sh will give pure chinese character error rate # A Chinese character error rate (CER) scoring script for Kaldi;
#
# By default the script assumes the input Chinese words/characters are 3 bytes UTF8 code \
# and continuous printable English/ASCII characters without space are treated as single token.
#
# When --useword2charmap option is applied, an input Chinese words to Chinese characters mapping table \
# (e.g. a word2char_map likes "195k_chinese_word2char_map" - 1st column of 195k_chinese_word2char_map contains chinese words, \
# 2nd to end column fields are corresponding seperated chinese characters) \
# is used for converting the corresponding Chinese words to seperate Chinese characters for scoring. \
# Please make sure the Chinese words in the mapping table is a superset of the Chinese words for decoding if you apply this option.
# begin configuration section. # begin configuration section.
cmd=run.pl cmd=run.pl
min_lmwt=9 min_lmwt=9
max_lmwt=20 max_lmwt=20
useword2charmap=
#end configuration section. #end configuration section.
[ -f ./path.sh ] && . ./path.sh [ -f ./path.sh ] && . ./path.sh
...@@ -19,6 +29,7 @@ if [ $# -ne 3 ]; then ...@@ -19,6 +29,7 @@ if [ $# -ne 3 ]; then
echo " --cmd (run.pl|queue.pl...) # specify how to run the sub-processes." echo " --cmd (run.pl|queue.pl...) # specify how to run the sub-processes."
echo " --min_lmwt <int> # minumum LM-weight for lattice rescoring " echo " --min_lmwt <int> # minumum LM-weight for lattice rescoring "
echo " --max_lmwt <int> # maximum LM-weight for lattice rescoring " echo " --max_lmwt <int> # maximum LM-weight for lattice rescoring "
echo " --useword2charmap word2charmap # use a chinese word to chinese characters mapping "
exit 1; exit 1;
fi fi
...@@ -61,15 +72,21 @@ filter_text <$data/text >$dir/scoring/text.filt ...@@ -61,15 +72,21 @@ filter_text <$data/text >$dir/scoring/text.filt
unset LC_ALL unset LC_ALL
#for Chinese character error rate #for Chinese character error rate
perl local/ext/hkust_word2ch_tran.pl $dir/scoring/text.filt > $dir/scoring/cchar.filt if [ -z $useword2charmap ]; then
# perl local/ext/hkust_word2ch_tran.pl --useword2charmap local/ext/195k_chinese_word2char_map $dir/scoring/text.filt > $dir/scoring/cchar.filt perl local/ext/hkust_word2ch_tran.pl $dir/scoring/text.filt > $dir/scoring/cchar.filt
# perl local/ext/hkust_word2ch_tran.pl --encodeoutput $dir/scoring/text.filt > $dir/scoring/cchar.filt # perl local/ext/hkust_word2ch_tran.pl --encodeoutput $dir/scoring/text.filt > $dir/scoring/cchar.filt
for lmwt in `seq $min_lmwt $max_lmwt`; do
for lmwt in `seq $min_lmwt $max_lmwt`; do perl local/ext/hkust_word2ch_tran.pl $dir/scoring/$lmwt.txt > $dir/scoring/${lmwt}.cchar
perl local/ext/hkust_word2ch_tran.pl $dir/scoring/$lmwt.txt > $dir/scoring/${lmwt}.cchar # perl local/ext/hkust_word2ch_tran.pl --encodeoutput $dir/scoring/$lmwt.txt > $dir/scoring/${lmwt}.cchar
# perl local/ext/hkust_word2ch_tran.pl --useword2charmap local/ext/195k_chinese_word2char_map $dir/scoring/$lmwt.txt > $dir/scoring/${lmwt}.cchar done
# perl local/ext/hkust_word2ch_tran.pl --encodeoutput $dir/scoring/$lmwt.txt > $dir/scoring/${lmwt}.cchar else
done perl local/ext/hkust_word2ch_tran.pl --useword2charmap $useword2charmap $dir/scoring/text.filt > $dir/scoring/cchar.filt
# perl local/ext/hkust_word2ch_tran.pl --useword2charmap local/ext/195k_chinese_word2char_map $dir/scoring/text.filt > $dir/scoring/cchar.filt
for lmwt in `seq $min_lmwt $max_lmwt`; do
perl local/ext/hkust_word2ch_tran.pl --useword2charmap $useword2charmap $dir/scoring/$lmwt.txt > $dir/scoring/${lmwt}.cchar
# perl local/ext/hkust_word2ch_tran.pl --useword2charmap local/ext/195k_chinese_word2char_map $dir/scoring/$lmwt.txt > $dir/scoring/${lmwt}.cchar
done
fi
export LC_ALL=C export LC_ALL=C
...@@ -81,4 +98,8 @@ $cmd LMWT=$min_lmwt:$max_lmwt $dir/scoring/log/score.LMWT.cer.log \ ...@@ -81,4 +98,8 @@ $cmd LMWT=$min_lmwt:$max_lmwt $dir/scoring/log/score.LMWT.cer.log \
compute-wer --text --mode=present \ compute-wer --text --mode=present \
ark:$dir/scoring/cchar.filt ark:$dir/scoring/LMWT.cchar ">&" $dir/cer_LMWT || exit 1; ark:$dir/scoring/cchar.filt ark:$dir/scoring/LMWT.cchar ">&" $dir/cer_LMWT || exit 1;
for lmwt in `seq $min_lmwt $max_lmwt`; do
sed 's/%WER /%CER /g' $dir/cer_${lmwt} > $dir/tmpf; mv $dir/tmpf $dir/cer_${lmwt};
done
exit 0 exit 0
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment