Commit ea29395f authored by Dan Povey's avatar Dan Povey
Browse files

Changed queue.pl to use qrsh not qsub -sync y (thus working around qsub bug);...

Changed queue.pl to use qrsh not qsub -sync y (thus working around qsub bug); other minor script changes

git-svn-id: https://svn.code.sf.net/p/kaldi/code/trunk@613 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8
parent 855b5849
......@@ -18,7 +18,7 @@
# To be run from one directory above this script.
perl -e 'while(<>){
if (m/WER (\S+)/ && (!defined $bestwer || $bestwer > $2)){ $bestwer = $2; $bestline=$_; } # kaldi "compute-wer" tool.
if (m/WER (\S+)/ && (!defined $bestwer || $bestwer > $1)){ $bestwer = $1; $bestline=$_; } # kaldi "compute-wer" tool.
elsif (m/ Mean\s+\|\s+\S+\s+\S+\s+\|\s+\S+\s+\S+\s+\S+\s+\S+\s+(\S+)\s+\S+\s+\|/
&& (!defined $bestwer || $bestwer > $1)){ $bestwer = $1; $bestline=$_; } } # sclite.
if (defined $bestline){ print $bestline; } '
......
......@@ -37,6 +37,9 @@ newlm=$newlang/G.fst
! cmp $oldlang/words.txt $newlang/words.txt && echo "Warning: vocabularies may be incompatible."
[ ! -f $oldlm ] && echo Missing file $oldlm && exit 1;
[ ! -f $newlm ] && echo Missing file $newlm && exit 1;
[ ! ls $indir/lat.*.gz >/dev/null ] && echo "No lattices input directory $indir" && exit 1;
oldlmcommand="fstproject --project_output=true $oldlm |"
newlmcommand="fstproject --project_output=true $newlm |"
......@@ -55,7 +58,7 @@ if [ "$mode" == 4 ]; then
fi
rm $dir/.error 2>/dev/null
rm $outdir/.error 2>/dev/null
for lat in $indir/lat.*.gz; do
number=`basename $lat | cut -d. -f2`;
......@@ -65,7 +68,7 @@ for lat in $indir/lat.*.gz; do
$cmd $outdir/rescorelm.$number.log \
lattice-lmrescore --lm-scale=-1.0 "ark:gunzip -c $lat|" "$oldlmcommand" ark:- \| \
lattice-lmrescore --lm-scale=1.0 ark:- "$newlmcommand" "ark,t:|gzip -c>$newlat" \
|| touch $dir/.error &
|| touch $outdir/.error &
;;
2) # 2 is equivalent to 1, but using more basic operations, combined.
$cmd $outdir/rescorelm.$number.log \
......@@ -76,7 +79,7 @@ for lat in $indir/lat.*.gz; do
lattice-scale --acoustic-scale=-1 --lm-scale=-1 ark:- ark:- \| \
lattice-compose ark:- "fstproject --project_output=true $newlm |" ark:- \| \
lattice-determinize ark:- ark:- \| \
gzip -c \>$newlat || touch $dir/.error &
gzip -c \>$newlat || touch $outdir/.error &
;;
3) # 3 is "exact" in that we remove the old LM scores accepting any path
# through G.fst (which is what we want as that happened in lattice
......@@ -90,7 +93,7 @@ for lat in $indir/lat.*.gz; do
lattice-scale --acoustic-scale=-1 --lm-scale=-1 ark:- ark:- \| \
lattice-compose --phi-label=$phi ark:- $newlm ark:- \| \
lattice-determinize ark:- ark:- \| \
gzip -c \>$newlat || touch $dir/.error &
gzip -c \>$newlat || touch $outdir/.error &
;;
4) # 4 is also exact (like 3), but instead of subtracting the old LM-scores,
# it removes the old graph scores entirely and adds in the lexicon,
......@@ -106,13 +109,13 @@ for lat in $indir/lat.*.gz; do
lattice-compose --phi-label=$phi ark:- $newlm ark:- \| \
lattice-add-trans-probs --transition-scale=1.0 --self-loop-scale=0.1 \
$mdl ark:- ark:- \| \
gzip -c \>$newlat || touch $dir/.error &
gzip -c \>$newlat || touch $outdir/.error &
;;
esac
done
wait
[ -f $dir/.error ] && echo Error doing LM rescoring && exit 1
[ -f $outdir/.error ] && echo Error doing LM rescoring && exit 1
rm $outdir/Ldet.fst 2>/dev/null
scripts/score_lats.sh $outdir $newlang/words.txt $data
......@@ -56,6 +56,11 @@ loopscale=0.1
# (note: the [[ ]] brackets make the || type operators work (inside [ ], we
# would have to use -o instead), -f means file exists, and -ot means older than).
required="$lang/L.fst $lang/G.fst $lang/phones_disambig.txt $lang/words.txt $lang/silphones.csl $model $tree"
for f in $required; do
[ ! -f $f ] && echo "mkgraph.sh: expected $f to exist" && exit 1;
done
mkdir -p $lang/tmp
if [[ ! -f $lang/tmp/LG.fst || $lang/tmp/LG.fst -ot $lang/G.fst || \
$lang/tmp/LG.fst -ot $lang/L_disambig.fst ]]; then
......
......@@ -14,8 +14,8 @@ use Cwd;
# . path.sh
# ( my-prog '--opt=foo bar' foo | other-prog baz ) >& some.log
# EOF
# qsub -sync y -j y -o /cur/location/some.log /cur/location/q/some.sh && exit 0;
# qsub -sync y -j y -o /cur/location/some.log /cur/location/q/some.sh
# qrsh -j y -now no -o /cur/location/some.log /cur/location/q/some.sh && exit 0;
# qrsh -j y -now no -o /cur/location/some.log /cur/location/q/some.sh
#
# What this means is it creates a .sh file to put the command in,
# along with changing the directory and setting the path. It then runs
......@@ -73,7 +73,7 @@ $shfile = "$dir/$base";
open(S, ">$shfile") || die "Could not write to script file $shfile";
`chmod +x $shfile`;
$qsub_cmd = "qsub -sync y -j y -o $logfile $qsub_opts $shfile >>$dir/queue.log 2>&1";
$qsub_cmd = "qrsh -j y -now no -o $logfile $qsub_opts $shfile";
#
# Write to the script file, and close it.
#
......@@ -95,21 +95,6 @@ close(S) || die "Could not close script file $shfile";
#
system "$qsub_cmd";
if ($? == 0) { exit(0); }
$errmsgs = `cat $dir/queue.log`;
if ($errmsgs =~ m/containes/) { # the error message "range_list containes no elements"
# seems to be encountered due to a bug in grid engine... since this appears to be
# intermittent, we try a bunch of times, with sleeps in between, if this happens.
print STDERR "Command writing to $logfile failed, apparently due to queue bug " .
" (range_list containes no elements)... will try again a few times.\n";
$delay = 60; # one minute delay initially.
for ($x = 1; $x < 10; $x++) {
print STDERR "[$x/10]";
sleep($delay);
$delay += 60*5; # Add 5 minutes to the delay.
system "$qsub_cmd";
if ($? == 0) { exit(0); }
}
}
print STDERR "Command writing to $logfile failed; trying again\n";
system "mv $logfile $logfile.bak";
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment