Commit 48f1cfef authored by Karel Vesely's avatar Karel Vesely
Browse files

trunk,nnet1: adding tool 'cuda-gpu-available', which checks if GPU is ready to use.



git-svn-id: https://svn.code.sf.net/p/kaldi/code/trunk@5178 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8
parent b3c92d19
......@@ -75,12 +75,12 @@ echo $nj > $dir/num_jobs
# Select default locations to model files (if not already set externally)
[ -z "$nnet" ] && nnet=$srcdir/final.nnet
[ -z "$model" ] && model=$srcdir/final.mdl
[ -z "$feature_transform" ] && feature_transform=$srcdir/final.feature_transform
[ -z "$feature_transform" -a -e $srcdir/final.feature_transform ] && feature_transform=$srcdir/final.feature_transform
#
[ -z "$class_frame_counts" -a -f $srcdir/prior_counts ] && class_frame_counts=$srcdir/prior_counts # priority,
[ -z "$class_frame_counts" ] && class_frame_counts=$srcdir/ali_train_pdf.counts
# Check that files exist
# Check that files exist,
for f in $sdata/1/feats.scp $nnet $model $feature_transform $class_frame_counts $graphdir/HCLG.fst; do
[ ! -f $f ] && echo "$0: missing file $f" && exit 1;
done
......@@ -107,6 +107,8 @@ feats="ark,s,cs:copy-feats scp:$sdata/JOB/feats.scp ark:- |"
[ ! -z "$cmvn_opts" ] && feats="$feats apply-cmvn $cmvn_opts --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp ark:- ark:- |"
# add-deltas (optional),
[ ! -z "$delta_opts" ] && feats="$feats add-deltas $delta_opts ark:- ark:- |"
# add-pytel transform (optional),
[ -e $D/pytel_transform.py ] && feats="$feats /bin/env python $D/pytel_transform.py |"
#
# Run the decoding in the queue,
......
......@@ -95,13 +95,12 @@ echo "# INFO"
echo "$0 : Pre-training Deep Belief Network as a stack of RBMs"
printf "\t dir : $dir \n"
printf "\t Train-set : $data \n"
echo
[ -e $dir/${nn_depth}.dbn ] && echo "$0 Skipping, already have $dir/${nn_depth}.dbn" && exit 0
# check if CUDA is compiled in,
if ! $skip_cuda_check; then
cuda-compiled || { echo 'CUDA was not compiled in, skipping! Check src/kaldi.mk and src/configure' && exit 1; }
fi
# check if CUDA compiled in and GPU is available,
if ! $skip_cuda_check; then cuda-gpu-available || exit 1; fi
mkdir -p $dir/log
......
......@@ -11,6 +11,7 @@ nnet_init= # select initialized MLP (override initialization)
nnet_proto= # select network prototype (initialize it)
proto_opts= # non-default options for 'make_nnet_proto.py'
feature_transform= # provide feature transform (=splice,rescaling,...) (don't build new one)
pytel_transform= # use external transform defined in python (BUT specific)
network_type=dnn # (dnn,cnn1d,cnn2d,lstm) select type of neural network
cnn_proto_opts= # extra options for 'make_cnn_proto.py'
#
......@@ -115,16 +116,15 @@ echo "$0 : Training Neural Network"
printf "\t dir : $dir \n"
printf "\t Train-set : $data $alidir \n"
printf "\t CV-set : $data_cv $alidir_cv \n"
echo
mkdir -p $dir/{log,nnet}
# skip when already trained
[ -e $dir/final.nnet ] && printf "\nSKIPPING TRAINING... ($0)\nnnet already trained : $dir/final.nnet ($(readlink $dir/final.nnet))\n\n" && exit 0
# check if CUDA is compiled in,
if ! $skip_cuda_check; then
cuda-compiled || { echo 'CUDA was not compiled in, skipping! Check src/kaldi.mk and src/configure' && exit 1; }
fi
# check if CUDA compiled in and GPU is available,
if ! $skip_cuda_check; then cuda-gpu-available || exit 1; fi
###### PREPARE ALIGNMENTS ######
echo
......@@ -212,6 +212,15 @@ fi
[ ! -z "$delta_opts" ] && echo "$delta_opts" >$dir/delta_opts
#
# optionally append python feature transform,
if [ ! -z "$pytel_transform" ]; then
cp $pytel_transform $dir/pytel_transform.py
{ echo; echo "### Comes from here: '$pytel_transform' ###"; } >> $dir/pytel_transform.py
pytel_transform=$dir/pytel_transform.py
feats_tr="$feats_tr /bin/env python $pytel_transform |"
feats_cv="$feats_cv /bin/env python $pytel_transform |"
fi
# get feature dim
echo "Getting feature dim : "
feat_dim=$(feat-to-dim --print-args=false "$feats_tr" -)
......@@ -391,7 +400,7 @@ fi
echo
echo "# RUNNING THE NN-TRAINING SCHEDULER"
steps/nnet/train_scheduler.sh \
--feature-transform $feature_transform \
${feature_transform:+ --feature-transform $feature_transform} \
--learn-rate $learn_rate \
--randomizer-seed $seed \
${train_opts} \
......
......@@ -58,10 +58,8 @@ for f in $data/feats.scp $alidir/{tree,final.mdl,ali.1.gz} $denlatdir/lat.scp $s
[ ! -f $f ] && echo "$0: no such file $f" && exit 1;
done
# check if CUDA is compiled in,
if ! $skip_cuda_check; then
cuda-compiled || { echo 'CUDA was not compiled in, skipping! Check src/kaldi.mk and src/configure' && exit 1; }
fi
# check if CUDA compiled in and GPU is available,
if ! $skip_cuda_check; then cuda-gpu-available || exit 1; fi
mkdir -p $dir/log
......
......@@ -59,10 +59,8 @@ for f in $data/feats.scp $alidir/{tree,final.mdl,ali.1.gz} $denlatdir/lat.scp $s
[ ! -f $f ] && echo "$0: no such file $f" && exit 1;
done
# check if CUDA is compiled in,
if ! $skip_cuda_check; then
cuda-compiled || { echo 'CUDA was not compiled in, skipping! Check src/kaldi.mk and src/configure' && exit 1; }
fi
# check if CUDA compiled in and GPU is available,
if ! $skip_cuda_check; then cuda-gpu-available || exit 1; fi
mkdir -p $dir/log
......
......@@ -15,7 +15,8 @@ BINFILES = nnet-train-frmshuff \
nnet-forward nnet-copy nnet-info nnet-concat \
transf-to-nnet cmvn-to-nnet nnet-initialize \
nnet-kl-hmm-acc nnet-kl-hmm-mat-to-component \
feat-to-post paste-post train-transitions
feat-to-post paste-post train-transitions \
cuda-gpu-available
OBJFILES =
......
// nnetbin/cuda-gpu-available.cc
// Copyright 2015 Brno University of Technology (author: Karel Vesely)
// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.
#include "base/kaldi-common.h"
#include "cudamatrix/cu-device.h"
#ifndef _MSC_VER
#include <unistd.h>
#include <errno.h>
#endif
using namespace kaldi;
int main(int argc, char *argv[]) try {
char hostname[100] = "UNKNOWN-HOSTNAME";
#ifndef _MSC_VER
if (gethostname(hostname, 100)) {
KALDI_WARN << "Cannot get hostname, " << strerror(errno);
}
#endif
std::cerr << "### IS CUDA GPU AVAILABLE? '"
<< hostname << "' ###" << std::endl;
#if HAVE_CUDA==1
CuDevice::Instantiate().SelectGpuId("yes");
std::cerr << "### HURRAY, WE GOT A CUDA GPU FOR COMPUTATION!!! ###" << std::endl;
return 0;
#else
std::cerr << "### CUDA WAS NOT COMPILED IN! ###" << std::endl
<< "To support CUDA, you must run 'configure' on a machine "
<< "that has the CUDA compiler 'nvcc' available.";
return 1;
#endif
} catch (const std::exception &e) {
// Error message 'e' already printed in base/kaldi-error.cc:175,
std::cerr << "### WE DID NOT GET A CUDA GPU!!! ###" << std::endl
<< "### If it's your 1st experiment with CUDA, try updating "
<< "'display drivers' and 'CUDA toolkit'." << std::endl
<< "### In other cases run 'nvidia-smi' in terminal "
<< "(gets installed with display drivers) :"
<< std::endl
<< "### - Check that you see your GPU."
<< std::endl
<< "### - Burnt GPUs indicate error or disappear from the list until reboot."
<< std::endl
<< "### - Check 'Memory-Usage' and 'GPU fan', the GPU may be taken by other process."
<< std::endl
<< "### - Check with NVidia web that your 'display driver' and 'CUDA toolkit' is not too old."
<< std::endl;
return 1;
}
......@@ -219,7 +219,7 @@ int main(int argc, char *argv[]) {
if (num_done == 0) return -1;
return 0;
} catch(const std::exception &e) {
KALDI_ERR << e.what();
std::cerr << e.what();
return -1;
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment