nnet-component-test.cc 27.7 KB
Newer Older
1
// nnet2/nnet-component-test.cc
2

3
// Copyright 2012-2014  Johns Hopkins University (author:  Daniel Povey)
4

5 6
// See ../../COPYING for clarification regarding multiple authors
//
7 8 9 10 11 12 13 14 15 16 17 18 19
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.

20
#include "nnet2/nnet-component.h"
21
#include "util/common-utils.h"
22
#include <unistd.h> // for sleep().
23 24

namespace kaldi {
25
namespace nnet2 {
26 27 28 29


void UnitTestGenericComponentInternal(const Component &component) {
  int32 input_dim = component.InputDim(),
30
      output_dim = component.OutputDim();
31 32

  KALDI_LOG << component.Info();
Dan Povey's avatar
Dan Povey committed
33
  
34
  CuVector<BaseFloat> objf_vec(output_dim); // objective function is linear function of output.
35 36
  objf_vec.SetRandn(); // set to Gaussian noise.
  
37
  int32 num_egs = 10 + Rand() % 5;
38
  CuMatrix<BaseFloat> input(num_egs, input_dim),
39 40
      output(num_egs, output_dim);
  input.SetRandn();
Dan Povey's avatar
Dan Povey committed
41
  
42
  int32 rand_seed = Rand();
43 44 45 46 47 48 49

  RandomComponent *rand_component =
      const_cast<RandomComponent*>(dynamic_cast<const RandomComponent*>(&component));
  if (rand_component != NULL) {
    srand(rand_seed);
    rand_component->ResetGenerator();
  }
50 51
  component.Propagate(input, 1, &output);
  {
52
    bool binary = (Rand() % 2 == 0);
53 54 55 56 57 58 59 60 61
    Output ko("tmpf", binary);
    component.Write(ko.Stream(), binary);
  }
  Component *component_copy;
  {
    bool binary_in;
    Input ki("tmpf", &binary_in);
    component_copy = Component::ReadNew(ki.Stream(), binary_in);
  }
62
  unlink("tmpf");
63 64
  
  { // Test backward derivative is correct.
65
    CuVector<BaseFloat> output_objfs(num_egs);
66 67 68
    output_objfs.AddMatVec(1.0, output, kNoTrans, objf_vec, 0.0);
    BaseFloat objf = output_objfs.Sum();

Dan Povey's avatar
Dan Povey committed
69
    
70
    CuMatrix<BaseFloat> output_deriv(output.NumRows(), output.NumCols());
71 72 73
    for (int32 i = 0; i < output_deriv.NumRows(); i++)
      output_deriv.Row(i).CopyFromVec(objf_vec);

74
    CuMatrix<BaseFloat> input_deriv(input.NumRows(), input.NumCols());
75

Dan Povey's avatar
Dan Povey committed
76
    
77 78
    CuMatrix<BaseFloat> empty_mat;
    CuMatrix<BaseFloat> &input_ref =
79 80 81 82
        (component_copy->BackpropNeedsInput() ? input : empty_mat),
        &output_ref =
        (component_copy->BackpropNeedsOutput() ? output : empty_mat);
    int32 num_chunks = 1;
Dan Povey's avatar
Dan Povey committed
83 84

    
85 86 87
    component_copy->Backprop(input_ref, output_ref,
                             output_deriv, num_chunks, NULL, &input_deriv);

88
    int32 num_ok = 0, num_bad = 0, num_tries = 10;
89 90
    KALDI_LOG << "Comparing feature gradients " << num_tries << " times.";
    for (int32 i = 0; i < num_tries; i++) {
91
      CuMatrix<BaseFloat> perturbed_input(input.NumRows(), input.NumCols());
92 93 94 95 96 97 98 99
      {
        RandomComponent *rand_component =
            const_cast<RandomComponent*>(dynamic_cast<const RandomComponent*>(&component));
        if (rand_component != NULL) {
          srand(rand_seed);
          rand_component->ResetGenerator();
        }
      }        
100
      perturbed_input.SetRandn();
101
      perturbed_input.Scale(1.0e-04); // scale by a small amount so it's like a delta.
102 103 104
      BaseFloat predicted_difference = TraceMatMat(perturbed_input,
                                                   input_deriv, kTrans);
      perturbed_input.AddMat(1.0, input); // now it's the input + a delta.
105 106
      { // Compute objf with perturbed input and make sure it matches
        // prediction.
107
        CuMatrix<BaseFloat> perturbed_output(output.NumRows(), output.NumCols());
108 109 110 111 112 113 114 115
        {
          RandomComponent *rand_component =
              const_cast<RandomComponent*>(dynamic_cast<const RandomComponent*>(&component));
          if (rand_component != NULL) {
            srand(rand_seed);
            rand_component->ResetGenerator();
          }
        }        
116
        component.Propagate(perturbed_input, 1, &perturbed_output);
117
        CuVector<BaseFloat> perturbed_output_objfs(num_egs);
118 119 120 121 122 123 124
        perturbed_output_objfs.AddMatVec(1.0, perturbed_output, kNoTrans,
                                         objf_vec, 0.0);
        BaseFloat perturbed_objf = perturbed_output_objfs.Sum(),
             observed_difference = perturbed_objf - objf;
        KALDI_LOG << "Input gradients: comparing " << predicted_difference
                  << " and " << observed_difference;
        if (fabs(predicted_difference - observed_difference) >
125 126
            0.15 * fabs((predicted_difference + observed_difference)/2) &&
            fabs(predicted_difference - observed_difference) > 1.0e-06) {
127 128 129 130 131 132 133 134 135
          KALDI_WARN << "Bad difference!";
          num_bad++;
        } else {
          num_ok++;
        }
      }
    }
    KALDI_LOG << "Succeeded for " << num_ok << " out of " << num_tries
              << " tries.";
136
    if (num_ok <= num_bad) {
137
      delete component_copy;
138 139
      KALDI_ERR << "Feature-derivative check failed";
    }
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
  }

  UpdatableComponent *ucomponent =
      dynamic_cast<UpdatableComponent*>(component_copy);

  if (ucomponent != NULL) { // Test parameter derivative is correct.

    int32 num_ok = 0, num_bad = 0, num_tries = 10;
    KALDI_LOG << "Comparing model gradients " << num_tries << " times.";
    for (int32 i = 0; i < num_tries; i++) {    
      UpdatableComponent *perturbed_ucomponent =
          dynamic_cast<UpdatableComponent*>(ucomponent->Copy()),
          *gradient_ucomponent =
          dynamic_cast<UpdatableComponent*>(ucomponent->Copy());
      KALDI_ASSERT(perturbed_ucomponent != NULL);
      gradient_ucomponent->SetZero(true); // set params to zero and treat as gradient.
156
      BaseFloat perturb_stddev = 5.0e-04;
157
      perturbed_ucomponent->PerturbParams(perturb_stddev);
158
      
159
      CuVector<BaseFloat> output_objfs(num_egs);
160 161 162
      output_objfs.AddMatVec(1.0, output, kNoTrans, objf_vec, 0.0);
      BaseFloat objf = output_objfs.Sum();

163
      CuMatrix<BaseFloat> output_deriv(output.NumRows(), output.NumCols());
164 165
      for (int32 i = 0; i < output_deriv.NumRows(); i++)
        output_deriv.Row(i).CopyFromVec(objf_vec);
166
      CuMatrix<BaseFloat> input_deriv; // (input.NumRows(), input.NumCols());
167 168 169 170 171 172 173 174 175 176

      int32 num_chunks = 1;

      // This will compute the parameter gradient.
      ucomponent->Backprop(input, output, output_deriv, num_chunks,
                           gradient_ucomponent, &input_deriv);

      // Now compute the perturbed objf.
      BaseFloat objf_perturbed;
      {
177
        CuMatrix<BaseFloat> output_perturbed; // (num_egs, output_dim);
178 179 180 181 182 183 184 185
        {
          RandomComponent *rand_component =
              const_cast<RandomComponent*>(dynamic_cast<const RandomComponent*>(&component));
          if (rand_component != NULL) {
            srand(rand_seed);
            rand_component->ResetGenerator();
          }
        }        
186
        perturbed_ucomponent->Propagate(input, 1, &output_perturbed);
187
        CuVector<BaseFloat> output_objfs_perturbed(num_egs);
188 189 190 191 192 193 194 195 196 197 198 199
        output_objfs_perturbed.AddMatVec(1.0, output_perturbed,
                                         kNoTrans, objf_vec, 0.0);
        objf_perturbed = output_objfs_perturbed.Sum();
      }

      BaseFloat delta_objf_observed = objf_perturbed - objf,
          delta_objf_predicted = (perturbed_ucomponent->DotProduct(*gradient_ucomponent) -
                                  ucomponent->DotProduct(*gradient_ucomponent));
      
      KALDI_LOG << "Model gradients: comparing " << delta_objf_observed
                << " and " << delta_objf_predicted;
      if (fabs(delta_objf_predicted - delta_objf_observed) >
200 201
          0.05 * (fabs(delta_objf_predicted + delta_objf_observed)/2) &&
          fabs(delta_objf_predicted - delta_objf_observed) > 1.0e-06) {
202 203 204 205 206 207 208 209
        KALDI_WARN << "Bad difference!";
        num_bad++;
      } else {
        num_ok++;
      }
      delete perturbed_ucomponent;
      delete gradient_ucomponent;
    }
210
    if (num_ok < num_bad) {
211
      delete component_copy;
212 213
      KALDI_ERR << "model-derivative check failed";
    }
214 215 216 217 218 219 220 221 222
  }
  delete component_copy; // No longer needed.
}


void UnitTestSigmoidComponent() {
  // We're testing that the gradients are computed correctly:
  // the input gradients and the model gradients.
  
223
  int32 input_dim = 10 + Rand() % 50;
224 225 226 227 228 229 230 231 232 233 234 235
  {
    SigmoidComponent sigmoid_component(input_dim);
    UnitTestGenericComponentInternal(sigmoid_component);
  }
  {
    SigmoidComponent sigmoid_component;
    sigmoid_component.InitFromString("dim=15");
    UnitTestGenericComponentInternal(sigmoid_component);
  }
}

template<class T>
236 237
void UnitTestGenericComponent(std::string extra_str = "") {
  // works if it has an initializer from int,
238 239 240 241 242
  // e.g. tanh, sigmoid.
  
  // We're testing that the gradients are computed correctly:
  // the input gradients and the model gradients.
  
243
  int32 input_dim = 10 + Rand() % 50;
244 245 246 247 248 249
  {
    T component(input_dim);
    UnitTestGenericComponentInternal(component);
  }
  {
    T component;
250
    component.InitFromString(static_cast<std::string>("dim=15 ") + extra_str);
251 252 253 254
    UnitTestGenericComponentInternal(component);
  }
}

255 256 257 258 259 260 261 262
void UnitTestMaxoutComponent() {
  // works if it has an initializer from int,
  // e.g. tanh, sigmoid.
  
  // We're testing that the gradients are computed correctly:
  // the input gradients and the model gradients.

  for (int32 i = 0; i < 5; i++) {
263 264
    int32 output_dim = 10 + Rand() % 20,
        group_size = 1 + Rand() % 10,
265 266 267 268 269 270 271 272 273 274 275 276
        input_dim = output_dim * group_size;
    
    MaxoutComponent component(input_dim, output_dim);
    UnitTestGenericComponentInternal(component);
  }

  {
    MaxoutComponent component;
    component.InitFromString("input-dim=15 output-dim=5");
    UnitTestGenericComponentInternal(component);
  }
}
277 278 279 280 281 282 283 284

void UnitTestPnormComponent() {
  // works if it has an initializer from int,
  // e.g. tanh, sigmoid.
  
  // We're testing that the gradients are computed correctly:
  // the input gradients and the model gradients.

285
  for (int32 i = 0; i < 5; i++) {
286 287
    int32 output_dim = 10 + Rand() % 20,
        group_size = 1 + Rand() % 10,
288
        input_dim = output_dim * group_size;
289
    BaseFloat p = 0.8 + 0.1 * (Rand() % 20);
290 291 292 293 294 295 296 297 298 299 300 301 302 303
    
    PnormComponent component(input_dim, output_dim, p);
    UnitTestGenericComponentInternal(component);
  }

  {
    PnormComponent component;
    component.InitFromString("input-dim=15 output-dim=5 p=3.0");
    UnitTestGenericComponentInternal(component);
  }
}



304 305 306
void UnitTestAffineComponent() {
  BaseFloat learning_rate = 0.01,
      param_stddev = 0.1, bias_stddev = 1.0;
307
  int32 input_dim = 5 + Rand() % 10, output_dim = 5 + Rand() % 10;
308 309
  {
    AffineComponent component;
310
    if (Rand() % 2 == 0) {
311 312 313 314 315 316 317 318 319
      component.Init(learning_rate, input_dim, output_dim,
                     param_stddev, bias_stddev);
    } else {
      Matrix<BaseFloat> mat(output_dim + 1, input_dim);
      mat.SetRandn();
      mat.Scale(param_stddev);
      WriteKaldiObject(mat, "tmpf", true);
      sleep(1);
      component.Init(learning_rate, "tmpf");
320
      unlink("tmpf");
321
    }
322 323 324 325 326 327 328 329 330 331
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "learning-rate=0.01 input-dim=10 output-dim=15 param-stddev=0.1";
    AffineComponent component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
}

332 333 334
void UnitTestDropoutComponent() {
  // We're testing that the gradients are computed correctly:
  // the input gradients and the model gradients.
335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352

  int32 num_fail = 0, num_tries = 4;
  for (int32 i = 0; i < num_tries; i++) {
    try {
      int32 input_dim = 10 + Rand() % 50;
      {
        DropoutComponent dropout_component(input_dim, 0.5, 0.3);
        UnitTestGenericComponentInternal(dropout_component);
      }
      {
        DropoutComponent dropout_component;
        dropout_component.InitFromString("dim=15 dropout-proportion=0.6 dropout-scale=0.1");
        UnitTestGenericComponentInternal(dropout_component);
      }
    } catch (...) {
      KALDI_WARN << "Ignoring test failure in UnitTestDropoutComponent().";
      num_fail++;
    }
353
  }
354 355
  if (num_fail >= num_tries/2) {
    KALDI_ERR << "Too many test failures.";
356 357 358 359 360 361
  }
}

void UnitTestAdditiveNoiseComponent() {
  // We're testing that the gradients are computed correctly:
  // the input gradients and the model gradients.
362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378

  int32 num_fail = 0, num_tries = 4;
  for (int32 i = 0; i < num_tries; i++) {
    try {
      int32 input_dim = 10 + Rand() % 50;
      {
        AdditiveNoiseComponent additive_noise_component(input_dim, 0.1);
        UnitTestGenericComponentInternal(additive_noise_component);
      }
      {
        AdditiveNoiseComponent additive_noise_component;
        additive_noise_component.InitFromString("dim=15 stddev=0.2");
        UnitTestGenericComponentInternal(additive_noise_component);
      }
    } catch (...) {
      KALDI_WARN << "Ignoring failure in AdditiveNoiseComponent test";
      num_fail++;
379
    }
380
  }
381 382 383
  if (num_fail >= num_tries/2) {
    KALDI_ERR << "Too many test failures.";
  }  
384 385 386 387
}


void UnitTestPiecewiseLinearComponent() {
388 389
  BaseFloat learning_rate = 0.01, max_change = 0.1 * (Rand() % 2);
  int32 dim = 5 + Rand() % 10, N = 3 + 2 * (Rand() % 5);
390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
  {
    PiecewiseLinearComponent component;
    component.Init(dim, N, learning_rate, max_change);
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "learning-rate=0.01 dim=10 N=5 max-change=0.01";
    PiecewiseLinearComponent component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
}



void UnitTestScaleComponent() {
406 407
  int32 dim = 1 + Rand() % 10;
  BaseFloat scale = 0.1 + Rand() % 3;
408 409
  {
    ScaleComponent component;
410
    if (Rand() % 2 == 0) {
411 412 413 414 415 416 417 418 419 420 421
      component.Init(dim, scale);
    } else {
      std::ostringstream str;
      str << "dim=" << dim << " scale=" << scale;
      component.InitFromString(str.str());
    }
    UnitTestGenericComponentInternal(component);
  }
}


422 423
void UnitTestAffineComponentPreconditioned() {
  BaseFloat learning_rate = 0.01,
424 425
      param_stddev = 0.1, bias_stddev = 1.0, alpha = 0.01,
      max_change = 100.0;
426
  int32 input_dim = 5 + Rand() % 10, output_dim = 5 + Rand() % 10;
427 428
  {
    AffineComponentPreconditioned component;
429
    if (Rand() % 2 == 0) {
430 431 432 433 434 435 436 437 438 439
      component.Init(learning_rate, input_dim, output_dim,
                     param_stddev, bias_stddev,
                     alpha, max_change);
    } else {
      Matrix<BaseFloat> mat(output_dim + 1, input_dim);
      mat.SetRandn();
      mat.Scale(param_stddev);
      WriteKaldiObject(mat, "tmpf", true);
      sleep(1);
      component.Init(learning_rate, alpha, max_change, "tmpf");
440
      unlink("tmpf");
441
    }
442 443 444 445 446 447 448 449 450 451 452
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "learning-rate=0.01 input-dim=16 output-dim=15 param-stddev=0.1 alpha=0.01";
    AffineComponentPreconditioned component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
}


453 454
void UnitTestAffineComponentPreconditionedOnline() {
  BaseFloat learning_rate = 0.01,
455
      param_stddev = 0.1, bias_stddev = 1.0, num_samples_history = 2000.0, alpha = 4.0,
456
      max_change_per_sample = 0.1, update_period = 1;
457 458
  int32 input_dim = 5 + Rand() % 10, output_dim = 5 + Rand() % 10,
      rank_in = 1 + Rand() % 5, rank_out = 1 + Rand() % 5;
459 460
  {
    AffineComponentPreconditionedOnline component;
461
    if (Rand() % 2 == 0) {
462 463
      component.Init(learning_rate, input_dim, output_dim,
                     param_stddev, bias_stddev,
464 465
                     rank_in, rank_out, update_period,
                     num_samples_history, alpha,
466
                     max_change_per_sample);
467 468 469 470 471 472
    } else {
      Matrix<BaseFloat> mat(output_dim + 1, input_dim);
      mat.SetRandn();
      mat.Scale(param_stddev);
      WriteKaldiObject(mat, "tmpf", true);
      sleep(1);
473
      component.Init(learning_rate, rank_in, rank_out,
474
                     update_period, num_samples_history, alpha,
475
                     max_change_per_sample, "tmpf");
476
      unlink("tmpf");
477 478 479 480
    }
    UnitTestGenericComponentInternal(component);
  }
  {
481
    const char *str = "learning-rate=0.01 input-dim=16 output-dim=15 param-stddev=0.1 num-samples-history=3000 alpha=2.0 update-period=1 rank-in=5 rank-out=6";
482 483 484 485 486 487 488
    AffineComponentPreconditionedOnline component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
}


489 490 491 492
void UnitTestAffineComponentModified() {
  BaseFloat learning_rate = 0.01,
      param_stddev = 0.1, bias_stddev = 1.0, length_cutoff = 10.0,
      max_change = 0.1;
493
  int32 input_dim = 5 + Rand() % 10, output_dim = 5 + Rand() % 10;
494 495
  {
    AffineComponentModified component;
496
    if (Rand() % 2 == 0) {
497 498 499 500 501 502 503 504 505 506
      component.Init(learning_rate, input_dim, output_dim,
                     param_stddev, bias_stddev,
                     length_cutoff, max_change);
    } else {
      Matrix<BaseFloat> mat(output_dim + 1, input_dim);
      mat.SetRandn();
      mat.Scale(param_stddev);
      WriteKaldiObject(mat, "tmpf", true);
      sleep(1);
      component.Init(learning_rate, length_cutoff, max_change, "tmpf");
507
      unlink("tmpf");
508 509 510 511 512 513 514 515 516 517 518 519
    }
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "learning-rate=0.01 input-dim=16 output-dim=15 param-stddev=0.1 cutoff-length=10.0 max-change=0.01";
    AffineComponentModified component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
}


520 521 522 523
void UnitTestAffinePreconInputComponent() {
  BaseFloat learning_rate = 0.01,
      param_stddev = 0.1, bias_stddev = 1.0,
      avg_samples = 100.0;
524
  int32 input_dim = 5 + Rand() % 10, output_dim = 5 + Rand() % 10;
525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541

  {
    AffinePreconInputComponent component;
    component.Init(learning_rate, input_dim, output_dim,
                   param_stddev, bias_stddev, avg_samples);
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "learning-rate=0.01 input-dim=10 output-dim=15 param-stddev=0.1 avg-samples=100";
    AffinePreconInputComponent component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
}

void UnitTestBlockAffineComponent() {
  BaseFloat learning_rate = 0.01,
542
      param_stddev = 0.1, bias_stddev = 0.1;
543 544 545
  int32 num_blocks = 1 + Rand() % 3,
         input_dim = num_blocks * (2 + Rand() % 4),
        output_dim = num_blocks * (2 + Rand() % 4);
546 547 548 549 550 551 552 553 554 555 556 557 558 559 560
  
  {
    BlockAffineComponent component;
    component.Init(learning_rate, input_dim, output_dim,
                   param_stddev, bias_stddev, num_blocks);
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "learning-rate=0.01 input-dim=10 output-dim=15 param-stddev=0.1 num-blocks=5";
    BlockAffineComponent component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
}

561 562 563
void UnitTestBlockAffineComponentPreconditioned() {
  BaseFloat learning_rate = 0.01,
      param_stddev = 0.1, bias_stddev = 1.0, alpha = 3.0;
564 565 566
  int32 num_blocks = 1 + Rand() % 3,
         input_dim = num_blocks * (2 + Rand() % 4),
        output_dim = num_blocks * (2 + Rand() % 4);
567 568 569 570 571 572 573 574 575 576 577 578 579 580 581
  
  {
    BlockAffineComponentPreconditioned component;
    component.Init(learning_rate, input_dim, output_dim,
                   param_stddev, bias_stddev, num_blocks, alpha);
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "learning-rate=0.01 input-dim=10 output-dim=15 param-stddev=0.1 num-blocks=5 alpha=3.0";
    BlockAffineComponentPreconditioned component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
}

582 583 584 585
void UnitTestMixtureProbComponent() {
  BaseFloat learning_rate = 0.01,
      diag_element = 0.8;
  std::vector<int32> sizes;
586
  int32 num_sizes = 1 + Rand() % 5; // allow 
587
  for (int32 i = 0; i < num_sizes; i++)
588
    sizes.push_back(2 + Rand() % 5); // TODO: change to 1 + Rand() % 5
589
  // and fix test errors.  May be issue in the code itself.
590 591 592 593 594 595 596 597 598 599 600 601 602 603 604
  
  
  {
    MixtureProbComponent component;
    component.Init(learning_rate, diag_element, sizes);
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "learning-rate=0.01 diag-element=0.9 dims=3:4:5";
    MixtureProbComponent component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
}

605 606 607

void UnitTestSumGroupComponent() {
  std::vector<int32> sizes;
608
  int32 num_sizes = 1 + Rand() % 5;
609
  for (int32 i = 0; i < num_sizes; i++)
610
    sizes.push_back(1 + Rand() % 5); 
611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
  
  {
    SumGroupComponent component;
    component.Init(sizes);
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "sizes=3:4:5";
    SumGroupComponent component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
}


626
void UnitTestDctComponent() {
627
  int32 m = 1 + Rand() % 4, n = 1 + Rand() % 4,
628
  dct_dim = m, dim = m * n;
629
  bool reorder = (Rand() % 2 == 0);
630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668
  {
    DctComponent component;
    component.Init(dim, dct_dim, reorder);
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "dim=10 dct-dim=5 reorder=true";
    DctComponent component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "dim=10 dct-dim=5 reorder=true dct-keep-dim=1";
    DctComponent component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "dim=10 dct-dim=5 reorder=true dct-keep-dim=2";
    DctComponent component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "dim=10 dct-dim=5 reorder=true dct-keep-dim=3";
    DctComponent component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
  {
    const char *str = "dim=10 dct-dim=5 reorder=true dct-keep-dim=4";
    DctComponent component;
    component.InitFromString(str);
    UnitTestGenericComponentInternal(component);
  }
}


void UnitTestFixedLinearComponent() {
669
  int32 m = 1 + Rand() % 4, n = 1 + Rand() % 4;
670
  {
671 672
    CuMatrix<BaseFloat> mat(m, n);
    mat.SetRandn();
673 674 675 676 677 678 679
    FixedLinearComponent component;
    component.Init(mat);
    UnitTestGenericComponentInternal(component);
  }
}


680
void UnitTestFixedAffineComponent() {
681
  int32 m = 15 + Rand() % 4, n = 15 + Rand() % 4;
682
  {
683 684
    CuMatrix<BaseFloat> mat(m, n);
    mat.SetRandn();
685 686 687 688 689 690
    FixedAffineComponent component;
    component.Init(mat);
    UnitTestGenericComponentInternal(component);
  }
}

691
void UnitTestFixedScaleComponent() {
692
  int32 m = 1 + Rand() % 20;
693 694 695 696 697 698 699 700 701 702
  {
    CuVector<BaseFloat> vec(m);
    vec.SetRandn();
    FixedScaleComponent component;
    component.Init(vec);
    UnitTestGenericComponentInternal(component);
  }
}

void UnitTestFixedBiasComponent() {
703
  int32 m = 1 + Rand() % 20;
704 705 706 707 708 709 710 711 712
  {
    CuVector<BaseFloat> vec(m);
    vec.SetRandn();
    FixedBiasComponent component;
    component.Init(vec);
    UnitTestGenericComponentInternal(component);
  }
}

713

714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765

void UnitTestParsing() {
  int32 i;
  BaseFloat f;
  bool b;
  std::vector<int32> v;
  std::string s = "x=y";
  KALDI_ASSERT(ParseFromString("foo", &s, &i) == false
               && s == "x=y");
  KALDI_ASSERT(ParseFromString("foo", &s, &f) == false
               && s == "x=y");
  KALDI_ASSERT(ParseFromString("foo", &s, &v) == false
               && s == "x=y");
  KALDI_ASSERT(ParseFromString("foo", &s, &b) == false
               && s == "x=y");
  {
    std::string s = "x=1";
    KALDI_ASSERT(ParseFromString("x", &s, &i) == true
                 && i == 1 && s == "");
    s = "a=b x=1";
    KALDI_ASSERT(ParseFromString("x", &s, &i) == true
                 && i == 1 && s == "a=b");
  }
  {
    std::string s = "foo=false";
    KALDI_ASSERT(ParseFromString("foo", &s, &b) == true
                 && b == false && s == "");
    s = "x=y foo=true a=b";
    KALDI_ASSERT(ParseFromString("foo", &s, &b) == true
                 && b == true && s == "x=y a=b");    
  }

  {
    std::string s = "foobar x=1";
    KALDI_ASSERT(ParseFromString("x", &s, &f) == true
                 && f == 1.0 && s == "foobar");
    s = "a=b x=1 bxy";
    KALDI_ASSERT(ParseFromString("x", &s, &f) == true
                 && f == 1.0 && s == "a=b bxy");
  }
  {
    std::string s = "x=1:2:3";
    KALDI_ASSERT(ParseFromString("x", &s, &v) == true
                 && v.size() == 3 && v[0] == 1 && v[1] == 2 && v[2] == 3
                 && s == "");
    s = "a=b x=1:2:3 c=d";
    KALDI_ASSERT(ParseFromString("x", &s, &v) == true
                 && f == 1.0 && s == "a=b c=d");
  }

}

766
void BasicDebugTestForSplice(bool output=false) {
767 768 769 770 771 772
  int32 C=5;
  int32 K=4, contextLen=1;
  int32 R=3+2 * contextLen;
 
  SpliceComponent *c = new SpliceComponent();
  c->Init(C, contextLen, contextLen, K);
773 774
  CuMatrix<BaseFloat> in(R, C), in_deriv(R, C);
  CuMatrix<BaseFloat> out(R, c->OutputDim());
775 776 777

  in.SetRandn();
  if (output)
778
    KALDI_LOG << in;
779 780 781 782

  c->Propagate(in, 1, &out);
  
  if (output) 
783
    KALDI_LOG << out;
784 785 786 787

  out.Set(1);
  
  if (K > 0) {
788
    CuSubMatrix<BaseFloat> k(out, 0, out.NumRows(), c->OutputDim() - K, K);
789 790 791 792
    k.Set(-2);
  }

  if (output)
793
    KALDI_LOG << out;
794 795
  
  int32 num_chunks = 1;
796
  c->Backprop(in, in, out, num_chunks, c, &in_deriv);
797 798
  
  if (output)
799
    KALDI_LOG << in_deriv;
Ho Yin Chan's avatar
Ho Yin Chan committed
800
  delete c;
801 802 803 804 805 806 807 808 809
}

void BasicDebugTestForSpliceMax(bool output=false) {
  int32 C=5;
  int32 contextLen=2;
  int32 R= 3 + 2*contextLen;
 
  SpliceMaxComponent *c = new SpliceMaxComponent();
  c->Init(C, contextLen, contextLen);
810 811
  CuMatrix<BaseFloat> in(R, C), in_deriv(R, C);
  CuMatrix<BaseFloat> out(R, c->OutputDim());
812 813 814 815 816 817 818 819 820
  
  in.SetRandn();
  if (output)
    KALDI_LOG << in;

  c->Propagate(in, 1, &out);
  
  if (output) 
    KALDI_LOG << out;
Ho Yin Chan's avatar
Ho Yin Chan committed
821

822 823 824 825 826 827 828 829 830 831 832 833
  out.Set(5.0);
  
  if (output)
    KALDI_LOG << out;
  
  int32 num_chunks = 1;
  c->Backprop(in, in, out, num_chunks, c, &in_deriv);
  
  if (output)
    KALDI_LOG << in_deriv;

  delete c;
834 835
}

836

837
} // namespace nnet2
838 839 840 841 842 843 844
} // namespace kaldi

#include "matrix/matrix-functions.h"


int main() {
  using namespace kaldi;
845
  using namespace kaldi::nnet2;
846

847 848 849 850

  for (int32 loop = 0; loop < 2; loop++) {
#if HAVE_CUDA == 1
    if (loop == 0)
851
      CuDevice::Instantiate().SelectGpuId("no"); // -1 means no GPU
852
    else
853
      CuDevice::Instantiate().SelectGpuId("optional"); // -2 .. automatic selection
854
#endif
Dan Povey's avatar
Dan Povey committed
855
    
856 857 858 859 860
    BasicDebugTestForSplice(true);
    BasicDebugTestForSpliceMax(true);
    for (int32 i = 0; i < 3; i++) {
      UnitTestGenericComponent<SigmoidComponent>();
      UnitTestGenericComponent<TanhComponent>();
861
      UnitTestGenericComponent<PowerComponent>("power=1.5");
862
      UnitTestGenericComponent<PowerComponent>("power=1.0");
863 864 865 866 867
      UnitTestGenericComponent<PermuteComponent>();
      UnitTestGenericComponent<SoftmaxComponent>();
      UnitTestGenericComponent<RectifiedLinearComponent>();
      UnitTestGenericComponent<SoftHingeComponent>();
      UnitTestGenericComponent<PowerExpandComponent>("higher-power-scale=0.1");
868
      UnitTestMaxoutComponent(); 
869 870
      UnitTestPnormComponent(); 
      UnitTestGenericComponent<NormalizeComponent>();
871 872 873 874 875 876 877 878
      UnitTestSigmoidComponent();
      UnitTestAffineComponent();
      UnitTestPiecewiseLinearComponent();
      UnitTestScaleComponent();
      UnitTestAffinePreconInputComponent();
      UnitTestBlockAffineComponent();
      UnitTestBlockAffineComponentPreconditioned();
      UnitTestMixtureProbComponent();
879
      UnitTestSumGroupComponent();
880 881 882
      UnitTestDctComponent();
      UnitTestFixedLinearComponent();
      UnitTestFixedAffineComponent();
883 884
      UnitTestFixedScaleComponent();
      UnitTestFixedBiasComponent();
885
      UnitTestAffineComponentPreconditioned();
886
      UnitTestAffineComponentPreconditionedOnline();
887
      UnitTestAffineComponentModified();
888 889
      UnitTestDropoutComponent();
      UnitTestAdditiveNoiseComponent();
890 891
      UnitTestParsing();
      if (loop == 0)
892
        KALDI_LOG << "Tests without GPU use succeeded.";
893
      else
894
        KALDI_LOG << "Tests with GPU use (if available) succeeded.";
895
    }
896
  }
897 898 899 900
#if HAVE_CUDA == 1
  CuDevice::Instantiate().PrintProfile();
#endif
  return 0;
901
}