analyze-counts.cc 4.88 KB
Newer Older
1
// bin/analyze-counts.cc
2

Karel Vesely's avatar
Karel Vesely committed
3
// Copyright 2012-2014 Brno University of Technology (Author: Karel Vesely)
4

5 6
// See ../../COPYING for clarification regarding multiple authors
//
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.

/** @brief Sums the pdf vectors to counts, this is used to obtain prior counts for hybrid decoding.
*/
#include "base/kaldi-common.h"
#include "util/common-utils.h"
Karel Vesely's avatar
Karel Vesely committed
24
#include "fst/fstlib.h"
25

Karel Vesely's avatar
Karel Vesely committed
26
#include <iomanip>
Karel Vesely's avatar
Karel Vesely committed
27 28
#include <algorithm>
#include <numeric>
Karel Vesely's avatar
Karel Vesely committed
29

30 31 32
int main(int argc, char *argv[]) {
  using namespace kaldi;
  typedef kaldi::int32 int32;
Dan Povey's avatar
Dan Povey committed
33
  typedef kaldi::uint64 uint64;
34 35
  try {
    const char *usage =
Dan Povey's avatar
Dan Povey committed
36 37
        "Computes element counts from integer vector table.\n"
        "(e.g. for example to get pdf-counts to estimate DNN-output priors, for data analysis)\n"
Karel Vesely's avatar
Karel Vesely committed
38 39
        "Verbosity : level 1 => print frequencies and histogram\n"
        "\n"
40
        "Usage:  analyze-counts  [options] <alignments-rspecifier> <counts-wxfilname>\n"
41
        "e.g.: \n"
Karel Vesely's avatar
Karel Vesely committed
42 43 44 45
        " analyze-counts ark:1.ali prior.counts\n"
        " Show phone counts by:\n"
        " ali-to-phone --per-frame=true ark:1.ali ark:- | analyze-counts --verbose=1 ark:- - >/dev/null\n";
    
46 47 48
    ParseOptions po(usage);
    
    bool binary = false;
Karel Vesely's avatar
Karel Vesely committed
49 50
    std::string symbol_table_filename = "";
    
51
    po.Register("binary", &binary, "write in binary mode");
Karel Vesely's avatar
Karel Vesely committed
52
    po.Register("symbol-table", &symbol_table_filename, "Read symbol table for display of counts");
Karel Vesely's avatar
Karel Vesely committed
53

54 55 56 57 58 59 60 61 62 63 64 65
    po.Read(argc, argv);

    if (po.NumArgs() != 2) {
      po.PrintUsage();
      exit(1);
    }

    std::string alignments_rspecifier = po.GetArg(1),
        wxfilename = po.GetArg(2);

    SequentialInt32VectorReader reader(alignments_rspecifier);

Karel Vesely's avatar
Karel Vesely committed
66 67
    // Get the counts
    std::vector<uint64> counts;
68 69 70 71 72 73 74 75 76 77
    int32 num_done = 0;
    for (; !reader.Done(); reader.Next()) {
      std::string key = reader.Key();
      std::vector<int32> alignment = reader.Value();

      for (size_t i = 0; i < alignment.size(); i++) {
        int32 value = alignment[i];
        if(value >= counts.size()) {
          counts.resize(value+1);
        }
Karel Vesely's avatar
Karel Vesely committed
78
        counts[value]++; // Accumulate
79 80 81 82 83
      }

      num_done++;
    }

Karel Vesely's avatar
Karel Vesely committed
84 85
    // We need at least one occurence for each tgt, so there is no nan during decoding
    std::vector<uint64> counts_nozero(counts);
86
    for(size_t i = 0; i < counts.size(); i++) {
Karel Vesely's avatar
Karel Vesely committed
87 88
      if(counts_nozero[i] == 0) {
        KALDI_WARN << "Zero count for element " << i << ", force setting to one."
Dan Povey's avatar
Dan Povey committed
89
                   << " This avoids divide-by-zero when we use the counts in decoding.";
Karel Vesely's avatar
Karel Vesely committed
90 91
        counts_nozero[i]++;
      }
92 93
    }

Karel Vesely's avatar
Karel Vesely committed
94
    // Write
95
    Output ko(wxfilename, binary);
Karel Vesely's avatar
Karel Vesely committed
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
    WriteIntegerVector(ko.Stream(), binary, counts_nozero);

    ////
    //// THE REST IS FOR ANALYSIS, IT GETS PRINTED TO LOG
    ////
    if (symbol_table_filename != "" || (kaldi::g_kaldi_verbose_level >= 1)) {

      // load the symbol table
      fst::SymbolTable *elem_syms = NULL;
      if (symbol_table_filename != "") {
          elem_syms = fst::SymbolTable::ReadText(symbol_table_filename);
          if (!elem_syms)
            KALDI_ERR << "Could not read symbol table from file " << symbol_table_filename;
      }
      
      // sort the counts
      std::vector<std::pair<int32,int32> > sorted_counts;
Dan Povey's avatar
Dan Povey committed
113
      for (int32 i = 0; i < counts.size(); i++) {
114
        sorted_counts.push_back(std::make_pair(static_cast<int32>(counts[i]), i));
Karel Vesely's avatar
Karel Vesely committed
115
      }
Karel Vesely's avatar
Karel Vesely committed
116 117 118 119 120 121 122 123 124 125 126 127 128
      std::sort(sorted_counts.begin(), sorted_counts.end());
      
      // print
      std::ostringstream os;
      int32 sum = std::accumulate(counts.begin(),counts.end(), 0);
      os << "Printing...\n### The sorted count table," << std::endl;
      os << "count\t(norm),\tid\t(symbol):" << std::endl;
      for (int32 i=0; i<sorted_counts.size(); i++) {
        os << sorted_counts[i].first << "\t(" 
           << static_cast<float>(sorted_counts[i].first) / sum << "),\t"
           << sorted_counts[i].second << "\t" 
           << (elem_syms != NULL ? std::string("(")+elem_syms->Find(sorted_counts[i].second)+")" : "")
           << std::endl;
Karel Vesely's avatar
Karel Vesely committed
129
      }
Karel Vesely's avatar
Karel Vesely committed
130 131 132 133
      os << "\n#total " << sum 
         << " (" << static_cast<float>(sum)/100/3600 << "h)" 
         << std::endl;
      KALDI_LOG << os.str();
Karel Vesely's avatar
Karel Vesely committed
134
    }
135 136

    KALDI_LOG << "Summed " << num_done << " int32 vectors to counts.";
Karel Vesely's avatar
Karel Vesely committed
137
    KALDI_LOG << "Counts written to " << wxfilename;
138 139 140 141 142 143 144 145
    return 0;
  } catch(const std::exception &e) {
    std::cerr << e.what();
    return -1;
  }
}