HashedDocConverter.h

Go to the documentation of this file.

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2013 Evangelos Anagnostopoulos
 * Copyright (C) 2013 Evangelos Anagnostopoulos
 */

#ifndef _HASHEDDOCCONVERTER__H__
#define _HASHEDDOCCONVERTER__H__

#include <shogun/lib/config.h>

#include <shogun/converter/Converter.h>
#include <shogun/features/Features.h>
#include <shogun/lib/Tokenizer.h>
#include <shogun/features/SparseFeatures.h>

namespace shogun
{
class CFeatures;
class CTokenizer;
class CConverter;
template<class T> class CSparseFeatures;

class CHashedDocConverter : public CConverter
{
public:
    CHashedDocConverter();

    CHashedDocConverter(int32_t hash_bits, bool normalize = false, int32_t n_grams = 1, int32_t skips = 0);

    CHashedDocConverter(CTokenizer* tzer, int32_t hash_bits, bool normalize = false, int32_t n_grams = 1,
        int32_t skips = 0);

    virtual ~CHashedDocConverter();

    virtual CFeatures* apply(CFeatures* features);

    SGSparseVector<float64_t> apply(SGVector<char> document);

    static index_t generate_ngram_hashes(SGVector<uint32_t>& hashes, index_t hashes_start, index_t len,
            SGVector<index_t>& ngram_hashes, int32_t num_bits, int32_t ngrams, int32_t tokens_to_skip);

    virtual const char* get_name() const;

    void set_normalization(bool normalize);

    void set_k_skip_n_grams(int32_t k, int32_t n);
protected:

    void init(CTokenizer* tzer, int32_t d, bool normalize, int32_t n_grams, int32_t skips);

    int32_t count_distinct_indices(CDynamicArray<uint32_t>& hashed_indices);

    SGSparseVector<float64_t> create_hashed_representation(CDynamicArray<uint32_t>& hashed_indices);

protected:

    int32_t num_bits;

    CTokenizer* tokenizer;

    bool should_normalize;

    int32_t ngrams;

    int32_t tokens_to_skip;
};
}

#endif