NGramTokenizer.h

Go to the documentation of this file.

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2013 Evangelos Anagnostopoulos
 * Copyright (C) 2013 Evangelos Anagnostopoulos
 */

#ifndef _NGRAMTOKENIZER__H__
#define _NGRAMTOKENIZER__H__

#include <shogun/lib/config.h>

#include <shogun/lib/Tokenizer.h>

namespace shogun
{
template <class T> class SGVector;

class CNGramTokenizer: public CTokenizer
{

public:
    CNGramTokenizer(int32_t ns=3);

    CNGramTokenizer(const CNGramTokenizer& orig);

    virtual ~CNGramTokenizer() {}

    virtual void set_text(SGVector<char> txt);

    virtual bool has_next();

    virtual index_t next_token_idx(index_t& start);

    virtual const char* get_name() const;

    virtual CNGramTokenizer* get_copy();

private:
    void init();

protected:

    int32_t n;

    index_t last_idx;
};
}
#endif  /* _NGRAMTOKENIZER__H__ */