NGramTokenizer.cpp

Go to the documentation of this file.

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2013 Evangelos Anagnostopoulos
 * Copyright (C) 2013 Evangelos Anagnostopoulos
 */

#include <shogun/lib/NGramTokenizer.h>
#include <shogun/lib/SGVector.h>
#include <shogun/base/Parameter.h>

namespace shogun
{

CNGramTokenizer::CNGramTokenizer(int32_t ns) : CTokenizer()
{
    n = ns;
    last_idx = 0;
    init();
}

CNGramTokenizer::CNGramTokenizer(const CNGramTokenizer& orig)
: CTokenizer(orig)
{
    CTokenizer::set_text(orig.text);
    n = orig.n;
    init();
}

void CNGramTokenizer::init()
{
    SG_ADD(&n, "n", "Size of n-grams",
        MS_NOT_AVAILABLE);
    SG_ADD(&last_idx, "last_idx", "Index of last token",
        MS_NOT_AVAILABLE);
}

void CNGramTokenizer::set_text(SGVector<char> txt)
{
    last_idx = 0;
    CTokenizer::set_text(txt);
}

const char* CNGramTokenizer::get_name() const
{
    return "NGramTokenizer";
}

bool CNGramTokenizer::has_next()
{
    return last_idx<=text.size()-n;
}

index_t CNGramTokenizer::next_token_idx(index_t& start)
{
    start = last_idx++;
    return start + n;
}

CNGramTokenizer* CNGramTokenizer::get_copy()
{
    CNGramTokenizer* t = new CNGramTokenizer(n);
    return t;
}
}