VwParser.h

Go to the documentation of this file.

/*
 * Copyright (c) 2009 Yahoo! Inc.  All rights reserved.  The copyrights
 * embodied in the content of this file are licensed under the BSD
 * (revised) open source license.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2011 Shashwat Lal Das
 * Adaptation of Vowpal Wabbit v5.1.
 * Copyright (C) 2011 Berlin Institute of Technology and Max-Planck-Society.
 */

#ifndef _VW_PARSER_H__
#define _VW_PARSER_H__

#include <shogun/lib/config.h>

#include <shogun/base/SGObject.h>
#include <shogun/io/SGIO.h>
#include <shogun/lib/Hash.h>
#include <shogun/classifier/vw/vw_common.h>
#include <shogun/classifier/vw/cache/VwCacheWriter.h>

namespace shogun
{
enum E_VW_PARSER_TYPE
{
    T_VW = 1,
    T_SVMLIGHT = 2,
    T_DENSE = 3
};

class CVwParser: public CSGObject
{
public:
    CVwParser();

    CVwParser(CVwEnvironment* env_to_use);

    virtual ~CVwParser();

    CVwEnvironment* get_env()
    {
        return env;
    }

    void set_env(CVwEnvironment* env_to_use)
    {
        if(env_to_use != env)
        {
            SG_REF(env_to_use);
            SG_UNREF(env);
            env = env_to_use;
        }
    }

    void set_cache_parameters(char * fname, EVwCacheType type = C_NATIVE)
    {
        init_cache(fname, type);
    }

    EVwCacheType get_cache_type()
    {
        return cache_type;
    }

    void set_write_cache(bool wr_cache)
    {
        write_cache = wr_cache;
        if (wr_cache)
            init_cache(NULL);
        else
            if (cache_writer)
                SG_UNREF(cache_writer);
    }

    bool get_write_cache()
    {
        return write_cache;
    }

    void set_mm(float64_t label)
    {
        env->min_label = CMath::min(env->min_label, label);
        if (label != FLT_MAX)
            env->max_label = CMath::max(env->max_label, label);
    }

    void noop_mm(float64_t label) { }

    void set_minmax(float64_t label)
    {
        set_mm(label);
    }

    int32_t read_features(CIOBuffer* buf, VwExample*& ex);

    int32_t read_svmlight_features(CIOBuffer* buf, VwExample*& ae);

    int32_t read_dense_features(CIOBuffer* buf, VwExample*& ae);

    virtual const char* get_name() const { return "VwParser"; }

protected:
    void init_cache(char * fname, EVwCacheType type = C_NATIVE);

    void feature_value(substring &s, v_array<substring>& name, float32_t &v);

    void tokenize(char delim, substring s, v_array<substring> &ret);

    inline char* safe_index(char *start, char v, char *max)
    {
        while (start != max && *start != v)
            start++;
        return start;
    }

public:
    hash_func_t hasher;

protected:
    CVwEnvironment* env;
    CVwCacheWriter* cache_writer;
    EVwCacheType cache_type;
    bool write_cache;

private:
    v_array<substring> channels;
    v_array<substring> words;
    v_array<substring> name;
};

}
#endif // _VW_PARSER_H__