SHOGUN  4.2.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
ProtobufFile.cpp
Go to the documentation of this file.
1 /*
2  * This program is free software; you can redistribute it and/or modify
3  * it under the terms of the GNU General Public License as published by
4  * the Free Software Foundation; either version 3 of the License, or
5  * (at your option) any later version.
6  *
7  * Written (W) 2013 Evgeniy Andreev (gsomix)
8  */
9 #include <shogun/io/ProtobufFile.h>
10 
11 #ifdef HAVE_PROTOBUF
12 
13 #include <shogun/io/SGIO.h>
14 
15 #include <shogun/lib/SGVector.h>
16 #include <shogun/lib/SGMatrix.h>
17 #include <shogun/lib/SGSparseVector.h>
18 #include <shogun/lib/SGString.h>
19 #include <google/protobuf/message.h>
20 
21 using namespace shogun;
22 
23 CProtobufFile::CProtobufFile()
24 {
25  init();
26 }
27 
28 CProtobufFile::CProtobufFile(FILE* f, const char* name) :
29  CFile(f, name)
30 {
31  init();
32 }
33 
34 CProtobufFile::CProtobufFile(const char* fname, char rw, const char* name) :
35  CFile(fname, rw, name)
36 {
37  init();
38 }
39 
40 CProtobufFile::~CProtobufFile()
41 {
42  SG_FREE(buffer);
43 }
44 
45 void CProtobufFile::init()
46 {
47  version=1;
48  message_size=1024*1024;
49 
50  buffer=SG_MALLOC(uint8_t, message_size*sizeof(uint32_t));
51 }
52 
53 #define GET_VECTOR(sg_type) \
54 void CProtobufFile::get_vector(sg_type*& vector, int32_t& len) \
55 { \
56  read_and_validate_global_header(ShogunVersion::VECTOR); \
57  VectorHeader data_header=read_vector_header(); \
58  len=data_header.len(); \
59  read_memory_block(vector, len, data_header.num_messages()); \
60 }
61 
62 GET_VECTOR(int8_t)
63 GET_VECTOR(uint8_t)
64 GET_VECTOR(char)
65 GET_VECTOR(int32_t)
66 GET_VECTOR(uint32_t)
67 GET_VECTOR(float32_t)
68 GET_VECTOR(float64_t)
69 GET_VECTOR(floatmax_t)
70 GET_VECTOR(int16_t)
71 GET_VECTOR(uint16_t)
72 GET_VECTOR(int64_t)
73 GET_VECTOR(uint64_t)
74 #undef GET_VECTOR
75 
76 #define GET_MATRIX(read_func, sg_type) \
77 void CProtobufFile::get_matrix(sg_type*& matrix, int32_t& num_feat, int32_t& num_vec) \
78 { \
79  read_and_validate_global_header(ShogunVersion::MATRIX); \
80  MatrixHeader data_header=read_matrix_header(); \
81  num_feat=data_header.num_cols(); \
82  num_vec=data_header.num_rows(); \
83  read_memory_block(matrix, num_feat*num_vec, data_header.num_messages()); \
84 }
85 
86 GET_MATRIX(read_char, int8_t)
87 GET_MATRIX(read_byte, uint8_t)
88 GET_MATRIX(read_char, char)
89 GET_MATRIX(read_int, int32_t)
90 GET_MATRIX(read_uint, uint32_t)
91 GET_MATRIX(read_short_real, float32_t)
92 GET_MATRIX(read_real, float64_t)
93 GET_MATRIX(read_long_real, floatmax_t)
94 GET_MATRIX(read_short, int16_t)
95 GET_MATRIX(read_word, uint16_t)
96 GET_MATRIX(read_long, int64_t)
97 GET_MATRIX(read_ulong, uint64_t)
98 #undef GET_MATRIX
99 
100 #define GET_NDARRAY(read_func, sg_type) \
101 void CProtobufFile::get_ndarray(sg_type*& array, int32_t*& dims, int32_t& num_dims) \
102 { \
103  SG_NOTIMPLEMENTED \
104 }
105 
106 GET_NDARRAY(read_byte, uint8_t)
107 GET_NDARRAY(read_char, char)
108 GET_NDARRAY(read_int, int32_t)
109 GET_NDARRAY(read_short_real, float32_t)
110 GET_NDARRAY(read_real, float64_t)
111 GET_NDARRAY(read_short, int16_t)
112 GET_NDARRAY(read_word, uint16_t)
113 #undef GET_NDARRAY
114 
115 #define GET_SPARSE_MATRIX(sg_type) \
116 void CProtobufFile::get_sparse_matrix( \
117  SGSparseVector<sg_type>*& matrix, int32_t& num_feat, int32_t& num_vec) \
118 { \
119  read_and_validate_global_header(ShogunVersion::SPARSE_MATRIX); \
120  SparseMatrixHeader data_header=read_sparse_matrix_header(); \
121  num_feat=data_header.num_features(); \
122  num_vec=data_header.num_vectors(); \
123  read_sparse_matrix(matrix, data_header); \
124 }
125 
126 GET_SPARSE_MATRIX(bool)
127 GET_SPARSE_MATRIX(int8_t)
128 GET_SPARSE_MATRIX(uint8_t)
129 GET_SPARSE_MATRIX(char)
130 GET_SPARSE_MATRIX(int32_t)
131 GET_SPARSE_MATRIX(uint32_t)
132 GET_SPARSE_MATRIX(float32_t)
133 GET_SPARSE_MATRIX(float64_t)
134 GET_SPARSE_MATRIX(floatmax_t)
135 GET_SPARSE_MATRIX(int16_t)
136 GET_SPARSE_MATRIX(uint16_t)
137 GET_SPARSE_MATRIX(int64_t)
138 GET_SPARSE_MATRIX(uint64_t)
139 #undef GET_SPARSE_MATRIX
140 
141 #define SET_VECTOR(sg_type) \
142 void CProtobufFile::set_vector(const sg_type* vector, int32_t len) \
143 { \
144  int32_t num_messages=compute_num_messages(len, sizeof(sg_type)); \
145  write_global_header(ShogunVersion::VECTOR); \
146  write_vector_header(len, num_messages); \
147  write_memory_block(vector, len, num_messages); \
148 }
149 
150 SET_VECTOR(int8_t)
151 SET_VECTOR(uint8_t)
152 SET_VECTOR(char)
153 SET_VECTOR(int32_t)
154 SET_VECTOR(uint32_t)
155 SET_VECTOR(int64_t)
156 SET_VECTOR(uint64_t)
157 SET_VECTOR(float32_t)
158 SET_VECTOR(float64_t)
159 SET_VECTOR(floatmax_t)
160 SET_VECTOR(int16_t)
161 SET_VECTOR(uint16_t)
162 #undef SET_VECTOR
163 
164 #define SET_MATRIX(sg_type) \
165 void CProtobufFile::set_matrix(const sg_type* matrix, int32_t num_feat, int32_t num_vec) \
166 { \
167  int32_t num_messages=compute_num_messages(num_feat*num_vec, sizeof(sg_type)); \
168  write_global_header(ShogunVersion::MATRIX); \
169  write_matrix_header(num_feat, num_vec, num_messages); \
170  write_memory_block(matrix, num_feat*num_vec, num_messages); \
171 }
172 
173 SET_MATRIX(int8_t)
174 SET_MATRIX(uint8_t)
175 SET_MATRIX(char)
176 SET_MATRIX(int32_t)
177 SET_MATRIX(uint32_t)
178 SET_MATRIX(int64_t)
179 SET_MATRIX(uint64_t)
180 SET_MATRIX(float32_t)
181 SET_MATRIX(float64_t)
182 SET_MATRIX(floatmax_t)
183 SET_MATRIX(int16_t)
184 SET_MATRIX(uint16_t)
185 #undef SET_MATRIX
186 
187 #define SET_SPARSE_MATRIX(sg_type) \
188 void CProtobufFile::set_sparse_matrix( \
189  const SGSparseVector<sg_type>* matrix, int32_t num_feat, int32_t num_vec) \
190 { \
191  write_global_header(ShogunVersion::SPARSE_MATRIX); \
192  write_sparse_matrix_header(matrix, num_feat, num_vec); \
193  write_sparse_matrix(matrix, num_vec); \
194 }
195 
196 SET_SPARSE_MATRIX(bool)
197 SET_SPARSE_MATRIX(int8_t)
198 SET_SPARSE_MATRIX(uint8_t)
199 SET_SPARSE_MATRIX(char)
200 SET_SPARSE_MATRIX(int32_t)
201 SET_SPARSE_MATRIX(uint32_t)
202 SET_SPARSE_MATRIX(int64_t)
203 SET_SPARSE_MATRIX(uint64_t)
204 SET_SPARSE_MATRIX(float32_t)
205 SET_SPARSE_MATRIX(float64_t)
206 SET_SPARSE_MATRIX(floatmax_t)
207 SET_SPARSE_MATRIX(int16_t)
208 SET_SPARSE_MATRIX(uint16_t)
209 #undef SET_SPARSE_MATRIX
210 
211 #define GET_STRING_LIST(sg_type) \
212 void CProtobufFile::get_string_list( \
213  SGString<sg_type>*& strings, int32_t& num_str, \
214  int32_t& max_string_len) \
215 { \
216  read_and_validate_global_header(ShogunVersion::STRING_LIST); \
217  StringListHeader data_header=read_string_list_header(); \
218  num_str=data_header.num_str(); \
219  max_string_len=data_header.max_string_len(); \
220  read_string_list(strings, data_header); \
221 }
222 
223 GET_STRING_LIST(int8_t)
224 GET_STRING_LIST(uint8_t)
225 GET_STRING_LIST(char)
226 GET_STRING_LIST(int32_t)
227 GET_STRING_LIST(uint32_t)
228 GET_STRING_LIST(int64_t)
229 GET_STRING_LIST(uint64_t)
230 GET_STRING_LIST(float32_t)
231 GET_STRING_LIST(float64_t)
232 GET_STRING_LIST(floatmax_t)
233 GET_STRING_LIST(int16_t)
234 GET_STRING_LIST(uint16_t)
235 #undef GET_STRING_LIST
236 
237 #define SET_STRING_LIST(sg_type) \
238 void CProtobufFile::set_string_list( \
239  const SGString<sg_type>* strings, int32_t num_str) \
240 { \
241  write_global_header(ShogunVersion::STRING_LIST); \
242  write_string_list_header(strings, num_str); \
243  write_string_list(strings, num_str); \
244 }
245 
246 SET_STRING_LIST(int8_t)
247 SET_STRING_LIST(uint8_t)
248 SET_STRING_LIST(char)
249 SET_STRING_LIST(int32_t)
250 SET_STRING_LIST(uint32_t)
251 SET_STRING_LIST(int64_t)
252 SET_STRING_LIST(uint64_t)
253 SET_STRING_LIST(float32_t)
254 SET_STRING_LIST(float64_t)
255 SET_STRING_LIST(floatmax_t)
256 SET_STRING_LIST(int16_t)
257 SET_STRING_LIST(uint16_t)
258 #undef SET_STRING_LIST
259 
260 void CProtobufFile::write_big_endian_uint(uint32_t number, uint8_t* array, uint32_t size)
261 {
262  if (size<4)
263  SG_ERROR("array is too small to write\n");
264 
265  array[0]=(number>>24)&0xffu;
266  array[1]=(number>>16)&0xffu;
267  array[2]=(number>>8)&0xffu;
268  array[3]=number&0xffu;
269 }
270 
271 uint32_t CProtobufFile::read_big_endian_uint(uint8_t* array, uint32_t size)
272 {
273  if (size<4)
274  SG_ERROR("array is too small to read\n");
275 
276  return (array[0]<<24) | (array[1]<<16) | (array[2]<<8) | array[3];
277 }
278 
279 int32_t CProtobufFile::compute_num_messages(uint64_t len, int32_t sizeof_type) const
280 {
281  uint32_t elements_in_message=message_size/sizeof_type;
282  uint32_t num_messages=len/elements_in_message;
283  if (len % elements_in_message > 0)
284  num_messages++;
285 
286  return num_messages;
287 }
288 
289 void CProtobufFile::read_and_validate_global_header(ShogunVersion_SGDataType type)
290 {
291  ShogunVersion header;
292  read_message(header);
293  REQUIRE(header.version()==version, "wrong version\n")
294  REQUIRE(header.data_type()==type, "wrong type\n")
295 }
296 
297 void CProtobufFile::write_global_header(ShogunVersion_SGDataType type)
298 {
299  ShogunVersion header;
300  header.set_version(version);
301  header.set_data_type(type);
302  write_message(header);
303 }
304 
305 VectorHeader CProtobufFile::read_vector_header()
306 {
307  VectorHeader data_header;
308  read_message(data_header);
309 
310  return data_header;
311 }
312 
313 SparseMatrixHeader CProtobufFile::read_sparse_matrix_header()
314 {
315  SparseMatrixHeader data_header;
316  read_message(data_header);
317 
318  return data_header;
319 }
320 
321 MatrixHeader CProtobufFile::read_matrix_header()
322 {
323  MatrixHeader data_header;
324  read_message(data_header);
325 
326  return data_header;
327 }
328 
329 StringListHeader CProtobufFile::read_string_list_header()
330 {
331  StringListHeader data_header;
332  read_message(data_header);
333 
334  return data_header;
335 }
336 
337 void CProtobufFile::write_vector_header(int32_t len, int32_t num_messages)
338 {
339  VectorHeader data_header;
340  data_header.set_len(len);
341  data_header.set_num_messages(num_messages);
342  write_message(data_header);
343 }
344 
345 void CProtobufFile::write_matrix_header(int32_t num_feat, int32_t num_vec, int32_t num_messages)
346 {
347  MatrixHeader data_header;
348  data_header.set_num_cols(num_feat);
349  data_header.set_num_rows(num_vec);
350  data_header.set_num_messages(num_messages);
351  write_message(data_header);
352 }
353 
354 #define WRITE_SPARSE_MATRIX_HEADER(sg_type) \
355 void CProtobufFile::write_sparse_matrix_header( \
356  const SGSparseVector<sg_type>* matrix, int32_t num_feat, int32_t num_vec) \
357 { \
358  SparseMatrixHeader data_header; \
359  data_header.set_num_features(num_feat); \
360  data_header.set_num_vectors(num_vec); \
361  for (int32_t i=0; i<num_vec; i++) \
362  { \
363  data_header.add_num_feat_entries(matrix[i].num_feat_entries); \
364  } \
365  \
366  write_message(data_header); \
367 }
368 
369 WRITE_SPARSE_MATRIX_HEADER(bool)
370 WRITE_SPARSE_MATRIX_HEADER(int8_t)
371 WRITE_SPARSE_MATRIX_HEADER(uint8_t)
372 WRITE_SPARSE_MATRIX_HEADER(char)
373 WRITE_SPARSE_MATRIX_HEADER(int32_t)
374 WRITE_SPARSE_MATRIX_HEADER(uint32_t)
375 WRITE_SPARSE_MATRIX_HEADER(int64_t)
376 WRITE_SPARSE_MATRIX_HEADER(uint64_t)
377 WRITE_SPARSE_MATRIX_HEADER(float32_t)
378 WRITE_SPARSE_MATRIX_HEADER(float64_t)
379 WRITE_SPARSE_MATRIX_HEADER(floatmax_t)
380 WRITE_SPARSE_MATRIX_HEADER(int16_t)
381 WRITE_SPARSE_MATRIX_HEADER(uint16_t)
382 #undef WRITE_SPARSE_MATRIX_HEADER
383 
384 #define WRITE_STRING_LIST_HEADER(sg_type) \
385 void CProtobufFile::write_string_list_header(const SGString<sg_type>* strings, int32_t num_str) \
386 { \
387  int32_t max_string_len=0; \
388  StringListHeader data_header; \
389  data_header.set_num_str(num_str); \
390  for (int32_t i=0; i<num_str; i++) \
391  { \
392  data_header.add_str_len(strings[i].slen); \
393  if (strings[i].slen>max_string_len) \
394  max_string_len=strings[i].slen; \
395  } \
396  data_header.set_max_string_len(max_string_len); \
397  write_message(data_header); \
398 }
399 
400 WRITE_STRING_LIST_HEADER(int8_t)
401 WRITE_STRING_LIST_HEADER(uint8_t)
402 WRITE_STRING_LIST_HEADER(char)
403 WRITE_STRING_LIST_HEADER(int32_t)
404 WRITE_STRING_LIST_HEADER(uint32_t)
405 WRITE_STRING_LIST_HEADER(int64_t)
406 WRITE_STRING_LIST_HEADER(uint64_t)
407 WRITE_STRING_LIST_HEADER(float32_t)
408 WRITE_STRING_LIST_HEADER(float64_t)
409 WRITE_STRING_LIST_HEADER(floatmax_t)
410 WRITE_STRING_LIST_HEADER(int16_t)
411 WRITE_STRING_LIST_HEADER(uint16_t)
412 #undef WRITE_STRING_LIST_HEADER
413 
414 void CProtobufFile::read_message(google::protobuf::Message& message)
415 {
416  uint32_t bytes_read=0;
417  uint32_t msg_size=0;
418 
419  // read size of message
420  bytes_read=fread(uint_buffer, sizeof(char), sizeof(uint32_t), file);
421  REQUIRE(bytes_read==sizeof(uint32_t), "IO error\n");
422  msg_size=read_big_endian_uint(uint_buffer, sizeof(uint32_t));
423  REQUIRE(msg_size>0, "message size should be more than zero\n");
424 
425  // read message
426  bytes_read=fread(buffer, sizeof(char), msg_size, file);
427  REQUIRE(bytes_read==msg_size, "IO error\n");
428 
429  // try to parse message from read data
430  REQUIRE(message.ParseFromArray(buffer, msg_size), "cannot parse header\n");
431 }
432 
433 void CProtobufFile::write_message(const google::protobuf::Message& message)
434 {
435  uint32_t bytes_write=0;
436  uint32_t msg_size=message.ByteSize();
437 
438  // write size of message
439  write_big_endian_uint(msg_size, uint_buffer, sizeof(uint32_t));
440  bytes_write=fwrite(uint_buffer, sizeof(char), sizeof(uint32_t), file);
441  REQUIRE(bytes_write==sizeof(uint32_t), "IO error\n");
442 
443  // write serialized message
444  message.SerializeToArray(buffer, msg_size);
445  bytes_write=fwrite(buffer, sizeof(char), msg_size, file);
446  REQUIRE(bytes_write==msg_size, "IO error\n");
447 }
448 
449 #define READ_MEMORY_BLOCK(chunk_type, sg_type) \
450 void CProtobufFile::read_memory_block(sg_type*& vector, uint64_t len, int32_t num_messages) \
451 { \
452  vector=SG_MALLOC(sg_type, len); \
453  \
454  chunk_type chunk; \
455  int32_t elements_in_message=message_size/sizeof(sg_type); \
456  for (int32_t i=0; i<num_messages; i++) \
457  { \
458  read_message(chunk); \
459  \
460  int32_t num_elements_to_read=0; \
461  if ((len-(i+1)*elements_in_message)<=0) \
462  num_elements_to_read=len-i*elements_in_message; \
463  else \
464  num_elements_to_read=elements_in_message; \
465  \
466  for (int32_t j=0; j<num_elements_to_read; j++) \
467  vector[j+i*elements_in_message]=chunk.data(j); \
468  } \
469 }
470 
471 READ_MEMORY_BLOCK(Int32Chunk, int8_t)
472 READ_MEMORY_BLOCK(UInt32Chunk, uint8_t)
473 READ_MEMORY_BLOCK(UInt32Chunk, char)
474 READ_MEMORY_BLOCK(Int32Chunk, int32_t)
475 READ_MEMORY_BLOCK(UInt32Chunk, uint32_t)
476 READ_MEMORY_BLOCK(Float32Chunk, float32_t)
477 READ_MEMORY_BLOCK(Float64Chunk, float64_t)
478 READ_MEMORY_BLOCK(Float64Chunk, floatmax_t)
479 READ_MEMORY_BLOCK(Int32Chunk, int16_t)
480 READ_MEMORY_BLOCK(UInt32Chunk, uint16_t)
481 READ_MEMORY_BLOCK(Int64Chunk, int64_t)
482 READ_MEMORY_BLOCK(UInt64Chunk, uint64_t)
483 #undef READ_MEMORY_BLOCK
484 
485 #define WRITE_MEMORY_BLOCK(chunk_type, sg_type) \
486 void CProtobufFile::write_memory_block(const sg_type* vector, uint64_t len, int32_t num_messages) \
487 { \
488  chunk_type chunk; \
489  int32_t elements_in_message=message_size/sizeof(sg_type); \
490  for (int32_t i=0; i<num_messages; i++) \
491  { \
492  \
493  int32_t num_elements_to_write=0; \
494  if ((len-(i+1)*elements_in_message)<=0) \
495  num_elements_to_write=len-i*elements_in_message; \
496  else \
497  num_elements_to_write=elements_in_message; \
498  \
499  for (int32_t j=0; j<num_elements_to_write; j++) \
500  chunk.add_data(vector[j+i*elements_in_message]); \
501  \
502  write_message(chunk); \
503  chunk.Clear(); \
504  } \
505 }
506 
507 WRITE_MEMORY_BLOCK(Int32Chunk, int8_t)
508 WRITE_MEMORY_BLOCK(UInt32Chunk, uint8_t)
509 WRITE_MEMORY_BLOCK(UInt32Chunk, char)
510 WRITE_MEMORY_BLOCK(Int32Chunk, int32_t)
511 WRITE_MEMORY_BLOCK(UInt64Chunk, uint32_t)
512 WRITE_MEMORY_BLOCK(Int64Chunk, int64_t)
513 WRITE_MEMORY_BLOCK(UInt64Chunk, uint64_t)
514 WRITE_MEMORY_BLOCK(Float32Chunk, float32_t)
515 WRITE_MEMORY_BLOCK(Float64Chunk, float64_t)
516 WRITE_MEMORY_BLOCK(Float64Chunk, floatmax_t)
517 WRITE_MEMORY_BLOCK(Int32Chunk, int16_t)
518 WRITE_MEMORY_BLOCK(UInt32Chunk, uint16_t)
519 #undef WRITE_MEMORY_BLOCK
520 
521 #define READ_SPARSE_MATRIX(chunk_type, sg_type) \
522 void CProtobufFile::read_sparse_matrix( \
523  SGSparseVector<sg_type>*& matrix, const SparseMatrixHeader& data_header) \
524 { \
525  matrix=SG_MALLOC(SGSparseVector<sg_type>, data_header.num_vectors()); \
526  \
527  UInt64Chunk feat_index_chunk; \
528  chunk_type entry_chunk; \
529  read_message(feat_index_chunk); \
530  read_message(entry_chunk); \
531  \
532  int32_t elements_in_message=message_size/sizeof(sg_type); \
533  int32_t buffer_counter=0; \
534  for (uint32_t i=0; i<data_header.num_vectors(); i++) \
535  { \
536  matrix[i]=SGSparseVector<sg_type>(data_header.num_feat_entries(i)); \
537  for (int32_t j=0; j<matrix[i].num_feat_entries; j++) \
538  { \
539  matrix[i].features[j].feat_index=feat_index_chunk.data(buffer_counter); \
540  matrix[i].features[j].entry=entry_chunk.data(buffer_counter); \
541  buffer_counter++; \
542  \
543  if (buffer_counter==elements_in_message) \
544  { \
545  read_message(feat_index_chunk); \
546  read_message(entry_chunk); \
547  buffer_counter=0; \
548  } \
549  } \
550  } \
551 }
552 
553 READ_SPARSE_MATRIX(BoolChunk, bool)
554 READ_SPARSE_MATRIX(Int32Chunk, int8_t)
555 READ_SPARSE_MATRIX(UInt32Chunk, uint8_t)
556 READ_SPARSE_MATRIX(UInt32Chunk, char)
557 READ_SPARSE_MATRIX(Int32Chunk, int32_t)
558 READ_SPARSE_MATRIX(UInt32Chunk, uint32_t)
559 READ_SPARSE_MATRIX(Float32Chunk, float32_t)
560 READ_SPARSE_MATRIX(Float64Chunk, float64_t)
561 READ_SPARSE_MATRIX(Float64Chunk, floatmax_t)
562 READ_SPARSE_MATRIX(Int32Chunk, int16_t)
563 READ_SPARSE_MATRIX(UInt32Chunk, uint16_t)
564 READ_SPARSE_MATRIX(Int64Chunk, int64_t)
565 READ_SPARSE_MATRIX(UInt64Chunk, uint64_t)
566 #undef READ_SPARSE_MATRIX
567 
568 #define WRITE_SPARSE_MATRIX(chunk_type, sg_type) \
569 void CProtobufFile::write_sparse_matrix( \
570  const SGSparseVector<sg_type>* matrix, int32_t num_vec) \
571 { \
572  UInt64Chunk feat_index_chunk; \
573  chunk_type entry_chunk; \
574  int32_t elements_in_message=message_size/sizeof(sg_type); \
575  int32_t buffer_counter=0; \
576  for (int32_t i=0; i<num_vec; i++) \
577  { \
578  for (int32_t j=0; j<matrix[i].num_feat_entries; j++) \
579  { \
580  feat_index_chunk.add_data(matrix[i].features[j].feat_index); \
581  entry_chunk.add_data(matrix[i].features[j].entry); \
582  buffer_counter++; \
583  \
584  if (buffer_counter==elements_in_message) \
585  { \
586  write_message(feat_index_chunk); \
587  write_message(entry_chunk); \
588  feat_index_chunk.Clear(); \
589  entry_chunk.Clear(); \
590  buffer_counter=0; \
591  } \
592  } \
593  } \
594  \
595  if (buffer_counter!=0) \
596  { \
597  write_message(feat_index_chunk); \
598  write_message(entry_chunk); \
599  } \
600 }
601 
602 WRITE_SPARSE_MATRIX(BoolChunk, bool)
603 WRITE_SPARSE_MATRIX(Int32Chunk, int8_t)
604 WRITE_SPARSE_MATRIX(UInt32Chunk, uint8_t)
605 WRITE_SPARSE_MATRIX(UInt32Chunk, char)
606 WRITE_SPARSE_MATRIX(Int32Chunk, int32_t)
607 WRITE_SPARSE_MATRIX(UInt64Chunk, uint32_t)
608 WRITE_SPARSE_MATRIX(Int64Chunk, int64_t)
609 WRITE_SPARSE_MATRIX(UInt64Chunk, uint64_t)
610 WRITE_SPARSE_MATRIX(Float32Chunk, float32_t)
611 WRITE_SPARSE_MATRIX(Float64Chunk, float64_t)
612 WRITE_SPARSE_MATRIX(Float64Chunk, floatmax_t)
613 WRITE_SPARSE_MATRIX(Int32Chunk, int16_t)
614 WRITE_SPARSE_MATRIX(UInt32Chunk, uint16_t)
615 #undef WRITE_SPARSE_MATRIX
616 
617 #define READ_STRING_LIST(chunk_type, sg_type) \
618 void CProtobufFile::read_string_list( \
619  SGString<sg_type>*& strings, const StringListHeader& data_header) \
620 { \
621  strings=SG_MALLOC(SGString<sg_type>, data_header.num_str()); \
622  \
623  chunk_type chunk; \
624  read_message(chunk); \
625  int32_t elements_in_message=message_size/sizeof(sg_type); \
626  int32_t buffer_counter=0; \
627  for (uint32_t i=0; i<data_header.num_str(); i++) \
628  { \
629  strings[i]=SGString<sg_type>(data_header.str_len(i)); \
630  for (int32_t j=0; j<strings[i].slen; j++) \
631  { \
632  strings[i].string[j]=chunk.data(buffer_counter); \
633  buffer_counter++; \
634  \
635  if (buffer_counter==elements_in_message) \
636  { \
637  read_message(chunk); \
638  buffer_counter=0; \
639  } \
640  } \
641  } \
642 }
643 
644 READ_STRING_LIST(Int32Chunk, int8_t)
645 READ_STRING_LIST(UInt32Chunk, uint8_t)
646 READ_STRING_LIST(UInt32Chunk, char)
647 READ_STRING_LIST(Int32Chunk, int32_t)
648 READ_STRING_LIST(UInt32Chunk, uint32_t)
649 READ_STRING_LIST(Float32Chunk, float32_t)
650 READ_STRING_LIST(Float64Chunk, float64_t)
651 READ_STRING_LIST(Float64Chunk, floatmax_t)
652 READ_STRING_LIST(Int32Chunk, int16_t)
653 READ_STRING_LIST(UInt32Chunk, uint16_t)
654 READ_STRING_LIST(Int64Chunk, int64_t)
655 READ_STRING_LIST(UInt64Chunk, uint64_t)
656 #undef READ_STRING_LIST
657 
658 #define WRITE_STRING_LIST(chunk_type, sg_type) \
659 void CProtobufFile::write_string_list( \
660  const SGString<sg_type>* strings, int32_t num_str) \
661 { \
662  chunk_type chunk; \
663  int32_t elements_in_message=message_size/sizeof(sg_type); \
664  int32_t buffer_counter=0; \
665  for (int32_t i=0; i<num_str; i++) \
666  { \
667  for (int32_t j=0; j<strings[i].slen; j++) \
668  { \
669  chunk.add_data(strings[i].string[j]); \
670  buffer_counter++; \
671  \
672  if (buffer_counter==elements_in_message) \
673  { \
674  write_message(chunk); \
675  chunk.Clear(); \
676  buffer_counter=0; \
677  } \
678  } \
679  } \
680  \
681  if (buffer_counter!=0) \
682  write_message(chunk); \
683 }
684 
685 WRITE_STRING_LIST(Int32Chunk, int8_t)
686 WRITE_STRING_LIST(UInt32Chunk, uint8_t)
687 WRITE_STRING_LIST(UInt32Chunk, char)
688 WRITE_STRING_LIST(Int32Chunk, int32_t)
689 WRITE_STRING_LIST(UInt64Chunk, uint32_t)
690 WRITE_STRING_LIST(Int64Chunk, int64_t)
691 WRITE_STRING_LIST(UInt64Chunk, uint64_t)
692 WRITE_STRING_LIST(Float32Chunk, float32_t)
693 WRITE_STRING_LIST(Float64Chunk, float64_t)
694 WRITE_STRING_LIST(Float64Chunk, floatmax_t)
695 WRITE_STRING_LIST(Int32Chunk, int16_t)
696 WRITE_STRING_LIST(UInt32Chunk, uint16_t)
697 #undef WRITE_STRING_LIST
698 
699 #endif /* HAVE_PROTOBUF */
SET_MATRIX
#define SET_MATRIX(sg_type)
Definition: ProtobufFile.cpp:164
GET_MATRIX
#define GET_MATRIX(read_func, sg_type)
Definition: ProtobufFile.cpp:76
GET_STRING_LIST
#define GET_STRING_LIST(sg_type)
Definition: ProtobufFile.cpp:211
SET_STRING_LIST
#define SET_STRING_LIST(sg_type)
Definition: ProtobufFile.cpp:237
SG_ERROR
#define SG_ERROR(...)
Definition: SGIO.h:129
shogun::CFile::file
FILE * file
Definition: File.h:505
REQUIRE
#define REQUIRE(x,...)
Definition: SGIO.h:206
GET_NDARRAY
#define GET_NDARRAY(read_func, sg_type)
Definition: ProtobufFile.cpp:100
GET_SPARSE_MATRIX
#define GET_SPARSE_MATRIX(sg_type)
Definition: ProtobufFile.cpp:115
GET_VECTOR
#define GET_VECTOR(sg_type)
Definition: ProtobufFile.cpp:53
READ_STRING_LIST
#define READ_STRING_LIST(chunk_type, sg_type)
Definition: ProtobufFile.cpp:617
WRITE_STRING_LIST
#define WRITE_STRING_LIST(chunk_type, sg_type)
Definition: ProtobufFile.cpp:658
SET_VECTOR
#define SET_VECTOR(sg_type)
Definition: ProtobufFile.cpp:141
float64_t
double float64_t
Definition: common.h:50
floatmax_t
long double floatmax_t
Definition: common.h:51
shogun::CFile
A File access base class.
Definition: File.h:34
WRITE_SPARSE_MATRIX
#define WRITE_SPARSE_MATRIX(chunk_type, sg_type)
Definition: ProtobufFile.cpp:568
float32_t
float float32_t
Definition: common.h:49
shogun
all of classes and functions are contained in the shogun namespace
Definition: class_list.h:18
WRITE_SPARSE_MATRIX_HEADER
#define WRITE_SPARSE_MATRIX_HEADER(sg_type)
Definition: ProtobufFile.cpp:354
WRITE_STRING_LIST_HEADER
#define WRITE_STRING_LIST_HEADER(sg_type)
Definition: ProtobufFile.cpp:384
WRITE_MEMORY_BLOCK
#define WRITE_MEMORY_BLOCK(chunk_type, sg_type)
Definition: ProtobufFile.cpp:485
READ_SPARSE_MATRIX
#define READ_SPARSE_MATRIX(chunk_type, sg_type)
Definition: ProtobufFile.cpp:521
READ_MEMORY_BLOCK
#define READ_MEMORY_BLOCK(chunk_type, sg_type)
Definition: ProtobufFile.cpp:449
SET_SPARSE_MATRIX
#define SET_SPARSE_MATRIX(sg_type)
Definition: ProtobufFile.cpp:187
SHOGUN Machine Learning Toolbox - Documentation