SHOGUN  v3.0.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ProtobufFile.cpp
Go to the documentation of this file.
1 /*
2  * This program is free software; you can redistribute it and/or modify
3  * it under the terms of the GNU General Public License as published by
4  * the Free Software Foundation; either version 3 of the License, or
5  * (at your option) any later version.
6  *
7  * Written (W) 2013 Evgeniy Andreev (gsomix)
8  */
9 #ifdef HAVE_PROTOBUF
10 
11 #include <shogun/io/ProtobufFile.h>
12 
13 #include <shogun/lib/SGVector.h>
14 #include <shogun/lib/SGMatrix.h>
15 #include <shogun/lib/SGSparseVector.h>
16 #include <shogun/lib/SGString.h>
17 
18 using namespace shogun;
19 
20 CProtobufFile::CProtobufFile()
21 {
22  init();
23 }
24 
25 CProtobufFile::CProtobufFile(FILE* f, const char* name) :
26  CFile(f, name)
27 {
28  init();
29 }
30 
31 CProtobufFile::CProtobufFile(const char* fname, char rw, const char* name) :
32  CFile(fname, rw, name)
33 {
34  init();
35 }
36 
37 CProtobufFile::~CProtobufFile()
38 {
39  SG_FREE(buffer);
40 }
41 
42 void CProtobufFile::init()
43 {
44  version=1;
45  message_size=1024*1024;
46 
47  buffer=SG_MALLOC(uint8_t, message_size*sizeof(uint32_t));
48 }
49 
50 #define GET_VECTOR(sg_type) \
51 void CProtobufFile::get_vector(sg_type*& vector, int32_t& len) \
52 { \
53  read_and_validate_global_header(ShogunVersion::VECTOR); \
54  VectorHeader data_header=read_vector_header(); \
55  len=data_header.len(); \
56  read_memory_block(vector, len, data_header.num_messages()); \
57 }
58 
59 GET_VECTOR(int8_t)
60 GET_VECTOR(uint8_t)
61 GET_VECTOR(char)
62 GET_VECTOR(int32_t)
63 GET_VECTOR(uint32_t)
64 GET_VECTOR(float32_t)
65 GET_VECTOR(float64_t)
66 GET_VECTOR(floatmax_t)
67 GET_VECTOR(int16_t)
68 GET_VECTOR(uint16_t)
69 GET_VECTOR(int64_t)
70 GET_VECTOR(uint64_t)
71 #undef GET_VECTOR
72 
73 #define GET_MATRIX(read_func, sg_type) \
74 void CProtobufFile::get_matrix(sg_type*& matrix, int32_t& num_feat, int32_t& num_vec) \
75 { \
76  read_and_validate_global_header(ShogunVersion::MATRIX); \
77  MatrixHeader data_header=read_matrix_header(); \
78  num_feat=data_header.num_cols(); \
79  num_vec=data_header.num_rows(); \
80  read_memory_block(matrix, num_feat*num_vec, data_header.num_messages()); \
81 }
82 
83 GET_MATRIX(read_char, int8_t)
84 GET_MATRIX(read_byte, uint8_t)
85 GET_MATRIX(read_char, char)
86 GET_MATRIX(read_int, int32_t)
87 GET_MATRIX(read_uint, uint32_t)
88 GET_MATRIX(read_short_real, float32_t)
89 GET_MATRIX(read_real, float64_t)
90 GET_MATRIX(read_long_real, floatmax_t)
91 GET_MATRIX(read_short, int16_t)
92 GET_MATRIX(read_word, uint16_t)
93 GET_MATRIX(read_long, int64_t)
94 GET_MATRIX(read_ulong, uint64_t)
95 #undef GET_MATRIX
96 
97 #define GET_NDARRAY(read_func, sg_type) \
98 void CProtobufFile::get_ndarray(sg_type*& array, int32_t*& dims, int32_t& num_dims) \
99 { \
100  SG_NOTIMPLEMENTED \
101 }
102 
103 GET_NDARRAY(read_byte, uint8_t)
104 GET_NDARRAY(read_char, char)
105 GET_NDARRAY(read_int, int32_t)
106 GET_NDARRAY(read_short_real, float32_t)
107 GET_NDARRAY(read_real, float64_t)
108 GET_NDARRAY(read_short, int16_t)
109 GET_NDARRAY(read_word, uint16_t)
110 #undef GET_NDARRAY
111 
112 #define GET_SPARSE_MATRIX(sg_type) \
113 void CProtobufFile::get_sparse_matrix( \
114  SGSparseVector<sg_type>*& matrix, int32_t& num_feat, int32_t& num_vec) \
115 { \
116  read_and_validate_global_header(ShogunVersion::SPARSE_MATRIX); \
117  SparseMatrixHeader data_header=read_sparse_matrix_header(); \
118  num_feat=data_header.num_features(); \
119  num_vec=data_header.num_vectors(); \
120  read_sparse_matrix(matrix, data_header); \
121 }
122 
123 GET_SPARSE_MATRIX(bool)
124 GET_SPARSE_MATRIX(int8_t)
125 GET_SPARSE_MATRIX(uint8_t)
126 GET_SPARSE_MATRIX(char)
127 GET_SPARSE_MATRIX(int32_t)
128 GET_SPARSE_MATRIX(uint32_t)
129 GET_SPARSE_MATRIX(float32_t)
130 GET_SPARSE_MATRIX(float64_t)
131 GET_SPARSE_MATRIX(floatmax_t)
132 GET_SPARSE_MATRIX(int16_t)
133 GET_SPARSE_MATRIX(uint16_t)
134 GET_SPARSE_MATRIX(int64_t)
135 GET_SPARSE_MATRIX(uint64_t)
136 #undef GET_SPARSE_MATRIX
137 
138 #define SET_VECTOR(sg_type) \
139 void CProtobufFile::set_vector(const sg_type* vector, int32_t len) \
140 { \
141  int32_t num_messages=compute_num_messages(len, sizeof(sg_type)); \
142  write_global_header(ShogunVersion::VECTOR); \
143  write_vector_header(len, num_messages); \
144  write_memory_block(vector, len, num_messages); \
145 }
146 
147 SET_VECTOR(int8_t)
148 SET_VECTOR(uint8_t)
149 SET_VECTOR(char)
150 SET_VECTOR(int32_t)
151 SET_VECTOR(uint32_t)
152 SET_VECTOR(int64_t)
153 SET_VECTOR(uint64_t)
154 SET_VECTOR(float32_t)
155 SET_VECTOR(float64_t)
156 SET_VECTOR(floatmax_t)
157 SET_VECTOR(int16_t)
158 SET_VECTOR(uint16_t)
159 #undef SET_VECTOR
160 
161 #define SET_MATRIX(sg_type) \
162 void CProtobufFile::set_matrix(const sg_type* matrix, int32_t num_feat, int32_t num_vec) \
163 { \
164  int32_t num_messages=compute_num_messages(num_feat*num_vec, sizeof(sg_type)); \
165  write_global_header(ShogunVersion::MATRIX); \
166  write_matrix_header(num_feat, num_vec, num_messages); \
167  write_memory_block(matrix, num_feat*num_vec, num_messages); \
168 }
169 
170 SET_MATRIX(int8_t)
171 SET_MATRIX(uint8_t)
172 SET_MATRIX(char)
173 SET_MATRIX(int32_t)
174 SET_MATRIX(uint32_t)
175 SET_MATRIX(int64_t)
176 SET_MATRIX(uint64_t)
177 SET_MATRIX(float32_t)
178 SET_MATRIX(float64_t)
179 SET_MATRIX(floatmax_t)
180 SET_MATRIX(int16_t)
181 SET_MATRIX(uint16_t)
182 #undef SET_MATRIX
183 
184 #define SET_SPARSE_MATRIX(sg_type) \
185 void CProtobufFile::set_sparse_matrix( \
186  const SGSparseVector<sg_type>* matrix, int32_t num_feat, int32_t num_vec) \
187 { \
188  write_global_header(ShogunVersion::SPARSE_MATRIX); \
189  write_sparse_matrix_header(matrix, num_feat, num_vec); \
190  write_sparse_matrix(matrix, num_vec); \
191 }
192 
193 SET_SPARSE_MATRIX(bool)
194 SET_SPARSE_MATRIX(int8_t)
195 SET_SPARSE_MATRIX(uint8_t)
196 SET_SPARSE_MATRIX(char)
197 SET_SPARSE_MATRIX(int32_t)
198 SET_SPARSE_MATRIX(uint32_t)
199 SET_SPARSE_MATRIX(int64_t)
200 SET_SPARSE_MATRIX(uint64_t)
201 SET_SPARSE_MATRIX(float32_t)
202 SET_SPARSE_MATRIX(float64_t)
203 SET_SPARSE_MATRIX(floatmax_t)
204 SET_SPARSE_MATRIX(int16_t)
205 SET_SPARSE_MATRIX(uint16_t)
206 #undef SET_SPARSE_MATRIX
207 
208 #define GET_STRING_LIST(sg_type) \
209 void CProtobufFile::get_string_list( \
210  SGString<sg_type>*& strings, int32_t& num_str, \
211  int32_t& max_string_len) \
212 { \
213  read_and_validate_global_header(ShogunVersion::STRING_LIST); \
214  StringListHeader data_header=read_string_list_header(); \
215  num_str=data_header.num_str(); \
216  max_string_len=data_header.max_string_len(); \
217  read_string_list(strings, data_header); \
218 }
219 
220 GET_STRING_LIST(int8_t)
221 GET_STRING_LIST(uint8_t)
222 GET_STRING_LIST(char)
223 GET_STRING_LIST(int32_t)
224 GET_STRING_LIST(uint32_t)
225 GET_STRING_LIST(int64_t)
226 GET_STRING_LIST(uint64_t)
227 GET_STRING_LIST(float32_t)
228 GET_STRING_LIST(float64_t)
229 GET_STRING_LIST(floatmax_t)
230 GET_STRING_LIST(int16_t)
231 GET_STRING_LIST(uint16_t)
232 #undef GET_STRING_LIST
233 
234 #define SET_STRING_LIST(sg_type) \
235 void CProtobufFile::set_string_list( \
236  const SGString<sg_type>* strings, int32_t num_str) \
237 { \
238  write_global_header(ShogunVersion::STRING_LIST); \
239  write_string_list_header(strings, num_str); \
240  write_string_list(strings, num_str); \
241 }
242 
243 SET_STRING_LIST(int8_t)
244 SET_STRING_LIST(uint8_t)
245 SET_STRING_LIST(char)
246 SET_STRING_LIST(int32_t)
247 SET_STRING_LIST(uint32_t)
248 SET_STRING_LIST(int64_t)
249 SET_STRING_LIST(uint64_t)
250 SET_STRING_LIST(float32_t)
251 SET_STRING_LIST(float64_t)
252 SET_STRING_LIST(floatmax_t)
253 SET_STRING_LIST(int16_t)
254 SET_STRING_LIST(uint16_t)
255 #undef SET_STRING_LIST
256 
257 void CProtobufFile::write_big_endian_uint(uint32_t number, uint8_t* array, uint32_t size)
258 {
259  if (size<4)
260  SG_ERROR("array is too small to write\n");
261 
262  array[0]=(number>>24)&0xffu;
263  array[1]=(number>>16)&0xffu;
264  array[2]=(number>>8)&0xffu;
265  array[3]=number&0xffu;
266 }
267 
268 uint32_t CProtobufFile::read_big_endian_uint(uint8_t* array, uint32_t size)
269 {
270  if (size<4)
271  SG_ERROR("array is too small to read\n");
272 
273  return (array[0]<<24) | (array[1]<<16) | (array[2]<<8) | array[3];
274 }
275 
276 int32_t CProtobufFile::compute_num_messages(uint64_t len, int32_t sizeof_type) const
277 {
278  uint32_t elements_in_message=message_size/sizeof_type;
279  uint32_t num_messages=len/elements_in_message;
280  if (len % elements_in_message > 0)
281  num_messages++;
282 
283  return num_messages;
284 }
285 
286 void CProtobufFile::read_and_validate_global_header(ShogunVersion_SGDataType type)
287 {
288  ShogunVersion header;
289  read_message(header);
290  REQUIRE(header.version()==version, "wrong version\n")
291  REQUIRE(header.data_type()==type, "wrong type\n")
292 }
293 
294 void CProtobufFile::write_global_header(ShogunVersion_SGDataType type)
295 {
296  ShogunVersion header;
297  header.set_version(version);
298  header.set_data_type(type);
299  write_message(header);
300 }
301 
302 VectorHeader CProtobufFile::read_vector_header()
303 {
304  VectorHeader data_header;
305  read_message(data_header);
306 
307  return data_header;
308 }
309 
310 SparseMatrixHeader CProtobufFile::read_sparse_matrix_header()
311 {
312  SparseMatrixHeader data_header;
313  read_message(data_header);
314 
315  return data_header;
316 }
317 
318 MatrixHeader CProtobufFile::read_matrix_header()
319 {
320  MatrixHeader data_header;
321  read_message(data_header);
322 
323  return data_header;
324 }
325 
326 StringListHeader CProtobufFile::read_string_list_header()
327 {
328  StringListHeader data_header;
329  read_message(data_header);
330 
331  return data_header;
332 }
333 
334 void CProtobufFile::write_vector_header(int32_t len, int32_t num_messages)
335 {
336  VectorHeader data_header;
337  data_header.set_len(len);
338  data_header.set_num_messages(num_messages);
339  write_message(data_header);
340 }
341 
342 void CProtobufFile::write_matrix_header(int32_t num_feat, int32_t num_vec, int32_t num_messages)
343 {
344  MatrixHeader data_header;
345  data_header.set_num_cols(num_feat);
346  data_header.set_num_rows(num_vec);
347  data_header.set_num_messages(num_messages);
348  write_message(data_header);
349 }
350 
351 #define WRITE_SPARSE_MATRIX_HEADER(sg_type) \
352 void CProtobufFile::write_sparse_matrix_header( \
353  const SGSparseVector<sg_type>* matrix, int32_t num_feat, int32_t num_vec) \
354 { \
355  SparseMatrixHeader data_header; \
356  data_header.set_num_features(num_feat); \
357  data_header.set_num_vectors(num_vec); \
358  for (int32_t i=0; i<num_vec; i++) \
359  { \
360  data_header.add_num_feat_entries(matrix[i].num_feat_entries); \
361  } \
362  \
363  write_message(data_header); \
364 }
365 
366 WRITE_SPARSE_MATRIX_HEADER(bool)
367 WRITE_SPARSE_MATRIX_HEADER(int8_t)
368 WRITE_SPARSE_MATRIX_HEADER(uint8_t)
369 WRITE_SPARSE_MATRIX_HEADER(char)
370 WRITE_SPARSE_MATRIX_HEADER(int32_t)
371 WRITE_SPARSE_MATRIX_HEADER(uint32_t)
372 WRITE_SPARSE_MATRIX_HEADER(int64_t)
373 WRITE_SPARSE_MATRIX_HEADER(uint64_t)
374 WRITE_SPARSE_MATRIX_HEADER(float32_t)
375 WRITE_SPARSE_MATRIX_HEADER(float64_t)
376 WRITE_SPARSE_MATRIX_HEADER(floatmax_t)
377 WRITE_SPARSE_MATRIX_HEADER(int16_t)
378 WRITE_SPARSE_MATRIX_HEADER(uint16_t)
379 #undef WRITE_SPARSE_MATRIX_HEADER
380 
381 #define WRITE_STRING_LIST_HEADER(sg_type) \
382 void CProtobufFile::write_string_list_header(const SGString<sg_type>* strings, int32_t num_str) \
383 { \
384  int32_t max_string_len=0; \
385  StringListHeader data_header; \
386  data_header.set_num_str(num_str); \
387  for (int32_t i=0; i<num_str; i++) \
388  { \
389  data_header.add_str_len(strings[i].slen); \
390  if (strings[i].slen>max_string_len) \
391  max_string_len=strings[i].slen; \
392  } \
393  data_header.set_max_string_len(max_string_len); \
394  write_message(data_header); \
395 }
396 
397 WRITE_STRING_LIST_HEADER(int8_t)
398 WRITE_STRING_LIST_HEADER(uint8_t)
399 WRITE_STRING_LIST_HEADER(char)
400 WRITE_STRING_LIST_HEADER(int32_t)
401 WRITE_STRING_LIST_HEADER(uint32_t)
402 WRITE_STRING_LIST_HEADER(int64_t)
403 WRITE_STRING_LIST_HEADER(uint64_t)
404 WRITE_STRING_LIST_HEADER(float32_t)
405 WRITE_STRING_LIST_HEADER(float64_t)
406 WRITE_STRING_LIST_HEADER(floatmax_t)
407 WRITE_STRING_LIST_HEADER(int16_t)
408 WRITE_STRING_LIST_HEADER(uint16_t)
409 #undef WRITE_STRING_LIST_HEADER
410 
411 void CProtobufFile::read_message(google::protobuf::Message& message)
412 {
413  uint32_t bytes_read=0;
414  uint32_t msg_size=0;
415 
416  // read size of message
417  bytes_read=fread(uint_buffer, sizeof(char), sizeof(uint32_t), file);
418  REQUIRE(bytes_read==sizeof(uint32_t), "IO error\n");
419  msg_size=read_big_endian_uint(uint_buffer, sizeof(uint32_t));
420  REQUIRE(msg_size>0, "message size should be more than zero\n");
421 
422  // read message
423  bytes_read=fread(buffer, sizeof(char), msg_size, file);
424  REQUIRE(bytes_read==msg_size, "IO error\n");
425 
426  // try to parse message from read data
427  REQUIRE(message.ParseFromArray(buffer, msg_size), "cannot parse header\n");
428 }
429 
430 void CProtobufFile::write_message(const google::protobuf::Message& message)
431 {
432  uint32_t bytes_write=0;
433  uint32_t msg_size=message.ByteSize();
434 
435  // write size of message
436  write_big_endian_uint(msg_size, uint_buffer, sizeof(uint32_t));
437  bytes_write=fwrite(uint_buffer, sizeof(char), sizeof(uint32_t), file);
438  REQUIRE(bytes_write==sizeof(uint32_t), "IO error\n");
439 
440  // write serialized message
441  message.SerializeToArray(buffer, msg_size);
442  bytes_write=fwrite(buffer, sizeof(char), msg_size, file);
443  REQUIRE(bytes_write==msg_size, "IO error\n");
444 }
445 
446 #define READ_MEMORY_BLOCK(chunk_type, sg_type) \
447 void CProtobufFile::read_memory_block(sg_type*& vector, uint64_t len, int32_t num_messages) \
448 { \
449  vector=SG_MALLOC(sg_type, len); \
450  \
451  chunk_type chunk; \
452  int32_t elements_in_message=message_size/sizeof(sg_type); \
453  for (int32_t i=0; i<num_messages; i++) \
454  { \
455  read_message(chunk); \
456  \
457  int32_t num_elements_to_read=0; \
458  if ((len-(i+1)*elements_in_message)<=0) \
459  num_elements_to_read=len-i*elements_in_message; \
460  else \
461  num_elements_to_read=elements_in_message; \
462  \
463  for (int32_t j=0; j<num_elements_to_read; j++) \
464  vector[j+i*elements_in_message]=chunk.data(j); \
465  } \
466 }
467 
468 READ_MEMORY_BLOCK(Int32Chunk, int8_t)
469 READ_MEMORY_BLOCK(UInt32Chunk, uint8_t)
470 READ_MEMORY_BLOCK(UInt32Chunk, char)
471 READ_MEMORY_BLOCK(Int32Chunk, int32_t)
472 READ_MEMORY_BLOCK(UInt32Chunk, uint32_t)
473 READ_MEMORY_BLOCK(Float32Chunk, float32_t)
474 READ_MEMORY_BLOCK(Float64Chunk, float64_t)
475 READ_MEMORY_BLOCK(Float64Chunk, floatmax_t)
476 READ_MEMORY_BLOCK(Int32Chunk, int16_t)
477 READ_MEMORY_BLOCK(UInt32Chunk, uint16_t)
478 READ_MEMORY_BLOCK(Int64Chunk, int64_t)
479 READ_MEMORY_BLOCK(UInt64Chunk, uint64_t)
480 #undef READ_MEMORY_BLOCK
481 
482 #define WRITE_MEMORY_BLOCK(chunk_type, sg_type) \
483 void CProtobufFile::write_memory_block(const sg_type* vector, uint64_t len, int32_t num_messages) \
484 { \
485  chunk_type chunk; \
486  int32_t elements_in_message=message_size/sizeof(sg_type); \
487  for (int32_t i=0; i<num_messages; i++) \
488  { \
489  \
490  int32_t num_elements_to_write=0; \
491  if ((len-(i+1)*elements_in_message)<=0) \
492  num_elements_to_write=len-i*elements_in_message; \
493  else \
494  num_elements_to_write=elements_in_message; \
495  \
496  for (int32_t j=0; j<num_elements_to_write; j++) \
497  chunk.add_data(vector[j+i*elements_in_message]); \
498  \
499  write_message(chunk); \
500  chunk.Clear(); \
501  } \
502 }
503 
504 WRITE_MEMORY_BLOCK(Int32Chunk, int8_t)
505 WRITE_MEMORY_BLOCK(UInt32Chunk, uint8_t)
506 WRITE_MEMORY_BLOCK(UInt32Chunk, char)
507 WRITE_MEMORY_BLOCK(Int32Chunk, int32_t)
508 WRITE_MEMORY_BLOCK(UInt64Chunk, uint32_t)
509 WRITE_MEMORY_BLOCK(Int64Chunk, int64_t)
510 WRITE_MEMORY_BLOCK(UInt64Chunk, uint64_t)
511 WRITE_MEMORY_BLOCK(Float32Chunk, float32_t)
512 WRITE_MEMORY_BLOCK(Float64Chunk, float64_t)
513 WRITE_MEMORY_BLOCK(Float64Chunk, floatmax_t)
514 WRITE_MEMORY_BLOCK(Int32Chunk, int16_t)
515 WRITE_MEMORY_BLOCK(UInt32Chunk, uint16_t)
516 #undef WRITE_MEMORY_BLOCK
517 
518 #define READ_SPARSE_MATRIX(chunk_type, sg_type) \
519 void CProtobufFile::read_sparse_matrix( \
520  SGSparseVector<sg_type>*& matrix, const SparseMatrixHeader& data_header) \
521 { \
522  matrix=SG_MALLOC(SGSparseVector<sg_type>, data_header.num_vectors()); \
523  \
524  UInt64Chunk feat_index_chunk; \
525  chunk_type entry_chunk; \
526  read_message(feat_index_chunk); \
527  read_message(entry_chunk); \
528  \
529  int32_t elements_in_message=message_size/sizeof(sg_type); \
530  int32_t buffer_counter=0; \
531  for (uint32_t i=0; i<data_header.num_vectors(); i++) \
532  { \
533  matrix[i]=SGSparseVector<sg_type>(data_header.num_feat_entries(i)); \
534  for (int32_t j=0; j<matrix[i].num_feat_entries; j++) \
535  { \
536  matrix[i].features[j].feat_index=feat_index_chunk.data(buffer_counter); \
537  matrix[i].features[j].entry=entry_chunk.data(buffer_counter); \
538  buffer_counter++; \
539  \
540  if (buffer_counter==elements_in_message) \
541  { \
542  read_message(feat_index_chunk); \
543  read_message(entry_chunk); \
544  buffer_counter=0; \
545  } \
546  } \
547  } \
548 }
549 
550 READ_SPARSE_MATRIX(BoolChunk, bool)
551 READ_SPARSE_MATRIX(Int32Chunk, int8_t)
552 READ_SPARSE_MATRIX(UInt32Chunk, uint8_t)
553 READ_SPARSE_MATRIX(UInt32Chunk, char)
554 READ_SPARSE_MATRIX(Int32Chunk, int32_t)
555 READ_SPARSE_MATRIX(UInt32Chunk, uint32_t)
556 READ_SPARSE_MATRIX(Float32Chunk, float32_t)
557 READ_SPARSE_MATRIX(Float64Chunk, float64_t)
558 READ_SPARSE_MATRIX(Float64Chunk, floatmax_t)
559 READ_SPARSE_MATRIX(Int32Chunk, int16_t)
560 READ_SPARSE_MATRIX(UInt32Chunk, uint16_t)
561 READ_SPARSE_MATRIX(Int64Chunk, int64_t)
562 READ_SPARSE_MATRIX(UInt64Chunk, uint64_t)
563 #undef READ_SPARSE_MATRIX
564 
565 #define WRITE_SPARSE_MATRIX(chunk_type, sg_type) \
566 void CProtobufFile::write_sparse_matrix( \
567  const SGSparseVector<sg_type>* matrix, int32_t num_vec) \
568 { \
569  UInt64Chunk feat_index_chunk; \
570  chunk_type entry_chunk; \
571  int32_t elements_in_message=message_size/sizeof(sg_type); \
572  int32_t buffer_counter=0; \
573  for (int32_t i=0; i<num_vec; i++) \
574  { \
575  for (int32_t j=0; j<matrix[i].num_feat_entries; j++) \
576  { \
577  feat_index_chunk.add_data(matrix[i].features[j].feat_index); \
578  entry_chunk.add_data(matrix[i].features[j].entry); \
579  buffer_counter++; \
580  \
581  if (buffer_counter==elements_in_message) \
582  { \
583  write_message(feat_index_chunk); \
584  write_message(entry_chunk); \
585  feat_index_chunk.Clear(); \
586  entry_chunk.Clear(); \
587  buffer_counter=0; \
588  } \
589  } \
590  } \
591  \
592  if (buffer_counter!=0) \
593  { \
594  write_message(feat_index_chunk); \
595  write_message(entry_chunk); \
596  } \
597 }
598 
599 WRITE_SPARSE_MATRIX(BoolChunk, bool)
600 WRITE_SPARSE_MATRIX(Int32Chunk, int8_t)
601 WRITE_SPARSE_MATRIX(UInt32Chunk, uint8_t)
602 WRITE_SPARSE_MATRIX(UInt32Chunk, char)
603 WRITE_SPARSE_MATRIX(Int32Chunk, int32_t)
604 WRITE_SPARSE_MATRIX(UInt64Chunk, uint32_t)
605 WRITE_SPARSE_MATRIX(Int64Chunk, int64_t)
606 WRITE_SPARSE_MATRIX(UInt64Chunk, uint64_t)
607 WRITE_SPARSE_MATRIX(Float32Chunk, float32_t)
608 WRITE_SPARSE_MATRIX(Float64Chunk, float64_t)
609 WRITE_SPARSE_MATRIX(Float64Chunk, floatmax_t)
610 WRITE_SPARSE_MATRIX(Int32Chunk, int16_t)
611 WRITE_SPARSE_MATRIX(UInt32Chunk, uint16_t)
612 #undef WRITE_SPARSE_MATRIX
613 
614 #define READ_STRING_LIST(chunk_type, sg_type) \
615 void CProtobufFile::read_string_list( \
616  SGString<sg_type>*& strings, const StringListHeader& data_header) \
617 { \
618  strings=SG_MALLOC(SGString<sg_type>, data_header.num_str()); \
619  \
620  chunk_type chunk; \
621  read_message(chunk); \
622  int32_t elements_in_message=message_size/sizeof(sg_type); \
623  int32_t buffer_counter=0; \
624  for (uint32_t i=0; i<data_header.num_str(); i++) \
625  { \
626  strings[i]=SGString<sg_type>(data_header.str_len(i)); \
627  for (int32_t j=0; j<strings[i].slen; j++) \
628  { \
629  strings[i].string[j]=chunk.data(buffer_counter); \
630  buffer_counter++; \
631  \
632  if (buffer_counter==elements_in_message) \
633  { \
634  read_message(chunk); \
635  buffer_counter=0; \
636  } \
637  } \
638  } \
639 }
640 
641 READ_STRING_LIST(Int32Chunk, int8_t)
642 READ_STRING_LIST(UInt32Chunk, uint8_t)
643 READ_STRING_LIST(UInt32Chunk, char)
644 READ_STRING_LIST(Int32Chunk, int32_t)
645 READ_STRING_LIST(UInt32Chunk, uint32_t)
646 READ_STRING_LIST(Float32Chunk, float32_t)
647 READ_STRING_LIST(Float64Chunk, float64_t)
648 READ_STRING_LIST(Float64Chunk, floatmax_t)
649 READ_STRING_LIST(Int32Chunk, int16_t)
650 READ_STRING_LIST(UInt32Chunk, uint16_t)
651 READ_STRING_LIST(Int64Chunk, int64_t)
652 READ_STRING_LIST(UInt64Chunk, uint64_t)
653 #undef READ_STRING_LIST
654 
655 #define WRITE_STRING_LIST(chunk_type, sg_type) \
656 void CProtobufFile::write_string_list( \
657  const SGString<sg_type>* strings, int32_t num_str) \
658 { \
659  chunk_type chunk; \
660  int32_t elements_in_message=message_size/sizeof(sg_type); \
661  int32_t buffer_counter=0; \
662  for (int32_t i=0; i<num_str; i++) \
663  { \
664  for (int32_t j=0; j<strings[i].slen; j++) \
665  { \
666  chunk.add_data(strings[i].string[j]); \
667  buffer_counter++; \
668  \
669  if (buffer_counter==elements_in_message) \
670  { \
671  write_message(chunk); \
672  chunk.Clear(); \
673  buffer_counter=0; \
674  } \
675  } \
676  } \
677  \
678  if (buffer_counter!=0) \
679  write_message(chunk); \
680 }
681 
682 WRITE_STRING_LIST(Int32Chunk, int8_t)
683 WRITE_STRING_LIST(UInt32Chunk, uint8_t)
684 WRITE_STRING_LIST(UInt32Chunk, char)
685 WRITE_STRING_LIST(Int32Chunk, int32_t)
686 WRITE_STRING_LIST(UInt64Chunk, uint32_t)
687 WRITE_STRING_LIST(Int64Chunk, int64_t)
688 WRITE_STRING_LIST(UInt64Chunk, uint64_t)
689 WRITE_STRING_LIST(Float32Chunk, float32_t)
690 WRITE_STRING_LIST(Float64Chunk, float64_t)
691 WRITE_STRING_LIST(Float64Chunk, floatmax_t)
692 WRITE_STRING_LIST(Int32Chunk, int16_t)
693 WRITE_STRING_LIST(UInt32Chunk, uint16_t)
694 #undef WRITE_STRING_LIST
695 
696 #endif /* HAVE_PROTOBUF */
SHOGUN Machine Learning Toolbox - Documentation