bes  Updated for version 3.20.8
Chunk.h
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of the BES
4 
5 // Copyright (c) 2016 OPeNDAP, Inc.
6 // Author: Nathan Potter <ndp@opendap.org>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23 
24 #ifndef _Chunk_h
25 #define _Chunk_h 1
26 
27 #include <string>
28 #include <utility>
29 #include <vector>
30 #include <memory>
31 #include "util.h"
32 
33 // This is used to track access to 'cloudydap' accesses in the S3 logs
34 // by adding a query string that will show up in those logs. This is
35 // activated by using a special BES context with the name 'cloudydap.'
36 #define S3_TRACKING_CONTEXT "cloudydap"
37 
38 namespace dmrpp {
39 
40 // Callback functions used by chunk readers
41 size_t chunk_header_callback(char *buffer, size_t size, size_t nitems, void *data);
42 size_t chunk_write_data(void *buffer, size_t size, size_t nmemb, void *data);
43 
50 class Chunk {
51 private:
52  std::string d_data_url;
53  std::string d_query_marker;
54  std::string d_byte_order;
55  unsigned long long d_size;
56  unsigned long long d_offset;
57 
58  std::vector<unsigned int> d_chunk_position_in_array;
59 
60  // These are used only during the libcurl callback;
61  // they are not duplicated by the copy ctor or assignment
62  // operator.
63 
80  bool d_read_buffer_is_mine;
81  unsigned long long d_bytes_read;
82  char *d_read_buffer;
83  unsigned long long d_read_buffer_size;
84  bool d_is_read;
85  bool d_is_inflated;
86  std::string d_response_content_type;
87 
88  // static const std::string tracking_context;
89 
90  friend class ChunkTest;
91  friend class DmrppCommonTest;
92  friend class MockChunk;
93 
94 protected:
95 
96  void _duplicate(const Chunk &bs)
97  {
98  // See above
99  d_read_buffer_is_mine = true;
100  d_bytes_read = 0;
101  d_read_buffer = nullptr;
102  d_read_buffer_size = 0;
103  d_is_read = false;
104  d_is_inflated = false;
105 
106  d_size = bs.d_size;
107  d_offset = bs.d_offset;
108  d_data_url = bs.d_data_url;
109  d_byte_order = bs.d_byte_order;
110  d_query_marker = bs.d_query_marker;
111  d_chunk_position_in_array = bs.d_chunk_position_in_array;
112  }
113 
114 public:
115 
125  Chunk() :
126  d_data_url(""), d_query_marker(""), d_byte_order(""), d_size(0), d_offset(0),
127  d_read_buffer_is_mine(true), d_bytes_read(0), d_read_buffer(nullptr),
128  d_read_buffer_size(0), d_is_read(false), d_is_inflated(false)
129  {
130  }
131 
142  Chunk(std::string data_url, std::string order, unsigned long long size, unsigned long long offset,
143  const std::string &pia_str = "") :
144  d_data_url(std::move(data_url)), d_query_marker(""),
145  d_byte_order(std::move(order)), d_size(size), d_offset(offset),
146  d_read_buffer_is_mine(true), d_bytes_read(0), d_read_buffer(nullptr),
147  d_read_buffer_size(0), d_is_read(false), d_is_inflated(false)
148  {
150  set_position_in_array(pia_str);
151  }
152 
163  Chunk(std::string data_url, std::string order, unsigned long long size,
164  unsigned long long offset, const std::vector<unsigned int> &pia_vec) :
165  d_data_url(std::move(data_url)), d_query_marker(""), d_byte_order(std::move(order)), d_size(size), d_offset(offset),
166  d_read_buffer_is_mine(true), d_bytes_read(0), d_read_buffer(nullptr),
167  d_read_buffer_size(0), d_is_read(false), d_is_inflated(false)
168  {
170  set_position_in_array(pia_vec);
171  }
172 
173  Chunk(const Chunk &h4bs)
174  {
175  _duplicate(h4bs);
176  }
177 
178  virtual ~Chunk()
179  {
180  if(d_read_buffer_is_mine)
181  delete[] d_read_buffer;
182  d_read_buffer = nullptr;
183  }
184 
189  Chunk &operator=(const Chunk &rhs)
190  {
191  if (this == &rhs) return *this;
192 
193  _duplicate(rhs);
194 
195  return *this;
196  }
197 
199  virtual std::string get_response_content_type() { return d_response_content_type; }
200 
202  void set_response_content_type(const std::string &ct) { d_response_content_type = ct; }
203 
205  virtual std::string get_byte_order() { return d_byte_order; }
206 
210  virtual unsigned long long get_size() const
211  {
212  return d_size;
213  }
214 
218  virtual unsigned long long get_offset() const
219  {
220  return d_offset;
221  }
222 
226  virtual std::string get_data_url() const;
227 
231  virtual void set_data_url(const std::string &data_url)
232  {
233  d_data_url = data_url;
234  }
235 
239  virtual unsigned long long get_bytes_read() const
240  {
241  return d_bytes_read;
242  }
243 
248  virtual void set_bytes_read(unsigned long long bytes_read)
249  {
250  d_bytes_read = bytes_read;
251  }
252 
267  virtual void set_rbuf_to_size()
268  {
269  if(d_read_buffer_is_mine)
270  delete[] d_read_buffer;
271 
272  d_read_buffer = new char[d_size];
273  d_read_buffer_size = d_size;
274  d_read_buffer_is_mine = true;
275  set_bytes_read(0);
276  }
277 
282  virtual char *get_rbuf()
283  {
284  return d_read_buffer;
285  }
286 
287 #if 0 // Superseded by Chunk::set_read_buffer(); - ndp 12/17/20
301  virtual void set_rbuf(char *buf, unsigned int buf_size)
302  {
303  if(d_read_buffer_is_mine)
304  delete[] d_read_buffer;
305 
306  d_read_buffer = buf;
307  d_read_buffer_size = buf_size;
308 
309  set_bytes_read(buf_size);
310  }
311 #endif
312 
325  char *buf,
326  unsigned long long buf_size,
327  unsigned long long bytes_read = 0,
328  bool assume_ownership = true ){
329 
330  if(d_read_buffer_is_mine)
331  delete[] d_read_buffer;
332 
333  d_read_buffer_is_mine = assume_ownership;
334  d_read_buffer = buf;
335  d_read_buffer_size = buf_size;
336 
337  set_bytes_read(bytes_read);
338  }
339 
343  virtual unsigned long long get_rbuf_size() const
344  {
345  return d_read_buffer_size;
346  }
347 
351  virtual const std::vector<unsigned int> &get_position_in_array() const
352  {
353  return d_chunk_position_in_array;
354  }
355 
357 
358  void set_position_in_array(const std::string &pia);
359  void set_position_in_array(const std::vector<unsigned int> &pia);
360 
361  virtual void read_chunk();
362 
363  virtual void inflate_chunk(bool deflate, bool shuffle, unsigned int chunk_size, unsigned int elem_width);
364 
365  virtual bool get_is_read() { return d_is_read; }
366  virtual void set_is_read(bool state) { d_is_read = state; }
367 
368  //virtual bool get_is_inflated() const { return d_is_inflated; }
369  //virtual void set_is_inflated(bool state) { d_is_inflated = state; }
370 
371  virtual std::string get_curl_range_arg_string();
372 
373  static void parse_chunk_position_in_array_string(const std::string &pia, std::vector<unsigned int> &pia_vect);
374 
375  virtual void dump(std::ostream & strm) const;
376 
377  virtual std::string to_string() const;
378 };
379 
380 #if 0
382 struct inflate_chunk_args {
383  Chunk *chunk;
384  bool deflate;
385  bool shuffle;
386  unsigned int chunk_size;
387  unsigned int elem_width;
388 
389  inflate_chunk_args(Chunk *c, bool d, bool s, unsigned int c_size, unsigned int e_size):
390  chunk(c), deflate(d), shuffle(s), chunk_size(c_size), elem_width(e_size) {}
391 };
392 
393 void *inflate_chunk(void *args);
394 #endif
395 
396 } // namespace dmrpp
397 
398 #endif // _Chunk_h
virtual void set_bytes_read(unsigned long long bytes_read)
Set the size of this Chunk's data block.
Definition: Chunk.h:248
virtual void dump(std::ostream &strm) const
Definition: Chunk.cc:646
Chunk(std::string data_url, std::string order, unsigned long long size, unsigned long long offset, const std::vector< unsigned int > &pia_vec)
Get a chunk initialized with values.
Definition: Chunk.h:163
virtual void set_data_url(const std::string &data_url)
Set the data url string for this Chunk's data block.
Definition: Chunk.h:231
virtual void read_chunk()
Definition: Chunk.cc:606
void add_tracking_query_param()
Modify this chunk's data URL so that it includes tracking info.
Definition: Chunk.cc:461
virtual std::string get_response_content_type()
Get the response type of the last response.
Definition: Chunk.h:199
virtual std::string get_byte_order()
Get the chunk byte order.
Definition: Chunk.h:205
virtual void inflate_chunk(bool deflate, bool shuffle, unsigned int chunk_size, unsigned int elem_width)
Decompress data in the chunk, managing the Chunk's data buffers.
Definition: Chunk.cc:530
virtual std::string get_curl_range_arg_string()
Returns a curl range argument. The libcurl requires a string argument for range-ge activitys,...
Definition: Chunk.cc:446
Chunk(std::string data_url, std::string order, unsigned long long size, unsigned long long offset, const std::string &pia_str="")
Get a chunk initialized with values.
Definition: Chunk.h:142
virtual std::string get_data_url() const
Get the data url string for this Chunk's data block.
Definition: Chunk.cc:669
Chunk & operator=(const Chunk &rhs)
Definition: Chunk.h:189
Chunk()
Get an empty chunk.
Definition: Chunk.h:125
virtual const std::vector< unsigned int > & get_position_in_array() const
Definition: Chunk.h:351
virtual void set_rbuf_to_size()
Allocates the internal read buffer to be d_size bytes.
Definition: Chunk.h:267
virtual unsigned long long get_offset() const
Get the offset to this Chunk's data block.
Definition: Chunk.h:218
virtual unsigned long long get_bytes_read() const
Get the number of bytes read so far for this Chunk.
Definition: Chunk.h:239
void set_position_in_array(const std::string &pia)
parse the chunk position string
Definition: Chunk.cc:395
virtual unsigned long long get_rbuf_size() const
Definition: Chunk.h:343
virtual unsigned long long get_size() const
Get the size of this Chunk's data block on disk.
Definition: Chunk.h:210
virtual char * get_rbuf()
Definition: Chunk.h:282
void set_read_buffer(char *buf, unsigned long long buf_size, unsigned long long bytes_read=0, bool assume_ownership=true)
Set the target read buffer for this chunk.
Definition: Chunk.h:324
void set_response_content_type(const std::string &ct)
Set the response type of the last response.
Definition: Chunk.h:202