bes  Updated for version 3.20.8
ArrayAggregateOnOuterDimension.cc
1 // This file is part of the "NcML Module" project, a BES module designed
3 // to allow NcML files to be used to be used as a wrapper to add
4 // AIS to existing datasets of any format.
5 //
6 // Copyright (c) 2010 OPeNDAP, Inc.
7 // Author: Michael Johnson <m.johnson@opendap.org>
8 //
9 // For more information, please also see the main website: http://opendap.org/
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26 //
27 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29 
30 #include "config.h"
31 
32 #include "ArrayAggregateOnOuterDimension.h"
33 #include "AggregationException.h"
34 
35 #include <DataDDS.h> // libdap::DataDDS
36 #include <Marshaller.h>
37 
38 // only NCML backlinks we want in this agg_util class.
39 #include "NCMLDebug.h" // BESDEBUG and throw macros
40 #include "NCMLUtil.h" // SAFE_DELETE, NCMLUtil::getVariableNoRecurse
41 #include "BESDebug.h"
42 #include "BESStopWatch.h"
43 
44 // BES debug channel we output to
45 static const string DEBUG_CHANNEL("agg_util");
46 
47 // Local flag for whether to print constraints, to help debugging
48 static const bool PRINT_CONSTRAINTS = false;
49 
50 extern BESStopWatch *bes_timing::elapsedTimeToReadStart;
51 extern BESStopWatch *bes_timing::elapsedTimeToTransmitStart;
52 
53 // Timeouts are now handled in/by the BES framework in BESInterface.
54 // jhrg 12/29/15
55 #undef USE_LOCAL_TIMEOUT_SCHEME
56 
57 namespace agg_util {
58 
60  const AMDList& memberDatasets, std::auto_ptr<ArrayGetterInterface>& arrayGetter, const Dimension& newDim) :
61  ArrayAggregationBase(proto, memberDatasets, arrayGetter) // no new dim yet in super chain
62  , _newDim(newDim)
63 {
64  BESDEBUG(DEBUG_CHANNEL, "ArrayAggregateOnOuterDimension: ctor called!" << endl);
65 
66  // Up the rank of the array using the new dimension as outer (prepend)
67  BESDEBUG(DEBUG_CHANNEL, "ArrayAggregateOnOuterDimension: adding new outer dimension: " << _newDim.name << endl);
68  prepend_dim(_newDim.size, _newDim.name);
69 }
70 
72  ArrayAggregationBase(proto), _newDim()
73 {
74  BESDEBUG(DEBUG_CHANNEL, "ArrayAggregateOnOuterDimension() copy ctor called!" << endl);
75  duplicate(proto);
76 }
77 
79 {
80  BESDEBUG(DEBUG_CHANNEL, "~ArrayAggregateOnOuterDimension() dtor called!" << endl);
81  cleanup();
82 }
83 
86 {
87  return new ArrayAggregateOnOuterDimension(*this);
88 }
89 
92 {
93  if (this != &rhs) {
94  cleanup();
95  ArrayAggregationBase::operator=(rhs);
96  duplicate(rhs);
97  }
98  return *this;
99 }
100 
101 // Set this to 0 to get the old behavior where the entire response
102 // (for this variable) is built in memory and then sent to the client.
103 #define PIPELINING 1
104 
125 bool ArrayAggregateOnOuterDimension::serialize(libdap::ConstraintEvaluator &eval, libdap::DDS &dds,
126  libdap::Marshaller &m, bool ce_eval)
127 {
128 
129  BESStopWatch sw;
130  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("ArrayAggregateOnOuterDimension::serialize", "");
131 
132  // Only continue if we are supposed to serialize this object at all.
133  if (!(send_p() || is_in_selection())) {
134  BESDEBUG_FUNC(DEBUG_CHANNEL, "Object not in output, skipping... name=" << name() << endl);
135  return true;
136  }
137 
138  bool status = false;
139 
140  delete bes_timing::elapsedTimeToReadStart;
141  bes_timing::elapsedTimeToReadStart = 0;
142 
143  if (!read_p()) {
144 
145  if (PRINT_CONSTRAINTS) {
146  BESDEBUG_FUNC(DEBUG_CHANNEL, "Constraints on this Array are:" << endl);
147  printConstraints(*this);
148  }
149 
150  // call subclass impl
152 
153  if (PRINT_CONSTRAINTS) {
154  BESDEBUG_FUNC(DEBUG_CHANNEL, "After transfer, constraints on the member template Array are: " << endl);
156  }
157 
158  // outer one is the first in iteration
159  const Array::dimension& outerDim = *(dim_begin());
160  BESDEBUG(DEBUG_CHANNEL,
161  "Aggregating datasets array with outer dimension constraints: " << " start=" << outerDim.start << " stride=" << outerDim.stride << " stop=" << outerDim.stop << endl);
162 
163  // Be extra sure we have enough datasets for the given request
164  if (static_cast<unsigned int>(outerDim.size) != getDatasetList().size()) {
165  // Not sure whose fault it was, but tell the author
166  THROW_NCML_PARSE_ERROR(-1, "The new outer dimension of the joinNew aggregation doesn't "
167  " have the same size as the number of datasets in the aggregation!");
168  }
169 
170 #if PIPELINING
171  // Prepare our output buffer for our constrained length
172  m.put_vector_start(length());
173 #else
174  reserve_value_capacity();
175 #endif
176  // this index pointing into the value buffer for where to write.
177  // The buffer has a stride equal to the _pSubArrayProto->length().
178 
179  // Keep this to do some error checking
180  int nextElementIndex = 0;
181 
182  // Traverse the dataset array respecting hyperslab
183  for (int i = outerDim.start; i <= outerDim.stop && i < outerDim.size; i += outerDim.stride) {
184  AggMemberDataset& dataset = *((getDatasetList())[i]);
185 
186  try {
187 #if USE_LOCAL_TIMEOUT_SCHEME
188  dds.timeout_on();
189 #endif
191  name(), dataset, getArrayGetterInterface(), DEBUG_CHANNEL);
192 #if USE_LOCAL_TIMEOUT_SCHEME
193  dds.timeout_off();
194 #endif
195 #if PIPELINING
196  delete bes_timing::elapsedTimeToTransmitStart;
197  bes_timing::elapsedTimeToTransmitStart = 0;
198  m.put_vector_part(pDatasetArray->get_buf(), getGranuleTemplateArray().length(), var()->width(),
199  var()->type());
200 #else
201  this->set_value_slice_from_row_major_vector(*pDatasetArray, nextElementIndex);
202 #endif
203 
204  pDatasetArray->clear_local_data();
205  }
206  catch (agg_util::AggregationException& ex) {
207  std::ostringstream oss;
208  oss << "Got AggregationException while streaming dataset index=" << i << " data for location=\""
209  << dataset.getLocation() << "\" The error msg was: " << std::string(ex.what());
210  THROW_NCML_PARSE_ERROR(-1, oss.str());
211  }
212 
213  // Jump forward by the amount we added.
214  nextElementIndex += getGranuleTemplateArray().length();
215  }
216 
217  // If we succeeded, we are at the end of the array!
218  NCML_ASSERT_MSG(nextElementIndex == length(), "Logic error:\n"
219  "ArrayAggregateOnOuterDimension::read(): "
220  "At end of aggregating, expected the nextElementIndex to be the length of the "
221  "aggregated array, but it wasn't!");
222 
223 #if PIPELINING
224  m.put_vector_end();
225  status = true;
226 #else
227  // Set the cache bit to avoid recomputing
228  set_read_p(true);
229 
230  delete bes_timing::elapsedTimeToTransmitStart;
231  bes_timing::elapsedTimeToTransmitStart = 0;
232  status = libdap::Array::serialize(eval, dds, m, ce_eval);
233 #endif
234  }
235  else {
236  status = libdap::Array::serialize(eval, dds, m, ce_eval);
237  }
238 
239  return status;
240 }
241 
243 // helpers
244 
245 void ArrayAggregateOnOuterDimension::duplicate(const ArrayAggregateOnOuterDimension& rhs)
246 {
247  _newDim = rhs._newDim;
248 }
249 
250 void ArrayAggregateOnOuterDimension::cleanup() throw ()
251 {
252 }
253 
254 /* virtual */
256 {
257  // transfer the constraints from this object into the subArray template
258  // skipping our first dim which is the new one and not in the subArray.
260  *this, // from this
261  true, // skip first dim in the copy since we handle it special
262  false, // also skip it in the toArray for the same reason.
263  true, // print debug
264  DEBUG_CHANNEL); // on this channel
265 }
266 
267 /* virtual */
268 // In this version of the code, I broke apart the call to
269 // agg_util::AggregationUtil::addDatasetArrayDataToAggregationOutputArray()
270 // into two calls: AggregationUtil::readDatasetArrayDataForAggregation()
271 // and this->set_value_slice_from_row_major_vector(). This
273 {
274  BESStopWatch sw;
275  if (BESDebug::IsSet(TIMING_LOG_KEY))
276  sw.start("ArrayAggregateOnOuterDimension::readConstrainedGranuleArraysAndAggregateDataHook", "");
277 
278  // outer one is the first in iteration
279  const Array::dimension& outerDim = *(dim_begin());
280  BESDEBUG(DEBUG_CHANNEL,
281  "Aggregating datasets array with outer dimension constraints: " << " start=" << outerDim.start << " stride=" << outerDim.stride << " stop=" << outerDim.stop << endl);
282 
283  // Be extra sure we have enough datasets for the given request
284  if (static_cast<unsigned int>(outerDim.size) != getDatasetList().size()) {
285  // Not sure whose fault it was, but tell the author
286  THROW_NCML_PARSE_ERROR(-1, "The new outer dimension of the joinNew aggregation doesn't "
287  " have the same size as the number of datasets in the aggregation!");
288  }
289 
290  // Prepare our output buffer for our constrained length
291  reserve_value_capacity();
292 
293  // this index pointing into the value buffer for where to write.
294  // The buffer has a stride equal to the _pSubArrayProto->length().
295  int nextElementIndex = 0;
296 
297  // Traverse the dataset array respecting hyperslab
298  for (int i = outerDim.start; i <= outerDim.stop && i < outerDim.size; i += outerDim.stride) {
299  AggMemberDataset& dataset = *((getDatasetList())[i]);
300 
301  try {
302  agg_util::AggregationUtil::addDatasetArrayDataToAggregationOutputArray(*this, // into the output buffer of this object
303  nextElementIndex, // into the next open slice
304  getGranuleTemplateArray(), // constraints template
305  name(), // aggvar name
306  dataset, // Dataset who's DDS should be searched
307  getArrayGetterInterface(), DEBUG_CHANNEL);
308 #if 0
309  // The code above is conceptually similar to this, but
310  // makes more efficient use of memory. jhrg8/18/15
312  name(), dataset, getArrayGetterInterface(), DEBUG_CHANNEL);
313 
314  this->set_value_slice_from_row_major_vector(*pDatasetArray, nextElementIndex);
315 #endif
316  }
317  catch (agg_util::AggregationException& ex) {
318  std::ostringstream oss;
319  oss << "Got AggregationException while streaming dataset index=" << i << " data for location=\""
320  << dataset.getLocation() << "\" The error msg was: " << std::string(ex.what());
321  THROW_NCML_PARSE_ERROR(-1, oss.str());
322  }
323 
324  // Jump forward by the amount we added.
325  nextElementIndex += getGranuleTemplateArray().length();
326  }
327 
328  // If we succeeded, we are at the end of the array!
329  NCML_ASSERT_MSG(nextElementIndex == length(), "Logic error:\n"
330  "ArrayAggregateOnOuterDimension::read(): "
331  "At end of aggregating, expected the nextElementIndex to be the length of the "
332  "aggregated array, but it wasn't!");
333 }
334 
335 }
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:160
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
const std::string & getLocation() const
static void addDatasetArrayDataToAggregationOutputArray(libdap::Array &oOutputArray, unsigned int atIndex, const libdap::Array &constrainedTemplateArray, const string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const string &debugChannel)
static void transferArrayConstraints(libdap::Array *pToArray, const libdap::Array &fromArray, bool skipFirstFromDim, bool skipFirstToDim, bool printDebug=false, const std::string &debugChannel="agg_util")
static libdap::Array * readDatasetArrayDataForAggregation(const libdap::Array &constrainedTemplateArray, const std::string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const std::string &debugChannel)
ArrayAggregateOnOuterDimension & operator=(const ArrayAggregateOnOuterDimension &rhs)
ArrayAggregateOnOuterDimension(const libdap::Array &proto, const AMDList &memberDatasets, std::auto_ptr< ArrayGetterInterface > &arrayGetter, const Dimension &newDim)
virtual bool serialize(libdap::ConstraintEvaluator &eval, libdap::DDS &dds, libdap::Marshaller &m, bool ce_eval)
virtual ArrayAggregateOnOuterDimension * ptr_duplicate()
const AMDList & getDatasetList() const
void printConstraints(const Array &fromArray)
const ArrayGetterInterface & getArrayGetterInterface() const
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...