36 #include "BESInternalError.h"
37 #include "BESInternalFatalError.h"
40 #include "TheBESKeys.h"
42 #include "BESStopWatch.h"
43 #include "BESIndent.h"
45 #include "DmrppNames.h"
47 #include "DmrppD4Group.h"
48 #include "DmrppArray.h"
49 #include "DmrppParserSax2.h"
50 #include "DmrppTypeFactory.h"
52 #include "SuperChunk.h"
54 #define prolog std::string("superchunky::").append(__func__).append("() - ")
61 void compute_super_chunks(
dmrpp::DmrppArray *array,
bool only_constrained, vector<SuperChunk *> &super_chunks){
64 auto chunk_dim_sizes = array->get_chunk_dimension_sizes();
69 auto currentSuperChunk =
new SuperChunk();
70 super_chunks.push_back(currentSuperChunk);
71 if(debug) cout <<
"SuperChunking array: "<< array->name() << endl;
73 for(
const auto &chunk:chunks){
74 bool was_added = currentSuperChunk->
add_chunk(chunk);
77 unsigned long long next_contiguous_chunk_offset = currentSuperChunk->get_offset() + currentSuperChunk->get_size();
78 unsigned long long gap_size;
79 bool is_behind =
false;
80 if(chunk->get_offset() > next_contiguous_chunk_offset){
81 gap_size = chunk->get_offset() - next_contiguous_chunk_offset;
85 gap_size = next_contiguous_chunk_offset - chunk->get_offset();
88 msg <<
"FOUND GAP chunk(offset: " << chunk->get_offset();
89 msg <<
" size: " << chunk->get_size() <<
")";
90 msg <<
" SuperChunk(ptr: " << (
void *) currentSuperChunk;
91 msg <<
" offset: " << currentSuperChunk->get_offset();
92 msg <<
" size: " << currentSuperChunk->get_size();
93 msg <<
" next_contiguous_chunk_offset: " << next_contiguous_chunk_offset <<
") ";
94 msg <<
" gap_size: " << gap_size;
95 msg <<
" bytes" << (is_behind?
" behind":
" beyond") <<
" target offset";
101 if(!currentSuperChunk->empty()){
102 currentSuperChunk =
new SuperChunk();
103 super_chunks.push_back(currentSuperChunk);
105 bool add_first_successful = currentSuperChunk->add_chunk(chunk);
106 if(!add_first_successful)
107 throw BESInternalError(
"ERROR: Failed to add first Chunk to a new SuperChunk."+
108 chunk->to_string() ,__FILE__,__LINE__);
113 if(currentSuperChunk->empty()) {
114 super_chunks.pop_back();
115 delete currentSuperChunk;
118 cout <<
"SuperChunk Inventory For Array: " << array->name() << endl;
119 for(
auto super_chunk: super_chunks) {
120 cout << super_chunk->to_string(
true) << endl;
124 void compute_super_chunks(libdap::BaseType *var,
bool only_constrained, vector<SuperChunk *> &super_chunks) {
125 if (var->is_simple_type())
127 if (var->is_constructor_type())
129 if (var->is_vector_type()) {
132 if(debug) cout <<
"Found DmrppArray: "<< array->name() << endl;
133 compute_super_chunks(array, only_constrained, super_chunks);
136 BESDEBUG(MODULE, prolog <<
"The variable: "<< var->name()
137 <<
" is not an instance of DmrppArray. SKIPPING"<< endl);
143 void inventory_super_chunks(libdap::BaseType *var,
bool only_constrained, vector<SuperChunk *> &super_chunks){
144 if(var->is_simple_type())
146 if(var->is_constructor_type())
148 if(var->is_vector_type()){
149 auto array =
dynamic_cast<DmrppArray*
>(var);
152 auto chunk_dim_sizes = array->get_chunk_dimension_sizes();
155 unsigned long long next_contiguous_chunk_offset = 0;
158 vector<vector<const Chunk *> *> super_chunks;
159 auto currentSuperChunk =
new vector<const Chunk *>();
160 super_chunks.push_back(currentSuperChunk);
162 if(debug) cout <<
"SuperChunking array: "<< array->name() << endl;
165 for(
auto chunk:chunks){
166 auto current_offset = chunk.get_offset();
167 auto current_size = chunk.
get_size();
171 if(current_offset!=next_contiguous_chunk_offset){
173 unsigned long long gap_size = current_offset - next_contiguous_chunk_offset;
175 cout <<
"FOUND GAP current_offset: " << current_offset <<
176 " nbytes: " << current_offset <<
177 " next_contiguous_chunk_offset: " << next_contiguous_chunk_offset <<
178 " gap_size: " << gap_size <<
179 " currentSuperChunk.size(): " << currentSuperChunk->size() << endl;
183 if(!currentSuperChunk->empty()){
184 currentSuperChunk =
new vector<const Chunk *>();
185 super_chunks.push_back(currentSuperChunk);
189 currentSuperChunk->push_back(&chunk);
190 next_contiguous_chunk_offset = current_offset + current_size;
194 if(currentSuperChunk->empty()) {
195 super_chunks.pop_back();
196 delete currentSuperChunk;
198 cout <<
"SuperChunk Inventory For Array: " << array->name() << endl;
199 unsigned long long sc_count=0;
200 for(
auto super_chunk: super_chunks) {
201 cout <<
" SuperChunk[" << sc_count++ <<
"] contains : " << super_chunk->size() <<
" chunks."
204 for (
auto chunk:*super_chunk) {
205 cout <<
" " << chunk->to_string() << endl;
212 cerr << prolog <<
" ERROR! The variable: "<< var->name()
213 <<
" is not an instance of DmrppArray. SKIPPING"<< endl;
220 void inventory_super_chunks(libdap::D4Group *group,
bool only_constrained, vector<SuperChunk *> &super_chunks){
223 auto gtr = group->grp_begin();
224 while(gtr!=group->grp_end()){
225 if(debug) cout <<
"Found Group: "<< (*gtr)->name() << endl;
226 inventory_super_chunks(*gtr++, only_constrained, super_chunks);
230 auto vtr = group->var_begin();
231 while(vtr!=group->var_end()){
232 if(debug) cout <<
"Found Variable: "<< (*vtr)->type_name() <<
" " << (*vtr)->name() << endl;
233 compute_super_chunks(*vtr++, only_constrained, super_chunks);
239 void inventory_super_chunks(DMRpp &dmr,
bool only_constrained, vector<SuperChunk *> &super_chunks){
240 inventory_super_chunks(dmr.root(), only_constrained, super_chunks);
245 ifstream dmrpp_ifs (dmrpp_filename);
246 if (dmrpp_ifs.is_open())
250 auto dmr =
new DMRpp(&factory,dmrpp_filename);
251 parser.
intern(dmrpp_ifs, dmr);
255 throw BESInternalFatalError(
"The provided file could not be opened. filename: '"+dmrpp_filename+
"'",__FILE__,__LINE__);
260 void inventory_super_chunks(
const string dmrpp_filename){
261 cout <<
"DMR++ file: " << dmrpp_filename << endl;
264 vector<SuperChunk *> super_chunks;
269 dmrpp::inventory_super_chunks(*dmr,
false, super_chunks);
272 cout <<
"DMR++ file: " << dmrpp_filename << endl;
273 cout <<
"Produced " << super_chunks.size() <<
" SuperChunks." << endl;
274 for(
auto super_chunk: super_chunks) {
275 cout << super_chunk->to_string(
true) << endl;
283 void dump_vars(libdap::D4Group *group){
285 auto gtr = group->grp_begin();
286 while(gtr!=group->grp_end()){
287 if(debug) cout <<
"Found Group: "<< (*gtr)->name() << endl;
292 auto vtr = group->var_begin();
293 while(vtr!=group->var_end()){
294 libdap::BaseType *bt = *vtr++;
300 void dump_vars(DMRpp &dmr){
301 dump_vars(dmr.root());
310 int main(
int argc,
char *argv[]) {
311 string bes_log_file(
"superchunky_bes.log");
317 string cache_effective_urls(
"false");
318 char *prefixCstr = getenv(
"prefix");
325 cout <<
"bes_log_file: " << bes_log_file << endl;
338 BESIndent::SetIndent(
"");
340 for(
auto i=1; i<argc; i++){
341 string dmrpp_filename(argv[i]);
344 dmrpp::DMRpp *dmrpp = dmrpp::get_dmrpp( dmrpp_filename);
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
virtual bool start(std::string name)
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
static TheBESKeys * TheKeys()
void set_key(const std::string &key, const std::string &val, bool addto=false)
allows the user to set key/value pairs from within the application.
static std::string ConfigFile
Provide a way to print the DMR++ response.
Extend libdap::Array so that a handler can read data using a DMR++ file.
virtual unsigned long long get_size(bool constrained=false)
Return the total number of elements in this Array.
virtual std::vector< std::shared_ptr< Chunk > > get_immutable_chunks() const
A const reference to the vector of chunks.
virtual unsigned long add_chunk(const std::string &data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::string &position_in_array="")
Add a new chunk as defined by an h4:byteStream element.
void intern(std::istream &f, libdap::DMR *dest_dmr)