bes  Updated for version 3.20.8
NgapApi.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of ngap_module, A C++ module that can be loaded in to
4 // the OPeNDAP Back-End Server (BES) and is able to handle remote requests.
5 
6 // Copyright (c) 2020 OPeNDAP, Inc.
7 // Author: Nathan Potter <ndp@opendap.org>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24 
25 #include "config.h"
26 
27 #include <cstdio>
28 #include <cstring>
29 #include <iostream>
30 #include <sstream>
31 #include <memory>
32 #include <time.h>
33 #include <curl/curl.h>
34 
35 #include <util.h>
36 #include <debug.h>
37 
38 #include "rapidjson/document.h"
39 #include "rapidjson/writer.h"
40 #include "rapidjson/prettywriter.h"
41 #include "rapidjson/stringbuffer.h"
42 #include "rapidjson/filereadstream.h"
43 
44 #include "BESError.h"
45 #include "BESNotFoundError.h"
46 #include "BESSyntaxUserError.h"
47 #include "BESDebug.h"
48 #include "BESUtil.h"
49 #include "BESStopWatch.h"
50 #include "BESLog.h"
51 #include "TheBESKeys.h"
52 #include "CurlUtils.h"
53 #include "url_impl.h"
54 #include "RemoteResource.h"
55 
56 #include "NgapApi.h"
57 #include "NgapNames.h"
58 #include "NgapError.h"
59 
60 using namespace std;
61 
62 #define prolog string("NgapApi::").append(__func__).append("() - ")
63 
64 namespace ngap {
65 
66  const string NGAP_PROVIDER_KEY("providers");
67  const string NGAP_COLLECTIONS_KEY("collections");
68  const string NGAP_CONCEPTS_KEY("concepts");
69  const string NGAP_GRANULES_KEY("granules");
70  const string DEFAULT_CMR_ENDPOINT_URL("https://cmr.earthdata.nasa.gov");
71  const string DEFAULT_CMR_SEARCH_ENDPOINT_PATH("/search/granules.umm_json_v1_4");
72 
73  const string CMR_PROVIDER("provider");
74  const string CMR_ENTRY_TITLE("entry_title");
75  const string CMR_COLLECTION_CONCEPT_ID("collection_concept_id");
76  const string CMR_GRANULE_UR("granule_ur");
77  const string CMR_URL_TYPE_GET_DATA("GET DATA");
78 
79  const string RJ_TYPE_NAMES[] = {
80  "kNullType",
81  "kFalseType",
82  "kTrueType",
83  "kObjectType",
84  "kArrayType",
85  "kStringType",
86  "kNumberType"
87  };
88 
89  const string AMS_EXPIRES_HEADER_KEY("X-Amz-Expires");
90  const string AWS_DATE_HEADER_KEY("X-Amz-Date");
91  // const string AWS_DATE_FORMAT("%Y%m%dT%H%MS"); // 20200624T175046Z
92  const string CLOUDFRONT_EXPIRES_HEADER_KEY("Expires");
93  const string INGEST_TIME_KEY("ingest_time");
94  const unsigned int REFRESH_THRESHOLD = 3600; // An hour
95 
96 
97  NgapApi::NgapApi() : d_cmr_hostname(DEFAULT_CMR_ENDPOINT_URL), d_cmr_search_endpoint_path(DEFAULT_CMR_SEARCH_ENDPOINT_PATH) {
98  bool found;
99  string cmr_hostnamer;
100  TheBESKeys::TheKeys()->get_value(NGAP_CMR_HOSTNAME_KEY, cmr_hostnamer, found);
101  if (found) {
102  d_cmr_hostname = cmr_hostnamer;
103  }
104 
105  string cmr_search_endpoint_path;
106  TheBESKeys::TheKeys()->get_value(NGAP_CMR_SEARCH_ENDPOINT_PATH_KEY, cmr_search_endpoint_path, found);
107  if (found) {
108  d_cmr_search_endpoint_path = cmr_search_endpoint_path;
109  }
110 
111 
112  }
113 
114  std::string NgapApi::get_cmr_search_endpoint_url(){
115  return BESUtil::assemblePath(d_cmr_hostname , d_cmr_search_endpoint_path);
116  }
117 
118 
119 #if 0
120  // OLD WAY (tokenization on '/' delimiter)
121  std::string
122  NgapApi::build_cmr_query_url(const std::string &restified_path) {
123  string data_access_url("");
124 
125  vector<string> tokens;
126  BESUtil::tokenize(restified_path, tokens);
127  if (tokens.empty()) {
128  throw BESSyntaxUserError(string("The specified NGAP API path '") + restified_path +
129  "' failed to tokenize: No tokens were returned.", __FILE__, __LINE__);
130  }
131  if(tokens.size()!=6){
132  stringstream msg;
133  msg << prolog << "The specified NGAP API path '" << restified_path << "' was tokenized, but ";
134  msg << tokens.size() << " token(s) were found where 6 are required.";
135  throw BESSyntaxUserError(msg.str() , __FILE__, __LINE__);
136  }
137 
138  // Check that the NGAP_PROVIDER_KEY token is present.
139  if (tokens[0] != NGAP_PROVIDER_KEY ) {
140  stringstream msg;
141  msg << prolog << "The specified path '" << restified_path << "'";
142  msg << "does not contain the required path element '" << NGAP_PROVIDER_KEY << "'";
143  throw BESSyntaxUserError(msg.str(), __FILE__, __LINE__);
144  }
145  // Check that the NGAP_COLLECTIONS_KEY or NGAP_CONCEPTS_KEY token is present.
146  if ((tokens[2] != NGAP_COLLECTIONS_KEY && tokens[2] != NGAP_CONCEPTS_KEY)) {
147  stringstream msg;
148  msg << prolog << "The specified path '" << restified_path << "'";
149  msg << "does not contain the expected path element '" << NGAP_COLLECTIONS_KEY << "' or it's alternate, '";
150  msg << NGAP_CONCEPTS_KEY << "'. One of these is required";
151  throw BESSyntaxUserError(msg.str(), __FILE__, __LINE__);
152  }
153  // Check that the NGAP_GRANULES_KEY token is present.
154  if (tokens[4] != NGAP_GRANULES_KEY) {
155  stringstream msg;
156  msg << prolog << "The specified path '" << restified_path << "'";
157  msg << "does not contain the required path element '" << NGAP_GRANULES_KEY << "'.";
158  throw BESSyntaxUserError(msg.str(), __FILE__, __LINE__);
159  }
160 
161  // Build the CMR query URL for the dataset
162  string cmr_url = get_cmr_search_endpoint_url() + "?";
163  {
164  // This easy handle is only created so we can use the curl_easy_escape() on the tokens
165  CURL *ceh = curl_easy_init();
166  char *esc_url_content;
167 
168  // Add provider
169  esc_url_content = curl_easy_escape(ceh, tokens[1].c_str(), tokens[1].size());
170  cmr_url += CMR_PROVIDER + "=" + esc_url_content + "&";
171  curl_free(esc_url_content);
172 
173  esc_url_content = curl_easy_escape(ceh, tokens[3].c_str(), tokens[3].size());
174  if (tokens[2] == NGAP_COLLECTIONS_KEY) {
175  // Add entry_title
176  cmr_url += CMR_ENTRY_TITLE + "=" + esc_url_content + "&";
177  }
178  else if(tokens[2] == NGAP_CONCEPTS_KEY){
179  // Add collection_concept_id
180  cmr_url += CMR_COLLECTION_CONCEPT_ID + "=" + esc_url_content + "&";
181  }
182  else {
183  // Bad inputs throw an exception.
184  curl_free(esc_url_content);
185  stringstream msg;
186  msg << prolog << "The specified path '" << restified_path << "'";
187  msg << "does not contain the expected path element '" << NGAP_COLLECTIONS_KEY << "' or it's alternate '";
188  msg << NGAP_CONCEPTS_KEY << "'. One of these is required";
189  throw BESSyntaxUserError(msg.str(), __FILE__, __LINE__);
190  }
191  curl_free(esc_url_content);
192 
193  esc_url_content = curl_easy_escape(ceh, tokens[5].c_str(), tokens[5].size());
194  cmr_url += CMR_GRANULE_UR + "=" + esc_url_content;
195  curl_free(esc_url_content);
196  curl_easy_cleanup(ceh);
197  }
198  return cmr_url;
199  }
200 #else
201 
202 
209 std::string NgapApi::build_cmr_query_url(const std::string &restified_path) {
210  string PROVIDERS_KEY("/providers/");
211  string COLLECTIONS_KEY("/collections/");
212  string CONCEPTS_KEY("/concepts/");
213  string GRANULES_KEY("/granules/");
214 
215  // Make sure it starts with a '/' (see key strings above)
216  string r_path = ( restified_path[0] != '/' ? "/" : "") + restified_path;
217 
218  size_t provider_index = r_path.find(PROVIDERS_KEY);
219  if(provider_index == string::npos){
220  stringstream msg;
221  msg << prolog << "The specified path '" << r_path << "'";
222  msg << " does not contain the required path element '" << NGAP_PROVIDER_KEY << "'";
223  throw BESSyntaxUserError(msg.str(), __FILE__, __LINE__);
224  }
225  if(provider_index != 0){
226  stringstream msg;
227  msg << prolog << "The specified path '" << r_path << "'";
228  msg << " has the path element '" << NGAP_PROVIDER_KEY << "' located in the incorrect position (";
229  msg << provider_index << ") expected 0.";
230  throw BESSyntaxUserError(msg.str(), __FILE__, __LINE__);
231  }
232  provider_index += PROVIDERS_KEY.length();
233 
234  bool use_collection_concept_id = false;
235  size_t collection_index = r_path.find(COLLECTIONS_KEY);
236  if(collection_index == string::npos) {
237  size_t concepts_index = r_path.find(CONCEPTS_KEY);
238  if (concepts_index == string::npos) {
239  stringstream msg;
240  msg << prolog << "The specified path '" << r_path << "'";
241  msg << " contains neither the '" << COLLECTIONS_KEY << "'";
242  msg << " nor the '" << CONCEPTS_KEY << "'";
243  msg << " one must be provided.";
244  throw BESSyntaxUserError(msg.str(), __FILE__, __LINE__);
245  }
246  collection_index = concepts_index;
247  use_collection_concept_id = true;
248  }
249  if(collection_index <= provider_index+1){ // The value of provider has to be at least 1 character
250  stringstream msg;
251  msg << prolog << "The specified path '" << r_path << "'";
252  msg << " has the path element '" << (use_collection_concept_id?CONCEPTS_KEY:COLLECTIONS_KEY) << "' located in the incorrect position (";
253  msg << collection_index << ") expected at least " << provider_index+1;
254  throw BESSyntaxUserError(msg.str(), __FILE__, __LINE__);
255  }
256  string provider = r_path.substr(provider_index,collection_index - provider_index);
257  collection_index += use_collection_concept_id?CONCEPTS_KEY.length():COLLECTIONS_KEY.length();
258 
259 
260  size_t granule_index = r_path.find(GRANULES_KEY);
261  if(granule_index == string::npos){
262  stringstream msg;
263  msg << prolog << "The specified path '" << r_path << "'";
264  msg << " does not contain the required path element '" << GRANULES_KEY << "'";
265  throw BESSyntaxUserError(msg.str(), __FILE__, __LINE__);
266  }
267  if(granule_index <= collection_index+1){ // The value of collection must have at least one character.
268  stringstream msg;
269  msg << prolog << "The specified path '" << r_path << "'";
270  msg << " has the path element '" << GRANULES_KEY << "' located in the incorrect position (";
271  msg << granule_index << ") expected at least " << collection_index+1;
272  throw BESSyntaxUserError(msg.str(), __FILE__, __LINE__);
273  }
274  string collection = r_path.substr(collection_index,granule_index - collection_index);
275  granule_index += GRANULES_KEY.length();
276 
277  // The granule value is the path terminus so it's every thing after the key
278  string granule = r_path.substr(granule_index);
279 
280  // Build the CMR query URL for the dataset
281  string cmr_url = get_cmr_search_endpoint_url() + "?";
282  {
283  // This easy handle is only created so we can use the curl_easy_escape() on the token values
284  CURL *ceh = curl_easy_init();
285  char *esc_url_content;
286 
287  // Add provider
288  esc_url_content = curl_easy_escape(ceh, provider.c_str(), provider.size());
289  cmr_url += CMR_PROVIDER + "=" + esc_url_content + "&";
290  curl_free(esc_url_content);
291 
292  esc_url_content = curl_easy_escape(ceh, collection.c_str(), collection.size());
293  if(use_collection_concept_id){
294  // Add collection_concept_id
295  cmr_url += CMR_COLLECTION_CONCEPT_ID + "=" + esc_url_content + "&";
296  }
297  else {
298  // Add entry_title
299  cmr_url += CMR_ENTRY_TITLE + "=" + esc_url_content + "&";
300 
301  }
302  curl_free(esc_url_content);
303 
304  esc_url_content = curl_easy_escape(ceh, granule.c_str(), granule.size());
305  cmr_url += CMR_GRANULE_UR + "=" + esc_url_content;
306  curl_free(esc_url_content);
307 
308  curl_easy_cleanup(ceh);
309  }
310  return cmr_url;
311 }
312 #endif
313 
324 std::string NgapApi::find_get_data_url_in_granules_umm_json_v1_4(const std::string &restified_path, rapidjson::Document &cmr_granule_response)
325 {
326 
327  string data_access_url("");
328 
329  rapidjson::Value &val = cmr_granule_response["hits"];
330  int hits = val.GetInt();
331  if (hits < 1) {
332  throw BESNotFoundError(string("The specified path '").append(restified_path).append(
333  "' does not identify a granule in CMR."), __FILE__, __LINE__);
334  }
335 
336  rapidjson::Value &items = cmr_granule_response["items"];
337  if (items.IsArray()) {
338  stringstream ss;
339  for (rapidjson::SizeType i = 0; i < items.Size(); i++) // Uses SizeType instead of size_t
340  ss << "items[" << i << "]: " << RJ_TYPE_NAMES[items[i].GetType()] << endl;
341 
342  BESDEBUG(MODULE, prolog << "items size: " << items.Size() << endl << ss.str() << endl);
343 
344  rapidjson::Value &items_obj = items[0];
345  rapidjson::GenericMemberIterator<false, rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator>> mitr = items_obj.FindMember(
346  "umm");
347 
348  rapidjson::Value &umm = mitr->value;
349  mitr = umm.FindMember("RelatedUrls");
350  if (mitr == umm.MemberEnd()) {
351  throw BESInternalError("Error! The umm/RelatedUrls object was not located!", __FILE__, __LINE__);
352  }
353  rapidjson::Value &related_urls = mitr->value;
354 
355  if (!related_urls.IsArray()) {
356  throw BESNotFoundError("Error! The RelatedUrls object in the CMR response is not an array!", __FILE__,
357  __LINE__);
358  }
359 
360  BESDEBUG(MODULE, prolog << " Found RelatedUrls array in CMR response." << endl);
361 
362  bool noSubtype;
363  for (rapidjson::SizeType i = 0; i < related_urls.Size() && data_access_url.empty(); i++) {
364  rapidjson::Value &obj = related_urls[i];
365  mitr = obj.FindMember("URL");
366  if (mitr == obj.MemberEnd()) {
367  stringstream err;
368  err << "Error! The umm/RelatedUrls[" << i << "] does not contain the URL object";
369  throw BESInternalError(err.str(), __FILE__, __LINE__);
370  }
371  rapidjson::Value &r_url = mitr->value;
372 
373  mitr = obj.FindMember("Type");
374  if (mitr == obj.MemberEnd()) {
375  stringstream err;
376  err << "Error! The umm/RelatedUrls[" << i << "] does not contain the Type object";
377  throw BESInternalError(err.str(), __FILE__, __LINE__);
378  }
379  rapidjson::Value &r_type = mitr->value;
380 
381  noSubtype = obj.FindMember("Subtype") == obj.MemberEnd();
382 #if 0
383  mitr = obj.FindMember("Description");
384  if(mitr == obj.MemberEnd()){
385  stringstream err;
386  err << "Error! The umm/RelatedUrls[" << i << "] does not contain the Description object";
387  throw BESInternalError(err.str(), __FILE__, __LINE__);
388  }
389  rapidjson::Value& r_desc = mitr->value;
390 #endif
391  BESDEBUG(MODULE, prolog << "RelatedUrl Object:" <<
392  " URL: '" << r_url.GetString() << "'" <<
393  " Type: '" << r_type.GetString() << "'" <<
394  " SubType: '" << (noSubtype ? "Absent" : "Present") << "'" << endl);
395 
396  if ((r_type.GetString() == CMR_URL_TYPE_GET_DATA) && noSubtype) {
397  data_access_url = r_url.GetString();
398  }
399  }
400  }
401 
402  if (data_access_url.empty()) {
403  throw BESInternalError(string("ERROR! Failed to locate a data access URL for the path: ") + restified_path,
404  __FILE__, __LINE__);
405  }
406 
407  return data_access_url;
408 }
409 
410 
411 
434  string NgapApi::convert_ngap_resty_path_to_data_access_url(
435  const std::string &restified_path,
436  const std::string &uid
437  ) {
438  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
439  string data_access_url("");
440 
441  string cmr_query_url = build_cmr_query_url(restified_path);
442 
443  BESDEBUG(MODULE, prolog << "CMR Request URL: " << cmr_query_url << endl);
444 
445  BESDEBUG(MODULE, prolog << "Building new RemoteResource." << endl);
446  http::RemoteResource cmr_query(cmr_query_url, uid);
447  {
448  BESStopWatch besTimer;
449  if (BESISDEBUG(MODULE) || BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()){
450  besTimer.start("CMR Query: " + cmr_query_url);
451  }
452  cmr_query.retrieveResource();
453  }
454  rapidjson::Document cmr_response = cmr_query.get_as_json();
455 
456  data_access_url = find_get_data_url_in_granules_umm_json_v1_4(restified_path, cmr_response);
457 
458  BESDEBUG(MODULE, prolog << "END (data_access_url: "<< data_access_url << ")" << endl);
459 
460  return data_access_url;
461  }
462 
463 
464 
465 
466  bool NgapApi::signed_url_is_expired(const http::url &signed_url)
467  {
468  bool is_expired;
469  time_t now;
470  time(&now); /* get current time; same as: timer = time(NULL) */
471  BESDEBUG(MODULE, prolog << "now: " << now << endl);
472 
473  time_t expires = now;
474  string cf_expires = signed_url.query_parameter_value(CLOUDFRONT_EXPIRES_HEADER_KEY);
475  string aws_expires = signed_url.query_parameter_value(AMS_EXPIRES_HEADER_KEY);
476  time_t ingest_time = signed_url.ingest_time();
477 
478  if(!cf_expires.empty()){ // CloudFront expires header?
479  expires = stoll(cf_expires);
480  BESDEBUG(MODULE, prolog << "Using "<< CLOUDFRONT_EXPIRES_HEADER_KEY << ": " << expires << endl);
481  }
482  else if(!aws_expires.empty()){
483  // AWS Expires header?
484  //
485  // By default we'll use the time we made the URL object, ingest_time
486  time_t start_time = ingest_time;
487  // But if there's an AWS Date we'll parse that and compute the time
488  // @TODO move to NgapApi::decompose_url() and add the result to the map
489  string aws_date = signed_url.query_parameter_value(AWS_DATE_HEADER_KEY);
490  if(!aws_date.empty()){
491  string date = aws_date; // 20200624T175046Z
492  string year = date.substr(0,4);
493  string month = date.substr(4,2);
494  string day = date.substr(6,2);
495  string hour = date.substr(9,2);
496  string minute = date.substr(11,2);
497  string second = date.substr(13,2);
498 
499  BESDEBUG(MODULE, prolog << "date: "<< date <<
500  " year: " << year << " month: " << month << " day: " << day <<
501  " hour: " << hour << " minute: " << minute << " second: " << second << endl);
502 
503  struct tm *ti = gmtime(&now);
504  ti->tm_year = stoll(year) - 1900;
505  ti->tm_mon = stoll(month) - 1;
506  ti->tm_mday = stoll(day);
507  ti->tm_hour = stoll(hour);
508  ti->tm_min = stoll(minute);
509  ti->tm_sec = stoll(second);
510 
511  BESDEBUG(MODULE, prolog << "ti->tm_year: "<< ti->tm_year <<
512  " ti->tm_mon: " << ti->tm_mon <<
513  " ti->tm_mday: " << ti->tm_mday <<
514  " ti->tm_hour: " << ti->tm_hour <<
515  " ti->tm_min: " << ti->tm_min <<
516  " ti->tm_sec: " << ti->tm_sec << endl);
517 
518 
519  start_time = mktime(ti);
520  BESDEBUG(MODULE, prolog << "AWS (computed) start_time: "<< start_time << endl);
521  }
522  expires = start_time + stoll(aws_expires);
523  BESDEBUG(MODULE, prolog << "Using "<< AMS_EXPIRES_HEADER_KEY << ": " << aws_expires <<
524  " (expires: " << expires << ")" << endl);
525  }
526  time_t remaining = expires - now;
527  BESDEBUG(MODULE, prolog << "expires_time: " << expires <<
528  " remaining_time: " << remaining <<
529  " refresh_threshold: " << REFRESH_THRESHOLD << endl);
530 
531  is_expired = remaining < REFRESH_THRESHOLD;
532  BESDEBUG(MODULE, prolog << "is_expired: " << (is_expired?"true":"false") << endl);
533 
534  return is_expired;
535  }
536 
537 
538 
539 #if 0
540  string NgapApi::convert_ngap_resty_path_to_data_access_url(string real_name){
541  string data_access_url("");
542 
543  vector<string> tokens;
544  BESUtil::tokenize(real_name,tokens);
545  if( tokens[0]!= NGAP_PROVIDER_KEY || tokens[2]!=NGAP_DATASETS_KEY || tokens[4]!=NGAP_GRANULES_KEY){
546  string err = (string) "The specified path " + real_name
547  + " does not conform to the NGAP request interface API.";
548  throw BESSyntaxUserError(err, __FILE__, __LINE__);
549  }
550 
551  string cmr_url = cmr_granule_search_endpoint_url + "?";
552  cmr_url += CMR_PROVIDER + "=" + tokens[1] + "&";
553  cmr_url += CMR_ENTRY_TITLE + "=" + tokens[3] + "&";
554  cmr_url += CMR_GRANULE_UR + "=" + tokens[5] ;
555  BESDEBUG( MODULE, prolog << "CMR Request URL: "<< cmr_url << endl );
556  rapidjson::Document cmr_response = ngap_curl::http_get_as_json(cmr_url);
557 
558  rapidjson::Value& val = cmr_response["hits"];
559  int hits = val.GetInt();
560  if(hits < 1){
561  string err = (string) "The specified path " + real_name
562  + " does not identify a thing we know about....";
563  throw BESNotFoundError(err, __FILE__, __LINE__);
564  }
565 
566  rapidjson::Value& items = cmr_response["items"];
567  if(items.IsArray()){
568  stringstream ss;
569  for (rapidjson::SizeType i = 0; i < items.Size(); i++) // Uses SizeType instead of size_t
570  ss << "items[" << i << "]: " << rjtype_names[items[i].GetType()] << endl;
571  BESDEBUG(MODULE,prolog << "items size: " << items.Size() << endl << ss.str() << endl);
572 
573  rapidjson::Value& items_obj = items[0];
574  rapidjson::GenericMemberIterator<false, rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator>> mitr = items_obj.FindMember("umm");
575 
576  rapidjson::Value& umm = mitr->value;
577  mitr = umm.FindMember("RelatedUrls");
578  rapidjson::Value& related_urls = mitr->value;
579 
580  if(!related_urls.IsArray()){
581  string err = (string) "Error! The RelatedUrls object in the CMR response is not an array!";
582  throw BESNotFoundError(err, __FILE__, __LINE__);
583  }
584 
585  BESDEBUG(MODULE,prolog << " Found RelatedUrls array in CMR response." << endl);
586 
587  bool noSubtype;
588 
589  for (rapidjson::SizeType i = 0; i < related_urls.Size() && data_access_url.empty(); i++) {
590  rapidjson::Value& obj = related_urls[i];
591  mitr = obj.FindMember("URL");
592  rapidjson::Value& r_url = mitr->value;
593  mitr = obj.FindMember("Type");
594  rapidjson::Value& r_type = mitr->value;
595  noSubtype = ((mitr = obj.FindMember("Subtype")) == obj.MemberEnd()) ? true : false;
596  mitr = obj.FindMember("Description");
597  rapidjson::Value& r_desc = mitr->value;
598  BESDEBUG(MODULE,prolog << "RelatedUrl Object:" <<
599  " URL: '" << r_url.GetString() << "'" <<
600  " Type: '" << r_type.GetString() << "'" <<
601  //" Subtype: '" << r_subtype.GetString() << "'" <<
602  " Description: '" << r_desc.GetString() << "'" << endl);
603 
604  if( (r_type.GetString() == CMR_URL_TYPE_GET_DATA) && noSubtype ){
605  data_access_url = r_url.GetString();
606  }
607  }
608 
609  }
610 
611  return data_access_url + ".dmrpp";
612 }
613 #endif
614 
615 #if 0
619  const rapidjson::Value&
620  NgapApi::get_children(const rapidjson::Value& obj) {
622 
623  itr = obj.FindMember("children");
624  bool result = itr != obj.MemberEnd();
625  string msg = prolog + (result?"Located":"FAILED to locate") + " the value 'children' in the object.";
626  BESDEBUG(MODULE, msg << endl);
627  if(!result){
628  throw NgapError(msg,__FILE__,__LINE__);
629  }
630 
631  const rapidjson::Value& children = itr->value;
632  result = children.IsArray();
633  msg = prolog + "The value 'children' is" + (result?"":" NOT") + " an array.";
634  BESDEBUG(MODULE, msg << endl);
635  if(!result){
636  throw NgapError(msg,__FILE__,__LINE__);
637  }
638  return children;
639  }
640 
644  const rapidjson::Value&
645  NgapApi::get_feed(const rapidjson::Document &ngap_doc){
646 
647  bool result = ngap_doc.IsObject();
648  string msg = prolog + "Json document is" + (result?"":" NOT") + " an object.";
649  BESDEBUG(MODULE, msg << endl);
650  if(!result){
651  throw NgapError(msg,__FILE__,__LINE__);
652  }
653 
654  //################### feed
655  rapidjson::Value::ConstMemberIterator itr = ngap_doc.FindMember("feed");
656  result = itr != ngap_doc.MemberEnd();
657  msg = prolog + (result?"Located":"FAILED to locate") + " the value 'feed'.";
658  BESDEBUG(MODULE, msg << endl);
659  if(!result){
660  throw NgapError(msg,__FILE__,__LINE__);
661  }
662 
663  const rapidjson::Value& feed = itr->value;
664  result = feed.IsObject();
665  msg = prolog + "The value 'feed' is" + (result?"":" NOT") + " an object.";
666  BESDEBUG(MODULE, msg << endl);
667  if(!result){
668  throw NgapError(msg,__FILE__,__LINE__);
669  }
670  return feed;
671  }
672 
676  const rapidjson::Value&
677  NgapApi::get_entries(const rapidjson::Document &ngap_doc){
678  bool result;
679  string msg;
680 
681  const rapidjson::Value& feed = get_feed(ngap_doc);
682 
683  rapidjson::Value::ConstMemberIterator itr = feed.FindMember("entry");
684  result = itr != feed.MemberEnd();
685  msg = prolog + (result?"Located":"FAILED to locate") + " the value 'entry'.";
686  BESDEBUG(MODULE, msg << endl);
687  if(!result){
688  throw NgapError(msg,__FILE__,__LINE__);
689  }
690 
691  const rapidjson::Value& entry = itr->value;
692  result = entry.IsArray();
693  msg = prolog + "The value 'entry' is" + (result?"":" NOT") + " an Array.";
694  BESDEBUG(MODULE, msg << endl);
695  if(!result){
696  throw NgapError(msg,__FILE__,__LINE__);
697  }
698  return entry;
699  }
700 
704  const rapidjson::Value&
705  NgapApi::get_temporal_group(const rapidjson::Document &ngap_doc){
706  rjson_utils ru;
707 
708  bool result;
709  string msg;
710  const rapidjson::Value& feed = get_feed(ngap_doc);
711 
712  //################### facets
713  rapidjson::Value::ConstMemberIterator itr = feed.FindMember("facets");
714  result = itr != feed.MemberEnd();
715  msg = prolog + (result?"Located":"FAILED to locate") + " the value 'facets'." ;
716  BESDEBUG(MODULE, msg << endl);
717  if(!result){
718  throw NgapError(msg,__FILE__,__LINE__);
719  }
720 
721  const rapidjson::Value& facets_obj = itr->value;
722  result = facets_obj.IsObject();
723  msg = prolog + "The value 'facets' is" + (result?"":" NOT") + " an object.";
724  BESDEBUG(MODULE, msg << endl);
725  if(!result){
726  throw NgapError(msg,__FILE__,__LINE__);
727  }
728 
729  const rapidjson::Value& facets = get_children(facets_obj);
730  for (rapidjson::SizeType i = 0; i < facets.Size(); i++) { // Uses SizeType instead of size_t
731  const rapidjson::Value& facet = facets[i];
732 
733  string facet_title = ru.getStringValue(facet,"title");
734  string temporal_title("Temporal");
735  if(facet_title == temporal_title){
736  msg = prolog + "Found Temporal object.";
737  BESDEBUG(MODULE, msg << endl);
738  return facet;
739  }
740  else {
741  msg = prolog + "The child of 'facets' with title '"+facet_title+"' does not match 'Temporal'";
742  BESDEBUG(MODULE, msg << endl);
743  }
744  }
745  msg = prolog + "Failed to locate the Temporal facet.";
746  BESDEBUG(MODULE, msg << endl);
747  throw NgapError(msg,__FILE__,__LINE__);
748 
749  } // NgapApi::get_temporal_group()
750 
754  const rapidjson::Value&
755  NgapApi::get_year_group(const rapidjson::Document &ngap_doc){
756  rjson_utils rju;
757  string msg;
758 
759  const rapidjson::Value& temporal_group = get_temporal_group(ngap_doc);
760  const rapidjson::Value& temporal_children = get_children(temporal_group);
761  for (rapidjson::SizeType j = 0; j < temporal_children.Size(); j++) { // Uses SizeType instead of size_t
762  const rapidjson::Value& temporal_child = temporal_children[j];
763 
764  string temporal_child_title = rju.getStringValue(temporal_child,"title");
765  string year_title("Year");
766  if(temporal_child_title == year_title){
767  msg = prolog + "Found Year object.";
768  BESDEBUG(MODULE, msg << endl);
769  return temporal_child;
770  }
771  else {
772  msg = prolog + "The child of 'Temporal' with title '"+temporal_child_title+"' does not match 'Year'";
773  BESDEBUG(MODULE, msg << endl);
774  }
775  }
776  msg = prolog + "Failed to locate the Year group.";
777  BESDEBUG(MODULE, msg << endl);
778  throw NgapError(msg,__FILE__,__LINE__);
779  }
780 
784  const rapidjson::Value&
785  NgapApi::get_month_group(const string r_year, const rapidjson::Document &ngap_doc){
786  rjson_utils rju;
787  string msg;
788 
789  const rapidjson::Value& year_group = get_year_group(ngap_doc);
790  const rapidjson::Value& years = get_children(year_group);
791  for (rapidjson::SizeType i = 0; i < years.Size(); i++) { // Uses SizeType instead of size_t
792  const rapidjson::Value& year_obj = years[i];
793 
794  string year_title = rju.getStringValue(year_obj,"title");
795  if(r_year == year_title){
796  msg = prolog + "Found Year object.";
797  BESDEBUG(MODULE, msg << endl);
798 
799  const rapidjson::Value& year_children = get_children(year_obj);
800  for (rapidjson::SizeType j = 0; j < year_children.Size(); j++) { // Uses SizeType instead of size_t
801  const rapidjson::Value& child = year_children[i];
802  string title = rju.getStringValue(child,"title");
803  string month_title("Month");
804  if(title == month_title){
805  msg = prolog + "Found Month object.";
806  BESDEBUG(MODULE, msg << endl);
807  return child;
808  }
809  else {
810  msg = prolog + "The child of 'Year' with title '"+title+"' does not match 'Month'";
811  BESDEBUG(MODULE, msg << endl);
812  }
813  }
814  }
815  else {
816  msg = prolog + "The child of 'Year' group with title '"+year_title+"' does not match the requested year ("+r_year+")";
817  BESDEBUG(MODULE, msg << endl);
818  }
819  }
820  msg = prolog + "Failed to locate the Year group.";
821  BESDEBUG(MODULE, msg << endl);
822  throw NgapError(msg,__FILE__,__LINE__);
823  }
824 
825  const rapidjson::Value&
826  NgapApi::get_month(const string r_month, const string r_year, const rapidjson::Document &ngap_doc){
827  rjson_utils rju;
828  stringstream msg;
829 
830  const rapidjson::Value& month_group = get_month_group(r_year,ngap_doc);
831  const rapidjson::Value& months = get_children(month_group);
832  for (rapidjson::SizeType i = 0; i < months.Size(); i++) { // Uses SizeType instead of size_t
833  const rapidjson::Value& month = months[i];
834  string month_id = rju.getStringValue(month,"title");
835  if(month_id == r_month){
836  msg.str("");
837  msg << prolog << "Located requested month ("<<r_month << ")";
838  BESDEBUG(MODULE, msg.str() << endl);
839  return month;
840  }
841  else {
842  msg.str("");
843  msg << prolog << "The month titled '"<<month_id << "' does not match the requested month ("<< r_month <<")";
844  BESDEBUG(MODULE, msg.str() << endl);
845  }
846  }
847  msg.str("");
848  msg << prolog << "Failed to locate request Year/Month.";
849  BESDEBUG(MODULE, msg.str() << endl);
850  throw NgapError(msg.str(),__FILE__,__LINE__);
851  }
852 
853  const rapidjson::Value&
854  NgapApi::get_day_group(const string r_month, const string r_year, const rapidjson::Document &ngap_doc){
855  rjson_utils rju;
856  stringstream msg;
857 
858  const rapidjson::Value& month = get_month(r_month, r_year, ngap_doc);
859  const rapidjson::Value& month_children = get_children(month);
860 
861  for (rapidjson::SizeType k = 0; k < month_children.Size(); k++) { // Uses SizeType instead of size_t
862  const rapidjson::Value& object = month_children[k];
863  string title = rju.getStringValue(object,"title");
864  string day_group_title = "Day";
865  if(title == day_group_title){
866  msg.str("");
867  msg << prolog << "Located Day group for year: " << r_year << " month: "<< r_month;
868  BESDEBUG(MODULE, msg.str() << endl);
869  return object;
870  }
871  }
872  msg.str("");
873  msg << prolog << "Failed to locate requested Day year: " << r_year << " month: "<< r_month;
874  BESDEBUG(MODULE, msg.str() << endl);
875  throw NgapError(msg.str(),__FILE__,__LINE__);
876  }
877 
878 
885  void
886  NgapApi::get_years(string collection_name, vector<string> &years_result){
887  rjson_utils rju;
888  // bool result;
889  string msg;
890 
891  string url = BESUtil::assemblePath(ngap_search_endpoint_url,"granules.json") + "?concept_id="+collection_name +"&include_facets=v2";
893  rju.getJsonDoc(url,doc);
894 
895  const rapidjson::Value& year_group = get_year_group(doc);
896  const rapidjson::Value& years = get_children(year_group);
897  for (rapidjson::SizeType k = 0; k < years.Size(); k++) { // Uses SizeType instead of size_t
898  const rapidjson::Value& year_obj = years[k];
899  string year = rju.getStringValue(year_obj,"title");
900  years_result.push_back(year);
901  }
902  } // NgapApi::get_years()
903 
904 
913  void
914  NgapApi::get_months(string collection_name, string r_year, vector<string> &months_result){
915  rjson_utils rju;
916 
917  stringstream msg;
918 
919  string url = BESUtil::assemblePath(ngap_search_endpoint_url,"granules.json")
920  + "?concept_id="+collection_name
921  +"&include_facets=v2"
922  +"&temporal_facet[0][year]="+r_year;
923 
925  rju.getJsonDoc(url,doc);
926  BESDEBUG(MODULE, prolog << "Got JSON Document: "<< endl << rju.jsonDocToString(doc) << endl);
927 
928  const rapidjson::Value& year_group = get_year_group(doc);
929  const rapidjson::Value& years = get_children(year_group);
930  if(years.Size() != 1){
931  msg.str("");
932  msg << prolog << "We expected to get back one year (" << r_year << ") but we got back " << years.Size();
933  BESDEBUG(MODULE, msg.str() << endl);
934  throw NgapError(msg.str(),__FILE__,__LINE__);
935  }
936 
937  const rapidjson::Value& year = years[0];
938  string year_title = rju.getStringValue(year,"title");
939  if(year_title != r_year){
940  msg.str("");
941  msg << prolog << "The returned year (" << year_title << ") does not match the requested year ("<< r_year << ")";
942  BESDEBUG(MODULE, msg.str() << endl);
943  throw NgapError(msg.str(),__FILE__,__LINE__);
944  }
945 
946  const rapidjson::Value& year_children = get_children(year);
947  if(year_children.Size() != 1){
948  msg.str("");
949  msg << prolog << "We expected to get back one child for the year (" << r_year << ") but we got back " << years.Size();
950  BESDEBUG(MODULE, msg.str() << endl);
951  throw NgapError(msg.str(),__FILE__,__LINE__);
952  }
953 
954  const rapidjson::Value& month_group = year_children[0];
955  string title = rju.getStringValue(month_group,"title");
956  if(title != string("Month")){
957  msg.str("");
958  msg << prolog << "We expected to get back a Month object, but we did not.";
959  BESDEBUG(MODULE, msg.str() << endl);
960  throw NgapError(msg.str(),__FILE__,__LINE__);
961  }
962 
963  const rapidjson::Value& months = get_children(month_group);
964  for (rapidjson::SizeType i = 0; i < months.Size(); i++) { // Uses SizeType instead of size_t
965  const rapidjson::Value& month = months[i];
966  string month_id = rju.getStringValue(month,"title");
967  months_result.push_back(month_id);
968  }
969  return;
970 
971  } // NgapApi::get_months()
972 
976  void
977  NgapApi::get_days(string collection_name, string r_year, string r_month, vector<string> &days_result){
978  rjson_utils rju;
979  stringstream msg;
980 
981  string url = BESUtil::assemblePath(ngap_search_endpoint_url,"granules.json")
982  + "?concept_id="+collection_name
983  +"&include_facets=v2"
984  +"&temporal_facet[0][year]="+r_year
985  +"&temporal_facet[0][month]="+r_month;
986 
987  rapidjson::Document ngap_doc;
988  rju.getJsonDoc(url,ngap_doc);
989  BESDEBUG(MODULE, prolog << "Got JSON Document: "<< endl << rju.jsonDocToString(ngap_doc) << endl);
990 
991  const rapidjson::Value& day_group = get_day_group(r_month, r_year, ngap_doc);
992  const rapidjson::Value& days = get_children(day_group);
993  for (rapidjson::SizeType i = 0; i < days.Size(); i++) { // Uses SizeType instead of size_t
994  const rapidjson::Value& day = days[i];
995  string day_id = rju.getStringValue(day,"title");
996  days_result.push_back(day_id);
997  }
998  }
999 
1000 
1001 
1005  void
1006  NgapApi::get_granule_ids(string collection_name, string r_year, string r_month, string r_day, vector<string> &granules_ids){
1007  rjson_utils rju;
1008  stringstream msg;
1009  rapidjson::Document ngap_doc;
1010 
1011  granule_search(collection_name, r_year, r_month, r_day, ngap_doc);
1012 
1013  const rapidjson::Value& entries = get_entries(ngap_doc);
1014  for (rapidjson::SizeType i = 0; i < entries.Size(); i++) { // Uses SizeType instead of size_t
1015  const rapidjson::Value& granule = entries[i];
1016  string day_id = rju.getStringValue(granule,"id");
1017  granules_ids.push_back(day_id);
1018  }
1019 
1020  }
1021 
1022 
1026  unsigned long
1027  NgapApi::granule_count(string collection_name, string r_year, string r_month, string r_day){
1028  stringstream msg;
1029  rapidjson::Document ngap_doc;
1030  granule_search(collection_name, r_year, r_month, r_day, ngap_doc);
1031  const rapidjson::Value& entries = get_entries(ngap_doc);
1032  return entries.Size();
1033  }
1034 
1039  void
1040  NgapApi::granule_search(string collection_name, string r_year, string r_month, string r_day, rapidjson::Document &result_doc){
1041  rjson_utils rju;
1042 
1043  string url = BESUtil::assemblePath(ngap_search_endpoint_url,"granules.json")
1044  + "?concept_id="+collection_name
1045  + "&include_facets=v2"
1046  + "&page_size=2000";
1047 
1048  if(!r_year.empty())
1049  url += "&temporal_facet[0][year]="+r_year;
1050 
1051  if(!r_month.empty())
1052  url += "&temporal_facet[0][month]="+r_month;
1053 
1054  if(!r_day.empty())
1055  url += "&temporal_facet[0][day]="+r_day;
1056 
1057  BESDEBUG(MODULE, prolog << "ngap Granule Search Request Url: : " << url << endl);
1058  rju.getJsonDoc(url,result_doc);
1059  BESDEBUG(MODULE, prolog << "Got JSON Document: "<< endl << rju.jsonDocToString(result_doc) << endl);
1060  }
1061 
1062 
1063 
1067  void
1068  NgapApi::get_granules(string collection_name, string r_year, string r_month, string r_day, vector<Granule *> &granules){
1069  stringstream msg;
1070  rapidjson::Document ngap_doc;
1071 
1072  granule_search(collection_name, r_year, r_month, r_day, ngap_doc);
1073 
1074  const rapidjson::Value& entries = get_entries(ngap_doc);
1075  for (rapidjson::SizeType i = 0; i < entries.Size(); i++) { // Uses SizeType instead of size_t
1076  const rapidjson::Value& granule_obj = entries[i];
1077  // rapidjson::Value grnl(granule_obj, ngap_doc.GetAllocator());
1078  Granule *g = new Granule(granule_obj);
1079  granules.push_back(g);
1080  }
1081 
1082  }
1083 
1084 
1085  void
1086  NgapApi::get_collection_ids(std::vector<std::string> &collection_ids){
1087  bool found = false;
1088  string key = NGAP_COLLECTIONS;
1089  TheBESKeys::TheKeys()->get_values(NGAP_COLLECTIONS, collection_ids, found);
1090  if(!found){
1091  throw BESInternalError(string("The '") +NGAP_COLLECTIONS
1092  + "' field has not been configured.", __FILE__, __LINE__);
1093  }
1094  }
1095 
1099  Granule* NgapApi::get_granule(string collection_name, string r_year, string r_month, string r_day, string granule_id)
1100  {
1101  vector<Granule *> granules;
1102  Granule *result = 0;
1103 
1104  get_granules(collection_name, r_year, r_month, r_day, granules);
1105  for(size_t i=0; i<granules.size() ;i++){
1106  string id = granules[i]->getName();
1107  BESDEBUG(MODULE, prolog << "Comparing granule id: " << granule_id << " to collection member id: " << id << endl);
1108  if( id == granule_id){
1109  result = granules[i];
1110  }
1111  else {
1112  delete granules[i];
1113  granules[i] = 0;
1114  }
1115  }
1116  return result;
1117  }
1118 #endif
1119 
1120 } // namespace ngap
1121 
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:160
exception thrown if internal error encountered
error thrown if the resource requested cannot be found
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
error thrown if there is a user syntax error in the request or any other user error
static void tokenize(const std::string &str, std::vector< std::string > &tokens, const std::string &delimiters="/")
Definition: BESUtil.cc:1057
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
Definition: BESUtil.cc:821
(Constant) member iterator for a JSON object value
Definition: document.h:177
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:339
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
void get_values(const std::string &s, std::vector< std::string > &vals, bool &found)
Retrieve the values of a given key, if set.
Definition: TheBESKeys.cc:370
rapidjson::Document get_as_json()
get_as_json() This function returns the cached resource parsed into a JSON document.
virtual std::string query_parameter_value(const std::string &key) const
Definition: url_impl.cc:201
GenericValue< UTF8<> > Value
GenericValue with UTF8 encoding.
Definition: document.h:2189
GenericDocument< UTF8<> > Document
GenericDocument with UTF8 encoding.
Definition: document.h:2585
RAPIDJSON_NAMESPACE_BEGIN typedef unsigned SizeType
Size type (for string lengths, array sizes, etc.)
Definition: rapidjson.h:384