bes  Updated for version 3.20.8
h5dmr.cc
Go to the documentation of this file.
1 // This file is part of hdf5_handler a HDF5 file handler for the OPeNDAP
2 // data server.
3 
4 // Copyright (c) 2007-2015 The HDF Group, Inc. and OPeNDAP, Inc.
5 //
6 // This is free software; you can redistribute it and/or modify it under the
7 // terms of the GNU Lesser General Public License as published by the Free
8 // Software Foundation; either version 2.1 of the License, or (at your
9 // option) any later version.
10 //
11 // This software is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 // License for more details.
15 //
16 // You should have received a copy of the GNU Lesser General Public
17 // License along with this library; if not, write to the Free Software
18 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
22 // Suite 203, Champaign, IL 61820
23 
36 // the correct DAP4 DMR layout(group's variables first and then the group).
39 
40 #include <sstream>
41 #include "config_hdf5.h"
42 
43 #include <InternalErr.h>
44 #include <BESDebug.h>
45 
46 #include <mime_util.h>
47 
48 #include "hdf5_handler.h"
49 #include "HDF5Int32.h"
50 #include "HDF5UInt32.h"
51 #include "HDF5UInt16.h"
52 #include "HDF5Int16.h"
53 #include "HDF5Byte.h"
54 #include "HDF5Array.h"
55 #include "HDF5Str.h"
56 #include "HDF5Float32.h"
57 #include "HDF5Float64.h"
58 #include "HDF5Url.h"
59 #include "HDF5Structure.h"
60 
61 // The HDF5CFUtil.h includes the utility function obtain_string_after_lastslash.
62 #include "HDF5CFUtil.h"
63 #include "h5dmr.h"
64 
65 using namespace std;
66 using namespace libdap;
69 
70 
72 static DS_t dt_inst;
73 
75 void map_h5_attrs_to_dap4(hid_t oid,D4Group* d4g,BaseType* d4b,Structure * d4s,int flag);
76 
77 #if 0
83 // \param par_grp DAP4 parent group
93 
94 //bool depth_first(hid_t pid, char *gname, DMR & dmr, D4Group* par_grp, const char *fname)
95 bool depth_first(hid_t pid, char *gname, D4Group* par_grp, const char *fname)
96 {
97  BESDEBUG("h5",
98  ">depth_first() for dmr "
99  << " pid: " << pid
100  << " gname: " << gname
101  << " fname: " << fname
102  << endl);
103 
105  int slinkindex = 0;
106 
107  H5G_info_t g_info;
108  hsize_t nelems = 0;
109 
111  if(H5Gget_info(pid,&g_info) <0) {
112  string msg =
113  "h5_dmr handler: counting hdf5 group elements error for ";
114  msg += gname;
115  throw InternalErr(__FILE__, __LINE__, msg);
116  }
117 
118  nelems = g_info.nlinks;
119 
120  ssize_t oname_size = 0;
121 
122  // Iterate through the file to see the members of the group from the root.
123  for (hsize_t i = 0; i < nelems; i++) {
124 
125  vector <char>oname;
126 
127  // Query the length of object name.
128  oname_size =
129  H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
130  (size_t)DODS_NAMELEN, H5P_DEFAULT);
131  if (oname_size <= 0) {
132  string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
133  msg += gname;
134  throw InternalErr(__FILE__, __LINE__, msg);
135  }
136 
137  // Obtain the name of the object
138  oname.resize((size_t) oname_size + 1);
139 
140  if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
141  (size_t)(oname_size+1), H5P_DEFAULT) < 0){
142  string msg =
143  "h5_dmr handler: Error getting the hdf5 object name from the group: ";
144  msg += gname;
145  throw InternalErr(__FILE__, __LINE__, msg);
146  }
147 
148  // Check if it is the hard link or the soft link
149  H5L_info_t linfo;
150  if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
151  string msg = "hdf5 link name error from: ";
152  msg += gname;
153  throw InternalErr(__FILE__, __LINE__, msg);
154  }
155 
156  // Information of soft links are stored as attributes
157  if(linfo.type == H5L_TYPE_SOFT) {
158  slinkindex++;
159  size_t val_size = linfo.u.val_size;
160  get_softlink(par_grp,pid,&oname[0],slinkindex,val_size);
161  //get_softlink(par_grp,pid,gname,&oname[0],slinkindex,val_size);
162  continue;
163  }
164 
165  // Ignore external links
166  if(linfo.type == H5L_TYPE_EXTERNAL)
167  continue;
168 
169  // Obtain the object type, such as group or dataset.
170  H5O_info_t oinfo;
171 
172  if (H5Oget_info_by_idx(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
173  i, &oinfo, H5P_DEFAULT)<0) {
174  string msg = "h5_dmr handler: Error obtaining the info for the object";
175  msg += string(oname.begin(),oname.end());
176  throw InternalErr(__FILE__, __LINE__, msg);
177  }
178 
179  H5O_type_t obj_type = oinfo.type;
180 
181  switch (obj_type) {
182 
183  case H5O_TYPE_GROUP:
184  {
185 
186  // Obtain the full path name
187  string full_path_name =
188  string(gname) + string(oname.begin(),oname.end()-1) + "/";
189 
190  BESDEBUG("h5", "=depth_first dmr ():H5G_GROUP " << full_path_name
191  << endl);
192 
193  vector <char>t_fpn;
194  t_fpn.resize(full_path_name.length()+1);
195  copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
196  t_fpn[full_path_name.length()] = '\0';
197 
198  hid_t cgroup = H5Gopen(pid, &t_fpn[0],H5P_DEFAULT);
199  if (cgroup < 0){
200  throw InternalErr(__FILE__, __LINE__, "h5_dmr handler: H5Gopen() failed.");
201  }
202 
203  string grp_name = string(oname.begin(),oname.end()-1);
204 
205  // Check the hard link loop and break the loop if it exists.
206  string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
207  if (oid == "") {
208  try {
209  D4Group* tem_d4_cgroup = new D4Group(grp_name);
210  // Map the HDF5 cgroup attributes to DAP4 group attributes.
211  // Note the last flag of map_h5_attrs_to_dap4 must be 0 for the group attribute mapping.
212  map_h5_attrs_to_dap4(cgroup,tem_d4_cgroup,NULL,NULL,0);
213 
214  // Add this new DAP4 group
215  par_grp->add_group_nocopy(tem_d4_cgroup);
216 
217  // Continue searching the objects under this group
218  //depth_first(cgroup, &t_fpn[0], dmr, tem_d4_cgroup,fname);
219  depth_first(cgroup, &t_fpn[0], tem_d4_cgroup,fname);
220  }
221  catch(...) {
222  H5Gclose(cgroup);
223  throw;
224  }
225  }
226  else {
227  // This group has been visited.
228  // Add the attribute table with the attribute name as HDF5_HARDLINK.
229  // The attribute value is the name of the group when it is first visited.
230  D4Group* tem_d4_cgroup = new D4Group(string(grp_name));
231 
232  // Note attr_str_c is the DAP4 attribute string datatype
233  D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
234 
235  d4_hlinfo->add_value(obj_paths.get_name(oid));
236  tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
237  par_grp->add_group_nocopy(tem_d4_cgroup);
238 
239  }
240 
241  if (H5Gclose(cgroup) < 0){
242  throw InternalErr(__FILE__, __LINE__, "Could not close the group.");
243  }
244  break;
245  }
246 
247  case H5O_TYPE_DATASET:
248  {
249 
250  // Obtain the absolute path of the HDF5 dataset
251  string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
252 
253  // TOOOODOOOO
254  // Obtain the hdf5 dataset handle stored in the structure dt_inst.
255  // All the metadata information in the handler is stored in dt_inst.
256  // Work on this later, redundant for dmr since dataset is opened twice. KY 2015-07-01
257  // Note: depth_first is for building DMR of an HDF5 file that doesn't use dim. scale.
258  // so passing the last parameter as false.
259  get_dataset(pid, full_path_name, &dt_inst,false);
260 
261  // Here we open the HDF5 dataset again to use the dataset id for dataset attributes.
262  // This is not necessary for DAP2 since DAS and DDS are separated.
263  hid_t dset_id = -1;
264  if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
265  string msg = "cannot open the HDF5 dataset ";
266  msg += full_path_name;
267  throw InternalErr(__FILE__, __LINE__, msg);
268  }
269 
270  try {
271  read_objects(par_grp, full_path_name, fname,dset_id);
272  }
273  catch(...) {
274  H5Dclose(dset_id);
275  throw;
276  }
277  if(H5Dclose(dset_id)<0) {
278  string msg = "cannot close the HDF5 dataset ";
279  msg += full_path_name;
280  throw InternalErr(__FILE__, __LINE__, msg);
281  }
282  }
283  break;
284 
285  case H5O_TYPE_NAMED_DATATYPE:
286  // ignore the named datatype
287  break;
288  default:
289  break;
290  }// switch(obj_type)
291  } // for i is 0 ... nelems
292 
293  BESDEBUG("h5", "<depth_first() for dmr" << endl);
294  return true;
295 }
296 #endif
302 // \param par_grp DAP4 parent group
314 
315 
316 // The reason to use breadth_first is that the DMR representation needs to show the dimension names and the variables under the group first and then the group names.
317 // So we use this search. In the future, we may just use the breadth_first search for all cases.??
318 //bool breadth_first(hid_t pid, char *gname, DMR & dmr, D4Group* par_grp, const char *fname,bool use_dimscale)
319 bool breadth_first(hid_t pid, char *gname, D4Group* par_grp, const char *fname,bool use_dimscale)
320 {
321  BESDEBUG("h5",
322  ">breadth_first() for dmr "
323  << " pid: " << pid
324  << " gname: " << gname
325  << " fname: " << fname
326  << endl);
327 
329  int slinkindex = 0;
330 
331  // Obtain the number of objects in this group
332  H5G_info_t g_info;
333  hsize_t nelems = 0;
334  if(H5Gget_info(pid,&g_info) <0) {
335  string msg =
336  "h5_dmr handler: counting hdf5 group elements error for ";
337  msg += gname;
338  throw InternalErr(__FILE__, __LINE__, msg);
339  }
340 
341  nelems = g_info.nlinks;
342 
343  ssize_t oname_size;
344 
345  // First iterate through the HDF5 datasets under the group.
346  for (hsize_t i = 0; i < nelems; i++) {
347 
348  vector <char>oname;
349 
350  // Query the length of object name.
351  oname_size =
352  H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
353  (size_t)DODS_NAMELEN, H5P_DEFAULT);
354  if (oname_size <= 0) {
355  string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
356  msg += gname;
357  throw InternalErr(__FILE__, __LINE__, msg);
358  }
359 
360  // Obtain the name of the object
361  oname.resize((size_t) oname_size + 1);
362 
363  if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
364  (size_t)(oname_size+1), H5P_DEFAULT) < 0){
365  string msg =
366  "h5_dmr handler: Error getting the hdf5 object name from the group: ";
367  msg += gname;
368  throw InternalErr(__FILE__, __LINE__, msg);
369  }
370 
371  // Check if it is the hard link or the soft link
372  H5L_info_t linfo;
373  if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
374  string msg = "hdf5 link name error from: ";
375  msg += gname;
376  throw InternalErr(__FILE__, __LINE__, msg);
377  }
378 
379  // Information of soft links are stored as attributes
380  if(linfo.type == H5L_TYPE_SOFT) {
381  slinkindex++;
382 
383  // Size of a soft link value
384  size_t val_size = linfo.u.val_size;
385  get_softlink(par_grp,pid,&oname[0],slinkindex,val_size);
386  continue;
387  }
388 
389  // Ignore external links
390  if(linfo.type == H5L_TYPE_EXTERNAL)
391  continue;
392 
393  // Obtain the object type, such as group or dataset.
394  H5O_info_t oinfo;
395 
396  if (H5Oget_info_by_idx(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
397  i, &oinfo, H5P_DEFAULT)<0) {
398  string msg = "h5_dmr handler: Error obtaining the info for the object";
399  msg += string(oname.begin(),oname.end());
400  throw InternalErr(__FILE__, __LINE__, msg);
401  }
402 
403  H5O_type_t obj_type = oinfo.type;
404 
405  if(H5O_TYPE_DATASET == obj_type) {
406 
407  // Obtain the absolute path of the HDF5 dataset
408  string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
409 
410  // TOOOODOOOO
411  // Obtain the hdf5 dataset handle stored in the structure dt_inst.
412  // All the metadata information in the handler is stored in dt_inst.
413  // Work on this later, redundant for dmr since dataset is opened twice. KY 2015-07-01
414  // Dimension scale is handled in this routine. So need to keep it. KY 2020-06-10
415  bool is_pure_dim = false;
416  get_dataset(pid, full_path_name, &dt_inst,use_dimscale,is_pure_dim);
417 
418  if(false == is_pure_dim) {
419  hid_t dset_id = -1;
420  if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
421  string msg = "cannot open the HDF5 dataset ";
422  msg += full_path_name;
423  throw InternalErr(__FILE__, __LINE__, msg);
424  }
425 
426  try {
427  read_objects(par_grp, full_path_name, fname,dset_id);
428  }
429  catch(...) {
430  H5Dclose(dset_id);
431  throw;
432  }
433  if(H5Dclose(dset_id)<0) {
434  string msg = "cannot close the HDF5 dataset ";
435  msg += full_path_name;
436  throw InternalErr(__FILE__, __LINE__, msg);
437  }
438  }
439  else {
440  //Need to add this pure dimension to the corresponding DAP4 group
441  D4Dimensions *d4_dims = par_grp->dims();
442  string d4dim_name = string(oname.begin(),oname.end()-1);
443  D4Dimension *d4_dim = d4_dims->find_dim(d4dim_name);
444  if(d4_dim == NULL) {
445  d4_dim = new D4Dimension(d4dim_name,dt_inst.nelmts);
446  d4_dims->add_dim_nocopy(d4_dim);
447  }
448  BESDEBUG("h5", "<h5dmr.cc: pure dimension: dataset name." << d4dim_name << endl);
449  if(H5Tclose(dt_inst.type)<0) {
450  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
451  }
452  }
453 
454  }
455  }
456 
457  // The attributes of this group. Doing this order to follow ncdump's way (variable,attribute then groups)
458  map_h5_attrs_to_dap4(pid,par_grp,NULL,NULL,0);
459 
460  // Then HDF5 child groups
461  for (hsize_t i = 0; i < nelems; i++) {
462 
463  vector <char>oname;
464 
465  // Query the length of object name.
466  oname_size =
467  H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
468  (size_t)DODS_NAMELEN, H5P_DEFAULT);
469  if (oname_size <= 0) {
470  string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
471  msg += gname;
472  throw InternalErr(__FILE__, __LINE__, msg);
473  }
474 
475  // Obtain the name of the object
476  oname.resize((size_t) oname_size + 1);
477 
478  if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
479  (size_t)(oname_size+1), H5P_DEFAULT) < 0){
480  string msg =
481  "h5_dmr handler: Error getting the hdf5 object name from the group: ";
482  msg += gname;
483  throw InternalErr(__FILE__, __LINE__, msg);
484  }
485 
486  // Check if it is the hard link or the soft link
487  H5L_info_t linfo;
488  if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
489  string msg = "hdf5 link name error from: ";
490  msg += gname;
491  throw InternalErr(__FILE__, __LINE__, msg);
492  }
493 
494  // Information of soft links are handled already, the softlinks need to be ignored, otherwise
495  // the group it links will be mapped again in the block of if obj_type is H5O_TYPE_GROUP
496  if(linfo.type == H5L_TYPE_SOFT) {
497  continue;
498  }
499 
500  // Ignore external links
501  if(linfo.type == H5L_TYPE_EXTERNAL)
502  continue;
503 
504  // Obtain the object type, such as group or dataset.
505  H5O_info_t oinfo;
506 
507  if (H5Oget_info_by_idx(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
508  i, &oinfo, H5P_DEFAULT)<0) {
509  string msg = "h5_dmr handler: Error obtaining the info for the object in the breadth_first.";
510  throw InternalErr(__FILE__, __LINE__, msg);
511  }
512 
513  H5O_type_t obj_type = oinfo.type;
514 
515 
516  if(obj_type == H5O_TYPE_GROUP) {
517 
518  // Obtain the full path name
519  string full_path_name =
520  string(gname) + string(oname.begin(),oname.end()-1) + "/";
521 
522  BESDEBUG("h5", "=breadth_first dmr ():H5G_GROUP " << full_path_name
523  << endl);
524 
525  vector <char>t_fpn;
526  t_fpn.resize(full_path_name.length()+1);
527  copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
528  t_fpn[full_path_name.length()] = '\0';
529 
530  hid_t cgroup = H5Gopen(pid, &t_fpn[0],H5P_DEFAULT);
531  if (cgroup < 0){
532  throw InternalErr(__FILE__, __LINE__, "h5_dmr handler: H5Gopen() failed.");
533  }
534 
535  string grp_name = string(oname.begin(),oname.end()-1);
536 
537  // Check the hard link loop and break the loop if it exists.
538  string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
539  if (oid == "") {
540  try {
541  D4Group* tem_d4_cgroup = new D4Group(grp_name);
542 
543  // Add this new DAP4 group
544  par_grp->add_group_nocopy(tem_d4_cgroup);
545 
546  // Continue searching the objects under this group
547  breadth_first(cgroup, &t_fpn[0], tem_d4_cgroup,fname,use_dimscale);
548  }
549  catch(...) {
550  H5Gclose(cgroup);
551  throw;
552  }
553  }
554  else {
555  // This group has been visited.
556  // Add the attribute table with the attribute name as HDF5_HARDLINK.
557  // The attribute value is the name of the group when it is first visited.
558  D4Group* tem_d4_cgroup = new D4Group(string(grp_name));
559 
560  // Note attr_str_c is the DAP4 attribute string datatype
561  D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
562 
563  d4_hlinfo->add_value(obj_paths.get_name(oid));
564  tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
565  par_grp->add_group_nocopy(tem_d4_cgroup);
566  }
567 
568  if (H5Gclose(cgroup) < 0){
569  throw InternalErr(__FILE__, __LINE__, "Could not close the group.");
570  }
571  }// end if
572  } // for i is 0 ... nelems
573 
574  BESDEBUG("h5", "<breadth_first() " << endl);
575  return true;
576 }
577 
593 //
594 void
595 read_objects( D4Group * d4_grp, const string &varname, const string &filename, const hid_t dset_id)
596 {
597 
598  switch (H5Tget_class(dt_inst.type)) {
599 
600  // HDF5 compound maps to DAP structure.
601  case H5T_COMPOUND:
602  read_objects_structure(d4_grp, varname, filename,dset_id);
603  break;
604 
605  case H5T_ARRAY:
606  H5Tclose(dt_inst.type);
607  throw InternalErr(__FILE__, __LINE__, "Currently don't support accessing data of Array datatype when array datatype is not inside the compound.");
608 
609  default:
610  read_objects_base_type(d4_grp,varname, filename,dset_id);
611  break;
612  }
613  // We must close the datatype obtained in the get_dataset routine since this is the end of reading DDS.
614  if(H5Tclose(dt_inst.type)<0) {
615  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
616  }
617 }
618 
633 //
634 
635 //void
636 //read_objects_base_type(DMR & dmr, D4Group * d4_grp,const string & varname,
637 void
638 read_objects_base_type(D4Group * d4_grp,const string & varname,
639  const string & filename,hid_t dset_id)
640 {
641 
642  // Obtain the relative path of the variable name under the leaf group
643  string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
644 
645  // Get a base type. It should be an HDF5 atomic datatype
646  // datatype.
647  BaseType *bt = Get_bt(newvarname, varname,filename, dt_inst.type,true);
648  if (!bt) {
649  throw
650  InternalErr(__FILE__, __LINE__,
651  "Unable to convert hdf5 datatype to dods basetype");
652  }
653 
654  // First deal with scalar data.
655  if (dt_inst.ndims == 0) {
656  // transform the DAP2 to DAP4 for this DAP base type and add it to d4_grp
657  bt->transform_to_dap4(d4_grp,d4_grp);
658  // Get it back - this may return null because the underlying type
659  // may have no DAP2 manifestation.
660  BaseType* new_var = d4_grp->var(bt->name());
661  if(new_var){
662  // Map the HDF5 dataset attributes to DAP4
663  map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
664  // If this variable is a hardlink, stores the HARDLINK info. as an attribute.
665  map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
666  }
667  delete bt;
668  bt = 0;
669  }
670  else {
671  // Next, deal with Array data. This 'else clause' runs to
672  // the end of the method.
673  HDF5Array *ar = new HDF5Array(newvarname, filename, bt);
674  delete bt; bt = 0;
675 
676  // set number of elements and variable name values.
677  // This essentially stores in the struct.
678  ar->set_memneed(dt_inst.need);
679  ar->set_numdim(dt_inst.ndims);
680  ar->set_numelm((int) (dt_inst.nelmts));
681  ar->set_varpath(varname);
682 
683 
684  // If we have dimension names(dimension scale is used.),we will see if we can add the names.
685  int dimnames_size = 0;
686  if((unsigned int)((int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
687  {
688  delete ar;
689  throw
690  InternalErr(__FILE__, __LINE__,
691  "number of dimensions: overflow");
692  }
693  dimnames_size = (int)(dt_inst.dimnames.size());
694 //cerr<<"dimnames_size is "<<dimnames_size <<endl;
695 //cerr<<"ndims is "<<dt_inst.ndims <<endl;
696 
697  if(dimnames_size ==dt_inst.ndims) {
698 
699  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++) {
700  if(dt_inst.dimnames[dim_index] !="")
701  ar->append_dim(dt_inst.size[dim_index],dt_inst.dimnames[dim_index]);
702  else
703  ar->append_dim(dt_inst.size[dim_index]);
704  // D4dimension has to have a name. If no name, no D4dimension(from comments libdap4: Array.cc)
705  }
706  dt_inst.dimnames.clear();
707  }
708  else {
709  // For DAP4, no need to add dimension if no dimension name
710  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++)
711  ar->append_dim(dt_inst.size[dim_index]);
712  }
713 
714  // We need to transform dimension info. to DAP4 group
715  BaseType* new_var = NULL;
716  try {
717  new_var = ar->h5dims_transform_to_dap4(d4_grp,dt_inst.dimnames_path);
718  }
719  catch(...) {
720  delete ar;
721  throw;
722  }
723 
724  // clear DAP4 dimnames_path vector
725  dt_inst.dimnames_path.clear();
726 
727  // Map HDF5 dataset attributes to DAP4
728  map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
729 
730  // If this is a hardlink, map the Hardlink info. as an DAP4 attribute.
731  map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
732 #if 0
733  // Test the attribute
734  D4Attribute *test_attr = new D4Attribute("DAP4_test",attr_str_c);
735  test_attr->add_value("test_grp_attr");
736  new_var->attributes()->add_attribute_nocopy(test_attr);
737 #endif
738  // Add this var to DAP4 group.
739  d4_grp->add_var_nocopy(new_var);
740  delete ar; ar = 0;
741  }
742  BESDEBUG("h5", "<read_objects_base_type(dmr)" << endl);
743 
744 }
745 
759 void
760 read_objects_structure(D4Group *d4_grp, const string & varname,
761  const string & filename,hid_t dset_id)
762 {
763  // Obtain the relative path of the variable name under the leaf group
764  string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
765 
766  // Map HDF5 compound datatype to Structure
767  Structure *structure = Get_structure(newvarname, varname,filename, dt_inst.type,true);
768 
769  try {
770  BESDEBUG("h5", "=read_objects_structure(): Dimension is "
771  << dt_inst.ndims << endl);
772 
773  if (dt_inst.ndims != 0) { // Array of Structure
774  BESDEBUG("h5", "=read_objects_structure(): array of size " <<
775  dt_inst.nelmts << endl);
776  BESDEBUG("h5", "=read_objects_structure(): memory needed = " <<
777  dt_inst.need << endl);
778 
779  // Create the Array of structure.
780  HDF5Array *ar = new HDF5Array(newvarname, filename, structure);
781  delete structure; structure = 0;
782 
783 
784  // These parameters are used in the data read function.
785  ar->set_memneed(dt_inst.need);
786  ar->set_numdim(dt_inst.ndims);
787  ar->set_numelm((int) (dt_inst.nelmts));
788  ar->set_length((int) (dt_inst.nelmts));
789  ar->set_varpath(varname);
790 
791  // If having dimension names, add the dimension names to DAP.
792  int dimnames_size = 0;
793  if((unsigned int)((int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
794  {
795  delete ar;
796  throw
797  InternalErr(__FILE__, __LINE__,
798  "number of dimensions: overflow");
799  }
800  dimnames_size = (int)(dt_inst.dimnames.size());
801 
802 
803  if(dimnames_size ==dt_inst.ndims) {
804  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++) {
805  if(dt_inst.dimnames[dim_index] !="")
806  ar->append_dim(dt_inst.size[dim_index],dt_inst.dimnames[dim_index]);
807  else
808  ar->append_dim(dt_inst.size[dim_index]);
809  }
810  dt_inst.dimnames.clear();
811  }
812  else {
813  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++)
814  ar->append_dim(dt_inst.size[dim_index]);
815 
816  }
817 
818  // We need to transform dimension info. to DAP4 group
819  BaseType* new_var = ar->h5dims_transform_to_dap4(d4_grp,dt_inst.dimnames_path);
820  dt_inst.dimnames_path.clear();
821 
822  // Map HDF5 dataset attributes to DAP4
823  map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
824 
825  // If this is a hardlink, map the Hardlink info. as an DAP4 attribute.
826  map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
827 
828  // Add this var to DAP4 group
829  if(new_var)
830  d4_grp->add_var_nocopy(new_var);
831  delete ar; ar = 0;
832  }// end if
833  else {// A scalar structure
834 
835  structure->set_is_dap4(true);
836  map_h5_attrs_to_dap4(dset_id,NULL,NULL,structure,2);
837  map_h5_dset_hardlink_to_d4(dset_id,varname,NULL,structure,2);
838  if(structure)
839  d4_grp->add_var_nocopy(structure);
840  }
841  } // try Structure
842  catch (...) {
843  delete structure;
844  throw;
845  }
846 }
847 
848 
862 //
863 
864 void map_h5_attrs_to_dap4(hid_t h5_objid,D4Group* d4g,BaseType* d4b,Structure * d4s,int flag) {
865 
866  // Get the object info
867  H5O_info_t obj_info;
868  if (H5Oget_info(h5_objid, &obj_info) <0) {
869  string msg = "Fail to obtain the HDF5 object info. .";
870  throw InternalErr(__FILE__, __LINE__, msg);
871  }
872 
873  // Obtain the number of attributes
874  int num_attr = obj_info.num_attrs;
875  if (num_attr < 0 ) {
876  string msg = "Fail to get the number of attributes for the HDF5 object. ";
877  throw InternalErr(__FILE__, __LINE__,msg);
878  }
879 
880  string print_rep;
881  vector<char>temp_buf;
882 
883  bool ignore_attr = false;
884  hid_t attr_id = -1;
885  for (int j = 0; j < num_attr; j++) {
886 
887  // Obtain attribute information.
888  DSattr_t attr_inst;
889 
890  // Ignore the attributes of which the HDF5 datatype
891  // cannot be mapped to DAP4. The ignored attribute datatypes can be found
892  // at function get_attr_info in h5get.cc.
893  attr_id = get_attr_info(h5_objid, j, true,&attr_inst, &ignore_attr);
894  if (true == ignore_attr) {
895  H5Aclose(attr_id);
896  continue;
897  }
898 
899  // Get the corresponding DAP data type of the HDF5 datatype.
900  // The following line doesn't work in HDF5 1.10.
901 #if 0
902  //hid_t ty_id = attr_inst.type;
903 #endif
904  hid_t ty_id = H5Aget_type(attr_id);
905  if(ty_id <0) {
906  H5Aclose(attr_id);
907  throw InternalErr(__FILE__, __LINE__, "Cannot retrieve HDF5 attribute datatype successfully.");
908  }
909 
910  string dap_type = get_dap_type(ty_id,true);
911 
912  // Need to have DAP4 representation of the attribute type
913  D4AttributeType dap4_attr_type = daptype_strrep_to_dap4_attrtype(dap_type);
914 
915  // We encounter an unsupported DAP4 attribute type.
916  if(attr_null_c == dap4_attr_type) {
917  H5Tclose(ty_id);
918  H5Aclose(attr_id);
919  throw InternalErr(__FILE__, __LINE__, "unsupported DAP4 attribute type");
920  }
921 
922  string attr_name = attr_inst.name;
923  BESDEBUG("h5", "arttr_name= " << attr_name << endl);
924 
925  // Create the DAP4 attribute mapped from HDF5
926  D4Attribute *d4_attr = new D4Attribute(attr_name,dap4_attr_type);
927 
928  // We have to handle variable length string differently.
929  if (H5Tis_variable_str(ty_id)) {
930  write_vlen_str_attrs(attr_id,ty_id,&attr_inst,d4_attr,NULL,true);
931  }// if (H5Tis_variable_str(ty_id)
932  else {
933 
934  vector<char> value;
935  value.resize(attr_inst.need + sizeof(char));
936  //value.resize(attr_inst.need);
937  BESDEBUG("h5", "arttr_inst.need=" << attr_inst.need << endl);
938 
939  // Need to obtain the memtype since we still find BE data.
940  hid_t memtype = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
941  // Read HDF5 attribute data.
942  if (H5Aread(attr_id, memtype, (void *) (&value[0])) < 0) {
943  delete d4_attr;
944  throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
945  }
946  H5Aclose(memtype);
947 
948  // For scalar data, just read data once.
949  if (attr_inst.ndims == 0) {
950  for (int loc = 0; loc < (int) attr_inst.nelmts; loc++) {
951  print_rep = print_attr(ty_id, loc, &value[0]);
952  if (print_rep.c_str() != NULL) {
953  d4_attr->add_value(print_rep);
954  }
955  }
956 
957  }
958  else {// The number of dimensions is > 0
959 
960  // Get the attribute datatype size
961  int elesize = (int) H5Tget_size(ty_id);
962  if (elesize == 0) {
963  H5Tclose(ty_id);
964  H5Aclose(attr_id);
965  delete d4_attr;
966  throw InternalErr(__FILE__, __LINE__, "unable to get attibute size");
967  }
968 
969  // Due to the implementation of print_attr, the attribute value will be
970  // written one by one.
971  char *tempvalue = &value[0];
972 
973  // Write this value. the "loc" can always be set to 0 since
974  // tempvalue will be moved to the next value.
975  for( hsize_t temp_index = 0; temp_index < attr_inst.nelmts; temp_index ++) {
976  //print_rep = print_attr(ty_id, 0, (void*)&value[0]);
977  print_rep = print_attr(ty_id, 0, tempvalue);
978  if (print_rep.c_str() != NULL) {
979 
980  BESDEBUG("h5", "print_rep= " << print_rep << endl);
981 
982  d4_attr->add_value(print_rep);
983  tempvalue = tempvalue + elesize;
984  BESDEBUG("h5",
985  "tempvalue= " << tempvalue
986  << "elesize=" << elesize
987  << endl);
988 
989  }
990  else {
991  H5Tclose(ty_id);
992  H5Aclose(attr_id);
993  delete d4_attr;
994  throw InternalErr(__FILE__, __LINE__, "unable to convert attibute value to DAP");
995  }
996  }//for(hsize_t temp_index=0; .....
997  } // if attr_inst.ndims != 0
998  }
999  if(H5Tclose(ty_id) < 0) {
1000  H5Aclose(attr_id);
1001  delete d4_attr;
1002  throw InternalErr(__FILE__, __LINE__, "unable to close HDF5 type id");
1003  }
1004  if (H5Aclose(attr_id) < 0) {
1005  delete d4_attr;
1006  throw InternalErr(__FILE__, __LINE__, "unable to close attibute id");
1007  }
1008 
1009  if(0 == flag) // D4group
1010  d4g->attributes()->add_attribute_nocopy(d4_attr);
1011  else if (1 == flag) // HDF5 dataset with atomic datatypes
1012  d4b->attributes()->add_attribute_nocopy(d4_attr);
1013  else if ( 2 == flag) // HDF5 dataset with compound datatype
1014  d4s->attributes()->add_attribute_nocopy(d4_attr);
1015  else {
1016  stringstream sflag;
1017  sflag << flag;
1018  string msg ="The add_dap4_attr flag has to be either 0,1 or 2.";
1019  msg+="The current flag is "+sflag.str();
1020  delete d4_attr;
1021  throw InternalErr(__FILE__, __LINE__, msg);
1022  }
1023  } // for (int j = 0; j < num_attr; j++)
1024 
1025  return;
1026 }
1027 
1041 
1042 
1043 void map_h5_dset_hardlink_to_d4(hid_t h5_dsetid,const string & full_path, BaseType* d4b,Structure * d4s,int flag) {
1044 
1045  // Obtain the unique object number info. If no hardlinks, empty string will return.
1046  string oid = get_hardlink_dmr(h5_dsetid, full_path);
1047 
1048  // Find that this is a hardlink,add the hardlink info to a DAP4 attribute.
1049  if(false == oid.empty()) {
1050 
1051  D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
1052  d4_hlinfo->add_value(obj_paths.get_name(oid));
1053 
1054  if (1 == flag)
1055  d4b->attributes()->add_attribute_nocopy(d4_hlinfo);
1056  else if ( 2 == flag)
1057  d4s->attributes()->add_attribute_nocopy(d4_hlinfo);
1058  else
1059  delete d4_hlinfo;
1060  }
1061 
1062 }
1063 
1076 void get_softlink(D4Group* par_grp, hid_t h5obj_id, const string & oname, int index, size_t val_size)
1077 {
1078  BESDEBUG("h5", "dap4 >get_softlink():" << oname << endl);
1079 
1080  ostringstream oss;
1081  oss << string("HDF5_SOFTLINK");
1082  oss << "_";
1083  oss << index;
1084  string temp_varname = oss.str();
1085 
1086 
1087  BESDEBUG("h5", "dap4->get_softlink():" << temp_varname << endl);
1088  D4Attribute *d4_slinfo = new D4Attribute;
1089  d4_slinfo->set_name(temp_varname);
1090 
1091  // Make the type as a container
1092  d4_slinfo->set_type(attr_container_c);
1093 
1094  string softlink_name = "linkname";
1095 
1096  D4Attribute *softlink_src = new D4Attribute(softlink_name,attr_str_c);
1097  softlink_src->add_value(oname);
1098 
1099  d4_slinfo->attributes()->add_attribute_nocopy(softlink_src);
1100  string softlink_value_name ="LINKTARGET";
1101 
1102  // Get the link target information. We always return the link value in a string format.
1103  D4Attribute *softlink_tgt = 0;
1104 
1105  try {
1106  vector<char> buf;
1107  buf.resize(val_size + 1);
1108 
1109  // get link target name
1110  if (H5Lget_val(h5obj_id, oname.c_str(), (void*) &buf[0], val_size + 1, H5P_DEFAULT) < 0) {
1111  throw InternalErr(__FILE__, __LINE__, "unable to get link value");
1112  }
1113  softlink_tgt = new D4Attribute(softlink_value_name, attr_str_c);
1114  string link_target_name = string(buf.begin(), buf.end());
1115  softlink_tgt->add_value(link_target_name);
1116 
1117  d4_slinfo->attributes()->add_attribute_nocopy(softlink_tgt);
1118  }
1119  catch (...) {
1120  delete softlink_tgt;
1121  throw;
1122  }
1123 
1124  par_grp->attributes()->add_attribute_nocopy(d4_slinfo);
1125 }
1126 
1127 
1140 string get_hardlink_dmr( hid_t h5obj_id, const string & oname) {
1141 
1142  BESDEBUG("h5", "dap4->get_hardlink_dmr():" << oname << endl);
1143 
1144  // Get the object info
1145  H5O_info_t obj_info;
1146  if (H5Oget_info(h5obj_id, &obj_info) <0) {
1147  throw InternalErr(__FILE__, __LINE__, "H5Oget_info() failed.");
1148  }
1149 
1150  // If the reference count is greater than 1,that means
1151  // hard links are found. return the original object name this
1152  // hard link points to.
1153 
1154  if (obj_info.rc >1) {
1155 
1156  ostringstream oss;
1157  oss << hex << obj_info.addr;
1158  string objno = oss.str();
1159 
1160  BESDEBUG("h5", "dap4->get_hardlink_dmr() objno=" << objno << endl);
1161 
1162  // Add this hard link to the map.
1163  // obj_paths is a global variable defined at the beginning of this file.
1164  // it is essentially a id to obj name map. See HDF5PathFinder.h.
1165  if (!obj_paths.add(objno, oname)) {
1166  return objno;
1167  }
1168  else {
1169  return "";
1170  }
1171  }
1172  else {
1173  return "";
1174  }
1175 
1176 }
A class for handling all types of array in HDF5 for the default option.
This class provides a way to map HDF5 byte to DAP Byte for the default option.
This file includes several helper functions for translating HDF5 to CF-compliant.
A class for mapping HDF5 32-bit float to DAP for the default option.
A class for mapping HDF5 64-bit float to DAP for the default option.
A class for HDF5 signed 16 bit integer type.
This class provides a way to map HDF5 32 bit integer to DAP Int32 for the default option.
This class that translates HDF5 string into DAP string for the default option.
This class converts HDF5 compound type into DAP structure for the default option.
This class provides a way to map unsigned HDF5 16 bit integer to DAP UInt16 for the default option.
This class provides a way to map unsigned HDF5 32 bit integer to DAP UInt32.
This class generates DAP URL type for the default option.
void set_numdim(int ndims)
remembers number of dimensions of this array.
Definition: HDF5Array.cc:1451
void set_numelm(int nelms)
remembers number of elements in this array.
Definition: HDF5Array.cc:1455
void set_memneed(size_t need)
remembers memory size needed.
Definition: HDF5Array.cc:1447
std::string get_name(std::string id)
bool add(std::string id, const std::string name)
void depth_first(hid_t pid, const char *gname, DAS &das)
Definition: h5das.cc:62
void read_objects(DAS &das, const string &varname, hid_t oid, int num_attr)
Definition: h5das.cc:295
void read_objects_base_type(DDS &dds_table, const string &varname, const string &filename)
Definition: h5dds.cc:257
void read_objects_structure(DDS &dds_table, const string &varname, const string &filename)
Definition: h5dds.cc:311
void map_h5_attrs_to_dap4(hid_t oid, D4Group *d4g, BaseType *d4b, Structure *d4s, int flag)
A function that map HDF5 attributes to DAP4.
Definition: h5dmr.cc:864
HDF5PathFinder obj_paths
A variable for remembering visited paths to break cyclic HDF5 groups.
Definition: h5dmr.cc:68
bool breadth_first(hid_t pid, char *gname, D4Group *par_grp, const char *fname, bool use_dimscale)
Definition: h5dmr.cc:319
void get_softlink(D4Group *par_grp, hid_t h5obj_id, const string &oname, int index, size_t val_size)
Definition: h5dmr.cc:1076
Data structure and retrieval processing header for the default option.
string print_attr(hid_t type, int loc, void *sm_buf)
Definition: h5get.cc:706
string get_dap_type(hid_t type, bool is_dap4)
Definition: h5get.cc:285
void get_dataset(hid_t pid, const string &dname, DS_t *dt_inst_ptr)
Definition: h5get.cc:435
hid_t get_attr_info(hid_t dset, int index, bool is_dap4, DSattr_t *attr_inst_ptr, bool *ignore_attr_ptr)
Definition: h5get.cc:84
The main header of the HDF5 OPeNDAP handler.
const int DODS_NAMELEN
Maximum length of variable or attribute name(default option only).
Definition: hdf5_handler.h:65
A structure for DDS generation.
Definition: hdf5_handler.h:71
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:89
hsize_t need
Space needed.
Definition: hdf5_handler.h:91
hid_t type
HDF5 data set id.
Definition: hdf5_handler.h:79
int size[DODS_MAX_RANK]
Size of each dimension.
Definition: hdf5_handler.h:85
int ndims
HDF5 data space id.
Definition: hdf5_handler.h:83
A structure for DAS generation.
Definition: hdf5_handler.h:94
char name[DODS_NAMELEN]
Name of HDF5 group or dataset.
Definition: hdf5_handler.h:96
int ndims
Number of dimensions.
Definition: hdf5_handler.h:100
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:104
hsize_t need
Memory space needed to hold nelmts type.
Definition: hdf5_handler.h:106