31 #include "config_hdf5.h"
37 #include "InternalErr.h"
43 #include "ObjMemCache.h"
49 BaseType *HDF5CFArray::ptr_duplicate()
55 bool HDF5CFArray::read()
58 BESDEBUG(
"h5",
"Coming to HDF5CFArray read "<<endl);
62 if((NULL == HDF5RequestHandler::get_lrdata_mem_cache()) &&
63 NULL == HDF5RequestHandler::get_srdata_mem_cache()){
64 read_data_NOT_from_mem_cache(
false,NULL);
69 short use_cache_flag = 0;
72 if(HDF5RequestHandler::get_srdata_mem_cache() != NULL) {
73 if(((cvtype == CV_EXIST) && (islatlon !=
true)) || (cvtype == CV_NONLATLON_MISS)
74 || (cvtype == CV_FILLINDEX) ||(cvtype == CV_MODIFY) ||(cvtype == CV_SPECIAL)){
76 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype)==
true)
82 if(use_cache_flag !=1) {
84 if(HDF5RequestHandler::get_lrdata_mem_cache() != NULL) {
89 if(HDF5RequestHandler::get_common_cache_dirs() ==
false) {
90 if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
91 || (cvtype == CV_EXIST && islatlon ==
true)) {
96 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype)==
true)
104 if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
105 || (cvtype == CV_EXIST && islatlon ==
true)) {
107 vector<string> cur_lrd_non_cache_dir_list;
108 HDF5RequestHandler::get_lrd_non_cache_dir_list(cur_lrd_non_cache_dir_list);
111 if( (cur_lrd_non_cache_dir_list.size() == 0) ||
112 (
"" == check_str_sect_in_list(cur_lrd_non_cache_dir_list,filename,
'/'))) {
115 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype)==
true)
121 vector<string> cur_lrd_var_cache_file_list;
122 HDF5RequestHandler::get_lrd_var_cache_file_list(cur_lrd_var_cache_file_list);
123 if(cur_lrd_var_cache_file_list.size() >0){
128 if(
true == check_var_cache_files(cur_lrd_var_cache_file_list,filename,varname)){
135 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype)==
true)
143 if(0 == use_cache_flag)
144 read_data_NOT_from_mem_cache(
false,NULL);
150 if( 3 == use_cache_flag){
151 vector<string> cur_cache_dlist;
152 HDF5RequestHandler::get_lrd_cache_dir_list(cur_cache_dlist);
153 string cache_dir = check_str_sect_in_list(cur_cache_dlist,filename,
'/');
155 cache_key = cache_dir + varname;
157 cache_key = filename + varname;
165 cache_key = filename + varname;
167 handle_data_with_mem_cache(dtype,total_elems,use_cache_flag,cache_key);
180 vector<hsize_t> hoffset;
181 vector<hsize_t>hcount;
182 vector<hsize_t>hstep;
186 throw InternalErr (__FILE__, __LINE__,
187 "The number of dimension of the variable is <=0 for an array.");
193 hoffset.resize(rank);
196 nelms = format_constraint (&offset[0], &step[0], &count[0]);
197 for (
int i = 0; i <rank; i++) {
198 hoffset[i] = (hsize_t) offset[i];
199 hcount[i] = (hsize_t) count[i];
200 hstep[i] = (hsize_t) step[i];
210 bool data_from_disk_cache =
false;
211 bool data_to_disk_cache =
false;
214 bool use_disk_cache = valid_disk_cache();
218 if(
true == use_disk_cache) {
220 BESDEBUG(
"h5",
"Coming to use disk cache "<<endl);
222 unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
223 string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
224 string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
226 string cache_fname=HDF5CFUtil::obtain_cache_fname(diskcache_prefix,filename,varname);
227 cache_fpath = diskcache_dir +
"/"+ cache_fname;
229 int temp_total_elems = 1;
230 for (
unsigned int i = 0; i <dimsizes.size();i++)
231 temp_total_elems = temp_total_elems*dimsizes[i];
232 short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
234 int expected_file_size = dtype_size *temp_total_elems;
237 if(
true == disk_cache->get_data_from_cache(cache_fpath, expected_file_size,fd)) {
239 vector<size_t> offset_size_t;
240 offset_size_t.resize(rank);
241 for(
int i = 0; i <rank;i++)
242 offset_size_t[i] = (
size_t)offset[i];
243 size_t offset_1st = INDEX_nD_TO_1D(dimsizes,offset_size_t);
246 for (
int i = 0; i < rank; i++)
247 end[i] = offset[i] +(count[i]-1)*step[i];
248 size_t offset_last = INDEX_nD_TO_1D(dimsizes,end);
253 size_t total_read = dtype_size*(offset_last-offset_1st+1);
255 off_t fpos = lseek(fd,dtype_size*offset_1st,SEEK_SET);
263 data_from_disk_cache = obtain_cached_data(disk_cache,cache_fpath,fd, step,count,total_read,dtype_size);
267 if(
true == data_from_disk_cache)
270 data_to_disk_cache =
true;
276 bool pass_fileid = HDF5RequestHandler::get_pass_fileid();
277 if(
false == pass_fileid) {
278 if ((fileid = H5Fopen(filename.c_str(),H5F_ACC_RDONLY,H5P_DEFAULT))<0) {
280 eherr <<
"HDF5 File " << filename
281 <<
" cannot be opened. "<<endl;
282 throw InternalErr (__FILE__, __LINE__, eherr.str ());
286 if ((dsetid = H5Dopen(fileid,varname.c_str(),H5P_DEFAULT))<0) {
287 HDF5CFUtil::close_fileid(fileid,pass_fileid);
289 eherr <<
"HDF5 dataset " << varname
290 <<
" cannot be opened. "<<endl;
291 throw InternalErr (__FILE__, __LINE__, eherr.str ());
294 if ((dspace = H5Dget_space(dsetid))<0) {
297 HDF5CFUtil::close_fileid(fileid,pass_fileid);
299 eherr <<
"Space id of the HDF5 dataset " << varname
300 <<
" cannot be obtained. "<<endl;
301 throw InternalErr (__FILE__, __LINE__, eherr.str ());
304 if (H5Sselect_hyperslab(dspace, H5S_SELECT_SET,
305 &hoffset[0], &hstep[0],
306 &hcount[0], NULL) < 0) {
310 HDF5CFUtil::close_fileid(fileid,pass_fileid);
312 eherr <<
"The selection of hyperslab of the HDF5 dataset " << varname
314 throw InternalErr (__FILE__, __LINE__, eherr.str ());
317 mspace = H5Screate_simple(rank, &hcount[0],NULL);
321 HDF5CFUtil::close_fileid(fileid,pass_fileid);
323 eherr <<
"The creation of the memory space of the HDF5 dataset " << varname
325 throw InternalErr (__FILE__, __LINE__, eherr.str ());
329 if ((dtypeid = H5Dget_type(dsetid)) < 0) {
334 HDF5CFUtil::close_fileid(fileid,pass_fileid);
336 eherr <<
"Obtaining the datatype of the HDF5 dataset " << varname
338 throw InternalErr (__FILE__, __LINE__, eherr.str ());
342 if ((memtype = H5Tget_native_type(dtypeid, H5T_DIR_ASCEND))<0) {
348 HDF5CFUtil::close_fileid(fileid,pass_fileid);
350 eherr <<
"Obtaining the memory type of the HDF5 dataset " << varname
352 throw InternalErr (__FILE__, __LINE__, eherr.str ());
360 if(
true == add_mem_cache) {
366 HDF5CFUtil::close_fileid(fileid,pass_fileid);
367 throw InternalErr(__FILE__,__LINE__,
"The memory data cache buffer needs to be set");
369 read_ret= H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf);
375 HDF5CFUtil::close_fileid(fileid,pass_fileid);
376 throw InternalErr(__FILE__,__LINE__,
"Cannot read the data to the buffer.");
392 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
400 HDF5CFUtil::close_fileid(fileid,pass_fileid);
402 eherr <<
"Cannot read the HDF5 dataset " << varname
403 <<
" with the type of H5T_NATIVE_CHAR "<<endl;
404 throw InternalErr (__FILE__, __LINE__, eherr.str ());
409 newval.resize(nelms);
411 for (
int counter = 0; counter < nelms; counter++)
412 newval[counter] = (
short) (val[counter]);
414 set_value ((dods_int16 *) &newval[0], nelms);
416 if(
true == data_to_disk_cache) {
418 BESDEBUG(
"h5",
"writing data to disk cache "<<endl);
419 write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,2,val,nelms);
427 HDF5CFUtil::close_fileid(fileid,pass_fileid);
429 eherr <<
"write data to cache failed.";
430 throw InternalErr (__FILE__, __LINE__, eherr.str ());
451 size_t dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
453 val.resize(nelms*dtype_size);
455 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
462 HDF5CFUtil::close_fileid(fileid,pass_fileid);
464 eherr <<
"Cannot read the HDF5 dataset " << varname
465 <<
" with the type of H5T_NATIVE_UCHAR "<<endl;
466 throw InternalErr (__FILE__, __LINE__, eherr.str ());
473 if(
true == data_to_disk_cache) {
474 BESDEBUG(
"h5",
"writing data to disk cache "<<endl);
476 write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,dtype_size,val,nelms);
484 HDF5CFUtil::close_fileid(fileid,pass_fileid);
486 eherr <<
"write data to cache failed.";
487 throw InternalErr (__FILE__, __LINE__, eherr.str ());
503 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
511 HDF5CFUtil::close_fileid(fileid,pass_fileid);
514 eherr <<
"Cannot read the HDF5 dataset " << varname
515 <<
" with the type of H5T_NATIVE_SHORT "<<endl;
516 throw InternalErr (__FILE__, __LINE__, eherr.str ());
519 set_value ((dods_int16 *) &val[0], nelms);
526 vector<unsigned short> val;
528 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
536 HDF5CFUtil::close_fileid(fileid,pass_fileid);
538 eherr <<
"Cannot read the HDF5 dataset " << varname
539 <<
" with the type of H5T_NATIVE_USHORT "<<endl;
540 throw InternalErr (__FILE__, __LINE__, eherr.str ());
543 set_value ((dods_uint16 *) &val[0], nelms);
552 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
559 HDF5CFUtil::close_fileid(fileid,pass_fileid);
561 eherr <<
"Cannot read the HDF5 dataset " << varname
562 <<
" with the type of H5T_NATIVE_INT "<<endl;
563 throw InternalErr (__FILE__, __LINE__, eherr.str ());
566 set_value ((dods_int32 *) &val[0], nelms);
572 vector<unsigned int>val;
574 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
581 HDF5CFUtil::close_fileid(fileid,pass_fileid);
583 eherr <<
"Cannot read the HDF5 dataset " << varname
584 <<
" with the type of H5T_NATIVE_UINT "<<endl;
585 throw InternalErr (__FILE__, __LINE__, eherr.str ());
588 set_value ((dods_uint32 *) &val[0], nelms);
598 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
605 HDF5CFUtil::close_fileid(fileid,pass_fileid);
607 eherr <<
"Cannot read the HDF5 dataset " << varname
608 <<
" with the type of H5T_NATIVE_FLOAT "<<endl;
609 throw InternalErr (__FILE__, __LINE__, eherr.str ());
612 set_value ((dods_float32 *) &val[0], nelms);
622 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
630 HDF5CFUtil::close_fileid(fileid,pass_fileid);
632 eherr <<
"Cannot read the HDF5 dataset " << varname
633 <<
" with the type of H5T_NATIVE_DOUBLE "<<endl;
634 throw InternalErr (__FILE__, __LINE__, eherr.str ());
637 set_value ((dods_float64 *) &val[0], nelms);
645 size_t ty_size = H5Tget_size(dtypeid);
652 HDF5CFUtil::close_fileid(fileid,pass_fileid);
654 eherr <<
"Cannot obtain the size of the fixed size HDF5 string of the dataset "
656 throw InternalErr (__FILE__, __LINE__, eherr.str ());
659 vector <char> strval;
660 strval.resize(nelms*ty_size);
661 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(
void*)&strval[0]);
669 HDF5CFUtil::close_fileid(fileid,pass_fileid);
671 eherr <<
"Cannot read the HDF5 dataset " << varname
672 <<
" with the type of the fixed size HDF5 string "<<endl;
673 throw InternalErr (__FILE__, __LINE__, eherr.str ());
676 string total_string(strval.begin(),strval.end());
678 vector <string> finstrval;
679 finstrval.resize(nelms);
680 for (
int i = 0; i<nelms; i++)
681 finstrval[i] = total_string.substr(i*ty_size,ty_size);
688 if ((
true == HDF5RequestHandler::get_drop_long_string()) &&
689 ty_size > NC_JAVA_STR_SIZE_LIMIT) {
690 for (
int i = 0; i<nelms; i++)
693 set_value(finstrval,nelms);
694 total_string.clear();
701 size_t ty_size = H5Tget_size(memtype);
708 HDF5CFUtil::close_fileid(fileid,pass_fileid);
710 eherr <<
"Cannot obtain the size of the fixed size HDF5 string of the dataset "
712 throw InternalErr (__FILE__, __LINE__, eherr.str ());
714 vector <char> strval;
715 strval.resize(nelms*ty_size);
716 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(
void*)&strval[0]);
724 HDF5CFUtil::close_fileid(fileid,pass_fileid);
726 eherr <<
"Cannot read the HDF5 dataset " << varname
727 <<
" with the type of the HDF5 variable length string "<<endl;
728 throw InternalErr (__FILE__, __LINE__, eherr.str ());
731 vector<string>finstrval;
732 finstrval.resize(nelms);
733 char*temp_bp = &strval[0];
734 char*onestring = NULL;
735 for (
int i =0;i<nelms;i++) {
736 onestring = *(
char**)temp_bp;
738 finstrval[i] =string(onestring);
745 if (
false == strval.empty()) {
746 herr_t ret_vlen_claim;
747 ret_vlen_claim = H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(
void*)&strval[0]);
748 if (ret_vlen_claim < 0){
754 HDF5CFUtil::close_fileid(fileid,pass_fileid);
756 eherr <<
"Cannot reclaim the memory buffer of the HDF5 variable length string of the dataset "
758 throw InternalErr (__FILE__, __LINE__, eherr.str ());
766 if (
true == HDF5RequestHandler::get_drop_long_string()) {
767 bool drop_long_str =
false;
768 for (
int i =0;i<nelms;i++) {
769 if(finstrval[i].size() >NC_JAVA_STR_SIZE_LIMIT){
770 drop_long_str =
true;
774 if (drop_long_str ==
true) {
775 for (
int i =0;i<nelms;i++)
779 set_value(finstrval,nelms);
791 HDF5CFUtil::close_fileid(fileid,pass_fileid);
793 eherr <<
"Cannot read the HDF5 dataset " << varname
794 <<
" with the unsupported HDF5 datatype"<<endl;
795 throw InternalErr (__FILE__, __LINE__, eherr.str ());
804 HDF5CFUtil::close_fileid(fileid,pass_fileid);
809 bool HDF5CFArray::valid_disk_cache() {
811 bool ret_value =
false;
812 if(
true == HDF5RequestHandler::get_use_disk_cache()) {
814 BESDEBUG(
"h5",
"Coming to disk cache "<<endl);
816 if(dtype == H5CHAR || dtype ==H5UCHAR || dtype==H5INT16 || dtype ==H5UINT16 ||
817 dtype == H5INT32 || dtype ==H5UINT32 || dtype ==H5FLOAT32 || dtype==H5FLOAT64 ||
818 dtype == H5INT64 || dtype ==H5UINT64){
820 BESDEBUG(
"h5",
"Coming to disk cache datatype block"<<endl);
822 string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
823 string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
824 long diskcache_size = HDF5RequestHandler::get_disk_cache_size();
826 if((
"" == diskcache_dir)||(
""==diskcache_prefix)||(diskcache_size <=0))
827 throw InternalErr (__FILE__, __LINE__,
"Either the cached dir is empty or the prefix is NULL or the cache size is not set.");
830 if(stat(diskcache_dir.c_str(),&sb) !=0) {
831 string err_mesg=
"The cached directory " + diskcache_dir;
832 err_mesg = err_mesg +
" doesn't exist. ";
833 throw InternalErr(__FILE__,__LINE__,err_mesg);
836 if(
true == S_ISDIR(sb.st_mode)) {
837 if(access(diskcache_dir.c_str(),R_OK|W_OK|X_OK) == -1) {
838 string err_mesg=
"The cached directory " + diskcache_dir;
839 err_mesg = err_mesg +
" can NOT be read,written or executable.";
840 throw InternalErr(__FILE__,__LINE__,err_mesg);
844 string err_mesg=
"The cached directory " + diskcache_dir;
845 err_mesg = err_mesg +
" is not a directory.";
846 throw InternalErr(__FILE__,__LINE__,err_mesg);
851 short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
853 if(
true == HDF5RequestHandler::get_disk_cache_comp_data()){
854 BESDEBUG(
"h5",
"Compression disk cache key is true"<<endl);
855 ret_value = valid_disk_cache_for_compressed_data(dtype_size);
856 BESDEBUG(
"h5",
"variable disk cache passes the compression parameter check"<<endl);
859 BESDEBUG(
"h5",
"Compression disk cache key is NOT set, disk cache key is true."<<endl);
869 bool HDF5CFArray:: valid_disk_cache_for_compressed_data(
short dtype_size)
const {
871 bool ret_value =
false;
875 size_t total_byte = total_elems*dtype_size;
877 if((comp_ratio < HDF5RequestHandler::get_disk_comp_threshold())
878 && (total_elems*dtype_size >= HDF5RequestHandler::get_disk_var_size())) {
879 if(
true == HDF5RequestHandler::get_disk_cache_float_only_comp()) {
880 if(dtype==H5FLOAT32 || dtype == H5FLOAT64)
890 bool HDF5CFArray::obtain_cached_data(
HDF5DiskCache *disk_cache,
const string & cache_fpath,
int fd,vector<int> &cd_step, vector<int>&cd_count,
size_t total_read,
short dtype_size) {
892 ssize_t ret_read_val = -1;
895 buf.resize(total_read);
898 if((-1 == ret_read_val) || (ret_read_val != (ssize_t)total_read)) {
903 unsigned int nele_to_read = 1;
904 for(
int i = 0; i<rank;i++)
905 nele_to_read *=cd_count[i];
907 if(nele_to_read == (total_read/dtype_size)) {
913 vector<int>cd_start(rank,0);
914 vector<size_t>cd_pos(rank,0);
915 int nelms_to_send = 1;
916 for(
int i = 0; i <rank; i++)
917 nelms_to_send = nelms_to_send*cd_count[i];
924 vector<int>total_val;
925 total_val.resize(total_read/dtype_size);
926 memcpy(&total_val[0],(
void*)&buf[0],total_read);
928 vector<int>final_val;
942 vector<short>final_val;
954 set_value((dods_int16*)&final_val[0],nelms_to_send);
962 vector<unsigned char>total_val;
963 total_val.resize(total_read/dtype_size);
964 memcpy(&total_val[0],(
void*)&buf[0],total_read);
966 vector<unsigned char>final_val;
967 subset<unsigned char>(
980 vector<unsigned char>final_val;
981 subset<unsigned char>(
993 set_value ((dods_byte *) &final_val[0], nelms_to_send);
1000 vector<short>total_val;
1001 total_val.resize(total_read/dtype_size);
1002 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1004 vector<short>final_val;
1018 vector<short>final_val;
1031 set_value ((dods_int16 *) &final_val[0], nelms_to_send);
1038 vector<unsigned short>total_val;
1039 total_val.resize(total_read/dtype_size);
1040 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1042 vector<unsigned short>final_val;
1043 subset<unsigned short>(
1056 vector<unsigned short>final_val;
1057 subset<unsigned short>(
1069 set_value ((dods_uint16 *) &final_val[0], nelms_to_send);
1076 vector<int>total_val;
1077 total_val.resize(total_read/dtype_size);
1078 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1080 vector<int>final_val;
1095 vector<int>final_val;
1109 set_value ((dods_int32 *) &final_val[0], nelms_to_send);
1116 vector<unsigned int>total_val;
1117 total_val.resize(total_read/dtype_size);
1118 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1120 vector<unsigned int>final_val;
1121 subset<unsigned int>(
1134 vector<unsigned int>final_val;
1135 subset<unsigned int>(
1147 set_value ((dods_uint32 *) &final_val[0], nelms_to_send);
1154 vector<unsigned int>total_val;
1155 total_val.resize(total_read/dtype_size);
1156 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1158 vector<unsigned int>final_val;
1159 subset<unsigned int>(
1172 vector<long long >final_val;
1185 set_value ((dods_int64 *) &final_val[0], nelms_to_send);
1194 vector<unsigned int>total_val;
1195 total_val.resize(total_read/dtype_size);
1196 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1198 vector<unsigned int>final_val;
1199 subset<unsigned int>(
1212 vector<unsigned long long >final_val;
1213 subset<unsigned long long >(
1225 set_value ((dods_uint64 *) &final_val[0], nelms_to_send);
1233 vector<float>total_val;
1234 total_val.resize(total_read/dtype_size);
1235 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1237 vector<float>final_val;
1251 vector<float>final_val;
1265 set_value ((dods_float32 *) &final_val[0], nelms_to_send);
1271 vector<double>total_val;
1272 total_val.resize(total_read/dtype_size);
1273 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1275 vector<double>final_val;
1288 vector<double>final_val;
1301 set_value ((dods_float64 *) &final_val[0], nelms_to_send);
1305 throw InternalErr (__FILE__, __LINE__,
"unsupported data type.");
1315 HDF5CFArray::write_data_to_cache(hid_t dset_id, hid_t , hid_t , hid_t memtype,
1316 const string& cache_fpath,
short dtype_size,
const vector<char> &buf,
int nelms) {
1318 unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
1319 string disk_cache_dir = HDF5RequestHandler::get_disk_cache_dir();
1320 string disk_cache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
1322 int total_nelem = 1;
1323 for(
int i = 0; i <rank; i++)
1324 total_nelem = total_nelem*dimsizes[i];
1328 if(H5CHAR == dtype) {
1330 vector<short>newval;
1331 newval.resize(total_nelem);
1332 if(total_nelem == nelms) {
1333 for (
int i = 0; i < total_nelem;i++)
1334 newval[i] = (
short)buf[i];
1335 disk_cache->write_cached_data2(cache_fpath,
sizeof(
short)*total_nelem,(
const void*)&newval[0]);
1339 val2.resize(total_nelem);
1340 if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, &val2[0])<0)
1341 throw InternalErr (__FILE__, __LINE__,
"Cannot read the whole HDF5 dataset for the disk cache.");
1342 for (
int i = 0; i < total_nelem;i++)
1343 newval[i] = (
short)val2[i];
1344 disk_cache->write_cached_data2(cache_fpath,
sizeof(
short)*total_nelem,(
const void*)&newval[0]);
1348 if(total_nelem == nelms) {
1349 disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)&buf[0]);
1352 val.resize(dtype_size*total_nelem);
1353 if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, &val[0])<0)
1354 throw InternalErr (__FILE__, __LINE__,
"Cannot read the whole SDS for cache.");
1356 disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)&val[0]);
1362 void HDF5CFArray::read_data_from_mem_cache(
void*buf) {
1367 int nelms = format_constraint (&offset[0], &step[0], &count[0]);
1369 vector<int>at_pos(at_ndims,0);
1370 for (
int i = 0; i< rank; i++)
1371 at_pos[i] = at_offset[i];
1379 vector<unsigned char> val;
1381 subset<unsigned char>(
1394 set_value ((dods_byte *) &val[0], nelms);
1406 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1408 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1418 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1419 ostringstream eherr;
1420 eherr <<
"Cannot read the HDF5 dataset " << varname
1421 <<
" with the type of H5T_NATIVE_CHAR "<<endl;
1422 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1426 vector<short>newval;
1427 newval.resize(nelms);
1429 for (
int counter = 0; counter < nelms; counter++)
1430 newval[counter] = (
short) (val[counter]);
1432 set_value ((dods_int16 *) &newval[0], nelms);
1443 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1445 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1455 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1457 ostringstream eherr;
1458 eherr <<
"Cannot read the HDF5 dataset " << varname
1459 <<
" with the type of H5T_NATIVE_SHORT "<<endl;
1460 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1463 set_value ((dods_int16 *) &val[0], nelms);
1470 vector<unsigned short> val;
1473 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1475 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1479 if (rank > 0) H5Sclose(mspace);
1484 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1485 ostringstream eherr;
1486 eherr <<
"Cannot read the HDF5 dataset " << varname
1487 <<
" with the type of H5T_NATIVE_USHORT "<<endl;
1488 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1491 set_value ((dods_uint16 *) &val[0], nelms);
1501 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1503 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1512 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1513 ostringstream eherr;
1514 eherr <<
"Cannot read the HDF5 dataset " << varname
1515 <<
" with the type of H5T_NATIVE_INT "<<endl;
1516 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1519 set_value ((dods_int32 *) &val[0], nelms);
1525 vector<unsigned int>val;
1528 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1530 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1540 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1541 ostringstream eherr;
1542 eherr <<
"Cannot read the HDF5 dataset " << varname
1543 <<
" with the type of H5T_NATIVE_UINT "<<endl;
1544 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1547 set_value ((dods_uint32 *) &val[0], nelms);
1558 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1560 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1569 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1570 ostringstream eherr;
1571 eherr <<
"Cannot read the HDF5 dataset " << varname
1572 <<
" with the type of H5T_NATIVE_FLOAT "<<endl;
1573 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1576 set_value ((dods_float32 *) &val[0], nelms);
1587 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1589 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1598 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1599 ostringstream eherr;
1600 eherr <<
"Cannot read the HDF5 dataset " << varname
1601 <<
" with the type of H5T_NATIVE_DOUBLE "<<endl;
1602 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1605 set_value ((dods_float64 *) &val[0], nelms);
1620 BaseType* HDF5CFArray::h5cfdims_transform_to_dap4(D4Group *grp) {
1624 Array *dest =
static_cast<HDF5CFArray*
>(ptr_duplicate());
1630 for (Array::Dim_iter d = dest->dim_begin(), e = dest->dim_end(); d != e; ++d) {
1631 if (
false == (*d).name.empty()) {
1633 D4Group *temp_grp = grp;
1634 D4Dimension *d4_dim = NULL;
1637 D4Dimensions *temp_dims = temp_grp->dims();
1640 d4_dim = temp_dims->find_dim((*d).name);
1646 if(temp_grp->get_parent())
1647 temp_grp =
static_cast<D4Group*
>(temp_grp->get_parent());
1656 bool d4_dim_null = ((d4_dim==NULL)?
true:
false);
1661 if(d4_dim_null ==
true) {
1663 d4_dim =
new D4Dimension((*d).name, (*d).size);
1664 D4Dimensions * dims = grp->dims();
1665 dims->add_dim_nocopy(d4_dim);
1671 dest->set_is_dap4(
true);
1680 HDF5CFArray::format_constraint (
int *offset,
int *step,
int *count)
1686 Dim_iter p = dim_begin ();
1688 while (p != dim_end ()) {
1690 int start = dimension_start (p,
true);
1691 int stride = dimension_stride (p,
true);
1692 int stop = dimension_stop (p,
true);
1698 oss <<
"Array/Grid hyperslab start point "<< start <<
1699 " is greater than stop point " << stop <<
".";
1700 throw Error(malformed_expr, oss.str());
1705 count[id] = ((stop - start) / stride) + 1;
1709 "=format_constraint():"
1710 <<
"id=" <<
id <<
" offset=" << offset[
id]
1711 <<
" step=" << step[
id]
1712 <<
" count=" << count[
id]
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
include the entry functions to execute the handlers
virtual void unlock_and_close(const std::string &target)
virtual void purge_file(const std::string &file)
Purge a single file from the cache.
virtual void read_data_NOT_from_mem_cache(bool add_cache, void *buf)
static HDF5DiskCache * get_instance(const long, const std::string &, const std::string &)
Helper functions for generating DAS attributes and a function to check BES Key.
static ssize_t read_buffer_from_file(int fd, void *buf, size_t)
Getting a subset of a variable.