bes  Updated for version 3.20.8
BESUncompressCache.cc
1 
2 // This file is part of bes, A C++ back-end server implementation framework
3 // for the OPeNDAP Data Access Protocol.
4 
5 // Copyright (c) 2015 OPeNDAP, Inc
6 // Author: Nathan Potter <npotter@opendap.org>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 
22 #include "config.h"
23 
24 #include <string>
25 #include <fstream>
26 #include <sstream>
27 #include <sys/stat.h>
28 
29 #include "BESInternalError.h"
30 #include "BESUtil.h"
31 #include "BESDebug.h"
32 #include "TheBESKeys.h"
33 #include "BESUncompressCache.h"
34 
35 using std::endl;
36 using std::string;
37 
38 BESUncompressCache *BESUncompressCache::d_instance = 0;
39 bool BESUncompressCache::d_enabled = true;
40 
41 const string BESUncompressCache::DIR_KEY = "BES.UncompressCache.dir";
42 const string BESUncompressCache::PREFIX_KEY = "BES.UncompressCache.prefix";
43 const string BESUncompressCache::SIZE_KEY = "BES.UncompressCache.size";
44 
45 #define MODULE "cache"
46 #define prolog std::string("BESUncompressCache::").append(__func__).append("() - ")
47 
48 unsigned long BESUncompressCache::getCacheSizeFromConfig()
49 {
50  bool found;
51  string size;
52  unsigned long size_in_megabytes = 0;
53  TheBESKeys::TheKeys()->get_value(SIZE_KEY, size, found);
54  if (found) {
55  std::istringstream iss(size);
56  iss >> size_in_megabytes;
57  }
58  else {
59  string msg = prolog+ "The BES Key " + SIZE_KEY
60  + " is not set! It MUST be set to utilize the decompression cache. ";
61  BESDEBUG( MODULE, msg << endl);
62  throw BESInternalError(msg, __FILE__, __LINE__);
63  }
64  return size_in_megabytes;
65 }
66 
67 string BESUncompressCache::getCacheDirFromConfig()
68 {
69  bool found;
70  string subdir = "";
71  TheBESKeys::TheKeys()->get_value(DIR_KEY, subdir, found);
72 
73  if (!found) {
74  string msg = prolog + "The BES Key " + DIR_KEY
75  + " is not set! It MUST be set to utilize the decompression cache. ";
76  BESDEBUG( MODULE, msg << endl);
77  throw BESInternalError(msg, __FILE__, __LINE__);
78  }
79 
80  return subdir;
81 }
82 
83 string BESUncompressCache::getCachePrefixFromConfig()
84 {
85  bool found;
86  string prefix = "";
87  TheBESKeys::TheKeys()->get_value(PREFIX_KEY, prefix, found);
88  if (found) {
89  prefix = BESUtil::lowercase(prefix);
90  }
91  else {
92  string msg = prolog + "The BES Key " + PREFIX_KEY
93  + " is not set! It MUST be set to utilize the decompression cache. ";
94  BESDEBUG( MODULE, msg << endl);
95  throw BESInternalError(msg, __FILE__, __LINE__);
96  }
97 
98  return prefix;
99 }
100 
128 string BESUncompressCache::get_cache_file_name(const string &src, bool mangle)
129 {
130  string cache_file_name = src;
131 
132  if (mangle) {
133  string::size_type last_dot = cache_file_name.rfind('.');
134  if (last_dot != string::npos) {
135  cache_file_name = cache_file_name.substr(0, last_dot);
136  }
137  }
138  cache_file_name = BESFileLockingCache::get_cache_file_name(cache_file_name);
139 
140  BESDEBUG( MODULE, prolog << "cache_file_name: '" << cache_file_name << "'" << endl);
141 
142  return cache_file_name;
143 }
144 
145 BESUncompressCache::BESUncompressCache()
146 {
147  BESDEBUG( MODULE, prolog << "BEGIN" << endl);
148 
149  d_enabled = true;
150  d_dimCacheDir = getCacheDirFromConfig();
151  d_dimCacheFilePrefix = getCachePrefixFromConfig();
152  d_maxCacheSize = getCacheSizeFromConfig();
153 
154  BESDEBUG( MODULE, prolog << "Cache configuration params: " << d_dimCacheDir << ", " << d_dimCacheFilePrefix << ", " << d_maxCacheSize << endl);
155 
156  initialize(d_dimCacheDir, d_dimCacheFilePrefix, d_maxCacheSize);
157 
158  BESDEBUG( MODULE, prolog << "END" << endl);
159 
160 }
161 BESUncompressCache::BESUncompressCache(const string &data_root_dir, const string &cache_dir, const string &prefix,
162  unsigned long long size)
163 {
164  BESDEBUG( MODULE, prolog << "BEGIN" << endl);
165  d_enabled = true;
166 
167  d_dataRootDir = data_root_dir;
168  d_dimCacheDir = cache_dir;
169  d_dimCacheFilePrefix = prefix;
170  d_maxCacheSize = size;
171 
172  initialize(d_dimCacheDir, d_dimCacheFilePrefix, d_maxCacheSize);
173 
174  BESDEBUG( MODULE, prolog << "END" << endl);
175 }
176 
178 BESUncompressCache::get_instance(const string &data_root_dir, const string &cache_dir, const string &result_file_prefix,
179  unsigned long long max_cache_size)
180 {
181  if (d_enabled && d_instance == 0) {
182  if (dir_exists(cache_dir)) {
183  d_instance = new BESUncompressCache(data_root_dir, cache_dir, result_file_prefix, max_cache_size);
184  d_enabled = d_instance->cache_enabled();
185  if(!d_enabled){
186  delete d_instance;
187  d_instance = NULL;
188  BESDEBUG( MODULE, prolog << "Cache is DISABLED"<< endl);
189  }
190  else {
191  #ifdef HAVE_ATEXIT
192  atexit(delete_instance);
193  #endif
194  BESDEBUG( MODULE, prolog << "Cache is ENABLED"<< endl);
195  }
196  }
197  }
198  return d_instance;
199 }
200 
206 {
207  if (d_enabled && d_instance == 0) {
208  d_instance = new BESUncompressCache();
209  d_enabled = d_instance->cache_enabled();
210  if(!d_enabled){
211  delete d_instance;
212  d_instance = NULL;
213  BESDEBUG( MODULE, prolog << "Cache is DISABLED"<< endl);
214  }
215  else {
216 #ifdef HAVE_ATEXIT
217  atexit(delete_instance);
218 #endif
219  BESDEBUG( MODULE, prolog << "Cache is ENABLED"<< endl);
220  }
221  }
222 
223  return d_instance;
224 }
225 
226 BESUncompressCache::~BESUncompressCache() {}
227 
240 bool BESUncompressCache::is_valid(const string &cache_file_name, const string &local_id)
241 {
242  // If the cached response is zero bytes in size, it's not valid.
243  // (hmmm...)
244  string datasetFileName = BESUtil::assemblePath(d_dataRootDir, local_id, true);
245 
246  off_t entry_size = 0;
247  time_t entry_time = 0;
248  struct stat buf;
249  if (stat(cache_file_name.c_str(), &buf) == 0) {
250  entry_size = buf.st_size;
251  entry_time = buf.st_mtime;
252  }
253  else {
254  return false;
255  }
256 
257  if (entry_size == 0) return false;
258 
259  time_t dataset_time = entry_time;
260  if (stat(datasetFileName.c_str(), &buf) == 0) {
261  dataset_time = buf.st_mtime;
262  }
263 
264  // Trick: if the d_dataset is not a file, stat() returns error and
265  // the times stay equal and the code uses the cache entry.
266 
267  // TODO Fix this so that the code can get a LMT from the correct handler.
268  // TODO Consider adding a getLastModified() method to the libdap::DDS object to support this
269  // TODO The DDS may be expensive to instantiate - I think the handler may be a better location
270  // for an LMT method, if we can access the handler when/where needed.
271  if (dataset_time > entry_time) return false;
272 
273  return true;
274 }
275 
void initialize(const std::string &cache_dir, const std::string &prefix, unsigned long long size)
Initialize an instance of FileLockingCache.
static bool dir_exists(const std::string &dir)
virtual std::string get_cache_file_name(const std::string &src, bool mangle=true)
exception thrown if internal error encountered
static BESUncompressCache * get_instance()
virtual std::string get_cache_file_name(const std::string &src, bool mangle=true)
Build the name of file that will holds the uncompressed data from 'src' in the cache.
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:200
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
Definition: BESUtil.cc:821
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:339
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71