HDK
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
FS_DiskCache.h
Go to the documentation of this file.
1 /*
2  * PROPRIETARY INFORMATION. This software is proprietary to
3  * Side Effects Software Inc., and is not to be reproduced,
4  * transmitted, or disclosed in any way without written permission.
5  *
6  * NAME: FS_DiskCache.h (UT Library, C++)
7  *
8  * COMMENTS:
9  */
10 
11 #ifndef __FS_DiskCache__
12 #define __FS_DiskCache__
13 
14 #include "FS_API.h"
15 #include <SYS/SYS_AtomicInt.h>
16 #include <UT/UT_StringHolder.h>
17 #include <UT/UT_WorkBuffer.h>
18 #include <UT/UT_Access.h>
19 
20 class UT_JSONWriter;
21 
22 /// A disk cache can store data on disk. The cache is designed to be
23 /// thread-safe, even across multiple processes.
24 ///
25 /// This class is intended to be a singleton.
27 {
28 public:
29  /// The key is composed of two main parts:
30  /// 1) The value is used to uniquely identify the data in the file. This
31  /// value is hashed to compute the filename in the cache and can be
32  /// arbitrarily long.
33  /// 2) The suffix is a short identifier - The first 32 characters of this
34  /// suffix are appended to the value hash to create the filename.
35  /// The key will automatically add the current major.minor release of the
36  /// software (but not the build).
37  class FS_API Key
38  {
39  public:
41  const UT_StringRef &suffix = UT_StringRef())
42  {
43  computeFilename(value, suffix);
44  }
45  const UT_StringHolder &filename() const { return myFilename; }
46  private:
47  void computeFilename(const UT_StringRef &value,
48  const UT_StringRef &suffix);
49  UT_StringHolder myFilename;
50  };
51 
52  /// The Accessor class is used to access items in the cache. When an
53  /// accessor is in write-mode, the thread/process will have unique access
54  /// to the cache item until the Accessor is destructed. Multiple
55  /// threads/processes can have read-only access to the item.
57  {
58  public:
60  : myCache(nullptr)
61  , myFilename()
62  , myBytesWritten(0)
63  , myFd(-1)
64  , myAccess(0)
65  , myLocked(false)
66  , myStoreInCache(false)
67  {
68  }
70  {
71  clear();
72  }
73 
74  /// Clear the accessor. This will close any file descriptors and
75  /// release any locks.
76  void clear();
77 
78  /// Write data to the file. This method can be called multiple times.
79  bool write(const void *data, exint len);
80 
81  /// Convenience methods to write data to the cache
82  bool write(const UT_WorkBuffer &buf)
83  { return write(buf.buffer(), buf.length()); }
84 
85  /// Return the number of bytes written
86  exint bytesWritten() const { return myBytesWritten; }
87 
88  /// Read the contents of the stream.
89  bool read(UT_WorkBuffer &contents) const;
90 
91  /// Whether the accessor is valid
92  bool isValid() const { return myFd >= 0; }
93 
94  /// Whether the accessor was opened as read-only
95  bool isRead() const { return myAccess & R_OK; }
96 
97  /// Whether the accessor was opened for writing
98  bool isWrite() const { return myAccess & W_OK; }
99 
100  /// Return the full path of the file in the cache
101  const UT_StringHolder &filename() const { return myFilename; }
102 
103  /// Normally, accessors that write data are counted to the cache
104  /// storage. This can disable this behaviour.
105  void storeInCache(bool v) { myStoreInCache = v; }
106 
107  void dump() const;
108  void dump(UT_JSONWriter &w) const;
109 
110  private:
111  /// Return the size of the read stream (or 0 if there was an error)
112  exint fileSize() const;
113 
114  /// @private - used solely by the FS_DiskCache to initialize
115  friend class FS_DiskCache; // Provide access to open()
116 
117  const FS_DiskCache *myCache; // Parent cache
118  UT_StringHolder myFilename;
119  exint myBytesWritten;
120  int myFd; // File descriptor
121  int myAccess; // Access mode
122  bool myLocked;
123  bool myStoreInCache;
124  };
125 
126  /// Statistics for the cache
127  class FS_API Stats
128  {
129  public:
131  {
132  fullClear();
133  }
134  ~Stats() {}
135 
136  void clearStats()
137  {
138  myCacheHits = 0;
139  myCacheMiss = 0;
140  myCacheError = 0;
141  myFileCount = 0;
142  myDiskSize = 0;
143  myPurgeCount = 0;
144  }
145  void fullClear()
146  {
147  clearStats();
148  myMaxSize = 0;
149  myMaxLogSize = 0;
150  myCreateTime = 0;
151  myLastRecompute = 0;
152  }
153 
154  /// Merge stats from a different set of stats
155  void merge(const Stats &s)
156  {
157  myCacheHits += s.myCacheHits;
158  myCacheMiss += s.myCacheMiss;
159  myCacheError += s.myCacheError;
160  myFileCount += s.myFileCount;
161  myDiskSize += s.myDiskSize;
162  myPurgeCount += s.myPurgeCount;
163  }
164 
165  /// @{
166  /// Access statistics
167  int64 cacheHits() const { return myCacheHits; }
168  int64 cacheMiss() const { return myCacheMiss; }
169  int64 cacheError() const { return myCacheError; }
170  int64 purgeCount() const { return myPurgeCount; }
171  int64 fileCount() const { return myFileCount; }
172  int64 diskSize() const { return myDiskSize; }
173  int64 maxSize() const { return myMaxSize; }
174  int64 maxLogSize() const { return myMaxLogSize; }
175  time_t createTime() const { return myCreateTime; }
176  time_t lastRecompute() const { return myLastRecompute; }
177  /// @}
178 
179  /// Load stats from a given file descriptor
180  bool loadFrom(const Accessor &fd);
181  /// Write stats to the file descriptor
182  bool saveTo(Accessor &a) const;
183 
184  /// Update stats with a cache hit
185  void addCacheHit() { myCacheHits++; }
186 
187  /// Update stats from the given accessor
188  void addToCache(const Accessor &a);
189 
190  /// Purge from the given accessor
191  int64 rmFromCache(const Accessor &a, const char *filepath);
192 
193  /// Reset the file count and size
194  void reset(int64 nfiles, int64 size)
195  {
196  myFileCount = nfiles;
197  myDiskSize = size;
198  }
199 
200  void dump() const;
201  void dump(UT_JSONWriter &w) const;
202  private:
203  int64 myCacheHits; // Cache hits
204  int64 myCacheMiss; // Cache misses
205  int64 myCacheError; // Cache errors
206  int64 myFileCount; // Files in cache
207  int64 myPurgeCount; // Files removed from cache
208  int64 myDiskSize; // Size of cache
209 
210  friend class FS_DiskCache;
211  UT_StringHolder myCachePath; // Path to the cache
212  int64 myMaxSize; // Maximum cache size
213  int64 myMaxLogSize; // Maximum size for log file
214  time_t myCreateTime; // Time cache was created
215  time_t myLastRecompute; // Time cache was last recomputed
216  };
217 
219  ~FS_DiskCache();
220 
221  /// Check to see if the cache is valid
222  bool isValid() const { return myPath.isstring(); }
223 
224  /// Initialize the disk cache from a given path. If the path doesn't
225  /// exist, the cache will be created.
226  bool initialize(const UT_StringHolder &path);
227 
228  /// Set the maximum data size of the cache
229  bool setMaxSize(int64 size_in_bytes = 256*1024*1024);
230 
231  /// Set the maximum log size of the cache
232  bool setMaxLogSize(int64 size_in_bytes = 1024*1024);
233 
234  /// Return the maximum size of the cache
235  int64 maxSize() const { return myMaxSize; }
236 
237  /// Return the maximum size of the log file
238  int64 maxLogSize() const { return myMaxLogSize; }
239 
240  /// Get statistics for the cache
241  bool stats(Stats &stats) const;
242 
243  /// Return the location of the path
244  const UT_StringHolder &location() const { return myPath; }
245 
246  /// Reset the cache
247  bool recompute();
248 
249  /// Clear the cache contents
250  bool clearCache() const;
251 
252  /// Reset the cache entirely
253  bool clobberCache();
254 
255  /// Possible return values for the @c insert() function
257  {
258  INSERT_CREATE, ///< New item, accessor has write-only file
259  INSERT_EXIST, ///< Existing item, accessor has read-only file
260  INSERT_ERROR, ///< Unknown error, accessor will be invalid
261  };
262 
263  /// Similar to UT_ConcurrentHash, this method will try to insert an item
264  /// into the cache. If the item exists in the cache, @c insert() will
265  /// return false and the @c Accessor will have read-only access to the
266  /// item.
267  InsertStatus insert(Accessor &access, const Key &key);
268 
269  /// Find an item in the disk cache. If found, the Accessor will have
270  /// read-only access to the item. The item cannot be deleted from the
271  /// cache while it's referenced.
272  bool find(Accessor &access, const Key &key) const;
273 
274  /// Purge items from a cache to meet size requirements using LRU
275  bool purge(bool force=false) const;
276 
277  /// Dump debug information about the cache
278  void dump() const;
279  bool dump(UT_JSONWriter &w) const;
280 
281  void printLog(const char *fmt, ...) const
283 
284  template<typename... Args>
285  void log(const char *fmt, const Args &...args) const
286  {
287  UT_WorkBuffer msg;
288  msg.format(fmt, args...);
289  printLog("%s", msg.buffer());
290  }
291 
292 private:
293  void cacheFilePath(Accessor &a, const Key &k,
294  const FS_DiskCache *cache) const;
295 
296  void initStats(Stats &s) const
297  {
298  s.myCachePath = myPath;
299  s.myMaxSize = myMaxSize;
300  s.myMaxLogSize = myMaxLogSize;
301  s.myCreateTime = myCreateTime;
302  s.myLastRecompute = myLastRecompute;
303  }
304  void statPath(const char *filename, UT_WorkBuffer &path) const;
305  void statPath(int bucket, UT_WorkBuffer &path) const;
306  void statPath(const Accessor &a, UT_WorkBuffer &path) const
307  { statPath(a.filename(), path); }
308 
309  friend class Accessor; // Accessor needs to call cacheMiss()
310  void cacheHit(const Accessor &a) const;
311  void cacheMiss(const Accessor &a) const;
312  void cachePurge(const Accessor &a, const char *filepath) const;
313 
314 
315  void setDefaults();
316  bool saveConfig() const;
317  bool loadConfig();
318  bool initSubdir(const UT_StringRef &path) const;
319  bool saveStats(const char *path, const Stats &s) const;
320 
321  InsertStatus writeLock(Accessor &access, const char *path,
322  bool must_create, bool in_cache) const;
323  bool readLock(Accessor &access, const char *path) const;
324 
325  UT_StringHolder myPath;
326  int64 myMaxSize;
327  int64 myMaxLogSize;
328  time_t myCreateTime;
329  time_t myLastRecompute;
330  mutable SYS_AtomicCounter myCurrSize;
331 };
332 
333 #endif
void merge(const Stats &s)
Merge stats from a different set of stats.
Definition: FS_DiskCache.h:155
bool setMaxSize(int64 size_in_bytes=256 *1024 *1024)
Set the maximum data size of the cache.
GT_API const UT_StringHolder filename
SYS_FORCE_INLINE exint length() const
void reset(int64 nfiles, int64 size)
Reset the file count and size.
Definition: FS_DiskCache.h:194
exint bytesWritten() const
Return the number of bytes written.
Definition: FS_DiskCache.h:86
int64 cacheHits() const
Definition: FS_DiskCache.h:167
bool setMaxLogSize(int64 size_in_bytes=1024 *1024)
Set the maximum log size of the cache.
const GLdouble * v
Definition: glcorearb.h:836
GLsizei const GLchar *const * path
Definition: glcorearb.h:3340
void printLog(const char *fmt,...) const SYS_PRINTF_CHECK_ATTRIBUTE(2
SYS_FORCE_INLINE const char * buffer() const
GLboolean GLboolean GLboolean GLboolean a
Definition: glcorearb.h:1221
void addCacheHit()
Update stats with a cache hit.
Definition: FS_DiskCache.h:185
Class which writes ASCII or binary JSON streams.
Definition: UT_JSONWriter.h:32
int64 cacheError() const
Definition: FS_DiskCache.h:169
void read(T &in, bool &v)
Definition: ImfXdr.h:611
bool isWrite() const
Whether the accessor was opened for writing.
Definition: FS_DiskCache.h:98
GLsizeiptr size
Definition: glcorearb.h:663
SYS_API double log(double x)
Definition: SYS_FPUMath.h:87
long long int64
Definition: SYS_Types.h:107
time_t lastRecompute() const
Definition: FS_DiskCache.h:176
InsertStatus
Possible return values for the insert() function.
Definition: FS_DiskCache.h:256
int64 cacheMiss() const
Definition: FS_DiskCache.h:168
void storeInCache(bool v)
Definition: FS_DiskCache.h:105
int64 diskSize() const
Definition: FS_DiskCache.h:172
int64 exint
Definition: SYS_Types.h:116
GLuint GLint GLboolean GLint GLenum access
Definition: glcorearb.h:2221
#define SYS_PRINTF_CHECK_ATTRIBUTE(string_index, first_to_check)
Definition: SYS_Types.h:426
New item, accessor has write-only file.
Definition: FS_DiskCache.h:258
bool find(Accessor &access, const Key &key) const
const UT_StringHolder & location() const
Return the location of the path.
Definition: FS_DiskCache.h:244
const UT_StringHolder & filename() const
Definition: FS_DiskCache.h:45
GLboolean * data
Definition: glcorearb.h:130
OPENVDB_API void initialize()
Global registration of basic types.
Definition: logging.h:318
Statistics for the cache.
Definition: FS_DiskCache.h:127
bool isRead() const
Whether the accessor was opened as read-only.
Definition: FS_DiskCache.h:95
int64 maxLogSize() const
Return the maximum size of the log file.
Definition: FS_DiskCache.h:238
Unknown error, accessor will be invalid.
Definition: FS_DiskCache.h:260
bool clobberCache()
Reset the cache entirely.
int64 maxLogSize() const
Definition: FS_DiskCache.h:174
bool isValid() const
Whether the accessor is valid.
Definition: FS_DiskCache.h:92
GLenum GLuint GLenum GLsizei const GLchar * buf
Definition: glcorearb.h:2539
const UT_StringHolder & filename() const
Return the full path of the file in the cache.
Definition: FS_DiskCache.h:101
GLsizei const GLfloat * value
Definition: glcorearb.h:823
size_t format(const char *fmt, const Args &...args)
void dump() const
Dump debug information about the cache.
bool recompute()
Reset the cache.
InsertStatus insert(Accessor &access, const Key &key)
int64 purgeCount() const
Definition: FS_DiskCache.h:170
time_t createTime() const
Definition: FS_DiskCache.h:175
bool purge(bool force=false) const
Purge items from a cache to meet size requirements using LRU.
int64 maxSize() const
Return the maximum size of the cache.
Definition: FS_DiskCache.h:235
bool stats(Stats &stats) const
Get statistics for the cache.
friend class Accessor
Definition: FS_DiskCache.h:309
GLubyte GLubyte GLubyte GLubyte w
Definition: glcorearb.h:856
int64 fileCount() const
Definition: FS_DiskCache.h:171
#define const
Definition: zconf.h:214
bool clearCache() const
Clear the cache contents.
void write(T &out, bool v)
Definition: ImfXdr.h:332
FS_DiskCache(const UT_StringHolder &path=UT_StringHolder())
bool isValid() const
Check to see if the cache is valid.
Definition: FS_DiskCache.h:222
int64 maxSize() const
Definition: FS_DiskCache.h:173
#define FS_API
Definition: FS_API.h:10
Key(const UT_StringRef &value=UT_StringRef(), const UT_StringRef &suffix=UT_StringRef())
Definition: FS_DiskCache.h:40
bool write(const UT_WorkBuffer &buf)
Convenience methods to write data to the cache.
Definition: FS_DiskCache.h:82
Existing item, accessor has read-only file.
Definition: FS_DiskCache.h:259