HDK
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
FS_DiskCache.h
Go to the documentation of this file.
1 /*
2  * PROPRIETARY INFORMATION. This software is proprietary to
3  * Side Effects Software Inc., and is not to be reproduced,
4  * transmitted, or disclosed in any way without written permission.
5  *
6  * NAME: FS_DiskCache.h (UT Library, C++)
7  *
8  * COMMENTS:
9  */
10 
11 #ifndef __FS_DiskCache__
12 #define __FS_DiskCache__
13 
14 #include "FS_API.h"
15 #include <UT/UT_Access.h>
16 #include <UT/UT_NonCopyable.h>
17 #include <UT/UT_StringHolder.h>
18 #include <UT/UT_WorkBuffer.h>
19 #include <SYS/SYS_AtomicInt.h>
20 
21 class UT_JSONWriter;
22 
23 /// A disk cache can store data on disk. The cache is designed to be
24 /// thread-safe, even across multiple processes.
25 ///
26 /// This class is intended to be a singleton.
28 {
29 public:
30  /// The key is composed of two main parts:
31  /// 1) The value is used to uniquely identify the data in the file. This
32  /// value is hashed to compute the filename in the cache and can be
33  /// arbitrarily long.
34  /// 2) The suffix is a short identifier - The first 32 characters of this
35  /// suffix are appended to the value hash to create the filename.
36  /// The key will automatically add the current major.minor release of the
37  /// software (but not the build).
38  class FS_API Key
39  {
40  public:
42  const UT_StringRef &suffix = UT_StringRef())
43  {
44  computeFilename(value, suffix);
45  }
46  const UT_StringHolder &filename() const { return myFilename; }
47  private:
48  void computeFilename(const UT_StringRef &value,
49  const UT_StringRef &suffix);
50  UT_StringHolder myFilename;
51  };
52 
53  /// The Accessor class is used to access items in the cache. When an
54  /// accessor is in write-mode, the thread/process will have unique access
55  /// to the cache item until the Accessor is destructed. Multiple
56  /// threads/processes can have read-only access to the item.
58  {
59  public:
61  : myCache(nullptr)
62  , myFilename()
63  , myBytesWritten(0)
64  , myFd(-1)
65  , myAccess(0)
66  , myLocked(false)
67  , myStoreInCache(false)
68  {
69  }
71  {
72  clear();
73  }
74 
76 
77  /// Clear the accessor. This will close any file descriptors and
78  /// release any locks.
79  void clear();
80 
81  /// Write data to the file. This method can be called multiple times.
82  bool write(const void *data, exint len);
83 
84  /// Convenience methods to write data to the cache
86  { return write(buf.buffer(), buf.length()); }
87 
88  /// Return the number of bytes written
89  exint bytesWritten() const { return myBytesWritten; }
90 
91  /// Read the contents of the stream.
92  bool read(UT_WorkBuffer &contents) const;
93 
94  /// Whether the accessor is valid
95  bool isValid() const { return myFd >= 0; }
96 
97  /// Whether the accessor was opened as read-only
98  bool isRead() const { return myAccess & R_OK; }
99 
100  /// Whether the accessor was opened for writing
101  bool isWrite() const { return myAccess & W_OK; }
102 
103  /// Return the full path of the file in the cache
104  const UT_StringHolder &filename() const { return myFilename; }
105 
106  /// Normally, accessors that write data are counted to the cache
107  /// storage. This can disable this behaviour.
108  void storeInCache(bool v) { myStoreInCache = v; }
109 
110  void dump() const;
111  void dump(UT_JSONWriter &w) const;
112 
113  private:
114  /// Return the size of the read stream (or 0 if there was an error)
115  exint fileSize() const;
116 
117  /// @private - used solely by the FS_DiskCache to initialize
118  friend class FS_DiskCache; // Provide access to open()
119 
120  const FS_DiskCache *myCache; // Parent cache
121  UT_StringHolder myFilename;
122  exint myBytesWritten;
123  int myFd; // File descriptor
124  int myAccess; // Access mode
125  bool myLocked;
126  bool myStoreInCache;
127  };
128 
129  /// Statistics for the cache
130  class FS_API Stats
131  {
132  public:
134  {
135  fullClear();
136  }
137 
138  void clearStats()
139  {
140  myCacheHits = 0;
141  myCacheMiss = 0;
142  myCacheError = 0;
143  myFileCount = 0;
144  myDiskSize = 0;
145  myPurgeCount = 0;
146  }
147  void fullClear()
148  {
149  clearStats();
150  myMaxSize = 0;
151  myMaxLogSize = 0;
152  myCreateTime = 0;
153  myLastRecompute = 0;
154  }
155 
156  /// Merge stats from a different set of stats
157  void merge(const Stats &s)
158  {
159  myCacheHits += s.myCacheHits;
160  myCacheMiss += s.myCacheMiss;
161  myCacheError += s.myCacheError;
162  myFileCount += s.myFileCount;
163  myDiskSize += s.myDiskSize;
164  myPurgeCount += s.myPurgeCount;
165  }
166 
167  /// @{
168  /// Access statistics
169  int64 cacheHits() const { return myCacheHits; }
170  int64 cacheMiss() const { return myCacheMiss; }
171  int64 cacheError() const { return myCacheError; }
172  int64 purgeCount() const { return myPurgeCount; }
173  int64 fileCount() const { return myFileCount; }
174  int64 diskSize() const { return myDiskSize; }
175  int64 maxSize() const { return myMaxSize; }
176  int64 maxLogSize() const { return myMaxLogSize; }
177  time_t createTime() const { return myCreateTime; }
178  time_t lastRecompute() const { return myLastRecompute; }
179  /// @}
180 
181  /// Load stats from a given file descriptor
182  bool loadFrom(const Accessor &fd);
183  /// Write stats to the file descriptor
184  bool saveTo(Accessor &a) const;
185 
186  /// Update stats with a cache hit
187  void addCacheHit() { myCacheHits++; }
188 
189  /// Update stats from the given accessor
190  void addToCache(const Accessor &a);
191 
192  /// Purge from the given accessor
193  int64 rmFromCache(const Accessor &a, const char *filepath);
194 
195  /// Reset the file count and size
196  void reset(int64 nfiles, int64 size)
197  {
198  myFileCount = nfiles;
199  myDiskSize = size;
200  }
201 
202  void dump() const;
203  void dump(UT_JSONWriter &w) const;
204  private:
205  int64 myCacheHits; // Cache hits
206  int64 myCacheMiss; // Cache misses
207  int64 myCacheError; // Cache errors
208  int64 myFileCount; // Files in cache
209  int64 myPurgeCount; // Files removed from cache
210  int64 myDiskSize; // Size of cache
211 
212  friend class FS_DiskCache;
213  UT_StringHolder myCachePath; // Path to the cache
214  int64 myMaxSize; // Maximum cache size
215  int64 myMaxLogSize; // Maximum size for log file
216  time_t myCreateTime; // Time cache was created
217  time_t myLastRecompute; // Time cache was last recomputed
218  };
219 
221  exint default_size_in_bytes = 0);
222  ~FS_DiskCache();
223 
224  /// Check to see if the cache is valid
225  bool isValid() const { return myPath.isstring(); }
226 
227  /// Initialize the disk cache from a given path. If the path doesn't
228  /// exist, the cache will be created.
229  bool initialize(const UT_StringHolder &path);
230 
231  /// Set the maximum data size of the cache
232  bool setMaxSize(int64 size_in_bytes = 256*1024*1024);
233 
234  /// Set the maximum log size of the cache
235  bool setMaxLogSize(int64 size_in_bytes = 1024*1024);
236 
237  /// Return the maximum size of the cache
238  int64 maxSize() const { return myMaxSize; }
239 
240  /// Return the maximum size of the log file
241  int64 maxLogSize() const { return myMaxLogSize; }
242 
243  /// Get statistics for the cache
244  bool stats(Stats &stats) const;
245 
246  /// Return the location of the path
247  const UT_StringHolder &location() const { return myPath; }
248 
249  /// Reset the cache
250  bool recompute();
251 
252  /// Clear the cache contents
253  bool clearCache() const;
254 
255  /// Reset the cache entirely
256  bool clobberCache();
257 
258  /// Possible return values for the @c insert() function
260  {
261  INSERT_CREATE, ///< New item, accessor has write-only file
262  INSERT_EXIST, ///< Existing item, accessor has read-only file
263  INSERT_ERROR, ///< Unknown error, accessor will be invalid
264  };
265 
266  /// Similar to UT_ConcurrentHash, this method will try to insert an item
267  /// into the cache. If the item exists in the cache, @c insert() will
268  /// return false and the @c Accessor will have read-only access to the
269  /// item.
270  InsertStatus insert(Accessor &access, const Key &key);
271 
272  /// Find an item in the disk cache. If found, the Accessor will have
273  /// read-only access to the item. The item cannot be deleted from the
274  /// cache while it's referenced.
275  bool find(Accessor &access, const Key &key) const;
276 
277  /// Purge items from a cache to meet size requirements using LRU
278  bool purge(bool force=false) const;
279 
280  /// Dump debug information about the cache
281  void dump() const;
282  bool dump(UT_JSONWriter &w) const;
283 
284  void printLog(const char *fmt, ...) const
286 
287  template<typename... Args>
288  void log(const char *fmt, const Args &...args) const
289  {
290  UT_WorkBuffer msg;
291  msg.format(fmt, args...);
292  printLog("%s", msg.buffer());
293  }
294 
295 private:
296  void cacheFilePath(Accessor &a, const Key &k,
297  const FS_DiskCache *cache) const;
298 
299  void initStats(Stats &s) const
300  {
301  s.myCachePath = myPath;
302  s.myMaxSize = myMaxSize;
303  s.myMaxLogSize = myMaxLogSize;
304  s.myCreateTime = myCreateTime;
305  s.myLastRecompute = myLastRecompute;
306  }
307  void statPath(const char *filename, UT_WorkBuffer &path) const;
308  void statPath(int bucket, UT_WorkBuffer &path) const;
309  void statPath(const Accessor &a, UT_WorkBuffer &path) const
310  { statPath(a.filename(), path); }
311 
312  friend class Accessor; // Accessor needs to call cacheMiss()
313  void cacheHit(const Accessor &a) const;
314  void cacheMiss(const Accessor &a) const;
315  void cachePurge(const Accessor &a, const char *filepath) const;
316 
317 
318  void setDefaults(exint default_size);
319  bool saveConfig() const;
320  bool loadConfig();
321  bool initSubdir(const UT_StringRef &path) const;
322  bool saveStats(const char *path, const Stats &s) const;
323 
324  InsertStatus writeLock(Accessor &access, const char *path,
325  bool must_create, bool in_cache) const;
326  bool readLock(Accessor &access, const char *path) const;
327 
328  UT_StringHolder myPath;
329  int64 myMaxSize;
330  int64 myMaxLogSize;
331  time_t myCreateTime;
332  time_t myLastRecompute;
333  mutable SYS_AtomicCounter myCurrSize;
334 };
335 
336 #endif
void merge(const Stats &s)
Merge stats from a different set of stats.
Definition: FS_DiskCache.h:157
GLenum GLuint GLenum GLsizei const GLchar * buf
Definition: glcorearb.h:2540
bool setMaxSize(int64 size_in_bytes=256 *1024 *1024)
Set the maximum data size of the cache.
GT_API const UT_StringHolder filename
void reset(int64 nfiles, int64 size)
Reset the file count and size.
Definition: FS_DiskCache.h:196
exint bytesWritten() const
Return the number of bytes written.
Definition: FS_DiskCache.h:89
int64 cacheHits() const
Definition: FS_DiskCache.h:169
void
Definition: png.h:1083
bool setMaxLogSize(int64 size_in_bytes=1024 *1024)
Set the maximum log size of the cache.
const GLdouble * v
Definition: glcorearb.h:837
GLsizei const GLchar *const * path
Definition: glcorearb.h:3341
void printLog(const char *fmt,...) const SYS_PRINTF_CHECK_ATTRIBUTE(2
int64 exint
Definition: SYS_Types.h:125
SYS_FORCE_INLINE const char * buffer() const
GLboolean GLboolean GLboolean GLboolean a
Definition: glcorearb.h:1222
GLdouble s
Definition: glad.h:3009
void addCacheHit()
Update stats with a cache hit.
Definition: FS_DiskCache.h:187
Class which writes ASCII or binary JSON streams.
Definition: UT_JSONWriter.h:37
int64 cacheError() const
Definition: FS_DiskCache.h:171
OIIO_FORCEINLINE vbool4 insert(const vbool4 &a, bool val)
Helper: substitute val for a[i].
Definition: simd.h:3436
void read(T &in, bool &v)
Definition: ImfXdr.h:502
bool isWrite() const
Whether the accessor was opened for writing.
Definition: FS_DiskCache.h:101
time_t lastRecompute() const
Definition: FS_DiskCache.h:178
InsertStatus
Possible return values for the insert() function.
Definition: FS_DiskCache.h:259
int64 cacheMiss() const
Definition: FS_DiskCache.h:170
void storeInCache(bool v)
Definition: FS_DiskCache.h:108
int64 diskSize() const
Definition: FS_DiskCache.h:174
GLuint GLint GLboolean GLint GLenum access
Definition: glcorearb.h:2222
#define SYS_PRINTF_CHECK_ATTRIBUTE(string_index, first_to_check)
Definition: SYS_Types.h:447
New item, accessor has write-only file.
Definition: FS_DiskCache.h:261
#define UT_NON_COPYABLE(CLASS)
Define deleted copy constructor and assignment operator inside a class.
long long int64
Definition: SYS_Types.h:116
const UT_StringHolder & location() const
Return the location of the path.
Definition: FS_DiskCache.h:247
const UT_StringHolder & filename() const
Definition: FS_DiskCache.h:46
OPENVDB_API void initialize()
Global registration of native Grid, Transform, Metadata and Point attribute types. Also initializes blosc (if enabled).
Definition: logging.h:294
Statistics for the cache.
Definition: FS_DiskCache.h:130
bool isRead() const
Whether the accessor was opened as read-only.
Definition: FS_DiskCache.h:98
int64 maxLogSize() const
Return the maximum size of the log file.
Definition: FS_DiskCache.h:241
Unknown error, accessor will be invalid.
Definition: FS_DiskCache.h:263
bool clobberCache()
Reset the cache entirely.
int64 maxLogSize() const
Definition: FS_DiskCache.h:176
bool isValid() const
Whether the accessor is valid.
Definition: FS_DiskCache.h:95
const UT_StringHolder & filename() const
Return the full path of the file in the cache.
Definition: FS_DiskCache.h:104
GLsizeiptr size
Definition: glcorearb.h:664
size_t format(const char *fmt, const Args &...args)
void dump() const
Dump debug information about the cache.
bool recompute()
Reset the cache.
SIM_API const UT_StringHolder force
int64 purgeCount() const
Definition: FS_DiskCache.h:172
time_t createTime() const
Definition: FS_DiskCache.h:177
bool purge(bool force=false) const
Purge items from a cache to meet size requirements using LRU.
int64 maxSize() const
Return the maximum size of the cache.
Definition: FS_DiskCache.h:238
FS_DiskCache(const UT_StringHolder &path=UT_StringHolder(), exint default_size_in_bytes=0)
bool stats(Stats &stats) const
Get statistics for the cache.
**If you just want to fire and args
Definition: thread.h:609
friend class Accessor
Definition: FS_DiskCache.h:312
GLubyte GLubyte GLubyte GLubyte w
Definition: glcorearb.h:857
Definition: core.h:1131
int64 fileCount() const
Definition: FS_DiskCache.h:173
#define const
Definition: zconf.h:214
bool clearCache() const
Clear the cache contents.
OIIO_FORCEINLINE T log(const T &v)
Definition: simd.h:7688
void write(T &out, bool v)
Definition: ImfXdr.h:287
bool isValid() const
Check to see if the cache is valid.
Definition: FS_DiskCache.h:225
int64 maxSize() const
Definition: FS_DiskCache.h:175
GLuint64 GLenum GLint fd
Definition: RE_OGL.h:262
Definition: format.h:895
#define FS_API
Definition: FS_API.h:10
FMT_CONSTEXPR auto find(Ptr first, Ptr last, T value, Ptr &out) -> bool
Definition: core.h:2089
Key(const UT_StringRef &value=UT_StringRef(), const UT_StringRef &suffix=UT_StringRef())
Definition: FS_DiskCache.h:41
Existing item, accessor has read-only file.
Definition: FS_DiskCache.h:262