HDK
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
StreamCompression.h
Go to the documentation of this file.
1 ///////////////////////////////////////////////////////////////////////////
2 //
3 // Copyright (c) 2012-2017 DreamWorks Animation LLC
4 //
5 // All rights reserved. This software is distributed under the
6 // Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )
7 //
8 // Redistributions of source code must retain the above copyright
9 // and license notice and the following restrictions and disclaimer.
10 //
11 // * Neither the name of DreamWorks Animation nor the names of
12 // its contributors may be used to endorse or promote products derived
13 // from this software without specific prior written permission.
14 //
15 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY INDIRECT, INCIDENTAL,
20 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 // IN NO EVENT SHALL THE COPYRIGHT HOLDERS' AND CONTRIBUTORS' AGGREGATE
27 // LIABILITY FOR ALL CLAIMS REGARDLESS OF THEIR BASIS EXCEED US$250.00.
28 //
29 ///////////////////////////////////////////////////////////////////////////
30 
31 /// @file points/StreamCompression.h
32 ///
33 /// @author Dan Bailey
34 ///
35 /// @brief Convenience wrappers to using Blosc and reading and writing of Paged data.
36 ///
37 /// Blosc is most effective with large (> ~256KB) blocks of data. Writing the entire
38 /// data block contiguously would provide the most optimal compression, however would
39 /// limit the ability to use delayed-loading as the whole block would be required to
40 /// be loaded from disk at once. To balance these two competing factors, Paging is used
41 /// to write out blocks of data that are a reasonable size for Blosc. These Pages are
42 /// loaded lazily, tracking the input stream pointers and creating Handles that reference
43 /// portions of the buffer. When the Page buffer is accessed, the data will be read from
44 /// the stream.
45 
46 #ifndef OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
47 #define OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
48 
49 #include <openvdb/io/io.h>
50 #include <tbb/spin_mutex.h>
51 #include <memory>
52 #include <string>
53 
54 
55 class TestStreamCompression;
56 
57 namespace openvdb {
59 namespace OPENVDB_VERSION_NAME {
60 namespace compression {
61 
62 
63 // This is the minimum number of bytes below which Blosc compression is not used to
64 // avoid unecessary computation, as Blosc offers minimal compression until this limit
65 static const int BLOSC_MINIMUM_BYTES = 48;
66 
67 // This is the minimum number of bytes below which the array is padded with zeros up
68 // to this number of bytes to allow Blosc to perform compression with small arrays
69 static const int BLOSC_PAD_BYTES = 128;
70 
71 
72 /// @brief Returns true if compression is available
74 
75 /// @brief Retrieves the uncompressed size of buffer when uncompressed
76 ///
77 /// @param buffer the compressed buffer
78 OPENVDB_API size_t bloscUncompressedSize(const char* buffer);
79 
80 /// @brief Compress into the supplied buffer.
81 ///
82 /// @param compressedBuffer the buffer to compress
83 /// @param compressedBytes number of compressed bytes
84 /// @param bufferBytes the number of bytes in compressedBuffer available to be filled
85 /// @param uncompressedBuffer the uncompressed buffer to compress
86 /// @param uncompressedBytes number of uncompressed bytes
87 OPENVDB_API void bloscCompress(char* compressedBuffer, size_t& compressedBytes,
88  const size_t bufferBytes, const char* uncompressedBuffer, const size_t uncompressedBytes);
89 
90 /// @brief Compress and return the heap-allocated compressed buffer.
91 ///
92 /// @param buffer the buffer to compress
93 /// @param uncompressedBytes number of uncompressed bytes
94 /// @param compressedBytes number of compressed bytes (written to this variable)
95 /// @param resize the compressed buffer will be exactly resized to remove the
96 /// portion used for Blosc overhead, for efficiency this can be
97 /// skipped if it is known that the resulting buffer is temporary
98 OPENVDB_API std::unique_ptr<char[]> bloscCompress(const char* buffer,
99  const size_t uncompressedBytes, size_t& compressedBytes, const bool resize = true);
100 
101 /// @brief Convenience wrapper to retrieve the compressed size of buffer when compressed
102 ///
103 /// @param buffer the uncompressed buffer
104 /// @param uncompressedBytes number of uncompressed bytes
105 OPENVDB_API size_t bloscCompressedSize(const char* buffer, const size_t uncompressedBytes);
106 
107 /// @brief Decompress into the supplied buffer. Will throw if decompression fails or
108 /// uncompressed buffer has insufficient space in which to decompress.
109 ///
110 /// @param uncompressedBuffer the uncompressed buffer to decompress into
111 /// @param expectedBytes the number of bytes expected once the buffer is decompressed
112 /// @param bufferBytes the number of bytes in uncompressedBuffer available to be filled
113 /// @param compressedBuffer the compressed buffer to decompress
114 OPENVDB_API void bloscDecompress(char* uncompressedBuffer, const size_t expectedBytes,
115  const size_t bufferBytes, const char* compressedBuffer);
116 
117 /// @brief Decompress and return the the heap-allocated uncompressed buffer.
118 ///
119 /// @param buffer the buffer to decompress
120 /// @param expectedBytes the number of bytes expected once the buffer is decompressed
121 /// @param resize the compressed buffer will be exactly resized to remove the
122 /// portion used for Blosc overhead, for efficiency this can be
123 /// skipped if it is known that the resulting buffer is temporary
124 OPENVDB_API std::unique_ptr<char[]> bloscDecompress(const char* buffer,
125  const size_t expectedBytes, const bool resize = true);
126 
127 
128 ////////////////////////////////////////
129 
130 
131 // 1MB = 1048576 Bytes
132 static const int PageSize = 1024 * 1024;
133 
134 
135 /// @brief Stores a variable-size, compressed, delayed-load Page of data
136 /// that is loaded into memory when accessed. Access to the Page is
137 /// thread-safe as loading and decompressing the data is protected by a mutex.
139 {
140 private:
141  struct Info
142  {
143  io::MappedFile::Ptr mappedFile;
145  std::streamoff filepos;
146  long compressedBytes;
147  long uncompressedBytes;
148  }; // Info
149 
150 public:
151  using Ptr = std::shared_ptr<Page>;
152 
153  Page() = default;
154 
155  /// @brief load the Page into memory
156  void load() const;
157 
158  /// @brief Uncompressed bytes of the Paged data, available
159  /// when the header has been read.
160  long uncompressedBytes() const;
161 
162  /// @brief Retrieves a data pointer at the specific @param index
163  /// @note Will force a Page load when called.
164  const char* buffer(const int index) const;
165 
166  /// @brief Read the Page header
167  void readHeader(std::istream&);
168 
169  /// @brief Read the Page buffers. If @a delayed is true, stream
170  /// pointers will be stored to load the data lazily.
171  void readBuffers(std::istream&, bool delayed);
172 
173  /// @brief Test if the data is out-of-core
174  bool isOutOfCore() const;
175 
176 private:
177  /// @brief Convenience method to store a copy of the supplied buffer
178  void copy(const std::unique_ptr<char[]>& temp, int pageSize);
179 
180  /// @brief Decompress and store the supplied data
181  void decompress(const std::unique_ptr<char[]>& temp);
182 
183  /// @brief Thread-safe loading of the data
184  void doLoad() const;
185 
186  std::unique_ptr<Info> mInfo = std::unique_ptr<Info>(new Info);
187  std::unique_ptr<char[]> mData;
188  tbb::spin_mutex mMutex;
189 }; // class Page
190 
191 
192 /// @brief A PageHandle holds a shared ptr to a Page and a specific stream
193 /// pointer to a point within the decompressed Page buffer
195 {
196 public:
197  using Ptr = std::shared_ptr<PageHandle>;
198 
199  /// @brief Create the page handle
200  /// @param page a shared ptr to the page that stores the buffer
201  /// @param index start position of the buffer to be read
202  /// @param size total size of the buffer to be read in bytes
203  PageHandle(const Page::Ptr& page, const int index, const int size);
204 
205  /// @brief Retrieve a reference to the stored page
206  Page& page();
207 
208  /// @brief Read and return the buffer, loading and decompressing
209  /// the Page if necessary.
210  std::unique_ptr<char[]> read();
211 
212 protected:
213  friend class ::TestStreamCompression;
214 
215 private:
216  Page::Ptr mPage;
217  int mIndex = -1;
218  int mSize = 0;
219 }; // class PageHandle
220 
221 
222 /// @brief A Paging wrapper to std::istream that is responsible for reading
223 /// from a given input stream and creating Page objects and PageHandles that
224 /// reference those pages for delayed reading.
226 {
227 public:
228  using Ptr = std::shared_ptr<PagedInputStream>;
229 
230  PagedInputStream() = default;
231 
232  explicit PagedInputStream(std::istream& is);
233 
234  /// @brief Size-only mode tags the stream as only reading size data.
235  void setSizeOnly(bool sizeOnly) { mSizeOnly = sizeOnly; }
236  bool sizeOnly() const { return mSizeOnly; }
237 
238  // @brief Set and get the input stream
239  std::istream& getInputStream() { assert(mIs); return *mIs; }
240  void setInputStream(std::istream& is) { mIs = &is; }
241 
242  /// @brief Creates a PageHandle to access the next @param n bytes of the Page.
243  PageHandle::Ptr createHandle(std::streamsize n);
244 
245  /// @brief Takes a @a pageHandle and updates the referenced page with the
246  /// current stream pointer position and if @a delayed is false performs
247  /// an immediate read of the data.
248  void read(PageHandle::Ptr& pageHandle, std::streamsize n, bool delayed = true);
249 
250 private:
251  int mByteIndex = 0;
252  int mUncompressedBytes = 0;
253  std::istream* mIs = nullptr;
254  Page::Ptr mPage;
255  bool mSizeOnly = false;
256 }; // class PagedInputStream
257 
258 
259 /// @brief A Paging wrapper to std::ostream that is responsible for writing
260 /// from a given output stream at intervals set by the PageSize. As Pages are
261 /// variable in size, they are flushed to disk as soon as sufficiently large.
263 {
264 public:
265  using Ptr = std::shared_ptr<PagedOutputStream>;
266 
268 
269  explicit PagedOutputStream(std::ostream& os);
270 
271  /// @brief Size-only mode tags the stream as only writing size data.
272  void setSizeOnly(bool sizeOnly) { mSizeOnly = sizeOnly; }
273  bool sizeOnly() const { return mSizeOnly; }
274 
275  /// @brief Set and get the output stream
276  std::ostream& getOutputStream() { assert(mOs); return *mOs; }
277  void setOutputStream(std::ostream& os) { mOs = &os; }
278 
279  /// @brief Writes the given @param str buffer of size @param n
280  PagedOutputStream& write(const char* str, std::streamsize n);
281 
282  /// @brief Manually flushes the current page to disk if non-zero
283  void flush();
284 
285 private:
286  /// @brief Compress the @param buffer of @param size bytes and write
287  /// out to the stream.
288  void compressAndWrite(const char* buffer, size_t size);
289 
290  /// @brief Resize the internal page buffer to @param size bytes
291  void resize(size_t size);
292 
293  std::unique_ptr<char[]> mData = std::unique_ptr<char[]>(new char[PageSize]);
294  std::unique_ptr<char[]> mCompressedData = nullptr;
295  size_t mCapacity = PageSize;
296  int mBytes = 0;
297  std::ostream* mOs = nullptr;
298  bool mSizeOnly = false;
299 }; // class PagedOutputStream
300 
301 
302 } // namespace compression
303 } // namespace OPENVDB_VERSION_NAME
304 } // namespace openvdb
305 
306 #endif // OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
307 
308 // Copyright (c) 2012-2017 DreamWorks Animation LLC
309 // All rights reserved. This software is distributed under the
310 // Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )
void setSizeOnly(bool sizeOnly)
Size-only mode tags the stream as only reading size data.
SharedPtr< MappedFile > Ptr
Definition: io.h:152
A Paging wrapper to std::istream that is responsible for reading from a given input stream and creati...
void read(T &in, bool &v)
Definition: ImfXdr.h:611
GLuint buffer
Definition: glcorearb.h:659
std::ostream & getOutputStream()
Set and get the output stream.
GLsizeiptr size
Definition: glcorearb.h:663
void setSizeOnly(bool sizeOnly)
Size-only mode tags the stream as only writing size data.
std::shared_ptr< T > SharedPtr
Definition: Types.h:130
A PageHandle holds a shared ptr to a Page and a specific stream pointer to a point within the decompr...
GLdouble n
Definition: glcorearb.h:2007
#define OPENVDB_API
Helper macros for defining library symbol visibility.
Definition: Platform.h:194
#define OPENVDB_VERSION_NAME
Definition: version.h:43
OPENVDB_API void bloscDecompress(char *uncompressedBuffer, const size_t expectedBytes, const size_t bufferBytes, const char *compressedBuffer)
Decompress into the supplied buffer. Will throw if decompression fails or uncompressed buffer has ins...
OPENVDB_API void bloscCompress(char *compressedBuffer, size_t &compressedBytes, const size_t bufferBytes, const char *uncompressedBuffer, const size_t uncompressedBytes)
Compress into the supplied buffer.
Stores a variable-size, compressed, delayed-load Page of data that is loaded into memory when accesse...
GLuint index
Definition: glcorearb.h:785
OPENVDB_API size_t bloscCompressedSize(const char *buffer, const size_t uncompressedBytes)
Convenience wrapper to retrieve the compressed size of buffer when compressed.
OPENVDB_API bool bloscCanCompress()
Returns true if compression is available.
A Paging wrapper to std::ostream that is responsible for writing from a given output stream at interv...
OPENVDB_API size_t bloscUncompressedSize(const char *buffer)
Retrieves the uncompressed size of buffer when uncompressed.
#define OPENVDB_USE_VERSION_NAMESPACE
Definition: version.h:71
void write(T &out, bool v)
Definition: ImfXdr.h:332