HDK
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Compression.h
Go to the documentation of this file.
1 ///////////////////////////////////////////////////////////////////////////
2 //
3 // Copyright (c) DreamWorks Animation LLC
4 //
5 // All rights reserved. This software is distributed under the
6 // Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )
7 //
8 // Redistributions of source code must retain the above copyright
9 // and license notice and the following restrictions and disclaimer.
10 //
11 // * Neither the name of DreamWorks Animation nor the names of
12 // its contributors may be used to endorse or promote products derived
13 // from this software without specific prior written permission.
14 //
15 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY INDIRECT, INCIDENTAL,
20 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 // IN NO EVENT SHALL THE COPYRIGHT HOLDERS' AND CONTRIBUTORS' AGGREGATE
27 // LIABILITY FOR ALL CLAIMS REGARDLESS OF THEIR BASIS EXCEED US$250.00.
28 //
29 ///////////////////////////////////////////////////////////////////////////
30 
31 #ifndef OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
32 #define OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
33 
34 #include <openvdb/Types.h>
35 #include <openvdb/MetaMap.h>
36 #include <openvdb/math/Math.h> // for negative()
37 #include "io.h" // for getDataCompression(), etc.
38 #include "DelayedLoadMetadata.h"
39 #include <algorithm>
40 #include <iostream>
41 #include <memory>
42 #include <string>
43 #include <vector>
44 
45 
46 namespace openvdb {
48 namespace OPENVDB_VERSION_NAME {
49 namespace io {
50 
51 /// @brief OR-able bit flags for compression options on input and output streams
52 /// @details
53 /// <dl>
54 /// <dt><tt>COMPRESS_NONE</tt>
55 /// <dd>On write, don't compress data.<br>
56 /// On read, the input stream contains uncompressed data.
57 ///
58 /// <dt><tt>COMPRESS_ZIP</tt>
59 /// <dd>When writing grids other than level sets or fog volumes, apply
60 /// ZLIB compression to internal and leaf node value buffers.<br>
61 /// When reading grids other than level sets or fog volumes, indicate that
62 /// the value buffers of internal and leaf nodes are ZLIB-compressed.<br>
63 /// ZLIB compresses well but is slow.
64 ///
65 /// <dt><tt>COMPRESS_ACTIVE_MASK</tt>
66 /// <dd>When writing a grid of any class, don't output a node's inactive values
67 /// if it has two or fewer distinct values. Instead, output minimal information
68 /// to permit the lossless reconstruction of inactive values.<br>
69 /// On read, nodes might have been stored without inactive values.
70 /// Where necessary, reconstruct inactive values from available information.
71 ///
72 /// <dt><tt>COMPRESS_BLOSC</tt>
73 /// <dd>When writing grids other than level sets or fog volumes, apply
74 /// Blosc compression to internal and leaf node value buffers.<br>
75 /// When reading grids other than level sets or fog volumes, indicate that
76 /// the value buffers of internal and leaf nodes are Blosc-compressed.<br>
77 /// Blosc is much faster than ZLIB and produces comparable file sizes.
78 /// </dl>
79 enum {
81  COMPRESS_ZIP = 0x1,
84 };
85 
86 /// Return a string describing the given compression flags.
88 
89 
90 ////////////////////////////////////////
91 
92 
93 /// @internal Per-node indicator byte that specifies what additional metadata
94 /// is stored to permit reconstruction of inactive values
95 enum {
96  /*0*/ NO_MASK_OR_INACTIVE_VALS, // no inactive vals, or all inactive vals are +background
97  /*1*/ NO_MASK_AND_MINUS_BG, // all inactive vals are -background
98  /*2*/ NO_MASK_AND_ONE_INACTIVE_VAL, // all inactive vals have the same non-background val
99  /*3*/ MASK_AND_NO_INACTIVE_VALS, // mask selects between -background and +background
100  /*4*/ MASK_AND_ONE_INACTIVE_VAL, // mask selects between backgd and one other inactive val
101  /*5*/ MASK_AND_TWO_INACTIVE_VALS, // mask selects between two non-background inactive vals
102  /*6*/ NO_MASK_AND_ALL_VALS // > 2 inactive vals, so no mask compression at all
103 };
104 
105 
106 template <typename ValueT, typename MaskT>
108 {
109  // Comparison function for values
110  static inline bool eq(const ValueT& a, const ValueT& b) {
111  return math::isExactlyEqual(a, b);
112  }
113 
115  const MaskT& valueMask, const MaskT& childMask,
116  const ValueT* srcBuf, const ValueT& background)
117  {
118  /// @todo Consider all values, not just inactive values?
119  inactiveVal[0] = inactiveVal[1] = background;
120  int numUniqueInactiveVals = 0;
121  for (typename MaskT::OffIterator it = valueMask.beginOff();
122  numUniqueInactiveVals < 3 && it; ++it)
123  {
124  const Index32 idx = it.pos();
125 
126  // Skip inactive values that are actually child node pointers.
127  if (childMask.isOn(idx)) continue;
128 
129  const ValueT& val = srcBuf[idx];
130  const bool unique = !(
131  (numUniqueInactiveVals > 0 && MaskCompress::eq(val, inactiveVal[0])) ||
132  (numUniqueInactiveVals > 1 && MaskCompress::eq(val, inactiveVal[1]))
133  );
134  if (unique) {
135  if (numUniqueInactiveVals < 2) inactiveVal[numUniqueInactiveVals] = val;
136  ++numUniqueInactiveVals;
137  }
138  }
139 
141 
142  if (numUniqueInactiveVals == 1) {
143  if (!MaskCompress::eq(inactiveVal[0], background)) {
144  if (MaskCompress::eq(inactiveVal[0], math::negative(background))) {
146  } else {
148  }
149  }
150  } else if (numUniqueInactiveVals == 2) {
152  if (!MaskCompress::eq(inactiveVal[0], background) && !MaskCompress::eq(inactiveVal[1], background)) {
153  // If neither inactive value is equal to the background, both values
154  // need to be saved, along with a mask that selects between them.
156 
157  } else if (MaskCompress::eq(inactiveVal[1], background)) {
158  if (MaskCompress::eq(inactiveVal[0], math::negative(background))) {
159  // If the second inactive value is equal to the background and
160  // the first is equal to -background, neither value needs to be saved,
161  // but save a mask that selects between -background and +background.
163  } else {
164  // If the second inactive value is equal to the background, only
165  // the first value needs to be saved, along with a mask that selects
166  // between it and the background.
168  }
169  } else if (MaskCompress::eq(inactiveVal[0], background)) {
170  if (MaskCompress::eq(inactiveVal[1], math::negative(background))) {
171  // If the first inactive value is equal to the background and
172  // the second is equal to -background, neither value needs to be saved,
173  // but save a mask that selects between -background and +background.
176  } else {
177  // If the first inactive value is equal to the background, swap it
178  // with the second value and save only that value, along with a mask
179  // that selects between it and the background.
182  }
183  }
184  } else if (numUniqueInactiveVals > 2) {
186  }
187  }
188 
190  ValueT inactiveVal[2];
191 };
192 
193 
194 ////////////////////////////////////////
195 
196 
197 /// @brief RealToHalf and its specializations define a mapping from
198 /// floating-point data types to analogous half float types.
199 template<typename T>
200 struct RealToHalf {
201  enum { isReal = false }; // unless otherwise specified, type T is not a floating-point type
202  using HalfT = T; // type T's half float analogue is T itself
203  static HalfT convert(const T& val) { return val; }
204 };
205 template<> struct RealToHalf<float> {
206  enum { isReal = true };
207  using HalfT = half;
208  static HalfT convert(float val) { return HalfT(val); }
209 };
210 template<> struct RealToHalf<double> {
211  enum { isReal = true };
212  using HalfT = half;
213  // A half can only be constructed from a float, so cast the value to a float first.
214  static HalfT convert(double val) { return HalfT(float(val)); }
215 };
216 template<> struct RealToHalf<Vec2s> {
217  enum { isReal = true };
218  using HalfT = Vec2H;
219  static HalfT convert(const Vec2s& val) { return HalfT(val); }
220 };
221 template<> struct RealToHalf<Vec2d> {
222  enum { isReal = true };
223  using HalfT = Vec2H;
224  // A half can only be constructed from a float, so cast the vector's elements to floats first.
225  static HalfT convert(const Vec2d& val) { return HalfT(Vec2s(val)); }
226 };
227 template<> struct RealToHalf<Vec3s> {
228  enum { isReal = true };
229  using HalfT = Vec3H;
230  static HalfT convert(const Vec3s& val) { return HalfT(val); }
231 };
232 template<> struct RealToHalf<Vec3d> {
233  enum { isReal = true };
234  using HalfT = Vec3H;
235  // A half can only be constructed from a float, so cast the vector's elements to floats first.
236  static HalfT convert(const Vec3d& val) { return HalfT(Vec3s(val)); }
237 };
238 
239 
240 /// Return the given value truncated to 16-bit float precision.
241 template<typename T>
242 inline T
244 {
245  return T(RealToHalf<T>::convert(val));
246 }
247 
248 
249 ////////////////////////////////////////
250 
251 
252 OPENVDB_API size_t zipToStreamSize(const char* data, size_t numBytes);
253 OPENVDB_API void zipToStream(std::ostream&, const char* data, size_t numBytes);
254 OPENVDB_API void unzipFromStream(std::istream&, char* data, size_t numBytes);
255 OPENVDB_API size_t bloscToStreamSize(const char* data, size_t valSize, size_t numVals);
256 OPENVDB_API void bloscToStream(std::ostream&, const char* data, size_t valSize, size_t numVals);
257 OPENVDB_API void bloscFromStream(std::istream&, char* data, size_t numBytes);
258 
259 /// @brief Read data from a stream.
260 /// @param is the input stream
261 /// @param data the contiguous array of data to read in
262 /// @param count the number of elements to read in
263 /// @param compression whether and how the data is compressed (either COMPRESS_NONE,
264 /// COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC)
265 /// @param metadata optional pointer to a DelayedLoadMetadata object that stores
266 /// the size of the compressed buffer
267 /// @param metadataOffset offset into DelayedLoadMetadata, ignored if pointer is null
268 /// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled
269 /// without Blosc support.
270 /// @details This default implementation is instantiated only for types
271 /// whose size can be determined by the sizeof() operator.
272 template<typename T>
273 inline void
274 readData(std::istream& is, T* data, Index count, uint32_t compression,
275  DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0))
276 {
277  const bool seek = data == nullptr;
278  if (seek) {
279  assert(!getStreamMetadataPtr(is) || getStreamMetadataPtr(is)->seekable());
280  }
281  const bool hasCompression = compression & (COMPRESS_BLOSC | COMPRESS_ZIP);
282 
283  if (metadata && seek && hasCompression) {
284  size_t compressedSize = metadata->getCompressedSize(metadataOffset);
285  is.seekg(compressedSize, std::ios_base::cur);
286  } else if (compression & COMPRESS_BLOSC) {
287  bloscFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
288  } else if (compression & COMPRESS_ZIP) {
289  unzipFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
290  } else if (seek) {
291  is.seekg(sizeof(T) * count, std::ios_base::cur);
292  } else {
293  is.read(reinterpret_cast<char*>(data), sizeof(T) * count);
294  }
295 }
296 
297 /// Specialization for std::string input
298 template<>
299 inline void
300 readData<std::string>(std::istream& is, std::string* data, Index count, uint32_t /*compression*/,
301  DelayedLoadMetadata* /*metadata*/, size_t /*metadataOffset*/)
302 {
303  for (Index i = 0; i < count; ++i) {
304  size_t len = 0;
305  is >> len;
306  //data[i].resize(len);
307  //is.read(&(data[i][0]), len);
308 
309  std::string buffer(len+1, ' ');
310  is.read(&buffer[0], len+1);
311  if (data != nullptr) data[i].assign(buffer, 0, len);
312  }
313 }
314 
315 /// HalfReader wraps a static function, read(), that is analogous to readData(), above,
316 /// except that it is partially specialized for floating-point types in order to promote
317 /// 16-bit half float values to full float. A wrapper class is required because
318 /// only classes, not functions, can be partially specialized.
319 template<bool IsReal, typename T> struct HalfReader;
320 /// Partial specialization for non-floating-point types (no half to float promotion)
321 template<typename T>
322 struct HalfReader</*IsReal=*/false, T> {
323  static inline void read(std::istream& is, T* data, Index count, uint32_t compression,
324  DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) {
325  readData(is, data, count, compression, metadata, metadataOffset);
326  }
327 };
328 /// Partial specialization for floating-point types
329 template<typename T>
330 struct HalfReader</*IsReal=*/true, T> {
331  using HalfT = typename RealToHalf<T>::HalfT;
332  static inline void read(std::istream& is, T* data, Index count, uint32_t compression,
333  DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) {
334  if (count < 1) return;
335  if (data == nullptr) {
336  // seek mode - pass through null pointer
337  readData<HalfT>(is, nullptr, count, compression, metadata, metadataOffset);
338  } else {
339  std::vector<HalfT> halfData(count); // temp buffer into which to read half float values
340  readData<HalfT>(is, reinterpret_cast<HalfT*>(&halfData[0]), count, compression,
341  metadata, metadataOffset);
342  // Copy half float values from the temporary buffer to the full float output array.
343  std::copy(halfData.begin(), halfData.end(), data);
344  }
345  }
346 };
347 
348 
349 template<typename T>
350 inline size_t
351 writeDataSize(const T *data, Index count, uint32_t compression)
352 {
353  if (compression & COMPRESS_BLOSC) {
354  return bloscToStreamSize(reinterpret_cast<const char*>(data), sizeof(T), count);
355  } else if (compression & COMPRESS_ZIP) {
356  return zipToStreamSize(reinterpret_cast<const char*>(data), sizeof(T) * count);
357  } else {
358  return sizeof(T) * count;
359  }
360 }
361 
362 
363 /// Specialization for std::string output
364 template<>
365 inline size_t
366 writeDataSize<std::string>(const std::string* data, Index count,
367  uint32_t /*compression*/) ///< @todo add compression
368 {
369  size_t size(0);
370  for (Index i = 0; i < count; ++i) {
371  const size_t len = data[i].size();
372  size += sizeof(size_t) + (len+1);
373  }
374  return size;
375 }
376 
377 
378 /// Write data to a stream.
379 /// @param os the output stream
380 /// @param data the contiguous array of data to write
381 /// @param count the number of elements to write out
382 /// @param compression whether and how to compress the data (either COMPRESS_NONE,
383 /// COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC)
384 /// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled
385 /// without Blosc support.
386 /// @details This default implementation is instantiated only for types
387 /// whose size can be determined by the sizeof() operator.
388 template<typename T>
389 inline void
390 writeData(std::ostream &os, const T *data, Index count, uint32_t compression)
391 {
392  if (compression & COMPRESS_BLOSC) {
393  bloscToStream(os, reinterpret_cast<const char*>(data), sizeof(T), count);
394  } else if (compression & COMPRESS_ZIP) {
395  zipToStream(os, reinterpret_cast<const char*>(data), sizeof(T) * count);
396  } else {
397  os.write(reinterpret_cast<const char*>(data), sizeof(T) * count);
398  }
399 }
400 
401 /// Specialization for std::string output
402 template<>
403 inline void
404 writeData<std::string>(std::ostream& os, const std::string* data, Index count,
405  uint32_t /*compression*/) ///< @todo add compression
406 {
407  for (Index i = 0; i < count; ++i) {
408  const size_t len = data[i].size();
409  os << len;
410  os.write(data[i].c_str(), len+1);
411  //os.write(&(data[i][0]), len );
412  }
413 }
414 
415 /// HalfWriter wraps a static function, write(), that is analogous to writeData(), above,
416 /// except that it is partially specialized for floating-point types in order to quantize
417 /// floating-point values to 16-bit half float. A wrapper class is required because
418 /// only classes, not functions, can be partially specialized.
419 template<bool IsReal, typename T> struct HalfWriter;
420 /// Partial specialization for non-floating-point types (no float to half quantization)
421 template<typename T>
422 struct HalfWriter</*IsReal=*/false, T> {
423  static inline size_t writeSize(const T* data, Index count, uint32_t compression) {
424  return writeDataSize(data, count, compression);
425  }
426  static inline void write(std::ostream& os, const T* data, Index count, uint32_t compression) {
427  writeData(os, data, count, compression);
428  }
429 };
430 /// Partial specialization for floating-point types
431 template<typename T>
432 struct HalfWriter</*IsReal=*/true, T> {
433  using HalfT = typename RealToHalf<T>::HalfT;
434  static inline size_t writeSize(const T* data, Index count, uint32_t compression) {
435  if (count < 1) return size_t(0);
436  // Convert full float values to half float, then output the half float array.
437  std::vector<HalfT> halfData(count);
438  for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<T>::convert(data[i]);
439  return writeDataSize<HalfT>(reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
440  }
441  static inline void write(std::ostream& os, const T* data, Index count, uint32_t compression) {
442  if (count < 1) return;
443  // Convert full float values to half float, then output the half float array.
444  std::vector<HalfT> halfData(count);
445  for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<T>::convert(data[i]);
446  writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
447  }
448 };
449 #ifdef _MSC_VER
450 /// Specialization to avoid double to float warnings in MSVC
451 template<>
452 struct HalfWriter</*IsReal=*/true, double> {
453  using HalfT = RealToHalf<double>::HalfT;
454  static inline size_t writeSize(const double* data, Index count, uint32_t compression)
455  {
456  if (count < 1) return size_t(0);
457  // Convert full float values to half float, then output the half float array.
458  std::vector<HalfT> halfData(count);
459  for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<double>::convert(data[i]);
460  return writeDataSize<HalfT>(reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
461  }
462  static inline void write(std::ostream& os, const double* data, Index count,
463  uint32_t compression)
464  {
465  if (count < 1) return;
466  // Convert full float values to half float, then output the half float array.
467  std::vector<HalfT> halfData(count);
468  for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<double>::convert(data[i]);
469  writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
470  }
471 };
472 #endif // _MSC_VER
473 
474 
475 ////////////////////////////////////////
476 
477 
478 /// Populate the given buffer with @a destCount values of type @c ValueT
479 /// read from the given stream, taking into account that the stream might
480 /// have been compressed via one of several supported schemes.
481 /// [Mainly for internal use]
482 /// @param is a stream from which to read data (possibly compressed,
483 /// depending on the stream's compression settings)
484 /// @param destBuf a buffer into which to read values of type @c ValueT
485 /// @param destCount the number of values to be stored in the buffer
486 /// @param valueMask a bitmask (typically, a node's value mask) indicating
487 /// which positions in the buffer correspond to active values
488 /// @param fromHalf if true, read 16-bit half floats from the input stream
489 /// and convert them to full floats
490 template<typename ValueT, typename MaskT>
491 inline void
492 readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
493  const MaskT& valueMask, bool fromHalf)
494 {
495  // Get the stream's compression settings.
496  auto meta = getStreamMetadataPtr(is);
497  const uint32_t compression = getDataCompression(is);
498  const bool maskCompressed = compression & COMPRESS_ACTIVE_MASK;
499 
500  const bool seek = (destBuf == nullptr);
501  assert(!seek || (!meta || meta->seekable()));
502 
503  // Get delayed load metadata if it exists
504 
505  DelayedLoadMetadata::Ptr delayLoadMeta;
506  uint64_t leafIndex(0);
507  if (seek && meta && meta->delayedLoadMeta()) {
508  delayLoadMeta =
509  meta->gridMetadata().getMetadata<DelayedLoadMetadata>("file_delayed_load");
510  leafIndex = meta->leaf();
511  }
512 
513  int8_t metadata = NO_MASK_AND_ALL_VALS;
515  // Read the flag that specifies what, if any, additional metadata
516  // (selection mask and/or inactive value(s)) is saved.
517  if (seek && !maskCompressed) {
518  is.seekg(/*bytes=*/1, std::ios_base::cur);
519  } else if (seek && delayLoadMeta) {
520  metadata = delayLoadMeta->getMask(leafIndex);
521  is.seekg(/*bytes=*/1, std::ios_base::cur);
522  } else {
523  is.read(reinterpret_cast<char*>(&metadata), /*bytes=*/1);
524  }
525  }
526 
527  ValueT background = zeroVal<ValueT>();
528  if (const void* bgPtr = getGridBackgroundValuePtr(is)) {
529  background = *static_cast<const ValueT*>(bgPtr);
530  }
531  ValueT inactiveVal1 = background;
532  ValueT inactiveVal0 =
533  ((metadata == NO_MASK_OR_INACTIVE_VALS) ? background : math::negative(background));
534 
535  if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL ||
536  metadata == MASK_AND_ONE_INACTIVE_VAL ||
537  metadata == MASK_AND_TWO_INACTIVE_VALS)
538  {
539  // Read one of at most two distinct inactive values.
540  if (seek) {
541  is.seekg(/*bytes=*/sizeof(ValueT), std::ios_base::cur);
542  } else {
543  is.read(reinterpret_cast<char*>(&inactiveVal0), /*bytes=*/sizeof(ValueT));
544  }
545  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
546  // Read the second of two distinct inactive values.
547  if (seek) {
548  is.seekg(/*bytes=*/sizeof(ValueT), std::ios_base::cur);
549  } else {
550  is.read(reinterpret_cast<char*>(&inactiveVal1), /*bytes=*/sizeof(ValueT));
551  }
552  }
553  }
554 
555  MaskT selectionMask;
556  if (metadata == MASK_AND_NO_INACTIVE_VALS ||
557  metadata == MASK_AND_ONE_INACTIVE_VAL ||
558  metadata == MASK_AND_TWO_INACTIVE_VALS)
559  {
560  // For use in mask compression (only), read the bitmask that selects
561  // between two distinct inactive values.
562  if (seek) {
563  is.seekg(/*bytes=*/selectionMask.memUsage(), std::ios_base::cur);
564  } else {
565  selectionMask.load(is);
566  }
567  }
568 
569  ValueT* tempBuf = destBuf;
570  std::unique_ptr<ValueT[]> scopedTempBuf;
571 
572  Index tempCount = destCount;
573 
574  if (maskCompressed && metadata != NO_MASK_AND_ALL_VALS
576  {
577  tempCount = valueMask.countOn();
578  if (!seek && tempCount != destCount) {
579  // If this node has inactive voxels, allocate a temporary buffer
580  // into which to read just the active values.
581  scopedTempBuf.reset(new ValueT[tempCount]);
582  tempBuf = scopedTempBuf.get();
583  }
584  }
585 
586  // Read in the buffer.
587  if (fromHalf) {
589  is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
590  } else {
591  readData<ValueT>(
592  is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
593  }
594 
595  // If mask compression is enabled and the number of active values read into
596  // the temp buffer is smaller than the size of the destination buffer,
597  // then there are missing (inactive) values.
598  if (!seek && maskCompressed && tempCount != destCount) {
599  // Restore inactive values, using the background value and, if available,
600  // the inside/outside mask. (For fog volumes, the destination buffer is assumed
601  // to be initialized to background value zero, so inactive values can be ignored.)
602  for (Index destIdx = 0, tempIdx = 0; destIdx < MaskT::SIZE; ++destIdx) {
603  if (valueMask.isOn(destIdx)) {
604  // Copy a saved active value into this node's buffer.
605  destBuf[destIdx] = tempBuf[tempIdx];
606  ++tempIdx;
607  } else {
608  // Reconstruct an unsaved inactive value and copy it into this node's buffer.
609  destBuf[destIdx] = (selectionMask.isOn(destIdx) ? inactiveVal1 : inactiveVal0);
610  }
611  }
612  }
613 }
614 
615 
616 template<typename ValueT, typename MaskT>
617 inline size_t
618 writeCompressedValuesSize(ValueT* srcBuf, Index srcCount,
619  const MaskT& valueMask, uint8_t maskMetadata, bool toHalf, uint32_t compress)
620 {
621  using NonConstValueT = typename std::remove_const<ValueT>::type;
622 
623  const bool maskCompress = compress & COMPRESS_ACTIVE_MASK;
624 
625  Index tempCount = srcCount;
626  ValueT* tempBuf = srcBuf;
627  std::unique_ptr<NonConstValueT[]> scopedTempBuf;
628 
629  if (maskCompress && maskMetadata != NO_MASK_AND_ALL_VALS) {
630 
631  tempCount = 0;
632 
633  Index64 onVoxels = valueMask.countOn();
634  if (onVoxels > Index64(0)) {
635  // Create a new array to hold just the active values.
636  scopedTempBuf.reset(new NonConstValueT[onVoxels]);
637  NonConstValueT* localTempBuf = scopedTempBuf.get();
638 
639  // Copy active values to a new, contiguous array.
640  for (typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
641  localTempBuf[tempCount] = srcBuf[it.pos()];
642  }
643 
644  tempBuf = scopedTempBuf.get();
645  }
646  }
647 
648  // Return the buffer size.
649  if (toHalf) {
650  return HalfWriter<RealToHalf<NonConstValueT>::isReal, NonConstValueT>::writeSize(
651  tempBuf, tempCount, compress);
652  } else {
653  return writeDataSize<NonConstValueT>(tempBuf, tempCount, compress);
654  }
655 }
656 
657 
658 /// Write @a srcCount values of type @c ValueT to the given stream, optionally
659 /// after compressing the values via one of several supported schemes.
660 /// [Mainly for internal use]
661 /// @param os a stream to which to write data (possibly compressed, depending
662 /// on the stream's compression settings)
663 /// @param srcBuf a buffer containing values of type @c ValueT to be written
664 /// @param srcCount the number of values stored in the buffer
665 /// @param valueMask a bitmask (typically, a node's value mask) indicating
666 /// which positions in the buffer correspond to active values
667 /// @param childMask a bitmask (typically, a node's child mask) indicating
668 /// which positions in the buffer correspond to child node pointers
669 /// @param toHalf if true, convert floating-point values to 16-bit half floats
670 template<typename ValueT, typename MaskT>
671 inline void
672 writeCompressedValues(std::ostream& os, ValueT* srcBuf, Index srcCount,
673  const MaskT& valueMask, const MaskT& childMask, bool toHalf)
674 {
675  // Get the stream's compression settings.
676  const uint32_t compress = getDataCompression(os);
677  const bool maskCompress = compress & COMPRESS_ACTIVE_MASK;
678 
679  Index tempCount = srcCount;
680  ValueT* tempBuf = srcBuf;
681  std::unique_ptr<ValueT[]> scopedTempBuf;
682 
683  int8_t metadata = NO_MASK_AND_ALL_VALS;
684 
685  if (!maskCompress) {
686  os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
687  } else {
688  // A valid level set's inactive values are either +background (outside)
689  // or -background (inside), and a fog volume's inactive values are all zero.
690  // Rather than write out all of these values, we can store just the active values
691  // (given that the value mask specifies their positions) and, if necessary,
692  // an inside/outside bitmask.
693 
694  const ValueT zero = zeroVal<ValueT>();
695  ValueT background = zero;
696  if (const void* bgPtr = getGridBackgroundValuePtr(os)) {
697  background = *static_cast<const ValueT*>(bgPtr);
698  }
699 
700  MaskCompress<ValueT, MaskT> maskCompressData(valueMask, childMask, srcBuf, background);
701  metadata = maskCompressData.metadata;
702 
703  os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
704 
705  if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL ||
706  metadata == MASK_AND_ONE_INACTIVE_VAL ||
707  metadata == MASK_AND_TWO_INACTIVE_VALS)
708  {
709  if (!toHalf) {
710  // Write one of at most two distinct inactive values.
711  os.write(reinterpret_cast<const char*>(&maskCompressData.inactiveVal[0]), sizeof(ValueT));
712  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
713  // Write the second of two distinct inactive values.
714  os.write(reinterpret_cast<const char*>(&maskCompressData.inactiveVal[1]), sizeof(ValueT));
715  }
716  } else {
717  // Write one of at most two distinct inactive values.
718  ValueT truncatedVal = static_cast<ValueT>(truncateRealToHalf(maskCompressData.inactiveVal[0]));
719  os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
720  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
721  // Write the second of two distinct inactive values.
722  truncatedVal = truncateRealToHalf(maskCompressData.inactiveVal[1]);
723  os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
724  }
725  }
726  }
727 
728  if (metadata == NO_MASK_AND_ALL_VALS) {
729  // If there are more than two unique inactive values, the entire input buffer
730  // needs to be saved (both active and inactive values).
731  /// @todo Save the selection mask as long as most of the inactive values
732  /// are one of two values?
733  } else {
734  // Create a new array to hold just the active values.
735  scopedTempBuf.reset(new ValueT[srcCount]);
736  tempBuf = scopedTempBuf.get();
737 
738  if (metadata == NO_MASK_OR_INACTIVE_VALS ||
739  metadata == NO_MASK_AND_MINUS_BG ||
740  metadata == NO_MASK_AND_ONE_INACTIVE_VAL)
741  {
742  // Copy active values to the contiguous array.
743  tempCount = 0;
744  for (typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
745  tempBuf[tempCount] = srcBuf[it.pos()];
746  }
747  } else {
748  // Copy active values to a new, contiguous array and populate a bitmask
749  // that selects between two distinct inactive values.
750  MaskT selectionMask;
751  tempCount = 0;
752  for (Index srcIdx = 0; srcIdx < srcCount; ++srcIdx) {
753  if (valueMask.isOn(srcIdx)) { // active value
754  tempBuf[tempCount] = srcBuf[srcIdx];
755  ++tempCount;
756  } else { // inactive value
757  if (MaskCompress<ValueT, MaskT>::eq(srcBuf[srcIdx], maskCompressData.inactiveVal[1])) {
758  selectionMask.setOn(srcIdx); // inactive value 1
759  } // else inactive value 0
760  }
761  }
762  assert(tempCount == valueMask.countOn());
763 
764  // Write out the mask that selects between two inactive values.
765  selectionMask.save(os);
766  }
767  }
768  }
769 
770  // Write out the buffer.
771  if (toHalf) {
772  HalfWriter<RealToHalf<ValueT>::isReal, ValueT>::write(os, tempBuf, tempCount, compress);
773  } else {
774  writeData(os, tempBuf, tempCount, compress);
775  }
776 }
777 
778 } // namespace io
779 } // namespace OPENVDB_VERSION_NAME
780 } // namespace openvdb
781 
782 #endif // OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
783 
784 // Copyright (c) DreamWorks Animation LLC
785 // All rights reserved. This software is distributed under the
786 // Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )
OPENVDB_API const void * getGridBackgroundValuePtr(std::ios_base &)
Return a pointer to the background value of the grid currently being read from or written to the give...
GLsizeiptr size
Definition: glew.h:1681
Store a buffer of data that can be optionally used during reading for faster delayed-load I/O perform...
bool isExactlyEqual(const T0 &a, const T1 &b)
Return true if a is exactly equal to b.
Definition: Math.h:415
void writeData(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition: Compression.h:390
T negative(const T &val)
Return the unary negation of the given value.
Definition: Math.h:108
void swap(UT::ArraySet< Key, MULTI, MAX_LOAD_FACTOR_256, Clearer, Hash, KeyEqual > &a, UT::ArraySet< Key, MULTI, MAX_LOAD_FACTOR_256, Clearer, Hash, KeyEqual > &b)
Definition: UT_ArraySet.h:1629
GLuint const GLfloat * val
Definition: glew.h:2794
OPENVDB_API void unzipFromStream(std::istream &, char *data, size_t numBytes)
void writeCompressedValues(std::ostream &os, ValueT *srcBuf, Index srcCount, const MaskT &valueMask, const MaskT &childMask, bool toHalf)
Definition: Compression.h:672
OPENVDB_API std::string compressionToString(uint32_t flags)
Return a string describing the given compression flags.
GLboolean GLboolean GLboolean GLboolean a
Definition: glew.h:9477
OPENVDB_API uint32_t getDataCompression(std::ios_base &)
Return a bitwise OR of compression option flags (COMPRESS_ZIP, COMPRESS_ACTIVE_MASK, etc.) specifying whether and how input data is compressed or output data should be compressed.
GLenum GLsizei GLsizei GLsizei GLsizei GLbitfield flags
Definition: glew.h:2864
#define OPENVDB_USE_VERSION_NAMESPACE
Definition: version.h:200
void read(T &in, bool &v)
Definition: ImfXdr.h:611
OPENVDB_API void bloscFromStream(std::istream &, char *data, size_t numBytes)
void readData(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Read data from a stream.
Definition: Compression.h:274
void readCompressedValues(std::istream &is, ValueT *destBuf, Index destCount, const MaskT &valueMask, bool fromHalf)
Definition: Compression.h:492
static size_t writeSize(const T *data, Index count, uint32_t compression)
Definition: Compression.h:423
OPENVDB_API size_t bloscToStreamSize(const char *data, size_t valSize, size_t numVals)
#define OPENVDB_API
Helper macros for defining library symbol visibility.
Definition: Platform.h:288
static bool eq(const ValueT &a, const ValueT &b)
Definition: Compression.h:110
size_t writeCompressedValuesSize(ValueT *srcBuf, Index srcCount, const MaskT &valueMask, uint8_t maskMetadata, bool toHalf, uint32_t compress)
Definition: Compression.h:618
T truncateRealToHalf(const T &val)
Return the given value truncated to 16-bit float precision.
Definition: Compression.h:243
GLuint buffer
Definition: glew.h:1680
GLint GLenum GLsizei GLint GLsizei const void * data
Definition: glew.h:1379
General-purpose arithmetic and comparison routines, most of which accept arbitrary value types (or at...
static void read(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Definition: Compression.h:323
static size_t writeSize(const T *data, Index count, uint32_t compression)
Definition: Compression.h:434
static void read(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Definition: Compression.h:332
GLdouble GLdouble GLdouble b
Definition: glew.h:9122
OPENVDB_API void bloscToStream(std::ostream &, const char *data, size_t valSize, size_t numVals)
I unique(I begin, I end)
Definition: pugixml.cpp:7195
GLsizei const GLchar *const * string
Definition: glew.h:1844
RealToHalf and its specializations define a mapping from floating-point data types to analogous half ...
Definition: Compression.h:200
OIIO_API bool copy(string_view from, string_view to, std::string &err)
GLuint GLuint GLsizei count
Definition: glew.h:1253
OPENVDB_API void zipToStream(std::ostream &, const char *data, size_t numBytes)
math::Vec3< half > Vec3H
Definition: Types.h:82
OPENVDB_API SharedPtr< StreamMetadata > getStreamMetadataPtr(std::ios_base &)
Return a shared pointer to an object that stores metadata (file format, compression scheme...
#define SIZE
Definition: simple.C:40
void write(T &out, bool v)
Definition: ImfXdr.h:332
size_t writeDataSize(const T *data, Index count, uint32_t compression)
Definition: Compression.h:351
MaskCompress(const MaskT &valueMask, const MaskT &childMask, const ValueT *srcBuf, const ValueT &background)
Definition: Compression.h:114
ImageBuf OIIO_API zero(ROI roi, int nthreads=0)
static void write(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition: Compression.h:441
#define OPENVDB_VERSION_NAME
The version namespace name for this library version.
Definition: version.h:146
Definition: half.h:91
OPENVDB_API uint32_t getFormatVersion(std::ios_base &)
Return the file format version number associated with the given input stream.
static void write(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition: Compression.h:426
OPENVDB_API size_t zipToStreamSize(const char *data, size_t numBytes)
type
Definition: core.h:528
GLenum GLsizei len
Definition: glew.h:7752
math::Vec2< half > Vec2H
Definition: Types.h:73