HDK
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
UT_PageArray.h
Go to the documentation of this file.
1 /*
2  * PROPRIETARY INFORMATION. This software is proprietary to
3  * Side Effects Software Inc., and is not to be reproduced,
4  * transmitted, or disclosed in any way without written permission.
5  *
6  * NAME: UT_PageArray.h (UT Library, C++)
7  *
8  * COMMENTS: An array class with special handling of constant pages and
9  * shared page data.
10  */
11 
12 #pragma once
13 
14 #ifndef __UT_PageArray__
15 #define __UT_PageArray__
16 
17 #include "UT_API.h"
18 #include "UT_Assert.h"
19 #include "UT_Defaults.h"
20 #include "UT_FixedVector.h"
21 #include "UT_SmallArray.h"
22 #include "UT_Storage.h"
23 #include <SYS/SYS_AtomicInt.h>
24 #include <SYS/SYS_Inline.h>
25 #include <SYS/SYS_Types.h>
26 #include <SYS/SYS_TypeTraits.h>
27 #include <stddef.h>
28 #include <stdlib.h>
29 #include <string.h>
30 
31 #ifdef WIN32
32 // Default template arguments on functions are legal. Don't error on them!
33 #pragma warning(disable: 4519)
34 #endif
35 
36 typedef exint UT_PageNum;
37 typedef exint UT_PageOff;
38 
39 class UT_MemoryCounter;
40 
41 /// @internal
42 /// W E L C O M E
43 /// to scenic
44 /// Template Hell
45 ///
46 /// Population: You
47 ///
48 /// If you're just using this class, hopefully you'll have a short and
49 /// not-so-painful visit!
50 ///
51 /// If, however, you're attempting to modify this class, BEWARE!
52 /// Any missteps can and likely will result in application of the
53 /// Doughnut Rules, so be sure to review them before visiting.
54 /// It may take several days to escape, so it may be best to just
55 /// revert and make absolutely sure that it works on all platforms
56 /// before committing again.
57 ///
58 /// This class is the result of a wide variety of needs for a wide variety
59 /// of use cases, most of which need both good performance and flexibility.
60 /// You may want to skip reading this gigantic comment, but whether it's
61 /// immediately or in a few months, you'll realize that skipping reading
62 /// this was a huge mistake.
63 ///
64 /// This array consists of a pointer to a page table, optionally a
65 /// tuple size (if TSIZE==-1), and optionally a UT_Storage indicating
66 /// the storage type (if DATA_T is void). The page table, if not NULL,
67 /// can be shared, and has the format: @code
68 /// +-------------------+
69 /// | ref count | (1 if unshared; >1 if shared)
70 /// +-------------------+
71 /// | array size | (number of real elements/tuples in the array)
72 /// +-------------------+
73 /// | array capacity | (number of allocated elements/tuples if all pages of the table were hardened)
74 /// +-------------------+
75 /// | page 0 pointer | (pointer to page if bit 0 is 0, else constant page representation)
76 /// +-------------------+
77 /// | page 1 pointer |
78 /// +-------------------+
79 /// | page 2 pointer |
80 /// +-------------------+
81 /// | ... |
82 /// +-------------------+
83 /// @endcode
84 ///
85 /// If a page pointer has bit 0 clear, (i.e. an address aligned to a 2-byte boundary),
86 /// it is a non-constant page, it can be shared, and has the format: @code
87 /// +-------------------+
88 /// | ref count | (1 if unshared; >1 if shared)
89 /// +-------------------+
90 /// | (unused) | (padding so that data starts on a 16-byte boundary)
91 /// +-------------------+
92 /// | tuple 0 |
93 /// +-------------------+
94 /// | tuple 1 |
95 /// +-------------------+
96 /// | tuple 2 |
97 /// +-------------------+
98 /// | ... |
99 /// +-------------------+
100 /// @endcode
101 ///
102 /// A normal capacity page contains 2^THEPAGEBITS tuples, e.g. the default
103 /// THEPAGEBITS==10 gives a page size of 1024 tuples. If there is exactly
104 /// one page, it may have a capacity that is any power of two up to
105 /// THEPAGEBITS, which can be checked by checking if the overall array has
106 /// capacity less than 2^THEPAGEBITS.
107 ///
108 /// If, instead, the page pointer has bit 0 set to 1, which would be an odd
109 /// address, all tuples in the page are the same. There are 3 ways this may
110 /// be represented:
111 ///
112 /// 1) If the size of a full tuple is strictly smaller than the size of a
113 /// pointer, (e.g. 7 bytes or fewer on a 64-bit platform), the constant value
114 /// will be stored inside the space that would have been a pointer, aligned
115 /// to the end of the pointer. That ensures that 4-byte tuples will be
116 /// aligned to 4 bytes, and 2- or 6-byte tuples will be aligned to 2 bytes.
117 /// This is most useful for fpreal32[1] and int32[1] (i.e. 4-byte scalars).
118 ///
119 /// 2) If the page pointer is equal to 1 (i.e. NULL apart from bit 0 being 1),
120 /// the value is all zero. This covers the extremely
121 /// common case of a zero default value, especially for fpreal32[3] and
122 /// fpreal64[3].
123 ///
124 /// 3) Otherwise, the page pointer, ignoring bit zero, points to a page
125 /// that contains just a single tuple.
126 ///
127 /// That covers the basic structure. It sounds so nice and simple...
128 /// but then there are the templates.
129 ///
130 /// DATA_T:
131 ///
132 /// If DATA_T is something other than void, that type is the type of each
133 /// component of each tuple. This is the only way that non-numeric types
134 /// are possible; DATA_T can also be a numeric type. If DATA_T is void,
135 /// myImpl contains a UT_Storage variable indicating what the data type is;
136 /// it must be a numeric type, i.e. it cannot be UT_Storage::INVALID.
137 /// The ...UnknownType functions switch on the UT_Storage to determine the
138 /// data type and call the corresponding function. If DATA_T is non-void,
139 /// it saves a lot of extra work at runtime. Because the codepaths that
140 /// will never be called due to DATA_T checks still need to compile with
141 /// DATA_T being void, NotVoidType ensures that there is some arbitrary type
142 /// in place so that the code will compile, even though it will never be
143 /// called. The same issue prevents using UT_ASSERT_COMPILETIME to verify
144 /// DATA_T, so UT_ASSERT_P is used instead.
145 /// myImpl is always structured so that if the type becomes known, by checking
146 /// getStorage(), this can be cast to have the now-known DATA_T set.
147 /// For example: @code
148 /// if (array.getStorage() == UT_Storage::REAL32)
149 /// array.castType<fpreal32>().someFunctionCall();
150 /// @endcode
151 ///
152 /// TSIZE:
153 ///
154 /// If TSIZE >= 1, it is the tuple size (the number of components in each
155 /// tuple), and is known at compile time, avoiding extra work at compile-time
156 /// that depends on the tuple size, especially if TSIZE is 1. Odds are that
157 /// for non-numeric types, TSIZE will almost always be 1, since it rarely
158 /// makes sense to be something else. If TSIZE is 0, something is almost
159 /// certainly wrong. If the tuple size is not known at compile-time, TSIZE
160 /// must be -1. In that case, myImpl contains an integer indicating the tuple
161 /// size. Like with DATA_T, because the codepaths that will never be called
162 /// due to TSIZE checks still need to compile with other TSIZE values, so
163 /// UT_ASSERT_COMPILETIME can't be used to verify TSIZE; UT_ASSERT_P is used
164 /// instead. There are a number of functions that require TSIZE to be the
165 /// tuple size, in order to avoid having to send an extra parameter and
166 /// incurring the slowdowns of their counterparts that assume that the tuple
167 /// size is what's passed in. If you modify one, REMEMBER TO MODIFY THE OTHER!
168 /// myImpl is always structured so that if the tuple size becomes
169 /// known, by checking the getTupleSize(), this can be cast to have the
170 /// now-known TSIZE set.
171 /// For example: @code
172 /// if (array.getTupleSize() == 3)
173 /// array.castTupleSize<3>().someFunctionCall();
174 /// @endcode
175 ///
176 /// TABLEHARDENED:
177 ///
178 /// Since page tables can be shared if all of the data are identical, modifying
179 /// the array in any way requires copying (hardening) the page table and may
180 /// require copying one or more shared pages or hardening constant pages.
181 /// Since every modification requires hardening the page table, it's useful
182 /// to harden the page table once and be able to skip checking whether it needs
183 /// hardening for all modifications after that. Calling hardenTable()
184 /// will ensure that the table is hardened, and return this with TABLEHARDENED
185 /// set to true, so that further calls on it to modify the array won't need to
186 /// check whether the table needs hardening. The page table should be
187 /// quite light compared to the data, so it's not the end of the world if
188 /// the table is hardened pre-emptively and there end up being no actual
189 /// modifications.
190 ///
191 /// PAGESHARDENED:
192 ///
193 /// When writing data in parallel to random locations, all pages that could
194 /// be written-to must be hardened in advance, to avoid a race condition on
195 /// the hardening of any page. Also, if it's known that most pages will
196 /// likely be hardened anyway when writing data, it'd be advantageous to
197 /// be able to harden all pages in advance and skip checking when making
198 /// modifications. The code for hardening pages just-in-time can
199 /// dramatically increase code size and time overhead. To avoid this,
200 /// hardenAllPages() will ensure that the table and all pages are hardened,
201 /// and will return this with TABLEHARDENED and PAGESHARDENED both set to true.
202 /// Any calls to modify page data made on that returned pointer won't check
203 /// whether any pages need hardening, because it will know that all pages are
204 /// already hardened. If you use this, be sure not to use the pointer
205 /// with PAGESHARDENED set to true after any call to tryCompressAllPages()
206 /// on the original array pointer.
207 ///
208 /// THEPAGEBITS:
209 ///
210 /// Full pages will contain 2^THEPAGEBITS tuples, e.g. THEPAGEBITS==10
211 /// means there will be 1024 tuples per full page.
212 ///
213 /// IDX_T:
214 ///
215 /// This allows templating on the array index type so that strict types
216 /// can be used, e.g. GA_Offset.
217 ///
218 /// FIXME BEFORE USING:
219 /// - Should make generic UT_R[OW]Handle.+ classes for easy accessing
220 /// of runtime-determined data type and tuple size data as tuples
221 /// with type and tuple size determined at compile-time.
222 /// - Make UT_PageNum and UT_PageOff strict types when strict types are
223 /// enabled.
224 /// - Add more functions for page-level access and block (bulk sequential)
225 /// access, for easy reading/writing to/from flat arrays.
226 /// @endinternal
227 template<typename DATA_T,exint TSIZE=1,bool TABLEHARDENED=false,bool PAGESHARDENED=false,exint THEPAGEBITS=10,typename IDX_T=exint>
229 {
230 protected:
231  /// This class is just to get code to compile when DATA_T is void,
232  /// by picking any type other than void if it is void.
233  template<typename T>
234  struct NotVoid
236 
238 
239  // These friend declarations are needed for calling protected functions on hardened/cast types.
240  template<typename DATA_T2,exint TSIZE2,bool TABLEHARDENED2,bool PAGESHARDENED2,exint THEPAGEBITS2,typename IDX_T2>
241  friend class UT_PageArray;
242 public:
244  typedef DATA_T DataType;
246  static const exint theTupleSize = TSIZE;
247  static const exint theSafeTupleSize = (TSIZE==-1) ? 3 : TSIZE;
248  static const exint thePageBits = THEPAGEBITS;
249  static const exint thePageSize = (exint(1)<<thePageBits);
250  static const exint thePageMask = thePageSize-1;
251  typedef IDX_T IndexType;
252 
253  /// The default constructor can only be used if the tuple size
254  /// and storage type are known at compile time.
256  {
257  UT_ASSERT_P(TSIZE >= 1);
259  myImpl.getPages() = NULL;
260  }
261 
262  /// This constructor can only be used if the storage type is
263  /// known at compile time, but the tuple size is not.
264  UT_PageArray(exint tuplesize)
265  {
266  UT_ASSERT_P(TSIZE == -1 || TSIZE == tuplesize);
268  // NOTE: This is one case where a tuple size of zero is okay.
269  UT_ASSERT_P(tuplesize >= 0);
270  myImpl.getPages() = NULL;
271  myImpl.setTupleSize(tuplesize);
272  }
273 
274  /// This constructor can only be used if the storage type is
275  /// known at compile time, but the tuple size is not.
277  {
278  UT_ASSERT_P(TSIZE >= 1);
280  UT_ASSERT_P(storage != UT_Storage::INVALID);
281  myImpl.getPages() = NULL;
282  myImpl.setStorage(storage);
283  }
284 
285  /// This constructor can only be used if the tuple size and
286  /// storage type are unknown at compile time.
288  {
289  UT_ASSERT_P(TSIZE == -1 || TSIZE == tuplesize);
291  // NOTE: This is one case where a tuple size of zero is okay.
292  UT_ASSERT_P(tuplesize >= 0);
293  UT_ASSERT_P(storage != UT_Storage::INVALID);
294  myImpl.getPages() = NULL;
295  myImpl.setStorage(storage);
296  myImpl.setTupleSize(tuplesize);
297  }
298 
299  /// This just references that's page table, unless TABLEHARDENED
300  /// or PAGESHARDENED. If TABLEHARDENED and !PAGESHARDENED,
301  /// the pages are referenced, and if PAGESHARDENED, the pages
302  /// are copied.
303  UT_PageArray(const ThisType &that)
304  {
305  myImpl.getPages() = NULL;
306  *this = that;
307  }
308 
309  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
311  {
312  myImpl.getPages() = NULL;
313  *this = that;
314  }
315 
318  {
319  clearAndDestroy();
320  }
321 
323  void clear()
324  {
325  if (myImpl.getPages())
326  myImpl.getPages()->setSize(IDX_T(0));
327  }
328 
330  {
331  if (!myImpl.getPages())
332  return;
333 
334  const exint tuplesize = myImpl.getTupleSize();
335 
336  // Just decrement the reference count to free the table safely
337  if (!SYSisSame<DATA_T,void>())
338  myImpl.getPages()->decRef(tuplesize);
339  else
340  {
341  const UT_Storage storage = myImpl.getStorage();
342  switch (storage)
343  {
344  case UT_Storage::INVALID:
345  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
346  return;
347  case UT_Storage::INT8:
348  castType<int8>().myImpl.getPages()->decRef(tuplesize); return;
349  case UT_Storage::INT16:
350  castType<int16>().myImpl.getPages()->decRef(tuplesize); return;
351  case UT_Storage::INT32:
352  castType<int32>().myImpl.getPages()->decRef(tuplesize); return;
353  case UT_Storage::INT64:
354  castType<int64>().myImpl.getPages()->decRef(tuplesize); return;
355  case UT_Storage::REAL16:
356  castType<fpreal16>().myImpl.getPages()->decRef(tuplesize); return;
357  case UT_Storage::REAL32:
358  castType<fpreal32>().myImpl.getPages()->decRef(tuplesize); return;
359  case UT_Storage::REAL64:
360  castType<fpreal64>().myImpl.getPages()->decRef(tuplesize); return;
361  }
362  }
363  myImpl.getPages() = NULL;
364  }
365 
366  /// NOTE: Compilers seem not to use the operator= below when the template
367  /// arguments match, so this operator= is necessary in order to
368  /// avoid the compiler silently using its implicitly-defined
369  /// assignment operator.
372  {
373  replace(that);
374  return *this;
375  }
376 
377  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
380  {
381  replace(that);
382  return *this;
383  }
384 
385  /// Completely replace this UT_PageArray with that, referencing
386  /// the pages or table, if allowed by the template arguments.
387  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
389  {
390  // NOTE: Careful if you add template parameter checks on this,
391  // because, e.g., a UT_PageArray<void,-1> could refer to the
392  // same exact array as a UT_PageArray<float 3>.
393  if ((void *)this == (const void *)&that)
394  return;
395  // NOTE: The storage and tuple size checks are so that if both
396  // tables are NULL, it still copies the storage and tuple
397  // size below.
398  if (!TABLEHARDENED &&
399  myImpl.getStorage() == that.myImpl.getStorage() &&
400  myImpl.getTupleSize() == that.myImpl.getTupleSize() &&
401  (void *)myImpl.getPages() == (const void *)that.myImpl.getPages())
402  return;
403 
404  clearAndDestroy();
405 
406  if (SYSisSame<DATA_T,void>())
407  myImpl.setStorage(that.myImpl.getStorage());
408  if (TSIZE == -1)
409  myImpl.setTupleSize(that.myImpl.getTupleSize());
410 
411  // May be able to share pages or table if matching tuple size and storage type
412  if ((SYSisSame<DATA_T,void>() || SYSisSame<DATA_T,SRC_DATA_T>()) && (TSIZE == -1 || TSIZE == SRC_TSIZE))
413  {
414  PageTable *pages = reinterpret_cast<PageTable *>(SYSconst_cast(that.myImpl.getPages()));
415  myImpl.getPages() = pages;
416  if (!pages)
417  return;
418  pages->incRef();
419  if (TABLEHARDENED || PAGESHARDENED || SRC_TABLEHARDENED || SRC_PAGESHARDENED)
420  {
421  forceHardenTable(pages->capacity());
422  if (PAGESHARDENED || SRC_PAGESHARDENED)
423  {
424  // If we don't cast to PAGESHARDENED==false, it won't do anything.
426  ->template hardenAllPages<false>();
427  }
428  }
429  }
430  else
431  {
432  // Can't share table or pages, so just use moveRange
433  IDX_T capacity = that.capacity();
434  IDX_T size = that.size();
435  forceHardenTable(capacity);
436  moveRange(that, IDX_T(0), IDX_T(0), size);
437  }
438  }
439 
440  /// Read element i's specified component (default component 0).
441  SYS_FORCE_INLINE NotVoidType operator()(IDX_T i,exint component=0) const
442  {
443  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
444  UT_ASSERT_P(TSIZE != 0);
445  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
446  UT_ASSERT_P(i >= IDX_T(0) && i < size());
447  UT_ASSERT_P(component >= 0 && component < myImpl.getTupleSize());
448 
449  UT_PageNum pagenum = pageNum(i);
450  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
451 
452  // If we know that the pages are hardened, we can omit
453  // the constant page case at compile time.
454  bool isconst = !PAGESHARDENED && page->isConstant();
455 
456  if (TSIZE >= 1)
457  {
458  if (!isconst)
459  {
460  UT_PageOff pageoff = pageOff(i);
461  return page->getFirstPtr()[TSIZE*pageoff + component];
462  }
463  return getConstant(page, component);
464  }
465  else
466  {
467  if (!isconst)
468  {
469  UT_PageOff pageoff = pageOff(i);
470  return page->getFirstPtr()[myImpl.getTupleSize()*pageoff + component];
471  }
472  return getConstant(page, component, myImpl.getTupleSize());
473  }
474  }
475 
476  /// This get function may seem redundant, given operator(), but it gives a
477  /// simple way to read from a non-const UT_PageArray without calling the
478  /// non-const operator() and potentially hardening pages. It just
479  /// explicitly adds const. It also allows data type conversion and
480  /// reading when the type is unknown at compile time.
481  /// It should hopefully be inlined in the no-conversion case;
482  /// it's less critical in the conversion case.
483  template<typename DEST_DATA_T=DATA_T>
484  SYS_FORCE_INLINE DEST_DATA_T get(IDX_T i,exint component=0) const
485  {
486  UT_ASSERT_P(i >= IDX_T(0) && i < size());
487  if (!(SYSisSame<DATA_T,void>()) && (SYSisSame<DEST_DATA_T,DATA_T>() || theStorage != UT_Storage::INVALID))
488  return UTconvertStorage<DEST_DATA_T>((*this)(i,component));
489 
490  return getUnknownType<DEST_DATA_T>(i,component);
491  }
492 
493  template<typename DEST_DATA_T=NotVoidType,exint DEST_TSIZE=theSafeTupleSize>
495  {
496  if (SYSisSame<DATA_T,void>())
497  {
499  SYS_CallIf<SYSisSame<DATA_T,void>()>::call([this,&dest,i](){
500  dest = getVectorUnknownType<DEST_DATA_T,DEST_TSIZE>(i);
501  });
502  return dest;
503  }
504  if (!SYSisSame<DEST_DATA_T,DATA_T>())
505  {
507  SYS_CallIf<!SYSisSame<DEST_DATA_T,DATA_T>()>::call([this,&dest,i](){
508  dest = convertVectorStorage<DEST_DATA_T>(getVector<NotVoidType,DEST_TSIZE>(i));
509  });
510  return dest;
511  }
512 
513  if (TSIZE==-1)
514  {
516  SYS_CallIf<TSIZE==-1>::call([this,&dest,i](){
517  dest = getVectorUnknownSize<DEST_DATA_T,DEST_TSIZE>(i);
518  });
519  return dest;
520  }
521  if (TSIZE!=DEST_TSIZE)
522  {
524  SYS_CallIf<TSIZE!=DEST_TSIZE>::call([this,&dest,i](){
525  dest = getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
526  });
527  return dest;
528  }
529 
530  UT_ASSERT_P(i >= IDX_T(0) && i < size());
531  UT_ASSERT_P(TSIZE==DEST_TSIZE);
532  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()));
533 
534  // Types and tuple sizes are known and match
535  UT_PageNum pagenum = pageNum(i);
536  // NOTE: We can dereference here because we're returning by value.
537  // getConstantPtr may return a pointer into this stack variable
538  // if the page is constant and inline.
539  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
540 
541  // If we know that the pages are hardened, we can omit
542  // the constant page case at compile time.
543  bool isconst = !PAGESHARDENED && page->isConstant();
544 
545  if (!isconst)
546  {
547  UT_PageOff pageoff = pageOff(i);
548  return ((const UT_FixedVector<DEST_DATA_T,DEST_TSIZE>*)page->getFirstPtr())[pageoff];
549  }
550  const NotVoidType *constdata = getConstantPtr(page);
551  if (!constdata)
552  return UT_FixedVector<DEST_DATA_T,DEST_TSIZE>(DEST_DATA_T(0));
553 
554  return *(const UT_FixedVector<DEST_DATA_T,DEST_TSIZE>*)constdata;
555  }
556 
557  /// Get a non-const reference to element i's specified component
558  /// (default component 0).
559  /// WARNING: DO NOT call this if DATA_T is void!
560  /// NOTE: Unlike set(), this must always harden, even if you're just
561  /// writing a value that is already there.
563  {
564  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
565  UT_ASSERT_P(TSIZE != 0);
566  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
567  UT_ASSERT_P(i >= IDX_T(0) && i < size());
568  UT_ASSERT_P(component >= 0 && component < myImpl.getTupleSize());
569 
570  // NOTE: This is a no-op if TABLEHARDENED is already true.
571  hardenTable();
572 
573  UT_PageNum pagenum = pageNum(i);
574  UT_PageOff pageoff = pageOff(i);
575  PageTable *pages = myImpl.getPages();
576  PageTableEntry *page = pages->getPPage(pagenum);
577 
578  if (!PAGESHARDENED)
579  {
580  if (page->isConstant())
581  {
582  if (TSIZE >= 1)
583  hardenConstantPage(pages, page);
584  else
585  hardenConstantPage(pages, page, myImpl.getTupleSize());
586  }
587  else if (page->isShared())
588  {
589  if (TSIZE >= 1)
590  hardenSharedPage(pages, page);
591  else
592  hardenSharedPage(pages, page, myImpl.getTupleSize());
593  }
594  }
595 
596  NotVoidType *data = page->getFirstPtr()
597  + pageoff*((TSIZE >= 1) ? TSIZE : myImpl.getTupleSize())
598  + component;
599 
600  return *data;
601  }
602 
603  /// component == 0 in this version
604  template<typename SRC_DATA_T>
605  SYS_FORCE_INLINE void set(IDX_T i,SRC_DATA_T v)
606  {
607  set(i, 0, v);
608  }
609  /// component == 0 in this version
610  template<typename SRC_DATA_T>
611  SYS_FORCE_INLINE void add(IDX_T i,SRC_DATA_T v)
612  {
613  add(i, 0, v);
614  }
615 
616  /// This looks gigantic, but it's mostly because of the different
617  /// behaviour for different template values. If
618  /// PAGESHARDENED is true, it'll skip checking for constant
619  /// or shared pages altogether.
620  template<typename SRC_DATA_T>
621  SYS_FORCE_INLINE void set(IDX_T i,exint component,SRC_DATA_T v)
622  {
623  op<0>(i, component, v);
624  }
625  template<typename SRC_DATA_T>
626  SYS_FORCE_INLINE void add(IDX_T i,exint component,SRC_DATA_T v)
627  {
628  op<1>(i, component, v);
629  }
630  template<int OP,typename SRC_DATA_T>
631  SYS_FORCE_INLINE void op(IDX_T i,exint component,SRC_DATA_T v)
632  {
633  UT_ASSERT_P(i >= IDX_T(0) && i < size());
634  UT_ASSERT_P(component >= 0 && component < getTupleSize());
635  UT_ASSERT_MSG_P(OP == 0 || OP == 1, "Only OP 0 (set) and 1 (add) are defined!");
636 
637  if (SYSisSame<DATA_T,void>())
638  {
639  opUnknownType<OP>(i, component, v);
640  return;
641  }
642 
643  // Make sure that v is the correct type before comparing.
644  NotVoidType vt(UTconvertStorage<NotVoidType>(v));
645 
646  UT_ASSERT_P(TSIZE != 0);
647  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
648 
649  hardenTable();
650 
651  UT_PageNum pagenum = pageNum(i);
652  UT_PageOff pageoff = pageOff(i);
653  PageTable *pages = myImpl.getPages();
654  PageTableEntry *page = pages->getPPage(pagenum);
655 
656  if (!PAGESHARDENED)
657  {
658  if (page->isConstant())
659  {
660  // If we're setting, check to see if it's a new value
661  if (OP == 0)
662  {
663  if (TSIZE >= 1)
664  {
665  if (getConstant(page, component) == vt)
666  return;
667  }
668  else
669  {
670  if (getConstant(page, component, myImpl.getTupleSize()) == vt)
671  return;
672  }
673  }
674  if (TSIZE >= 1)
675  hardenConstantPage(pages, page);
676  else
677  hardenConstantPage(pages, page, myImpl.getTupleSize());
678  }
679  else if (page->isShared())
680  {
681  // If we're setting, check to see if it's a new value
682  if (OP == 0)
683  {
684  NotVoidType *data = page->getFirstPtr()
685  + pageoff*((TSIZE >= 1) ? TSIZE : myImpl.getTupleSize())
686  + component;
687  if (*data == vt)
688  return;
689  }
690  if (TSIZE >= 1)
691  hardenSharedPage(pages, page);
692  else
693  hardenSharedPage(pages, page, myImpl.getTupleSize());
694  }
695  }
696 
697  NotVoidType *data = page->getFirstPtr()
698  + pageoff*((TSIZE >= 1) ? TSIZE : myImpl.getTupleSize())
699  + component;
700 
701  if (OP == 0)
702  *data = vt;
703  else if (OP == 1)
704  *data += vt;
705  else
706  {
707  UT_ASSERT_MSG_P(0,"Only OP 0 (set) and 1 (add) are defined!");
708  }
709  }
710 
711  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_INSTANTIATED>
713  {
714  opVector<0>(i, reinterpret_cast<const UT_FixedVector<SRC_DATA_T,SRC_TSIZE> &>(v));
715  }
716  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_INSTANTIATED>
718  {
719  opVector<1>(i, reinterpret_cast<const UT_FixedVector<SRC_DATA_T,SRC_TSIZE> &>(v));
720  }
721  /// This looks gigantic, but it's mostly because of the different
722  /// behaviour for different template values. If
723  /// PAGESHARDENED is true, it'll skip checking for constant
724  /// or shared pages altogether.
725  template<int OP,typename SRC_DATA_T,exint SRC_TSIZE>
727  {
728  UT_ASSERT_P(i >= IDX_T(0) && i < size());
729  UT_ASSERT_MSG_P(OP == 0 || OP == 1, "Only OP 0 (set) and 1 (add) are defined!");
730 
731  if (SYSisSame<DATA_T,void>())
732  {
733  SYS_CallIf<SYSisSame<DATA_T,void>()>::call([this,i,&v](){
734  opVectorUnknownType<OP>(i, v);
735  });
736  return;
737  }
738 
739  const exint tuplesize = myImpl.getTupleSize();
740  if (TSIZE == -1 && tuplesize == SRC_TSIZE)
741  {
742  SYS_CallIf<TSIZE == -1>::call([this,i,&v](){
743  this->castTupleSize<SRC_TSIZE>().template opVector<OP>(i, v);
744  });
745  return;
746  }
747  const exint mintuplesize = SYSmin(tuplesize, SRC_TSIZE);
748 
749  // Make sure that v is the correct type before comparing.
750  // Hopefully this will avoid copying if SRC_DATA_T is NotVoidType.
751  constexpr bool TYPE_MATCH = SYSisSame<SRC_DATA_T,NotVoidType>();
753  if (!TYPE_MATCH)
754  converted_v = convertVectorStorage<NotVoidType>(v);
755 
756  UT_ASSERT_P(TSIZE != 0);
757  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
758 
759  hardenTable();
760 
761  UT_PageNum pagenum = pageNum(i);
762  UT_PageOff pageoff = pageOff(i);
763  PageTable *pages = myImpl.getPages();
764  PageTableEntry *page = pages->getPPage(pagenum);
765 
766  if (!PAGESHARDENED)
767  {
768  if (page->isConstant())
769  {
770  // If we're setting, check to see if it's a new value
771  if (OP == 0)
772  {
773  const NotVoidType *data;
774  if (PageTableEntry::typeFitsInline(tuplesize))
775  data = page->getInlinePtr(tuplesize);
776  else
777  data = page->getMaskedPtr();
778  if (data)
779  {
780  if (TYPE_MATCH)
781  {
782  if (isEqual(data, v.data(), tuplesize))
783  return;
784  }
785  else
786  {
787  if (isEqual(data, converted_v.data(), tuplesize))
788  return;
789  }
790  }
791  else
792  {
793  if (TYPE_MATCH)
794  {
795  if (isZero(v.data(), tuplesize))
796  return;
797  }
798  else
799  {
800  if (isZero(converted_v.data(), tuplesize))
801  return;
802  }
803  }
804  }
805  if (TSIZE >= 1)
806  hardenConstantPage(pages, page);
807  else
808  hardenConstantPage(pages, page, tuplesize);
809  }
810  else if (page->isShared())
811  {
812  // If we're setting, check to see if it's a new value
813  if (OP == 0)
814  {
815  const NotVoidType *data = page->getFirstPtr()
816  + pageoff*((TSIZE >= 0) ? TSIZE : tuplesize);
817  if (TYPE_MATCH)
818  {
819  if (isEqual(data, v.data(), tuplesize))
820  return;
821  }
822  else
823  {
824  if (isEqual(data, converted_v.data(), tuplesize))
825  return;
826  }
827  }
828  if (TSIZE >= 1)
829  hardenSharedPage(pages, page);
830  else
831  hardenSharedPage(pages, page, tuplesize);
832  }
833  }
834 
835  NotVoidType *data = page->getFirstPtr() + pageoff*((TSIZE >= 0) ? TSIZE : tuplesize);
836  for (exint component = 0; component < mintuplesize; ++component)
837  {
838  if (OP == 0)
839  {
840  if (TYPE_MATCH)
841  data[component] = v[component];
842  else
843  data[component] = converted_v[component];
844  }
845  else if (OP == 1)
846  {
847  if (TYPE_MATCH)
848  data[component] += v[component];
849  else
850  data[component] += converted_v[component];
851  }
852  else
853  {
854  UT_ASSERT_MSG_P(0,"Only OP 0 (set) and 1 (add) are defined!");
855  }
856  }
857  }
858 
859  SYS_FORCE_INLINE IDX_T size() const
860  {
861  const PageTable *pages = myImpl.getPages();
862  if (!pages)
863  return IDX_T(0);
864  return pages->size();
865  }
866 
868  {
869  const PageTable *pages = myImpl.getPages();
870  if (!pages)
871  return IDX_T(0);
872  return pages->capacity();
873  }
874 
875  /// This helper function forces a (non-negative) capacity to be an
876  /// acceptable value: either
877  /// A) a power of two less than a full page in size, or
878  /// B) a multiple of the page size.
880  static IDX_T roundUpCapacity(IDX_T capacity)
881  {
882  UT_ASSERT_P(capacity >= IDX_T(0));
883 
884  // Round up to page size if more than half of one page.
885  if (capacity > IDX_T(thePageSize>>1))
886  return IDX_T((capacity + thePageMask) & ~thePageMask);
887 
888  // Round up to power of two if half a page or less.
889  // (Zero stays zero.)
890  if (capacity <= IDX_T(2))
891  return capacity;
892  return IDX_T(SYSmakePow2(exint(capacity)));
893  }
894 
895  void setCapacity(IDX_T newcapacity)
896  {
897  UT_ASSERT_P(newcapacity >= IDX_T(0));
898  newcapacity = roundUpCapacity(newcapacity);
899  if (newcapacity == capacity())
900  return;
901  if (newcapacity < size())
902  {
903  setSize(newcapacity);
904  }
905  forceHardenTable(newcapacity);
906  }
907 
908  void setCapacityIfNeeded(IDX_T newcapacity)
909  {
910  UT_ASSERT_P(newcapacity >= IDX_T(0));
911  if (newcapacity <= capacity())
912  return;
913  newcapacity = roundUpCapacity(newcapacity);
914  forceHardenTable(newcapacity);
915  }
916 
917  /// Sets the size of the array without initializing any added
918  /// elements.
919  void setSize(IDX_T newsize)
920  {
921  UT_ASSERT_P(newsize >= IDX_T(0));
922 
923  setCapacityIfNeeded(newsize);
924  hardenTable();
925  PageTable *pages = myImpl.getPages();
926  UT_ASSERT_P(pages || newsize == IDX_T(0));
927  if (pages)
928  pages->setSize(newsize);
929  }
930 
931  /// Sets the size of the array and initializes any new elements to initval.
932  /// This version of setSize only works if DATA_T is a numeric type.
933  /// If the tuple size is > 1, it will write initval to all components.
934  /// Include UT_PageArrayImpl.h to call this.
935  void setSize(IDX_T newsize, NotVoidType initval);
936 
937  /// Sets the size of the array and initializes any new elements
938  /// to initval.
939  /// This version of setSize only works if DATA_T is a numeric type
940  /// and TSIZE >= 1.
941  /// Include UT_PageArrayImpl.h to call this.
942  void setSize(IDX_T newsize, const UT_FixedVector<NotVoidType,theSafeTupleSize> &initval);
943 
944  /// Sets the size of the array and initializes any new elements to initval.
945  /// Include UT_PageArrayImpl.h to call this.
946  void setSize(IDX_T newsize, const UT_Defaults &initval);
947 
948  /// Set all of the values in the range [start,end) to v.
949  /// This will constant-compress pages if allowed and the pages are
950  /// completely overwritten with the given value.
951  /// This version of setConstant only works if DATA_T is a numeric type.
952  /// If the tuple size is > 1, it will write v to all components.
953  void setConstant(IDX_T start, IDX_T end, NotVoidType v);
954 
955  /// Set all of the values in the range [start,end) to v.
956  /// This will constant-compress pages if allowed and the pages are
957  /// completely overwritten with the given value.
958  /// This version of setConstant only works if DATA_T is a numeric type
959  /// and TSIZE >= 1.
961 
962  /// Set all of the values in the range [start,end) to v.
963  /// This will constant-compress pages if allowed and the pages are
964  /// completely overwritten with the given value.
965  void setConstant(IDX_T start, IDX_T end, const UT_Defaults &v);
966 
967  /// Changes the storage type and converts the data to the new type.
968  /// NOTE: This only makes sense if DATA_T is void and newstorage is
969  /// something other than INVALID.
970  /// Include UT_PageArrayImpl.h to call this.
971  void setStorage(const UT_Storage newstorage);
972 
973  /// Changes the tuple size, and if the new size is larger, pads with
974  /// values from the UT_Defaults.
975  /// NOTE: This only makes sense if TSIZE == -1.
976  /// Include UT_PageArrayImpl.h to call this.
977  void setTupleSize(exint newtuplesize, const UT_Defaults &v);
978 
979  /// Get the total of all memory referred-to by this.
980  /// NOTE: This does not handle shared memory correctly.
981  /// Include UT_PageArrayImpl.h to call this.
982  int64 getMemoryUsage(bool inclusive) const;
983 
984  /// Count memory usage using a UT_MemoryCounter in order to count
985  /// shared memory correctly.
986  /// If inclusive is true, the size of this object is counted,
987  /// else only memory owned by this object is counted.
988  /// If this is pointed to by the calling object, inclusive should be true.
989  /// If this is contained in the calling object, inclusive should be false.
990  /// (Its memory was already counted in the size of the calling object.)
991  /// Include UT_PageArrayImpl.h to call this.
992  void countMemory(UT_MemoryCounter &counter, bool inclusive) const;
993 
994  /// Copies the nelements from srcstart to deststart
995  /// The two ranges *are* allowed to overlap.
996  /// Include UT_PageArrayImpl.h to call this.
997  void moveRange(IDX_T srcstart, IDX_T deststart, IDX_T nelements);
998 
999  /// Copies the nelements from fromstart in src to tostart in this.
1000  /// The two ranges *are* allowed to overlap, even if &src==this.
1001  /// Include UT_PageArrayImpl.h to call this.
1002  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
1003  void moveRange(const UT_PageArray<SRC_DATA_T,SRC_TSIZE,SRC_TABLEHARDENED,SRC_PAGESHARDENED,THEPAGEBITS,IDX_T> &src, IDX_T srcstart, IDX_T deststart, IDX_T nelements);
1004 
1005  /// Swaps the nelements at astart with the nelements at bstart.
1006  /// The two ranges *must* be non-overlapping, else swapping is ill-defined.
1007  /// Include UT_PageArrayImpl.h to call this.
1008  void swapRange(IDX_T astart, IDX_T bstart, IDX_T nelements);
1009 
1010  /// Copies a single element from src to dest within this
1011  void copy(IDX_T dest, IDX_T src)
1012  {
1013  UT_ASSERT_P(dest >= IDX_T(0) && dest < size());
1014  UT_ASSERT_P(src >= IDX_T(0) && src < size());
1015 
1016  hardenTable();
1017 
1018  // Because no conversions are necessary, only the element size
1019  // in bytes is needed.
1020  const exint bytesize = (SYSisSame<DATA_T,void>() ? UTstorageSize(getStorage()) : sizeof(NotVoidType))
1021  * getTupleSize();
1022 
1023  UT_PageNum destpagenum = pageNum(dest);
1024  UT_PageNum srcpagenum = pageNum(src);
1025 
1026  PageTable *pages = myImpl.getPages();
1027  PageTableEntry *destpage = pages->getPPage(destpagenum);
1028  const PageTableEntry *srcpage = pages->getPPage(srcpagenum);
1029 
1030  const int8 *srcdata;
1031  if (srcpage->isConstant())
1032  {
1033  if (bytesize < sizeof(PageTableEntry))
1034  {
1035  // Copying within equal constant pages does nothing
1036  if (*srcpage == *destpage)
1037  return;
1038 
1039  srcdata = ((const int8 *)(srcpage+1)) - bytesize;
1040  }
1041  else if (srcpage->getUnmaskedPtrVoid() == destpage->getUnmaskedPtrVoid())
1042  {
1043  // Copying within equal constant pages does nothing
1044  return;
1045  }
1046  else
1047  {
1048  // NOTE: srcdata may be set to nullptr here.
1049  srcdata = (const int8 *)srcpage->getMaskedPtrVoid();
1050  }
1051  }
1052  else
1053  {
1054  srcdata = ((const int8*)srcpage->getFirstPtrVoid()) + pageOff(src)*bytesize;
1055  }
1056 
1057  int8 *destdata;
1058  if (destpage->isConstant())
1059  {
1060  if (bytesize < sizeof(PageTableEntry))
1061  destdata = ((int8 *)(destpage+1)) - bytesize;
1062  else
1063  {
1064  void *destmasked = destpage->getMaskedPtrVoid();
1065  if (destmasked == nullptr)
1066  {
1067  if (isZero(srcdata, bytesize))
1068  return;
1069  destdata = nullptr;
1070  }
1071  else
1072  destdata = (int8*)destmasked;
1073  }
1074  if (destdata != nullptr && isEqual(srcdata, destdata, bytesize))
1075  return;
1076 
1077  if (!SYSisSame<DATA_T,void>())
1078  hardenConstantPage(pages, destpage, getTupleSize());
1079  else
1080  {
1081  switch (getStorage())
1082  {
1083  case UT_Storage::INT8:
1084  {
1085  auto &a = castType<int8>();
1086  auto a_pages = a.myImpl.getPages();
1087  auto a_destpage = a_pages->getPPage(destpagenum);
1088  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1089  break;
1090  }
1091  case UT_Storage::INT16:
1092  case UT_Storage::REAL16:
1093  {
1094  auto &a = castType<int16>();
1095  auto a_pages = a.myImpl.getPages();
1096  auto a_destpage = a_pages->getPPage(destpagenum);
1097  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1098  break;
1099  }
1100  case UT_Storage::INT32:
1101  case UT_Storage::REAL32:
1102  {
1103  auto &a = castType<int32>();
1104  auto a_pages = a.myImpl.getPages();
1105  auto a_destpage = a_pages->getPPage(destpagenum);
1106  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1107  break;
1108  }
1109  case UT_Storage::INT64:
1110  case UT_Storage::REAL64:
1111  {
1112  auto &a = castType<int64>();
1113  auto a_pages = a.myImpl.getPages();
1114  auto a_destpage = a_pages->getPPage(destpagenum);
1115  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1116  break;
1117  }
1118  case UT_Storage::INVALID:
1119  UT_ASSERT_MSG(0, "Storage must be valid if DATA_T is void.");
1120  break;
1121  }
1122  }
1123  }
1124  else if (!PAGESHARDENED && destpage->isShared())
1125  {
1126  destdata = ((int8*)destpage->getFirstPtrVoid()) + pageOff(dest)*bytesize;
1127  if (srcdata)
1128  {
1129  if (isEqual(srcdata, destdata, bytesize))
1130  return;
1131  }
1132  else
1133  {
1134  if (isZero(destdata, bytesize))
1135  return;
1136  }
1137 
1138  if (!SYSisSame<DATA_T,void>())
1139  hardenSharedPage(pages, destpage, getTupleSize());
1140  else
1141  {
1142  switch (getStorage())
1143  {
1144  case UT_Storage::INT8:
1145  {
1146  auto &a = castType<int8>();
1147  auto a_pages = a.myImpl.getPages();
1148  auto a_destpage = a_pages->getPPage(destpagenum);
1149  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1150  break;
1151  }
1152  case UT_Storage::INT16:
1153  case UT_Storage::REAL16:
1154  {
1155  auto &a = castType<int16>();
1156  auto a_pages = a.myImpl.getPages();
1157  auto a_destpage = a_pages->getPPage(destpagenum);
1158  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1159  break;
1160  }
1161  case UT_Storage::INT32:
1162  case UT_Storage::REAL32:
1163  {
1164  auto &a = castType<int32>();
1165  auto a_pages = a.myImpl.getPages();
1166  auto a_destpage = a_pages->getPPage(destpagenum);
1167  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1168  break;
1169  }
1170  case UT_Storage::INT64:
1171  case UT_Storage::REAL64:
1172  {
1173  auto &a = castType<int64>();
1174  auto a_pages = a.myImpl.getPages();
1175  auto a_destpage = a_pages->getPPage(destpagenum);
1176  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1177  break;
1178  }
1179  case UT_Storage::INVALID:
1180  UT_ASSERT_MSG(0, "Storage must be valid if DATA_T is void.");
1181  break;
1182  }
1183  }
1184  }
1185  destdata = ((int8*)destpage->getFirstPtrVoid()) + pageOff(dest)*bytesize;
1186 
1187  for (const int8 *srcdataend = srcdata+bytesize; srcdata != srcdataend; ++srcdata, ++destdata)
1188  *destdata = *srcdata;
1189  }
1190 
1191  /// Copies nelements from srcstart in this to dest array
1192  /// Include UT_PageArrayImpl.h to call this.
1193  template<typename T>
1194  SYS_FORCE_INLINE void getRange(IDX_T srcstart, IDX_T nelements, T *dest) const
1195  {
1196  // Always cast to associated UT_FixedVector type, so that
1197  // we can just delegate to getVectorRange that always uses UT_FixedVector
1198  getVectorRange(srcstart, nelements, (typename UT_FixedVectorTraits<T>::FixedVectorType*)dest);
1199  }
1200 
1201  /// Copies nelements from src array to deststart in this
1202  /// Include UT_PageArrayImpl.h to call this.
1203  template<typename T>
1204  SYS_FORCE_INLINE void setRange(IDX_T deststart, IDX_T nelements, const T *src)
1205  {
1206  // Always cast to associated UT_FixedVector type, so that
1207  // we can just delegate to getVectorRange that always uses UT_FixedVector
1208  setVectorRange(deststart, nelements, (const typename UT_FixedVectorTraits<T>::FixedVectorType*)src);
1209  }
1210 
1211  /// Copies nelements from srcstart in this to dest array
1212  /// Include UT_PageArrayImpl.h to call this.
1213  template<typename DEST_DATA_T,exint DEST_TSIZE,bool DEST_INSTANTIATED>
1214  void getVectorRange(IDX_T srcstart, IDX_T nelements, UT_FixedVector<DEST_DATA_T,DEST_TSIZE,DEST_INSTANTIATED> *dest) const;
1215 
1216  /// Copies nelements from src array to deststart in this
1217  /// Include UT_PageArrayImpl.h to call this.
1218  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_INSTANTIATED>
1219  void setVectorRange(IDX_T deststart, IDX_T nelements, const UT_FixedVector<SRC_DATA_T,SRC_TSIZE,SRC_INSTANTIATED> *src);
1220 
1222  {
1223  return UT_PageNum(exint(i) >> thePageBits);
1224  }
1225  SYS_FORCE_INLINE static UT_PageNum numPages(IDX_T nelements)
1226  {
1227  return UT_PageNum((exint(nelements)+thePageMask) >> thePageBits);
1228  }
1230  {
1231  return UT_PageOff(exint(i) & thePageMask);
1232  }
1234  {
1235  return IDX_T(exint(page) << thePageBits);
1236  }
1237 
1238  /// Returns true iff the table isn't shared with any other UT_PageArray's.
1240  {
1241  // If TABLEHARDENED, assert that the page table hasn't become shared
1242  // after the fact! Nobody should share the page table after templating
1243  // on having hardened!
1244  UT_ASSERT_P(!TABLEHARDENED || !myImpl.getPages() || !myImpl.getPages()->isShared());
1245 
1246  return TABLEHARDENED || !myImpl.getPages() || !myImpl.getPages()->isShared();
1247  }
1248 
1249  /// This ensures that the table itself is not shared.
1250  /// It's a good idea to do this before writing, to avoid having to check
1251  /// every time. Remember to use the returned reference, since it
1252  /// records in the template that it doesn't need to be hardened again.
1253  /// It should be a no-op if TABLEHARDENED is already true.
1255  {
1256  if (!isTableHardened())
1257  {
1258  // Keep same capacity
1259  forceHardenTable(myImpl.getPages()->capacity());
1260  }
1261 
1263  }
1264 
1265  /// This forces all pages to be hardened, optionally including pages
1266  /// that are in the capacity but outside the size.
1267  template <bool including_capacity=false>
1269  {
1270  // Nothing to do if PAGESHARDENED
1271  PageTable *pages = myImpl.getPages();
1272  if (PAGESHARDENED || (pages == NULL))
1274 
1275  auto &hard = hardenTable();
1276 
1277  if (end < IDX_T(0))
1278  end = including_capacity ? pages->capacity() : pages->size();
1279 
1280  UT_ASSERT_P(start >= IDX_T(0));
1282 
1283  if (!SYSisSame<DATA_T,void>())
1284  {
1285  if (TSIZE >= 1)
1286  pages->hardenAllPages(start, end);
1287  else
1288  pages->hardenAllPages(start, end, myImpl.getTupleSize());
1289  }
1290  else
1291  {
1292  // Don't want to have to re-check that the table is hardened,
1293  // so call hardenAllPagesUnknownType on hard.
1294  hard.hardenAllPagesUnknownType(start, end);
1295  }
1296 
1298  }
1299 
1300  template <bool including_capacity=false>
1301  SYS_FORCE_INLINE void tryCompressAllPages(IDX_T start = IDX_T(0), IDX_T end = IDX_T(-1))
1302  {
1303  // Can't have compressed pages if PAGESHARDENED
1304  PageTable *pages = myImpl.getPages();
1305  if (PAGESHARDENED || (pages == NULL))
1306  return;
1307 
1308  auto &hard = hardenTable();
1309 
1310  if (end < IDX_T(0))
1311  end = including_capacity ? pages->capacity() : pages->size();
1312 
1313  UT_ASSERT_P(start >= IDX_T(0));
1315 
1316  if (!SYSisSame<DATA_T,void>())
1317  {
1318  if (TSIZE >= 1)
1319  pages->tryCompressAllPages(start, end);
1320  else
1321  pages->tryCompressAllPages(start, end, myImpl.getTupleSize());
1322  }
1323  else
1324  {
1325  // Don't want to have to re-check that the table is hardened,
1326  // so call hardenAllPagesUnknownType on hard.
1327  hard.tryCompressAllPagesUnknownType(start, end);
1328  }
1329  }
1330 
1331  /// This is for setting the DATA_T template after checking getStorage(),
1332  /// if DATA_T wasn't already known.
1333  /// @{
1334  template<typename DEST_DATA_T>
1336  castType() const
1337  {
1339  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()) || (getStorage() != UT_Storage::INVALID && getStorage() == UT_StorageNum<DEST_DATA_T>::theStorage));
1341  }
1342  template<typename DEST_DATA_T>
1345  {
1347  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()) || (getStorage() != UT_Storage::INVALID && getStorage() == UT_StorageNum<DEST_DATA_T>::theStorage));
1349  }
1350  /// @}
1351 
1352  /// This is for setting the TSIZE template after checking getTupleSize(),
1353  /// if TSIZE wasn't already known.
1354  /// @{
1355  template<exint DEST_TSIZE>
1358  {
1359  UT_ASSERT_P(TSIZE == -1 || TSIZE == DEST_TSIZE);
1360  UT_ASSERT_P(getTupleSize() == DEST_TSIZE);
1362  }
1363  template<exint DEST_TSIZE>
1366  {
1367  UT_ASSERT_P(TSIZE == -1 || TSIZE == DEST_TSIZE);
1368  UT_ASSERT_P(getTupleSize() == DEST_TSIZE);
1370  }
1371  /// @}
1372 
1374  {
1375  return myImpl.getTupleSize();
1376  }
1378  {
1379  return myImpl.getStorage();
1380  }
1381 
1382  /// Returns true iff the specified page is constant-compressed
1384  {
1385  UT_ASSERT_P(myImpl.getPages() != NULL);
1386  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1387  return myImpl.getPages()->getPPage(pagenum)->isConstant();
1388  }
1389 
1390  /// Returns true iff the entire array is constant & zero
1392  {
1393  // Zero capacity array is zero:
1394  if (myImpl.getPages() == NULL)
1395  return true;
1396 
1397  const PageTable *pages = myImpl.getPages();
1398 
1399  UT_PageNum npages = numPages(pages->capacity());
1400  for (UT_PageNum pagenum = 0; pagenum < npages; pagenum++)
1401  {
1402  const PageTableEntry *page = pages->getPPage(pagenum);
1403  if (!page->isConstantAndZeroSafe())
1404  return false;
1405  }
1406  return true;
1407  }
1408 
1409  /// Sets all components of all elements of the specified page to
1410  /// the given value.
1412  {
1413  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1414  UT_ASSERT_P(myImpl.getPages() != NULL);
1415  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1416 
1417  hardenTable();
1418 
1419  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1420  if (TSIZE >= 1)
1421  makeConstant(page, val);
1422  else
1423  makeConstant(page, val, myImpl.getTupleSize());
1424  }
1425 
1426  /// Sets all components of all elements of the specified page to
1427  /// the given value.
1428  template<typename SRC_DATA_T>
1430  {
1431  if (SYSisSame<DATA_T,void>())
1432  {
1433  SYS_CallIf<SYSisSame<DATA_T,void>()>::call([this,pagenum,&val](){
1434  setPageConstantUnknownType(pagenum, val);
1435  });
1436  return;
1437  }
1438 
1439  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1440 
1441  if (!SYSisSame<DATA_T,SRC_DATA_T>())
1442  {
1443  SYS_CallIf<!SYSisSame<DATA_T,SRC_DATA_T>()>::call([this,pagenum,&val](){
1445  for (exint i = 0; i < theSafeTupleSize; ++i)
1446  new_val[i] = UTconvertStorage<NotVoidType>(val[i]);
1447  setPageConstant(pagenum, new_val);
1448  });
1449  return;
1450  }
1451 
1452  UT_ASSERT_P((SYSisSame<DATA_T,SRC_DATA_T>()));
1453  UT_ASSERT_P(myImpl.getPages() != NULL);
1454  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1455  UT_ASSERT_P(TSIZE >= 1);
1456 
1457  hardenTable();
1458 
1459  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1460  makeConstant(page, val);
1461  }
1462 
1463  /// Sets all components of all elements of the specified page to
1464  /// the given values.
1465  /// NOTE: The length of values must be equal to the tuple size.
1466  template<typename SRC_DATA_T>
1467  SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const SRC_DATA_T *values)
1468  {
1469  if (SYSisSame<DATA_T,void>())
1470  {
1471  SYS_CallIf<SYSisSame<DATA_T,void>()>::call([this,pagenum,values](){
1472  setPageConstantUnknownType(pagenum, values);
1473  });
1474  return;
1475  }
1476 
1477  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1478 
1479  if (!SYSisSame<DATA_T,SRC_DATA_T>())
1480  {
1481  SYS_CallIf<!SYSisSame<DATA_T,SRC_DATA_T>()>::call([this,pagenum,values](){
1482  setPageConstantMismatchedType(pagenum, values);
1483  });
1484  return;
1485  }
1486 
1487  UT_ASSERT_P((SYSisSame<DATA_T,SRC_DATA_T>()));
1488  UT_ASSERT_P(myImpl.getPages() != NULL);
1489  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1490 
1491  hardenTable();
1492 
1493  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1494  // NOTE: The cast will never actually be hit unless DATA_T is SRC_DATA_T,
1495  // as asserted above. It's just to get this to compile in other cases.
1496  makeConstant(page, (const NotVoidType *)values, myImpl.getTupleSize());
1497  }
1498 
1499  /// Returns true iff the specified page is "hard",
1500  /// i.e. NOT constant-compressed and NOT shared.
1501  /// NOTE: The *table* may or may not be hard.
1503  {
1504  UT_ASSERT_P(myImpl.getPages() != NULL);
1505  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1506  if (PAGESHARDENED)
1507  return true;
1508  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1509  return !page->isConstant() && !page->isShared();
1510  }
1511 
1512  /// Hardens the specified page if it is constant-compressed or shared.
1513  /// It returns a pointer to the data, but it will only be the correct
1514  /// type if DATA_T is not void.
1516  {
1517  UT_ASSERT_P(myImpl.getPages() != NULL);
1518  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1519 
1520  hardenTable();
1521 
1522  PageTable *pages = myImpl.getPages();
1523  PageTableEntry *page = pages->getPPage(pagenum);
1524 
1525  if (PAGESHARDENED)
1526  return page->getFirstPtr();
1527 
1528  if (page->isConstant())
1529  hardenConstantPage(pages, page, myImpl.getTupleSize());
1530  else if (page->isShared())
1531  hardenSharedPage(pages, page, myImpl.getTupleSize());
1532  return page->getFirstPtr();
1533  }
1534 
1536  {
1537  UT_ASSERT_P(myImpl.getPages() != NULL);
1538  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1539 
1540  hardenTable();
1541 
1542  PageTable *pages = myImpl.getPages();
1543  PageTableEntry *page = pages->getPPage(pagenum);
1544 
1545  if (PAGESHARDENED)
1546  return page->getFirstPtr();
1547 
1548  if (page->isConstant())
1549  hardenConstantPageNoInit(pages, page, myImpl.getTupleSize());
1550  else if (page->isShared())
1551  hardenSharedPageNoInit(pages, page, myImpl.getTupleSize());
1552  return page->getFirstPtr();
1553  }
1554 
1555  /// If the specified page is constant and shared, (only relevant for
1556  /// nonzero tuples that are at least the size of a pointer), this
1557  /// keeps the page constant, but unshares it, so that the constant value
1558  /// can be modified.
1559  /// NOTE: This should ONLY be called on constant pages! Check
1560  /// isPageConstant(pagenum).
1562  {
1563  UT_ASSERT_P(myImpl.getPages() != NULL);
1564  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1565 
1566  hardenTable();
1567 
1568  PageTable *pages = myImpl.getPages();
1569  PageTableEntry *page = pages->getPPage(pagenum);
1570 
1571  UT_ASSERT_P(!PAGESHARDENED && page->isConstant());
1572 
1573  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1574  UT_ASSERT_P(TSIZE == -1 || TSIZE >= 0);
1575 
1576  const exint tuplesize = (TSIZE >= 0) ? TSIZE : myImpl.getTupleSize();
1577  if (PageTableEntry::typeFitsInline(tuplesize))
1578  return page->getInlinePtr(tuplesize);
1579 
1580  NotVoidType *src = page->getMaskedPtr();
1581  if (!page->isShared())
1582  return src;
1583 
1584  PageTableEntry newpage;
1585  newpage.alloc(UT_PageOff(1), tuplesize);
1586  NotVoidType *dest = newpage.getFirstPtr();
1587  memcpy(dest, src, tuplesize*sizeof(NotVoidType));
1588  newpage.setConstantBit();
1589  page->decRef();
1590  (*page) = newpage;
1591 
1592  return dest;
1593  }
1594 
1595  /// Checks if the specified page is all a single value,
1596  /// and if so, makes it a constant page.
1598  {
1599  // Can't have compressed pages if PAGESHARDENED
1600  PageTable *pages = myImpl.getPages();
1601  if (PAGESHARDENED || (pages == NULL))
1602  return;
1603 
1604  auto &hard = hardenTable();
1605 
1606  UT_ASSERT_P(pagenum >= UT_PageNum(0));
1607  UT_ASSERT_P(pagenum < numPages(pages->capacity()));
1608 
1609  if (!SYSisSame<DATA_T,void>())
1610  {
1611  if (TSIZE >= 1)
1612  pages->tryCompressPage(pagenum);
1613  else
1614  pages->tryCompressPage(pagenum, myImpl.getTupleSize());
1615  }
1616  else
1617  {
1618  // Don't want to have to re-check that the table is hardened,
1619  // so call hardenAllPagesUnknownType on hard.
1620  hard.tryCompressPageUnknownType(pagenum);
1621  }
1622  }
1623 
1624  /// Returns a pointer to the data of a page.
1625  /// WARNING: DO NOT call this if DATA_T is void! The pointer returned
1626  /// can depend on the type.
1627  /// This will return NULL if the page is constant compressed to zero.
1628  /// If it's non-NULL and isPageConstant(pagenum) returns true, there
1629  /// will only be one element (tuple) at that address.
1630  const NotVoidType *getPageData(UT_PageNum pagenum) const
1631  {
1632  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1633  UT_ASSERT_P(myImpl.getPages() != NULL);
1634  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1635  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1636  if (!page->isConstant())
1637  return page->getFirstPtr();
1638  if (TSIZE >= 1)
1639  return getConstantPtr(page);
1640  return getConstantPtr(page, 0, getTupleSize());
1641  }
1642 
1643  /// Returns a pointer to the data of a page.
1644  /// WARNING: DO NOT call this if DATA_T is void! The pointer returned
1645  /// can depend on the type.
1646  /// NOTE: DO NOT MODIFY THE DATA if there's any chance the page or the table
1647  /// might be shared. Check isTableHard() and isPageHard(pagenum).
1648  /// This will return NULL if the page is constant compressed to zero.
1649  /// If it's non-NULL and isPageConstant(pagenum) returns true, there
1650  /// will only be one element (tuple) at that address.
1652  {
1653  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1654  UT_ASSERT_P(myImpl.getPages() != nullptr);
1655  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1656  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1657  if (!page->isConstant())
1658  return page->getFirstPtr();
1659  if (TSIZE >= 1)
1660  return getConstantPtr(page);
1661  return getConstantPtr(page, 0, getTupleSize());
1662  }
1663 
1664  /// NOTE: Don't use this unless you know what you're doing!
1665  /// This is for the case where a non-POD type that requires destruction
1666  /// is masquerading as a POD type in here, e.g. GA_OffsetList as
1667  /// UT_FixedVector<int64,2>. When destroying a table, this first needs
1668  /// to be called. If it returns true, the table was shared,
1669  /// so it was just decRef'd, meaning that the data don't need to be
1670  /// destructed. If it returns false, the table was not shared,
1671  /// so the data need to be destructed before deleting the table.
1673  {
1674  if (TABLEHARDENED)
1675  return false;
1676 
1677  PageTable *table = myImpl.getPages();
1678  if (!table)
1679  return false;
1680 
1681  bool decremented = table->decRefIffShared();
1682  if (decremented)
1683  myImpl.getPages() = nullptr;
1684  return decremented;
1685  }
1686 
1687  /// NOTE: Don't use this unless you know what you're doing!
1688  /// This is for the case where a non-POD type that requires destruction
1689  /// is masquerading as a POD type in here, e.g. GA_OffsetList as
1690  /// UT_FixedVector<int64,2>. When destroying a page, this first needs
1691  /// to be called. If it returns true, the page was shared,
1692  /// so it was just decRef'd, meaning that the data don't need to be
1693  /// destructed. If it returns false, the page was not shared,
1694  /// so the data need to be destructed before deleting the page.
1696  {
1697  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1698  UT_ASSERT_P(TSIZE >= 1);
1699  UT_ASSERT_P(myImpl.getPages() != nullptr);
1700  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1701 
1702  if (PAGESHARDENED)
1703  return false;
1704 
1705  hardenTable();
1706 
1707  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1708  if (page->isConstant())
1709  {
1710  if (sizeof(NotVoidType)*TSIZE < sizeof(PageTableEntry))
1711  return false;
1712  }
1713  bool decremented = page->decRefIffShared();
1714  if (decremented)
1715  page->initZero();
1716  return decremented;
1717  }
1718 
1719  /// Only call this if DATA_T is void and you are absolutely, 100% CERTAIN
1720  /// that the storage type is int16, int32, or int64.
1721  /// NOTE: Really! Be careful!
1723  int64 getGuaranteedInt(IDX_T i) const
1724  {
1725  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1726  UT_ASSERT_P(TSIZE == 1);
1728 
1729  UT_PageNum pagenum = pageNum(i);
1730  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1731 
1732  // If we know that the pages are hardened, we can omit
1733  // the constant page case at compile time.
1734  bool isconst = !PAGESHARDENED && page->isConstant();
1735 
1736  const UT_Storage storage = getStorage();
1737  if (!isconst)
1738  {
1739  UT_PageOff pageoff = pageOff(i);
1740  const void *first = page->getFirstPtrVoid();
1741  switch (storage)
1742  {
1743  default:
1744  UT_ASSERT_MSG_P(0, "getGuaranteedInt() only supports int16, int32, and int64.");
1745  case UT_Storage::INT16:
1746  return (int64)((int16*)first)[pageoff];
1747  case UT_Storage::INT32:
1748  return (int64)((int32*)first)[pageoff];
1749  case UT_Storage::INT64:
1750  return ((int64*)first)[pageoff];
1751  }
1752  }
1753 
1754  // Special case for zero.
1755  if (!page->isConstantZero())
1756  return int64(0);
1757 
1758  // Special case for if the constant value is too big to fit in a PageTableEntry.
1759  // At the moment, PageTableEntry is 16 bytes on 64-bit platforms,
1760  // and 8 bytes on 32-bit platforms, so this case will not occur.
1761  if ((sizeof(PageTableEntry) < 9 && storage == UT_Storage::INT64) ||
1762  (sizeof(PageTableEntry) < 5 && storage == UT_Storage::INT32))
1763  {
1764  const void *v = page->getMaskedPtrVoid();
1765  if (sizeof(PageTableEntry) < 5 && storage == UT_Storage::INT32)
1766  {
1767  // This will not be run on 64-bit platforms
1768  return (int64)*(const int32 *)v;
1769  }
1770  return *(const int64 *)v;
1771  }
1772  else
1773  {
1774  // Any types that are strictly smaller than a pointer will be stored inline.
1775  // This is checked at compile time, so there'll be no op at runtime.
1776  if (sizeof(PageTableEntry) >= 9 && storage == UT_Storage::INT64)
1777  {
1778  return *page->template castType<int64>()->getInlinePtr(1);
1779  }
1780  if (sizeof(PageTableEntry) >= 5 && storage == UT_Storage::INT32)
1781  {
1782  return (int64)*page->template castType<int32>()->getInlinePtr(1);
1783  }
1784  return (int64)*page->template castType<int16>()->getInlinePtr(1);
1785  }
1786  }
1787 
1788  /// Returns true iff there is at least one NaN value in the given range.
1789  /// Include UT_PageArrayImpl.h to call this.
1790  bool hasNanInRange(IDX_T start, IDX_T end) const;
1791 
1792 protected:
1793  template<typename DEST_DATA_T=DATA_T>
1794  DEST_DATA_T getUnknownType(IDX_T i,exint component=0) const
1795  {
1796  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1797  UT_ASSERT_P(TSIZE != 0);
1798 
1799  // Make sure common work is extracted out where possible,
1800  // to avoid duplication.
1801 
1802  UT_PageNum pagenum = pageNum(i);
1803  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1804 
1805  // If we know that the pages are hardened, we can omit
1806  // the constant page case at compile time.
1807  bool isconst = !PAGESHARDENED && page->isConstant();
1808 
1809  const UT_Storage storage = getStorage();
1810  if (!isconst)
1811  {
1812  UT_PageOff pageoff = pageOff(i);
1813  const void *first = page->getFirstPtrVoid();
1814  exint i;
1815  if (TSIZE == 1)
1816  i = (exint)pageoff;
1817  else if (TSIZE > 1)
1818  i = TSIZE*pageoff + component;
1819  else
1820  i = myImpl.getTupleSize()*pageoff + component;
1821  switch (storage)
1822  {
1823  case UT_Storage::INVALID:
1824  UT_ASSERT_MSG_P(0, "Can't read a non-numeric type with a cast to a different type.");
1825  return DEST_DATA_T();
1826  case UT_Storage::INT8:
1827  return (DEST_DATA_T)((int8*)first)[i];
1828  case UT_Storage::INT16:
1829  return (DEST_DATA_T)((int16*)first)[i];
1830  case UT_Storage::INT32:
1831  return (DEST_DATA_T)((int32*)first)[i];
1832  case UT_Storage::INT64:
1833  return (DEST_DATA_T)((int64*)first)[i];
1834  case UT_Storage::REAL16:
1835  return (DEST_DATA_T)((fpreal16*)first)[i];
1836  case UT_Storage::REAL32:
1837  return (DEST_DATA_T)((fpreal32*)first)[i];
1838  case UT_Storage::REAL64:
1839  return (DEST_DATA_T)((fpreal64*)first)[i];
1840  }
1841  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1842  return DEST_DATA_T();
1843  }
1844 
1845  switch (storage)
1846  {
1847  case UT_Storage::INVALID:
1848  UT_ASSERT_MSG_P(0, "Can't read a non-numeric type with a cast to a different type.");
1849  return DEST_DATA_T();
1850  case UT_Storage::INT8:
1851  return (DEST_DATA_T)castType<int8>().getConstant(page->template castType<int8>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1852  case UT_Storage::INT16:
1853  return (DEST_DATA_T)castType<int16>().getConstant(page->template castType<int16>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1854  case UT_Storage::INT32:
1855  return (DEST_DATA_T)castType<int32>().getConstant(page->template castType<int32>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1856  case UT_Storage::INT64:
1857  return (DEST_DATA_T)castType<int64>().getConstant(page->template castType<int64>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1858  case UT_Storage::REAL16:
1859  return (DEST_DATA_T)castType<fpreal16>().getConstant(page->template castType<fpreal16>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1860  case UT_Storage::REAL32:
1861  return (DEST_DATA_T)castType<fpreal32>().getConstant(page->template castType<fpreal32>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1862  case UT_Storage::REAL64:
1863  return (DEST_DATA_T)castType<fpreal64>().getConstant(page->template castType<fpreal64>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1864  }
1865  return DEST_DATA_T();
1866  }
1867 
1868  template<int OP,typename SRC_DATA_T>
1869  void opUnknownType(IDX_T i,exint component,SRC_DATA_T v)
1870  {
1871  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1872  const UT_Storage storage = getStorage();
1873  switch (storage)
1874  {
1875  case UT_Storage::INVALID:
1876  UT_ASSERT_MSG_P(0, "Can't write a non-numeric type with opUnknownType.");
1877  return;
1878  case UT_Storage::INT8:
1879  castType<int8 >().template op<OP>(i,component, UTconvertStorage<int8>(v)); return;
1880  case UT_Storage::INT16:
1881  castType<int16 >().template op<OP>(i,component, UTconvertStorage<int16>(v)); return;
1882  case UT_Storage::INT32:
1883  castType<int32 >().template op<OP>(i,component, UTconvertStorage<int32>(v)); return;
1884  case UT_Storage::INT64:
1885  castType<int64 >().template op<OP>(i,component, UTconvertStorage<int64>(v)); return;
1886  case UT_Storage::REAL16:
1887  castType<fpreal16>().template op<OP>(i,component, (fpreal16)v); return;
1888  case UT_Storage::REAL32:
1889  castType<fpreal32>().template op<OP>(i,component, (fpreal32)v); return;
1890  case UT_Storage::REAL64:
1891  castType<fpreal64>().template op<OP>(i,component, (fpreal64)v); return;
1892  }
1893  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1894  }
1895  template<int OP,typename SRC_DATA_T,exint SRC_TSIZE>
1897  {
1898  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1899  const UT_Storage storage = getStorage();
1900  switch (storage)
1901  {
1902  case UT_Storage::INVALID:
1903  UT_ASSERT_MSG_P(0, "Can't write a non-numeric type with opVectorUnknownType.");
1904  return;
1905  case UT_Storage::INT8:
1906  castType<int8 >().template opVector<OP>(i,v); return;
1907  case UT_Storage::INT16:
1908  castType<int16 >().template opVector<OP>(i,v); return;
1909  case UT_Storage::INT32:
1910  castType<int32 >().template opVector<OP>(i,v); return;
1911  case UT_Storage::INT64:
1912  castType<int64 >().template opVector<OP>(i,v); return;
1913  case UT_Storage::REAL16:
1914  castType<fpreal16>().template opVector<OP>(i,v); return;
1915  case UT_Storage::REAL32:
1916  castType<fpreal32>().template opVector<OP>(i,v); return;
1917  case UT_Storage::REAL64:
1918  castType<fpreal64>().template opVector<OP>(i,v); return;
1919  }
1920  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1921  }
1922  template<typename DEST_DATA_T,exint DEST_TSIZE>
1924  {
1925  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1926 
1927  const UT_Storage storage = getStorage();
1928 
1929  // First, check for match on destination type
1931  return castType<DEST_DATA_T>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1932 
1933  // Then, try second-guess type (e.g. reading double as float or vice versa)
1935  return castType<typename UT_StorageNum<DEST_DATA_T>::SecondGuess>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1936 
1937  switch (storage)
1938  {
1939  case UT_Storage::INVALID:
1940  UT_ASSERT_MSG_P(0, "Can't read a non-numeric type with getVectorUnknownType.");
1941  break;
1942  case UT_Storage::INT8:
1943  return castType<int8 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1944  case UT_Storage::INT16:
1945  return castType<int16 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1946  case UT_Storage::INT32:
1947  return castType<int32 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1948  case UT_Storage::INT64:
1949  return castType<int64 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1950  case UT_Storage::REAL16:
1951  return castType<fpreal16>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1952  case UT_Storage::REAL32:
1953  return castType<fpreal32>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1954  case UT_Storage::REAL64:
1955  return castType<fpreal64>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1956  }
1957  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1958  return UT_FixedVector<DEST_DATA_T,DEST_TSIZE>(DEST_DATA_T(0));
1959  }
1960  template<typename DEST_DATA_T,exint DEST_TSIZE>
1962  {
1963  UT_ASSERT_P(TSIZE==-1);
1964  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()));
1965 
1966  const exint tuplesize = getTupleSize();
1967 
1968  // First, check for match on destination type
1969  if (tuplesize == DEST_TSIZE)
1970  return castTupleSize<DEST_TSIZE>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1971 
1972  // Then try small sizes
1973  if (tuplesize == 1)
1974  return castTupleSize<1>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
1975  if (tuplesize == 2)
1976  return castTupleSize<2>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
1977  if (tuplesize == 3)
1978  return castTupleSize<3>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
1979  if (tuplesize == 4)
1980  return castTupleSize<4>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
1981 
1982  // Mismatched large, unknown tuplesize (uncommon case)
1983  UT_FixedVector<DEST_DATA_T,DEST_TSIZE> v(DEST_DATA_T(0));
1984  const exint minsize = SYSmin(tuplesize,DEST_TSIZE);
1985  for (exint component = 0; component < minsize; ++component)
1986  v[component] = get<DEST_DATA_T>(i, component);
1987 
1988  return v;
1989  }
1990  template<typename DEST_DATA_T,exint DEST_TSIZE>
1992  {
1993  UT_ASSERT_P(TSIZE!=-1);
1994  UT_ASSERT_P(TSIZE!=DEST_TSIZE);
1995  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()));
1996 
1997  // Types and tuple sizes are known, but don't match
1998  UT_PageNum pagenum = pageNum(i);
1999  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
2000 
2001  // If we know that the pages are hardened, we can omit
2002  // the constant page case at compile time.
2003  bool isconst = !PAGESHARDENED && page->isConstant();
2004 
2005  const NotVoidType *srctuple;
2006  if (!isconst)
2007  {
2008  UT_PageOff pageoff = pageOff(i);
2009  srctuple = (const NotVoidType *)(((const UT_FixedVector<NotVoidType,theSafeTupleSize>*)page->getFirstPtr()) + pageoff);
2010  }
2011  else
2012  {
2013  srctuple = getConstantPtr(page);
2014  if (!srctuple)
2015  return UT_FixedVector<DEST_DATA_T,DEST_TSIZE>(DEST_DATA_T(0));
2016  }
2017 
2018  if (DEST_TSIZE < TSIZE)
2019  return *(const UT_FixedVector<DEST_DATA_T,DEST_TSIZE>*)srctuple;
2020 
2022  for (exint component = 0; component < TSIZE; ++component)
2023  dsttuple[component] = srctuple[component];
2024  for (exint component = TSIZE; component < DEST_TSIZE; ++component)
2025  dsttuple[component] = DEST_DATA_T(0);
2026  return dsttuple;
2027  }
2029  {
2030  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2031  const UT_Storage storage = getStorage();
2032  switch (storage)
2033  {
2034  case UT_Storage::INVALID:
2035  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2036  return;
2037  case UT_Storage::INT8:
2038  castType<int8>().template hardenAllPages<false>(start, end); return;
2039  case UT_Storage::INT16:
2040  castType<int16>().template hardenAllPages<false>(start, end); return;
2041  case UT_Storage::INT32:
2042  castType<int32>().template hardenAllPages<false>(start, end); return;
2043  case UT_Storage::INT64:
2044  castType<int64>().template hardenAllPages<false>(start, end); return;
2045  case UT_Storage::REAL16:
2046  castType<fpreal16>().template hardenAllPages<false>(start, end); return;
2047  case UT_Storage::REAL32:
2048  castType<fpreal32>().template hardenAllPages<false>(start, end); return;
2049  case UT_Storage::REAL64:
2050  castType<fpreal64>().template hardenAllPages<false>(start, end); return;
2051  }
2052  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2053  }
2054 
2056  {
2057  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2058  const UT_Storage storage = getStorage();
2059  switch (storage)
2060  {
2061  case UT_Storage::INVALID:
2062  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2063  return;
2064  case UT_Storage::INT8:
2065  castType<int8>().template tryCompressAllPages<false>(start, end); return;
2066  case UT_Storage::INT16:
2067  castType<int16>().template tryCompressAllPages<false>(start, end); return;
2068  case UT_Storage::INT32:
2069  castType<int32>().template tryCompressAllPages<false>(start, end); return;
2070  case UT_Storage::INT64:
2071  castType<int64>().template tryCompressAllPages<false>(start, end); return;
2072  case UT_Storage::REAL16:
2073  castType<fpreal16>().template tryCompressAllPages<false>(start, end); return;
2074  case UT_Storage::REAL32:
2075  castType<fpreal32>().template tryCompressAllPages<false>(start, end); return;
2076  case UT_Storage::REAL64:
2077  castType<fpreal64>().template tryCompressAllPages<false>(start, end); return;
2078  }
2079  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2080  }
2081 
2083  {
2084  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2085  const UT_Storage storage = getStorage();
2086  switch (storage)
2087  {
2088  case UT_Storage::INVALID:
2089  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2090  return;
2091  case UT_Storage::INT8:
2092  castType<int8>().tryCompressPage(pagenum); return;
2093  case UT_Storage::INT16:
2094  castType<int16>().tryCompressPage(pagenum); return;
2095  case UT_Storage::INT32:
2096  castType<int32>().tryCompressPage(pagenum); return;
2097  case UT_Storage::INT64:
2098  castType<int64>().tryCompressPage(pagenum); return;
2099  case UT_Storage::REAL16:
2100  castType<fpreal16>().tryCompressPage(pagenum); return;
2101  case UT_Storage::REAL32:
2102  castType<fpreal32>().tryCompressPage(pagenum); return;
2103  case UT_Storage::REAL64:
2104  castType<fpreal64>().tryCompressPage(pagenum); return;
2105  }
2106  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2107  }
2108 
2109  template<typename SRC_DATA_T>
2111  {
2112  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2113  const UT_Storage storage = getStorage();
2114  switch (storage)
2115  {
2116  case UT_Storage::INVALID:
2117  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2118  return;
2119  case UT_Storage::INT8:
2120  castType<int8>().setPageConstant(pagenum, val); return;
2121  case UT_Storage::INT16:
2122  castType<int16>().setPageConstant(pagenum, val); return;
2123  case UT_Storage::INT32:
2124  castType<int32>().setPageConstant(pagenum, val); return;
2125  case UT_Storage::INT64:
2126  castType<int64>().setPageConstant(pagenum, val); return;
2127  case UT_Storage::REAL16:
2128  castType<fpreal16>().setPageConstant(pagenum, val); return;
2129  case UT_Storage::REAL32:
2130  castType<fpreal32>().setPageConstant(pagenum, val); return;
2131  case UT_Storage::REAL64:
2132  castType<fpreal64>().setPageConstant(pagenum, val); return;
2133  }
2134  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2135  }
2136 
2137  template<typename SRC_DATA_T>
2138  void setPageConstantUnknownType(UT_PageNum pagenum, const SRC_DATA_T *values)
2139  {
2140  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2141  const UT_Storage storage = getStorage();
2142  switch (storage)
2143  {
2144  case UT_Storage::INVALID:
2145  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2146  return;
2147  case UT_Storage::INT8:
2148  castType<int8>().setPageConstant(pagenum, values); return;
2149  case UT_Storage::INT16:
2150  castType<int16>().setPageConstant(pagenum, values); return;
2151  case UT_Storage::INT32:
2152  castType<int32>().setPageConstant(pagenum, values); return;
2153  case UT_Storage::INT64:
2154  castType<int64>().setPageConstant(pagenum, values); return;
2155  case UT_Storage::REAL16:
2156  castType<fpreal16>().setPageConstant(pagenum, values); return;
2157  case UT_Storage::REAL32:
2158  castType<fpreal32>().setPageConstant(pagenum, values); return;
2159  case UT_Storage::REAL64:
2160  castType<fpreal64>().setPageConstant(pagenum, values); return;
2161  }
2162  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2163  }
2164 
2165  template<typename SRC_DATA_T>
2166  void setPageConstantMismatchedType(UT_PageNum pagenum, const SRC_DATA_T *values)
2167  {
2168  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2169  UT_ASSERT_P(!(SYSisSame<DATA_T,SRC_DATA_T>()));
2170 
2171  // This could be done directly, instead of converting
2172  // to a UT_SmallArray<DATA_T>, but this approach is simpler for now.
2173 
2174  UT_SmallArray<NotVoidType, ((TSIZE >= 1) ? TSIZE : 16)*sizeof(NotVoidType)> dest_values;
2175  const exint tuplesize = myImpl.getTupleSize();
2176  dest_values.setSize(tuplesize);
2177 
2178  for (exint i = 0; i < tuplesize; ++i)
2179  dest_values(i) = UTconvertStorage<NotVoidType>(values[i]);
2180 
2181  setPageConstant(pagenum, dest_values.array());
2182  }
2183 
2185  {
2186  public:
2189  static const uintptr_t theConstantPageBit = 1;
2190 
2191  /// This is always valid to call.
2193  {
2194  return (uintptr_t(myData) & theConstantPageBit) != 0;
2195  }
2196  /// This is only valid to call if the type doesn't fit inline.
2198  {
2199  return (uintptr_t(myData) == theConstantPageBit);
2200  }
2201  /// This is always valid to call, and will return true
2202  /// iff this is either inline and all zero or
2203  /// using the constant zero optimization.
2205  {
2206  return (uintptr_t(myData) == theConstantPageBit)
2207  && (myPRefCount == nullptr);
2208  }
2209  /// This is only valid to call when DATA_T is non-void.
2210  SYS_FORCE_INLINE static bool typeFitsInline(exint tuplesize = TSIZE)
2211  {
2213  if (sizeof(PageTableEntry) <= sizeof(NotVoidType))
2214  return false;
2215  return (sizeof(NotVoidType)*((TSIZE >= 0) ? TSIZE : tuplesize) < sizeof(PageTableEntry));
2216  }
2217 
2218  /// This is always valid to call, but be aware that the parameter is the
2219  /// number of *bytes* in a tuple, not the number of components in a
2220  /// tuple.
2221  SYS_FORCE_INLINE bool isRefd(exint tuplebytes) const
2222  {
2223  if (!isConstant())
2224  return true;
2225 
2226  // This will be checked at compile time, when possible.
2227  if (sizeof(PageTableEntry) > ((TSIZE >= 0 && !SYSisSame<DATA_T,void>()) ? TSIZE*sizeof(NotVoidType) : tuplebytes))
2228  return false;
2229 
2230  return !isConstantZero();
2231  }
2232 
2233  /// NOTE: This should only be called when there's an actual allocated page,
2234  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2236  {
2237  UT_ASSERT_P(myPRefCount->relaxedLoad() > 0);
2238  return myPRefCount->relaxedLoad() != 1;
2239  }
2240 
2241  /// NOTE: This should only be called when there's an actual allocated page to decRef,
2242  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2244  {
2245  myPRefCount->add(1);
2246  }
2247 
2248  /// NOTE: This should only be called when there's an actual allocated page to decRef,
2249  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2251  {
2252  int64 new_count = myPRefCount->add(-1);
2253  UT_ASSERT_P(new_count >= 0);
2254  if (new_count == 0)
2255  {
2256  free((NotVoidType*)(uintptr_t(myData) & ~theConstantPageBit));
2257  free(myPRefCount);
2258 #if 0
2259  printf("Free page %d pages total\n", int(thePageTablePageCount.add(-1)));
2260  fflush(stdout);
2261 #endif
2262  }
2263  }
2264 
2265  /// NOTE: This should only be called when there's an actual allocated page,
2266  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2268  {
2269  UT_ASSERT_P(myPRefCount->relaxedLoad() > 0);
2270  return myPRefCount->relaxedLoad();
2271  }
2272 
2273  SYS_FORCE_INLINE void alloc(UT_PageOff nelements, exint tuplesize = TSIZE)
2274  {
2276  myData = (NotVoidType *)malloc(nelements*((TSIZE >= 0) ? TSIZE : tuplesize)*sizeof(NotVoidType));
2277  myPRefCount = (SYS_AtomicCounter *)malloc(sizeof(SYS_AtomicCounter));
2278  myPRefCount->relaxedStore(1);
2279 #if 0
2280  printf("Alloc page %d pages total\n", int(thePageTablePageCount.add(1)));
2281  fflush(stdout);
2282 #endif
2283  }
2284  void realloc(UT_PageOff sizetocopy, UT_PageOff newpagecapacity, exint tuplesize = TSIZE)
2285  {
2287  UT_ASSERT_P(sizetocopy >= 0 && sizetocopy <= newpagecapacity);
2288  UT_ASSERT_P(!isConstant());
2289 
2290  UT_ASSERT_P(tuplesize >= 0);
2291  // This is just to try to force the compiler to constant-value-optimize.
2292  if (TSIZE >= 0)
2293  tuplesize = TSIZE;
2294 
2295  UT_ASSERT_P(myPRefCount->relaxedLoad() > 0);
2296  if (myPRefCount->relaxedLoad() > 1)
2297  {
2298  // If shared, need to make a new page.
2299  PageTableEntry newpage;
2300  newpage.alloc(newpagecapacity, tuplesize);
2301  NotVoidType *__restrict newdata = newpage.myData;
2302  const NotVoidType *__restrict olddata = myData;
2303  // Yes, this loop is an inline memcpy, because memcpy won't inline,
2304  // but the compiler will optimize it if it's like this.
2305  sizetocopy *= tuplesize;
2306  while (sizetocopy)
2307  {
2308  *newdata = *olddata;
2309  ++newdata;
2310  ++olddata;
2311  --sizetocopy;
2312  }
2313  decRef();
2314  myData = newpage.myData;
2315  myPRefCount = newpage.myPRefCount;
2316  }
2317  else
2318  {
2319  // If not shared, we can potentially reuse the same memory.
2320  myData = (NotVoidType*)::realloc(myData, newpagecapacity*(tuplesize*sizeof(NotVoidType)));
2321  // NOTE: Reference count doesn't need to change, since it's 1.
2322  }
2323  }
2324 
2325  /// Atomically decrement the reference count iff the page is shared.
2326  /// See the description of decRefIffSharedPage() above for
2327  /// why this function exists.
2329  bool decRefIffShared(exint tuplesize = TSIZE)
2330  {
2331  if (isConstant())
2332  {
2333  UT_ASSERT_P(tuplesize >= 0);
2334  if (typeFitsInline(tuplesize) || isConstantZero())
2335  return false;
2336  }
2337  return decCounterIffAbove1(*myPRefCount);
2338  }
2339 
2340  /// Returns the data pointer, if not a constant page
2343  {
2344  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtr() is for non-constant pages");
2346 
2347  return myData;
2348  }
2349 
2350  /// Returns the data pointer, if not a constant page
2352  const NotVoidType *getFirstPtr() const
2353  {
2354  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtr() is for non-constant pages");
2356 
2357  return myData;
2358  }
2359 
2360  /// Returns the data pointer, if not a constant page
2363  {
2364  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtrVoid() is for non-constant pages");
2365 
2366  return myData;
2367  }
2368 
2369  /// Returns the data pointer, if not a constant page
2371  const void *getFirstPtrVoid() const
2372  {
2373  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtrVoid() is for non-constant pages");
2374 
2375  return myData;
2376  }
2377 
2378  /// Returns the data pointer, or whatever's in its place
2379  /// with bit 0 set to 1 if it's something constant.
2380  /// If it's constant, it might not be a flagged pointer.
2383  {
2384  return myData;
2385  }
2386 
2387  /// Returns the data pointer, or whatever's in its place
2388  /// with bit 0 set to 1 if it's something constant.
2389  /// If it's constant, it might not be a flagged pointer.
2391  const void *getFirstPtrVoidUnsafe() const
2392  {
2393  return myData;
2394  }
2395 
2396  /// Returns the data pointer, if an inline constant page
2399  {
2400  UT_ASSERT_MSG_P(isConstant() && typeFitsInline(tuplesize), "getInlinePtr() is for inline constant pages");
2402  return (NotVoidType *)(((int8 *)(this+1)) - (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize)));
2403  }
2404 
2405  /// Returns the data pointer, if an inline constant page
2407  const NotVoidType *getInlinePtr(exint tuplesize) const
2408  {
2409  UT_ASSERT_MSG_P(isConstant() && typeFitsInline(tuplesize), "getInlinePtr() is for inline constant pages");
2411  return (const NotVoidType *)(((const int8 *)(this+1)) - (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize)));
2412  }
2413 
2414  /// Returns the data pointer, if a non-inline constant page.
2415  /// Returns nullptr for a zero constant page.
2418  {
2419  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtr() is for constant pages");
2421 
2422  return (NotVoidType *)(uintptr_t(myData) & ~theConstantPageBit);
2423  }
2424 
2425  /// Returns the data pointer, if a non-inline constant page.
2426  /// Returns nullptr for a zero constant page.
2428  const NotVoidType *getMaskedPtr() const
2429  {
2430  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtr() is for constant pages");
2432 
2433  return (const NotVoidType *)(uintptr_t(myData) & ~theConstantPageBit);
2434  }
2435 
2436  /// Returns the data pointer, if a non-inline constant page.
2437  /// Returns nullptr for a zero constant page.
2440  {
2441  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtrVoid() is for constant pages");
2442 
2443  return (void *)(uintptr_t(myData) & ~theConstantPageBit);
2444  }
2445 
2446  /// Returns the data pointer, if a non-inline constant page.
2447  /// Returns nullptr for a zero constant page.
2449  const void *getMaskedPtrVoid() const
2450  {
2451  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtrVoid() is for constant pages");
2452 
2453  return (const void *)(uintptr_t(myData) & ~theConstantPageBit);
2454  }
2455 
2456  /// Initializes a page to constant zero, assuming that it hasn't been
2457  /// initialized yet.
2459  void initZero()
2460  {
2461  myData = (NotVoidType*)theConstantPageBit;
2462  myPRefCount = nullptr;
2463  }
2464 
2467  {
2468  myData = (NotVoidType *)(uintptr_t(myData) | theConstantPageBit);
2469  }
2470 
2471  void tryCompressPage(UT_PageOff pagesize, exint tuplesize = TSIZE)
2472  {
2473  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2474  UT_ASSERT_P(!isConstant());
2475 
2476  tuplesize = (TSIZE >= 0 ? TSIZE : tuplesize);
2477 
2478  const NotVoidType *const first = myData;
2479  const NotVoidType *const end = first + (TSIZE >= 0 ? TSIZE : tuplesize)*exint(pagesize);
2480  for (const NotVoidType *other = first+tuplesize; other < end; other += tuplesize)
2481  {
2482  for (exint component = 0; component < (TSIZE >= 0 ? TSIZE : tuplesize); ++component)
2483  {
2484  if (other[component] != first[component])
2485  return;
2486  }
2487  }
2488  // We have a constant page! Compress it!
2489 
2490  PageTableEntry newpage;
2491  newpage.initZero();
2492  const exint bytesize = sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize);
2493  // Value stored inline if it's strictly smaller than a pointer
2494  // The first check can be done at compile time, so it's only logically redundant,
2495  // not functionally redundant.
2496  if (sizeof(PageTableEntry) > sizeof(NotVoidType) && sizeof(PageTableEntry) > bytesize)
2497  {
2498  memcpy(((int8*)&newpage) + (sizeof(PageTableEntry) - bytesize),
2499  first, bytesize);
2500  }
2501  else
2502  {
2503  if (!isZero(first, (TSIZE >= 0 ? TSIZE : tuplesize)))
2504  {
2505  newpage.alloc(UT_PageOff(1), tuplesize);
2506  NotVoidType *newfirst = newpage.getFirstPtr();
2507  memcpy(newfirst, first, bytesize);
2508  newpage.setConstantBit();
2509  }
2510  }
2511  decRef();
2512  *this = newpage;
2513  }
2514 
2515  /// NOTE: This always excludes sizeof(*this), and is only valid for
2516  /// non-small pages, (okay if constant).
2518  {
2519  if (!isConstant())
2520  return sizeof(SYS_AtomicCounter) + tuplebytes*thePageSize;
2521  if (tuplebytes < sizeof(PageTableEntry) || isConstantZero())
2522  return 0;
2523  return sizeof(SYS_AtomicCounter) + tuplebytes;
2524  }
2525 
2526  template<typename DEST_DATA_T>
2529  castType() const
2530  {
2533  }
2534  template<typename DEST_DATA_T>
2538  {
2541  }
2542 
2543  /// NOTE: This just does a shallow comparison.
2545  bool operator==(const ThisType &that) const
2546  {
2547  return (myData == that.myData) && (myPRefCount == that.myPRefCount);
2548  }
2549  /// NOTE: This just does a shallow comparison.
2551  bool operator!=(const ThisType &that) const
2552  {
2553  return (myData != that.myData) || (myPRefCount != that.myPRefCount);
2554  }
2555 
2556  private:
2557  NotVoidType *myData;
2558  SYS_AtomicCounter *myPRefCount;
2559  };
2560 
2563  {
2564  exint cur_count = counter.relaxedLoad();
2565 
2566  // It should be rare for this to ever loop, because looping
2567  // means that something else referring to the same page has
2568  // done an incRef, decRef, or decRefIffShared on this same page,
2569  // between the relaxedLoad and the compare_swap calls, or on
2570  // subsequent loops, between the compare_swap calls.
2571  while (true)
2572  {
2573  if (cur_count == 1)
2574  return false; // Not shared, so don't decrement
2575  exint new_count = cur_count-1;
2576  exint prev_count = counter.compare_swap(cur_count, new_count);
2577  if (prev_count == cur_count)
2578  return true; // Successfully decremented
2579  cur_count = prev_count;
2580  }
2581  // Execution never gets here, because the only escapes from
2582  // the loops return.
2583  }
2584 
2585  SYS_FORCE_INLINE static NotVoidType getConstant(const PageTableEntry *const page,exint component,exint tuplesize = TSIZE)
2586  {
2587  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2588  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2589  UT_ASSERT_P(component >= 0 && component < tuplesize);
2590  UT_ASSERT_P(page->isConstant());
2591 
2592  // Any types whose tuples are strictly smaller than a pointer will be stored inline.
2593  // If a single one is at least as large as a pointer, it's
2594  // rejected at compile time, so there'll be no op at runtime in that case.
2595  if (PageTableEntry::typeFitsInline(tuplesize))
2596  {
2597  return page->getInlinePtr(tuplesize)[component];
2598  }
2599 
2600  const NotVoidType *masked = page->getMaskedPtr();
2601  if (!masked)
2602  return NotVoidType(0);
2603 
2604  // Other types have a 1-element page.
2605  return masked[component];
2606  }
2607  /// NOTE: It may seem like overkill to take a PageTableEntry *, but the
2608  /// returned pointer may be *inside* the pointer that the caller owns,
2609  /// so we need the caller's pointer, not a copy. getConstant doesn't
2610  /// have this issue, since it can use the local pointer and read the
2611  /// value.
2612  SYS_FORCE_INLINE static const NotVoidType *getConstantPtr(const PageTableEntry *page,exint component = 0,exint tuplesize = TSIZE)
2613  {
2614  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2615  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2616  UT_ASSERT_P(component >= 0 && component < tuplesize);
2617  UT_ASSERT_P(page->isConstant());
2618 
2619  // Any types whose tuples are strictly smaller than PageTableEntry will be stored inline.
2620  if (PageTableEntry::typeFitsInline(tuplesize))
2621  {
2622  return page->getInlinePtr(tuplesize) + component;
2623  }
2624 
2625  const NotVoidType *masked = page->getMaskedPtr();
2626  if (!masked)
2627  return nullptr;
2628 
2629  // Other types have a 1-element page.
2630  return masked + component;
2631  }
2632  /// NOTE: It may seem like overkill to take a PageTableEntry *, but the
2633  /// returned pointer may be *inside* the pointer that the caller owns,
2634  /// so we need the caller's pointer, not a copy. getConstant doesn't
2635  /// have this issue, since it can use the local pointer and read the
2636  /// value.
2637  SYS_FORCE_INLINE static NotVoidType *getConstantPtr(PageTableEntry *page,exint component = 0,exint tuplesize = TSIZE)
2638  {
2639  return SYSconst_cast(getConstantPtr((const PageTableEntry *)page, component, tuplesize));
2640  }
2642  {
2643  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2644  UT_ASSERT_P(TSIZE >= 1);
2645  UT_ASSERT_P(!PAGESHARDENED);
2646 
2647  if (PAGESHARDENED)
2648  return;
2649 
2651 
2652  // Unref existing non-constant page
2653  bool wasconst = page->isConstant();
2654  if (!wasconst)
2655  page->decRef();
2656 
2657  // Easy for inline case, checked at compile-time.
2658  if (sizeof(PageTableEntry) > sizeof(Tuple))
2659  {
2660  page->initZero();
2661  NotVoidType *tuple = page->getInlinePtr(theSafeTupleSize);
2662  *(Tuple *)tuple = val;
2663  return;
2664  }
2665 
2666  bool iszero = val.isZero();
2667 
2668  // In other cases, we may or may not have to unref constant page
2669  if (wasconst)
2670  {
2671  NotVoidType *masked = page->getMaskedPtr();
2672  if (!masked)
2673  {
2674  // Fairly common case: already zero, making zero.
2675  if (iszero)
2676  return;
2677  }
2678  else if (iszero || (page->isShared() && (*(const Tuple*)masked != val)))
2679  {
2680  // No longer need this old constant page
2681  page->decRef();
2682  }
2683  else if (!page->isShared())
2684  {
2685  // Reuse the unshared constant page
2686  *(Tuple*)masked = val;
2687  return;
2688  }
2689  else
2690  {
2691  // Already equal; nothing to do
2692  UT_ASSERT_P(*(const Tuple*)masked == val);
2693  return;
2694  }
2695  }
2696 
2697  if (iszero)
2698  {
2699  page->initZero();
2700  return;
2701  }
2702 
2703  // Need to allocate new constant page
2704  PageTableEntry newpage;
2705  newpage.alloc(UT_PageOff(1), theSafeTupleSize);
2706  NotVoidType *tuple = newpage.getFirstPtr();
2707  (*(Tuple*)tuple) = val;
2708  newpage.setConstantBit();
2709  (*page) = newpage;
2710  }
2711 
2712  static void makeConstant(PageTableEntry *page, const NotVoidType &val, exint tuplesize = TSIZE)
2713  {
2714  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2715  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2716  UT_ASSERT_P(!PAGESHARDENED);
2717  UT_ASSERT_P(tuplesize >= 1);
2718 
2719  if (PAGESHARDENED)
2720  return;
2721 
2722  // Unref existing non-constant page
2723  bool wasconst = page->isConstant();
2724  if (!wasconst)
2725  page->decRef();
2726 
2727  // Easy for inline case, partly checked at compile-time.
2728  if (PageTableEntry::typeFitsInline(tuplesize))
2729  {
2730  page->initZero();
2731  NotVoidType *tuple = page->getInlinePtr(tuplesize);
2732  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2733  tuple[i] = val;
2734  return;
2735  }
2736 
2737  bool iszero = isZero(val);
2738 
2739  // In other cases, we may or may not have to unref constant page
2740  if (wasconst)
2741  {
2742  NotVoidType *masked = page->getMaskedPtr();
2743  if (!masked)
2744  {
2745  // Fairly common case: already zero, making zero.
2746  if (iszero)
2747  return;
2748  }
2749  else if (iszero)
2750  {
2751  // No longer need this old constant page
2752  page->decRef();
2753  }
2754  else if (page->isShared())
2755  {
2756  bool equal = true;
2757  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2758  {
2759  if (masked[i] != val)
2760  {
2761  equal = false;
2762  break;
2763  }
2764  }
2765 
2766  if (equal)
2767  {
2768  // Already equal; nothing to do
2769  return;
2770  }
2771 
2772  // No longer need this old constant page
2773  page->decRef();
2774  }
2775  else
2776  {
2777  // Reuse the unshared constant page
2778  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2779  masked[i] = val;
2780  return;
2781  }
2782  }
2783 
2784  if (iszero)
2785  {
2786  page->initZero();
2787  return;
2788  }
2789 
2790  // Need to allocate new constant page
2791  PageTableEntry newpage;
2792  newpage.alloc(UT_PageOff(1), tuplesize);
2793  NotVoidType *tuple = newpage.getFirstPtr();
2794  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2795  tuple[i] = val;
2796  newpage.setConstantBit();
2797  (*page) = newpage;
2798  }
2799 
2800  static void makeConstant(PageTableEntry *page, const NotVoidType *values, exint tuplesize)
2801  {
2802  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2803  UT_ASSERT_P(TSIZE == -1 || TSIZE == tuplesize);
2804  UT_ASSERT_P(!PAGESHARDENED);
2805  UT_ASSERT_P(tuplesize >= 1);
2806 
2807  if (PAGESHARDENED)
2808  return;
2809 
2810  // Unref existing non-constant page
2811  bool wasconst = page->isConstant();
2812  if (!wasconst)
2813  page->decRef();
2814 
2815  // Easy for inline case, partly checked at compile-time.
2816  if (PageTableEntry::typeFitsInline(tuplesize))
2817  {
2818  page->initZero();
2819  NotVoidType *tuple = page->getInlinePtr(tuplesize);
2820  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2821  tuple[i] = values[i];
2822  return;
2823  }
2824 
2825  bool iszero = isZero(values, (TSIZE >= 0 ? TSIZE : tuplesize));
2826 
2827  // In other cases, we may or may not have to unref constant page
2828  if (wasconst)
2829  {
2830  NotVoidType *masked = page->getMaskedPtr();
2831  if (!masked)
2832  {
2833  // Fairly common case: already zero, making zero.
2834  if (iszero)
2835  return;
2836  }
2837  else if (iszero)
2838  {
2839  // No longer need this old constant page
2840  page->decRef();
2841  }
2842  else if (page->isShared())
2843  {
2844  bool equal = true;
2845  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2846  {
2847  if (masked[i] != values[i])
2848  {
2849  equal = false;
2850  break;
2851  }
2852  }
2853 
2854  if (equal)
2855  {
2856  // Already equal; nothing to do
2857  return;
2858  }
2859 
2860  // No longer need this old constant page
2861  page->decRef();
2862  }
2863  else
2864  {
2865  // Reuse the unshared constant page
2866  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2867  masked[i] = values[i];
2868  return;
2869  }
2870  }
2871 
2872  if (iszero)
2873  {
2874  page->initZero();
2875  return;
2876  }
2877 
2878  // Need to allocate new constant page
2879  PageTableEntry newpage;
2880  newpage.alloc(UT_PageOff(1), tuplesize);
2881  NotVoidType *tuple = newpage.getFirstPtr();
2882  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2883  tuple[i] = values[i];
2884  newpage.setConstantBit();
2885  (*page) = newpage;
2886  }
2887 
2888  /// This replaces dest with src. It references if possible and keeps
2889  /// dest constant if easy. If desttuplesize > srctuplesize, it only
2890  /// replaces the first srctuplesize components.
2891  template<typename SrcType>
2892  static void replacePage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destpagesize, UT_PageOff destpagecapacity);
2893 
2894  /// This replaces part of dest with part of src. It keeps
2895  /// dest constant if easy. If desttuplesize > srctuplesize, it only
2896  /// replaces the first srctuplesize components.
2897  /// This will *not* check to see if the whole page is being replaced,
2898  /// so if you want the src page to be referenced, or overwrite a
2899  /// constant value, when writing the whole page, use replacePage.
2900  template<typename SrcType>
2901  static void copyPartialPage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destoff, UT_PageOff srcoff, UT_PageOff ntuples, UT_PageOff destpagecapacity);
2902 
2903  /// This version of makeConstant copies a const page onto this page,
2904  /// converting type if necessary, and supporting a different tuple size,
2905  /// so long as the destination size either isn't larger or is constant.
2906  /// NOTE: This would be templated on the actual templates of UT_PageArray,
2907  /// but somehow, Visual C++ can't deduce the type, so the template type
2908  /// has to be specified by the caller, and one argument is much easier
2909  /// to specify on every call than 4.
2910  template<typename SrcType>
2911  static void makeConstantFrom(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize);
2912 
2913  /// This is just a helper function for writing a single source tuple (or NULL) to a block of tuple data, with conversion.
2914  /// NOTE: destpagedata must point to an *UNSHARED* page's data.
2915  template<typename SrcNotVoidType>
2916  static void fillNonConstWithConst(NotVoidType *destpagedata, NotVoidType *destpageend, const SrcNotVoidType *stuple, exint mintuplesize, exint desttupleextra);
2917 
2918  /// This is just a helper function for copying a non-constant source page to a non-constant destination page, with conversion.
2919  /// NOTE: destpagedata must point to an *UNSHARED* page's data.
2920  /// NOTE: The two data ranges must be *NON-OVERLAPPING*.
2921  template<typename SrcNotVoidType>
2922  static void copyNonConst(NotVoidType *destpagedata, const SrcNotVoidType *srcpagedata, exint desttuplesize, exint srctuplesize, UT_PageOff ntuples);
2923 
2924  /// This is just a helper function for checking whether two constant pages are equal, up to
2925  /// the smaller of the two, to see if writing is necessary.
2926  template<typename SrcNotVoidType>
2927  static bool isEqualConst(const NotVoidType *tuple, const SrcNotVoidType *stuple, exint mintuplesize);
2928 
2929  static void hardenConstantPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
2930  {
2931  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2932  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2933  UT_ASSERT_P(page->isConstant());
2934 
2935  PageTableEntry newpage;
2936  newpage.alloc(pagecapacity, tuplesize);
2937  NotVoidType *dest = newpage.getFirstPtr();
2938  bool isinline = PageTableEntry::typeFitsInline(tuplesize);
2939  if (isinline ? page->isConstantAndZeroSafe() : page->isConstantZero())
2940  {
2941  memset(dest, 0, (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize))*pagecapacity);
2942  }
2943  else
2944  {
2945  const NotVoidType *src;
2946  if (isinline)
2947  src = page->getInlinePtr(tuplesize);
2948  else
2949  src = page->getMaskedPtr();
2950 
2951  for (UT_PageOff i = 0; i < pagecapacity; ++i)
2952  {
2953  for (exint component = 0; component < tuplesize; ++component, ++dest)
2954  *dest = src[component];
2955  }
2956 
2957  if (!isinline)
2958  page->decRef();
2959  }
2960  (*page) = newpage;
2961  }
2962 
2963  static void hardenConstantPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
2964  {
2965  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2966  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2967  UT_ASSERT_P(page->isConstant());
2968 
2969  PageTableEntry newpage;
2970  newpage.alloc(pagecapacity, tuplesize);
2971  // Compile-time check, to speed up case where even one element doesn't fit inline.
2972  bool isinline = PageTableEntry::typeFitsInline(tuplesize);
2973  if (!isinline && !page->isConstantZero())
2974  page->decRef();
2975  (*page) = newpage;
2976  }
2977 
2978  static void hardenSharedPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
2979  {
2980  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2981  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2982  UT_ASSERT_P(!page->isConstant());
2983  // This may technically no longer be shared between checking and calling
2984  // due to other threads decRef'ing the page. It's okay to harden anyway.
2985  //UT_ASSERT_P(page->isShared());
2986 
2987  PageTableEntry newpage;
2988  newpage.alloc(pagecapacity, tuplesize);
2989  NotVoidType *dest = newpage.getFirstPtr();
2990  const NotVoidType *src = page->getFirstPtr();
2991  memcpy(dest, src, (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize))*pagecapacity);
2992  page->decRef();
2993  (*page) = newpage;
2994  }
2995 
2996  static void hardenSharedPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
2997  {
2998  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2999  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3000  UT_ASSERT_P(!page->isConstant());
3001  // This may technically no longer be shared between checking and calling
3002  // due to other threads decRef'ing the page. It's okay to harden anyway.
3003  //UT_ASSERT_P(page->isShared());
3004 
3005  PageTableEntry newpage;
3006  newpage.alloc(pagecapacity, tuplesize);
3007  page->decRef();
3008  (*page) = newpage;
3009  }
3010 
3012  {
3013  public:
3016 
3018  {
3019  UT_ASSERT_P(myCapacity > IDX_T(0));
3020  return reinterpret_cast<PageTableEntry*>(this+1);
3021  }
3023  {
3024  UT_ASSERT_P(myCapacity > IDX_T(0));
3025  return reinterpret_cast<const PageTableEntry *>(this+1);
3026  }
3027  // NOTE: Clang sometimes decides to ignore const references and pass by
3028  // value, so we have to return by pointer, in case the page
3029  // is constant and inlined, so that the address of the constant
3030  // value is accessible through this function.
3031  // Yes, many cases don't rely on this, but to reduce the risk of
3032  // bugs, the only way to access page pointers in a const PageTable
3033  // is to call this to get the page pointer address and explicitly
3034  // dereference, if valid in the given circumstance.
3036  {
3037  UT_ASSERT_P(myCapacity > IDX_T(i << THEPAGEBITS));
3038  return reinterpret_cast<const PageTableEntry *>(this+1) + i;
3039  }
3041  {
3042  UT_ASSERT_P(myCapacity > IDX_T(i << THEPAGEBITS));
3043  return reinterpret_cast<PageTableEntry*>(this+1) + i;
3044  }
3046  {
3047  UT_ASSERT_P(myRefCount.relaxedLoad() > 0);
3048  return myRefCount.relaxedLoad() != 1;
3049  }
3051  {
3052  return myRefCount.relaxedLoad();
3053  }
3054  /// NOTE: This is the size of the full array, not the number of pages.
3055  SYS_FORCE_INLINE IDX_T size() const
3056  {
3057  return mySize;
3058  }
3059  /// NOTE: This is the capacity of the full array, not the capacity of pages.
3061  {
3062  return myCapacity;
3063  }
3064  /// NOTE: This will *assume* the capacity is large enough! The caller *must* check!
3065  /// NOTE: This is the size of the full array, not the number of pages.
3066  SYS_FORCE_INLINE void setSize(IDX_T newsize)
3067  {
3068  UT_ASSERT_P(newsize <= capacity());
3069  UT_ASSERT_MSG_P(myRefCount == 1, "The table must already be hardened before we modify it!");
3070  if (PAGESHARDENED && newsize > mySize)
3071  {
3072  UT_ASSERT_P(TSIZE >= 0);
3073  // Need to ensure pages between mySize and newsize are hardened.
3074  // Pages containing offsets less than or equal to mySize are
3075  // already guaranteed to be hardened.
3076  UT_PageNum startpage = numPages(mySize);
3077  UT_PageNum endpage = numPages(newsize);
3078  UT_PageOff pagecapacity(thePageSize);
3079  if (endpage == UT_PageNum(1) && exint(capacity()) < thePageSize)
3080  pagecapacity = capacity();
3081  PageTableEntry *page = getPPage(startpage);
3082  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3083  {
3084  if (page->isConstant())
3085  hardenConstantPage(page, pagecapacity);
3086  else if (page->isShared())
3087  hardenSharedPage(page, pagecapacity);
3088  }
3089  }
3090  mySize = newsize;
3091  }
3092  /// NOTE: This will *assume* the capacity is large enough! The caller *must* check!
3093  /// NOTE: This is the size of the full array, not the number of pages.
3094  SYS_FORCE_INLINE void setSize(IDX_T newsize, exint tuplesize)
3095  {
3096  UT_ASSERT_P(newsize <= capacity());
3097  UT_ASSERT_MSG_P(myRefCount == 1, "The table must already be hardened before we modify it!");
3098  if (PAGESHARDENED && newsize > mySize)
3099  {
3100  // Need to ensure pages between mySize and newsize are hardened.
3101  // Pages containing offsets less than or equal to mySize are
3102  // already guaranteed to be hardened.
3103  UT_PageNum startpage = numPages(mySize);
3104  UT_PageNum endpage = numPages(newsize);
3105  UT_PageOff pagecapacity(thePageSize);
3106  if (endpage == UT_PageNum(1) && exint(capacity()) < thePageSize)
3107  pagecapacity = capacity();
3108  PageTableEntry *page = getPPage(startpage);
3109  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3110  {
3111  if (page->isConstant())
3112  hardenConstantPage(page, pagecapacity, tuplesize);
3113  else if (page->isShared())
3114  hardenSharedPage(page, pagecapacity, tuplesize);
3115  }
3116  }
3117  mySize = newsize;
3118  }
3119  void fill(IDX_T start, IDX_T end, const NotVoidType &val);
3120  void fill(IDX_T start, IDX_T end, NotVoidType val, exint tuplesize);
3121  void fill(IDX_T start, IDX_T end, const NotVoidType *values, exint tuplesize);
3122  void fill(IDX_T start, IDX_T end, const UT_FixedVector<NotVoidType,theSafeTupleSize> &val);
3123 
3124  void incRef()
3125  {
3126  myRefCount.add(1);
3127  }
3128  void decRef(exint tuplesize)
3129  {
3130  const int64 new_count = myRefCount.add(-1);
3131  if (new_count != 0)
3132  {
3133  UT_ASSERT_P(new_count > 0);
3134  return;
3135  }
3136 
3137  UT_ASSERT_P(!(SYSisSame<DATA_T, void>()));
3138 
3139  const size_t bytesize = sizeof(NotVoidType)*((TSIZE >= 0) ? TSIZE : tuplesize);
3140 
3141  // If we're deleting the table, first, decRef all pages.
3142  UT_PageNum npages = numPages(myCapacity);
3143  PageTableEntry *page = getFirstPage();
3144  for (UT_PageNum p = 0; p < npages; ++p, ++page)
3145  {
3146  if (page->isRefd(bytesize))
3147  page->decRef();
3148  }
3149  free(this);
3150 #if 0
3151  printf("Free table %d pages (case 0); %d tables total\n",int(npages), int(thePageTableCount.add(-1)));
3152  fflush(stdout);
3153 #endif
3154  }
3155 
3156  void hardenAllPages(IDX_T start, IDX_T end, exint tuplesize = TSIZE)
3157  {
3158  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3159  UT_ASSERT_P(tuplesize >= 0);
3160  UT_PageNum startpage = pageNum(start);
3161  UT_PageNum endpage = numPages(end);
3162  UT_PageOff pagecapacity(thePageSize);
3163  if (endpage == UT_PageNum(1) && exint(capacity()) < thePageSize)
3164  pagecapacity = capacity();
3165  PageTableEntry *page = getPPage(startpage);
3166  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3167  {
3168  if (page->isConstant())
3169  hardenConstantPage(page, pagecapacity, tuplesize);
3170  else if (page->isShared())
3171  hardenSharedPage(page, pagecapacity, tuplesize);
3172  }
3173  }
3174 
3175  void tryCompressAllPages(IDX_T start, IDX_T end, exint tuplesize = TSIZE)
3176  {
3177  UT_ASSERT_P(tuplesize >= 0);
3178  UT_PageNum startpage = pageNum(start);
3179  UT_PageNum endpage = numPages(end);
3180  UT_PageNum npages = numPages(mySize);
3181  PageTableEntry *page = getPPage(startpage);
3182  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3183  {
3184  if (page->isConstant())
3185  continue;
3186 
3187  UT_PageOff pagesize = thePageSize;
3188  if (p+1 >= npages)
3189  {
3190  if (p+1 == npages)
3191  {
3192  UT_PageOff pagesizetemp = pageOff(mySize);
3193  if (pagesizetemp != UT_PageOff(0))
3194  pagesize = pagesizetemp;
3195  }
3196  else
3197  pagesize = UT_PageOff(0);
3198  }
3199  page->tryCompressPage(pagesize, tuplesize);
3200  }
3201  }
3202 
3203  void tryCompressPage(UT_PageNum pagenum, exint tuplesize = TSIZE)
3204  {
3205  UT_ASSERT_P(tuplesize >= 0);
3206  PageTableEntry *page = getPPage(pagenum);
3207  if (page->isConstant())
3208  return;
3209 
3210  UT_PageOff pagesize = thePageSize;
3211  UT_PageNum npages = numPages(mySize);
3212  if (pagenum+1 >= npages)
3213  {
3214  if (pagenum+1 == npages)
3215  {
3216  UT_PageOff pagesizetemp = pageOff(mySize);
3217  if (pagesizetemp != UT_PageOff(0))
3218  pagesize = pagesizetemp;
3219  }
3220  else
3221  pagesize = UT_PageOff(0);
3222  }
3223  page->tryCompressPage(pagesize, tuplesize);
3224  }
3225 
3226  static ThisType *alloc(UT_PageNum npages)
3227  {
3228  UT_ASSERT_MSG_P(npages > UT_PageNum(0), "Shouldn't have a zero-page page table; NULL should be used instead.");
3229  ThisType *p = (ThisType *)malloc(sizeof(ThisType) + npages*sizeof(PageTableEntry));
3230  //UT_ASSERT_P(_CrtIsValidHeapPointer(p));
3231  p->myRefCount.relaxedStore(1);
3232 #if 0
3233  printf("Alloc table %d pages; %d tables total\n",int(npages), int(thePageTableCount.add(1)));
3234  fflush(stdout);
3235 #endif
3236  return p;
3237  }
3238  static ThisType *harden(ThisType *pages, IDX_T newcapacity, exint tuplesize=TSIZE)
3239  {
3240  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3241 
3242  // If we're setting capacity to zero, free the table.
3243  if (newcapacity == IDX_T(0))
3244  {
3245  if (pages)
3246  pages->decRef(tuplesize);
3247  return NULL;
3248  }
3249 
3250  UT_ASSERT_P(tuplesize >= 0);
3251  UT_ASSERT_MSG_P(newcapacity == roundUpCapacity(newcapacity), "Caller should already have adjusted capacity");
3252 
3253  bool wasshared = pages && pages->isShared();
3254  UT_PageNum npages = numPages(newcapacity);
3255  IDX_T oldcapacity = pages ? pages->capacity() : IDX_T(0);
3256  UT_PageNum oldnpages = numPages(oldcapacity);
3257 
3258  // Caller must force capacity to be on a page boundary if more than one page, else power of 2.
3259  UT_ASSERT_P((newcapacity < IDX_T(thePageSize) && SYSisPow2(exint(newcapacity))) || newcapacity == IDX_T(exint(npages)<<thePageBits));
3260 
3261  const bool nonconstfirst = pages && !pages->getFirstPage()->isConstant();
3262 
3263  if (npages == UT_PageNum(1) && oldnpages == UT_PageNum(1) && !wasshared)
3264  {
3265  // Can reuse existing page table if we're staying at one page, but changing its capacity.
3266 
3267  // NOTE: The "newcapacity != oldcapacity" will almost always be true here, but
3268  // we need to make absolutely sure that we don't realloc a shared page when just
3269  // calling hardenTable()! If we didn't have this check, it could happen, because
3270  // between the reference count check in hardenTable() could return 2, and then
3271  // before the pages->isShared() call above, the reference count drops to 1,
3272  // so we have an already hardened table, but we can identify that we don't need
3273  // to do anything because the capacity won't be changing.
3274  // The reason we can't have hardenTable() realloc a shared page is that in
3275  // GA_PrimitiveList, it needs to incDataRef() on all GA_OffsetLists in a shared
3276  // page before ever hardening one.
3277  if (nonconstfirst && newcapacity != oldcapacity)
3278  {
3279  // Just need to realloc the page.
3280  UT_PageOff newpagecapacity = UT_PageOff(exint(newcapacity));
3281  UT_PageOff sizetocopy = SYSmin(UT_PageOff(exint(pages->mySize)), newpagecapacity);
3282  PageTableEntry *page = pages->getFirstPage();
3283  page->realloc(sizetocopy, newpagecapacity, tuplesize);
3284  }
3285  // Don't need to change anything but capacity (and possibly size)
3286  // if constant page, even if it's an allocated constant page.
3287 
3288  pages->myCapacity = newcapacity;
3289  if (pages->mySize > newcapacity)
3290  pages->mySize = newcapacity;
3291  return pages;
3292  }
3293 
3294  PageTable *table = PageTable::alloc(npages);
3295  table->myCapacity = newcapacity;
3296  if (pages)
3297  {
3298  UT_ASSERT_P(pages->getRefCount() > 0 && table->getRefCount() > 0);
3299  UT_ASSERT_P(pages->capacity() > IDX_T(0));
3300  UT_ASSERT_P(oldnpages > UT_PageNum(0));
3301  const bool wassmallfirstpage = pages->capacity() < IDX_T(thePageSize);
3302  const bool willbesmallfirstpage = newcapacity < IDX_T(thePageSize);
3303 
3304  UT_PageNum minnpages = SYSmin(npages, oldnpages);
3305  table->mySize = pages->mySize;
3306  memcpy(table+1, pages+1, (sizeof(PageTableEntry))*minnpages);
3307 
3308  UT_ASSERT_P(!(SYSisSame<DATA_T, void>()));
3309 
3310  const size_t bytesize = sizeof(NotVoidType)*((TSIZE >= 0) ? TSIZE : tuplesize);
3311 
3312  if (wasshared)
3313  {
3314  // Increment all of the reference counts of reused pages,
3315  // if we're unsharing a table.
3316  PageTableEntry *page = table->getFirstPage();
3317  for (UT_PageNum p = 0; p < minnpages; ++p, ++page)
3318  {
3319  if (page->isRefd(bytesize))
3320  page->incRef();
3321  }
3322  pages->decRef(tuplesize);
3323  }
3324  else
3325  {
3326  // pages isn't shared, so we can just decRef() any pages no
3327  // longer referenced and free pages. This avoids calling
3328  // incRef on all kept pages and decRef on all pages like the
3329  // above would do if pages had its reference count go to
3330  // zero.
3331  PageTableEntry *page = pages->getFirstPage() + npages;
3332  for (UT_PageNum p = npages; p < oldnpages; ++p, ++page)
3333  {
3334  if (page->isRefd(bytesize))
3335  page->decRef();
3336  }
3337  UT_ASSERT_MSG_P(!pages->isShared(), "This should still be unshared.");
3338  free(pages);
3339 #if 0
3340  printf("Free table %d pages (case 1); %d tables total\n",int(oldnpages), int(thePageTableCount.add(-1)));
3341  fflush(stdout);
3342 #endif
3343  }
3344  // Can't refer to pages beyond this point!
3345  UT_IF_ASSERT_P(pages = nullptr;)
3346 
3347  if (table->mySize > newcapacity)
3348  table->mySize = newcapacity;
3349 
3350  // NOTE: The "newcapacity != oldcapacity" is to make absolutely
3351  // sure that we don't realloc a shared page when just
3352  // calling hardenTable()! This is because in
3353  // GA_PrimitiveList, it needs to incDataRef() on all
3354  // GA_OffsetLists in a shared page before ever hardening one.
3355  if (nonconstfirst && (wassmallfirstpage || willbesmallfirstpage) && newcapacity != oldcapacity)
3356  {
3357  UT_PageOff newpagecapacity = SYSmin(UT_PageOff(exint(newcapacity)), UT_PageOff(thePageSize));
3358  UT_PageOff sizetocopy = SYSmin(UT_PageOff(exint(table->mySize)), newpagecapacity);
3359  PageTableEntry *page = table->getFirstPage();
3360  page->realloc(sizetocopy, newpagecapacity, tuplesize);
3361  }
3362  }
3363  else
3364  {
3365  oldnpages = 0;
3366  table->mySize = IDX_T(0);
3367  }
3368 
3369  // Set any new pages to be constant zero.
3370  PageTableEntry *page = table->getFirstPage() + oldnpages;
3371  for (UT_PageNum p = oldnpages; p < npages; ++p, ++page)
3372  {
3373  page->initZero();
3374  }
3375 
3376  if (PAGESHARDENED && wasshared)
3377  {
3378  if (TSIZE >= 1)
3379  table->hardenAllPages(IDX_T(0), table->mySize);
3380  else
3381  table->hardenAllPages(IDX_T(0), table->mySize, tuplesize);
3382  }
3383 
3384  return table;
3385  }
3386 
3387  /// Atomically decrement the reference count iff the page is shared.
3388  /// See the description of decRefIffSharedPage() above for
3389  /// why this function exists.
3391  {
3392  return decCounterIffAbove1(myRefCount);
3393  }
3394 
3395  private:
3396  // The constructor/destructor should never be called, since the class
3397  // can only be allocated with malloc.
3398  PageTable() { UT_ASSERT_P(0); }
3399  PageTable(const PageTable &) { UT_ASSERT_P(0); }
3400  ~PageTable() { UT_ASSERT_P(0); }
3401 
3402  SYS_AtomicCounter myRefCount;
3403  IDX_T mySize;
3404  IDX_T myCapacity;
3405  };
3406 
3407  /// NOTE: These have to be below the PageTable class definition, because
3408  /// for no apparent reason, even though you can have a regular local
3409  /// variable of type PageTable before, you can't have a function
3410  /// parameter of type PageTable before. *sigh*
3411  /// @{
3412  static void hardenConstantPage(PageTable *pages, PageTableEntry *page)
3413  {
3414  const exint arraycapacity(exint(pages->capacity()));
3415  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3416  hardenConstantPage(page, pagecapacity);
3417  }
3418  static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page)
3419  {
3420  const exint arraycapacity(exint(pages->capacity()));
3421  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3422  hardenConstantPageNoInit(page, pagecapacity);
3423  }
3424  static void hardenConstantPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
3425  {
3426  const exint arraycapacity(exint(pages->capacity()));
3427  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3428  hardenConstantPage(page, pagecapacity, tuplesize);
3429  }
3430  static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
3431  {
3432  const exint arraycapacity(exint(pages->capacity()));
3433  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3434  hardenConstantPageNoInit(page, pagecapacity, tuplesize);
3435  }
3436  static void hardenSharedPage(PageTable *pages, PageTableEntry *page)
3437  {
3438  const exint arraycapacity(exint(pages->capacity()));
3439  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3440  hardenSharedPage(page, pagecapacity);
3441  }
3442  static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page)
3443  {
3444  const exint arraycapacity(exint(pages->capacity()));
3445  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3446  hardenSharedPageNoInit(page, pagecapacity);
3447  }
3448  static void hardenSharedPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
3449  {
3450  const exint arraycapacity(exint(pages->capacity()));
3451  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3452  hardenSharedPage(page, pagecapacity, tuplesize);
3453  }
3454  static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
3455  {
3456  const exint arraycapacity(exint(pages->capacity()));
3457  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3458  hardenSharedPageNoInit(page, pagecapacity, tuplesize);
3459  }
3460  /// @}
3461 
3462  void forceHardenTable(IDX_T newcapacity)
3463  {
3464  if (SYSisSame<DATA_T,void>())
3465  {
3466  const UT_Storage storage = myImpl.getStorage();
3467  switch (storage)
3468  {
3469  case UT_Storage::INVALID:
3470  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
3471  return;
3472  case UT_Storage::INT8:
3473  castType<int8>().forceHardenTable(newcapacity); return;
3474  case UT_Storage::INT16:
3475  castType<int16>().forceHardenTable(newcapacity); return;
3476  case UT_Storage::INT32:
3477  castType<int32>().forceHardenTable(newcapacity); return;
3478  case UT_Storage::INT64:
3479  castType<int64>().forceHardenTable(newcapacity); return;
3480  case UT_Storage::REAL16:
3481  castType<fpreal16>().forceHardenTable(newcapacity); return;
3482  case UT_Storage::REAL32:
3483  castType<fpreal32>().forceHardenTable(newcapacity); return;
3484  case UT_Storage::REAL64:
3485  castType<fpreal64>().forceHardenTable(newcapacity); return;
3486  }
3487  return;
3488  }
3489 
3490  PageTable *&pages = myImpl.getPages();
3491  pages = PageTable::harden(pages, newcapacity, myImpl.getTupleSize());
3492  }
3493 
3494 protected:
3495  template<typename T>
3497  static bool isZero(const T &val)
3498  {
3499  // TODO: Find a way to check for zero that will still compile, but
3500  // that avoids the mess of dealing with memset for POD types.
3501  T zero;
3502  memset(&zero, 0, sizeof(T));
3503  return (val == zero);
3504  }
3505 
3506  template<typename T>
3508  static bool isZero(const T *val,exint tuplesize)
3509  {
3510  UT_ASSERT_P(tuplesize >= 1);
3511  if (tuplesize == 0)
3512  return true;
3513 
3514  // TODO: Find a way to check for zero that will still compile, but
3515  // that avoids the mess of dealing with memset for POD types.
3516  T zero;
3517  memset(&zero, 0, sizeof(T));
3518 
3519  bool iszero = (val[0] == zero);
3520  for (exint i = 1; i < tuplesize; ++i)
3521  iszero &= (val[i] == zero);
3522  return iszero;
3523  }
3524 
3525  template<typename T0,typename T1>
3527  static bool isEqual(const T0 *a,const T1 *b,exint tuplesize)
3528  {
3529  UT_ASSERT_P(tuplesize >= 1);
3530  UT_ASSERT_P(a != nullptr && b != nullptr);
3531 
3532  for (exint i = 0; i < tuplesize; ++i)
3533  {
3534  if (a[i] != b[i])
3535  return false;
3536  }
3537  return true;
3538  }
3539 
3540  template<typename DEST_TYPE,typename SRC_TYPE,exint SRC_TSIZE,bool SRC_INSTANTIATED>
3543  {
3544  if (SYSisSame<DEST_TYPE,SRC_TYPE>())
3545  return src;
3546 
3548  for (exint i = 0; i < SRC_TSIZE; ++i)
3549  dest[i] = UTconvertStorage<DEST_TYPE>(src[i]);
3550  return dest;
3551  }
3552 
3553 private:
3554  /// This class wraps the data of UT_PageArray, just so that if
3555  /// the tuple size and data type are known at compile-time, it doesn't
3556  /// need to store that information as an int and a UT_Storage.
3557  /// myPages is the same in all of them, but can't be extracted out,
3558  /// because the C++ standard, unfortunately, forbids 0-byte classes.
3559  /// @{
3560  struct ImplPageData
3561  {
3562  PageTable *myPages;
3563  };
3564 
3565  template<exint ITSIZE, typename IDATA_T>
3566  class Impl :
3567  private ImplPageData,
3568  private UT_TupleType<ITSIZE, IDATA_T>
3569  {
3570  public:
3571  SYS_FORCE_INLINE const PageTable *getPages() const { return ImplPageData::myPages; }
3572  SYS_FORCE_INLINE PageTable *&getPages() { return ImplPageData::myPages; }
3577  };
3578 
3579  Impl<TSIZE,DATA_T> myImpl;
3580  /// @}
3581 };
3582 
3583 #endif
void hardenAllPages(IDX_T start, IDX_T end, exint tuplesize=TSIZE)
SYS_FORCE_INLINE NotVoidType operator()(IDX_T i, exint component=0) const
Read element i's specified component (default component 0).
Definition: UT_PageArray.h:441
SYS_FORCE_INLINE constexpr const T * data() const noexcept
static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page)
GLint first
Definition: glcorearb.h:404
static void copyNonConst(NotVoidType *destpagedata, const SrcNotVoidType *srcpagedata, exint desttuplesize, exint srctuplesize, UT_PageOff ntuples)
SYS_FORCE_INLINE bool isPageConstant(UT_PageNum pagenum) const
Returns true iff the specified page is constant-compressed.
static void hardenConstantPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
SYS_FORCE_INLINE void opVector(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE > &v)
Definition: UT_PageArray.h:726
static SYS_FORCE_INLINE const NotVoidType * getConstantPtr(const PageTableEntry *page, exint component=0, exint tuplesize=TSIZE)
void SecondGuess
Definition: UT_Storage.h:74
SYS_FORCE_INLINE void setSize(IDX_T newsize)
static SYS_FORCE_INLINE bool decCounterIffAbove1(SYS_AtomicCounter &counter)
SYS_FORCE_INLINE const NotVoidType * getInlinePtr(exint tuplesize) const
Returns the data pointer, if an inline constant page.
SYS_FORCE_INLINE void * getFirstPtrVoid()
Returns the data pointer, if not a constant page.
UT_Storage
Definition: UT_Storage.h:26
static void hardenSharedPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
void hardenAllPagesUnknownType(IDX_T start, IDX_T end)
void tryCompressPage(UT_PageOff pagesize, exint tuplesize=TSIZE)
static ThisType * harden(ThisType *pages, IDX_T newcapacity, exint tuplesize=TSIZE)
static void hardenSharedPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
SYS_FORCE_INLINE bool isArrayZero() const
Returns true iff the entire array is constant & zero.
static const UT_Storage theStorage
Definition: UT_PageArray.h:245
static SYS_FORCE_INLINE NotVoidType getConstant(const PageTableEntry *const page, exint component, exint tuplesize=TSIZE)
NotVoid< DATA_T >::type NotVoidType
Definition: UT_PageArray.h:237
UT_PageArray(const ThisType &that)
Definition: UT_PageArray.h:303
void setTupleSize(exint newtuplesize, const UT_Defaults &v)
static ThisType * alloc(UT_PageNum npages)
void tryCompressPage(UT_PageNum pagenum, exint tuplesize=TSIZE)
SYS_FORCE_INLINE const NotVoidType * getMaskedPtr() const
void copy(IDX_T dest, IDX_T src)
Copies a single element from src to dest within this.
void tryCompressAllPagesUnknownType(IDX_T start, IDX_T end)
const GLdouble * v
Definition: glcorearb.h:836
SYS_FORCE_INLINE ~UT_PageArray()
Definition: UT_PageArray.h:317
SYS_FORCE_INLINE bool operator!=(const ThisType &that) const
NOTE: This just does a shallow comparison.
SYS_FORCE_INLINE bool isZero() const noexcept
void tryCompressAllPages(IDX_T start, IDX_T end, exint tuplesize=TSIZE)
GLuint start
Definition: glcorearb.h:474
SYS_FORCE_INLINE void clear()
Definition: UT_PageArray.h:323
SYS_FORCE_INLINE ThisType & operator=(const UT_PageArray< SRC_DATA_T, SRC_TSIZE, SRC_TABLEHARDENED, SRC_PAGESHARDENED, THEPAGEBITS, IDX_T > &that)
Definition: UT_PageArray.h:379
static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
void opVectorUnknownType(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE > &v)
SYS_FORCE_INLINE IDX_T size() const
Definition: UT_PageArray.h:859
SYS_FORCE_INLINE T * SYSconst_cast(const T *foo)
Definition: SYS_Types.h:126
SYS_FORCE_INLINE void add(IDX_T i, SRC_DATA_T v)
component == 0 in this version
Definition: UT_PageArray.h:611
UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVectorUnknownSize(IDX_T i) const
SYS_FORCE_INLINE bool isShared() const
void countMemory(UT_MemoryCounter &counter, bool inclusive) const
bool iszero(T a, T t)
Definition: ImathFun.h:135
SYS_FORCE_INLINE PageTableEntry * getFirstPage()
GLboolean GLboolean GLboolean GLboolean a
Definition: glcorearb.h:1221
static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page)
void setConstant(IDX_T start, IDX_T end, NotVoidType v)
void opUnknownType(IDX_T i, exint component, SRC_DATA_T v)
exint UT_PageNum
Definition: UT_PageArray.h:36
SYS_FORCE_INLINE const PageTableEntry * getPPage(UT_PageNum i) const
static void hardenConstantPage(PageTable *pages, PageTableEntry *page)
SYS_FORCE_INLINE UT_PageArray< DATA_T, TSIZE, true, true, THEPAGEBITS, IDX_T > & hardenAllPages(IDX_T start=IDX_T(0), IDX_T end=IDX_T(-1))
static const exint theTupleSize
Definition: UT_PageArray.h:246
SYS_FORCE_INLINE UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T >::PageTableEntry * castType()
SYS_FORCE_INLINE exint getTupleSize() const
DEST_DATA_T getUnknownType(IDX_T i, exint component=0) const
SYS_SelectType< T, float, SYS_IsSame< T, void >::value >::type type
Definition: UT_PageArray.h:235
SYS_FORCE_INLINE IDX_T size() const
NOTE: This is the size of the full array, not the number of pages.
SYS_FORCE_INLINE UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVector(IDX_T i) const
Definition: UT_PageArray.h:494
SYS_FORCE_INLINE void set(IDX_T i, SRC_DATA_T v)
component == 0 in this version
Definition: UT_PageArray.h:605
SYS_FORCE_INLINE UT_Storage getStorage() const
static SYS_FORCE_INLINE IDX_T roundUpCapacity(IDX_T capacity)
Definition: UT_PageArray.h:880
SYS_FORCE_INLINE bool isTableHardened() const
Returns true iff the table isn't shared with any other UT_PageArray's.
SYS_FORCE_INLINE exint getRefCount() const
png_uint_32 i
Definition: png.h:2877
SYS_FORCE_INLINE UT_PageArray< DATA_T, DEST_TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castTupleSize()
bool hasNanInRange(IDX_T start, IDX_T end) const
static SYS_FORCE_INLINE UT_FixedVector< DEST_TYPE, SRC_TSIZE > convertVectorStorage(const UT_FixedVector< SRC_TYPE, SRC_TSIZE, SRC_INSTANTIATED > &src)
UT_PageArray(const UT_PageArray< SRC_DATA_T, SRC_TSIZE, SRC_TABLEHARDENED, SRC_PAGESHARDENED, THEPAGEBITS, IDX_T > &that)
Definition: UT_PageArray.h:310
GLsizeiptr size
Definition: glcorearb.h:663
UT_PageArray(exint tuplesize, UT_Storage storage)
Definition: UT_PageArray.h:287
SYS_FORCE_INLINE void addVector(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE, SRC_INSTANTIATED > &v)
Definition: UT_PageArray.h:717
void tryCompressPageUnknownType(UT_PageNum pagenum)
#define UT_ASSERT_P(ZZ)
Definition: UT_Assert.h:101
void decRef(exint tuplesize)
SYS_FORCE_INLINE bool isShared() const
#define UT_IF_ASSERT_P(ZZ)
Definition: UT_Assert.h:128
void fill(IDX_T start, IDX_T end, const NotVoidType &val)
static bool isEqualConst(const NotVoidType *tuple, const SrcNotVoidType *stuple, exint mintuplesize)
signed char int8
Definition: SYS_Types.h:30
SYS_FORCE_INLINE void setConstantBit()
UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVectorMismatchedSize(IDX_T i) const
static SYS_FORCE_INLINE void call(FUNCTOR functor)
long long int64
Definition: SYS_Types.h:106
UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVectorUnknownType(IDX_T i) const
SYS_FORCE_INLINE exint getRefCount() const
SYS_FORCE_INLINE bool isConstantZero() const
This is only valid to call if the type doesn't fit inline.
SYS_FORCE_INLINE void setSize(IDX_T newsize, exint tuplesize)
static SYS_FORCE_INLINE UT_PageOff pageOff(IDX_T i)
UT_PageArray(UT_Storage storage)
Definition: UT_PageArray.h:276
SYS_FORCE_INLINE void tryCompressPage(UT_PageNum pagenum)
SYS_FORCE_INLINE ThisType & operator=(const ThisType &that)
Definition: UT_PageArray.h:371
static SYS_FORCE_INLINE bool typeFitsInline(exint tuplesize=TSIZE)
This is only valid to call when DATA_T is non-void.
SYS_FORCE_INLINE const PageTableEntry * getFirstPage() const
SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const UT_FixedVector< SRC_DATA_T, theSafeTupleSize > &val)
static SYS_FORCE_INLINE UT_PageNum numPages(IDX_T nelements)
static SYS_FORCE_INLINE bool isEqual(const T0 *a, const T1 *b, exint tuplesize)
int64 exint
Definition: SYS_Types.h:115
SYS_FORCE_INLINE void getRange(IDX_T srcstart, IDX_T nelements, T *dest) const
SYS_FORCE_INLINE bool isRefd(exint tuplebytes) const
SYS_FORCE_INLINE bool operator==(const ThisType &that) const
NOTE: This just does a shallow comparison.
SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const NotVoidType &val)
double fpreal64
Definition: SYS_Types.h:191
SYS_FORCE_INLINE IDX_T capacity() const
NOTE: This is the capacity of the full array, not the capacity of pages.
static SYS_FORCE_INLINE bool isZero(const T &val)
SYS_FORCE_INLINE IDX_T capacity() const
Definition: UT_PageArray.h:867
GLuint GLuint end
Definition: glcorearb.h:474
#define SYS_FORCE_INLINE
Definition: SYS_Inline.h:45
T compare_swap(T expected, T desired)
SYS_FORCE_INLINE NotVoidType * hardenPage(UT_PageNum pagenum)
UT_FixedVector< NotVoidType, theSafeTupleSize > Tuple
DATA_T DataType
Definition: UT_PageArray.h:244
SYS_FORCE_INLINE void op(IDX_T i, exint component, SRC_DATA_T v)
Definition: UT_PageArray.h:631
SYS_FORCE_INLINE NotVoidType * getInlinePtr(exint tuplesize)
Returns the data pointer, if an inline constant page.
void setCapacity(IDX_T newcapacity)
Definition: UT_PageArray.h:895
void clearAndDestroy()
Definition: UT_PageArray.h:329
SYS_FORCE_INLINE void setVector(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE, SRC_INSTANTIATED > &v)
Definition: UT_PageArray.h:712
GLenum GLsizei GLsizei GLint * values
Definition: glcorearb.h:1601
SYS_FORCE_INLINE void * getMaskedPtrVoid()
static SYS_FORCE_INLINE UT_PageOff pageStart(IDX_T page)
UT_PageArray(exint tuplesize)
Definition: UT_PageArray.h:264
SYS_FORCE_INLINE const void * getFirstPtrVoidUnsafe() const
GLboolean * data
Definition: glcorearb.h:130
static void copyPartialPage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destoff, UT_PageOff srcoff, UT_PageOff ntuples, UT_PageOff destpagecapacity)
SYS_FORCE_INLINE bool isConstantAndZeroSafe() const
void setPageConstantUnknownType(UT_PageNum pagenum, const UT_FixedVector< SRC_DATA_T, theSafeTupleSize > &val)
int int32
Definition: SYS_Types.h:34
SYS_FORCE_INLINE bool decRefIffShared(exint tuplesize=TSIZE)
static void makeConstant(PageTableEntry *page, const NotVoidType *values, exint tuplesize)
SYS_FORCE_INLINE NotVoidType * hardenPageNoInit(UT_PageNum pagenum)
GLboolean GLboolean GLboolean b
Definition: glcorearb.h:1221
SYS_FORCE_INLINE void tryCompressAllPages(IDX_T start=IDX_T(0), IDX_T end=IDX_T(-1))
static const exint thePageBits
Definition: UT_PageArray.h:248
SYS_FORCE_INLINE void add(IDX_T i, exint component, SRC_DATA_T v)
Definition: UT_PageArray.h:626
const NotVoidType * getPageData(UT_PageNum pagenum) const
exint UT_PageOff
Definition: UT_PageArray.h:37
static const exint theSafeTupleSize
Definition: UT_PageArray.h:247
SYS_FORCE_INLINE void incRef()
void realloc(UT_PageOff sizetocopy, UT_PageOff newpagecapacity, exint tuplesize=TSIZE)
static void makeConstantFrom(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize)
PageTableEntry ThisType
SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const SRC_DATA_T *values)
void moveRange(IDX_T srcstart, IDX_T deststart, IDX_T nelements)
static const exint thePageMask
Definition: UT_PageArray.h:250
void getVectorRange(IDX_T srcstart, IDX_T nelements, UT_FixedVector< DEST_DATA_T, DEST_TSIZE, DEST_INSTANTIATED > *dest) const
static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
static const uintptr_t theConstantPageBit
SYS_FORCE_INLINE int64 getMemoryUsage(exint tuplebytes) const
SYS_AtomicInt< int32 > SYS_AtomicCounter
Definition: SYS_AtomicInt.h:86
SYS_FORCE_INLINE const UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castType() const
void setSize(IDX_T newsize)
Definition: UT_PageArray.h:919
#define UT_ASSERT_MSG_P(ZZ, MM)
Definition: UT_Assert.h:104
SYS_FORCE_INLINE const void * getMaskedPtrVoid() const
SYS_FORCE_INLINE const UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T >::PageTableEntry * castType() const
void swapRange(IDX_T astart, IDX_T bstart, IDX_T nelements)
SYS_FORCE_INLINE void * getFirstPtrVoidUnsafe()
bool decRefIffSharedPage(UT_PageNum pagenum)
void setPageConstantUnknownType(UT_PageNum pagenum, const SRC_DATA_T *values)
SYS_FORCE_INLINE PageTableEntry * getPPage(UT_PageNum i)
void setStorage(const UT_Storage newstorage)
SYS_FORCE_INLINE void set(IDX_T i, exint component, SRC_DATA_T v)
Definition: UT_PageArray.h:621
static void hardenConstantPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
static void fillNonConstWithConst(NotVoidType *destpagedata, NotVoidType *destpageend, const SrcNotVoidType *stuple, exint mintuplesize, exint desttupleextra)
SYS_FORCE_INLINE NotVoidType * getMaskedPtr()
bool decRefIffSharedTable()
SYS_FORCE_INLINE const UT_PageArray< DATA_T, DEST_TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castTupleSize() const
SYS_FORCE_INLINE bool isConstant() const
This is always valid to call.
SYS_FORCE_INLINE void setRange(IDX_T deststart, IDX_T nelements, const T *src)
static void hardenSharedPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
SYS_FORCE_INLINE const void * getFirstPtrVoid() const
Returns the data pointer, if not a constant page.
GLuint GLfloat * val
Definition: glcorearb.h:1607
static SYS_FORCE_INLINE NotVoidType * getConstantPtr(PageTableEntry *page, exint component=0, exint tuplesize=TSIZE)
static SYS_FORCE_INLINE void makeConstant(PageTableEntry *page, const UT_FixedVector< NotVoidType, theSafeTupleSize > &val)
bool equal(T1 a, T2 b, T3 t)
Definition: ImathFun.h:143
static void makeConstant(PageTableEntry *page, const NotVoidType &val, exint tuplesize=TSIZE)
SYS_FORCE_INLINE UT_PageArray< DATA_T, TSIZE, true, PAGESHARDENED, THEPAGEBITS, IDX_T > & hardenTable()
static const exint thePageSize
Definition: UT_PageArray.h:249
static SYS_FORCE_INLINE bool isZero(const T *val, exint tuplesize)
SYS_FORCE_INLINE void alloc(UT_PageOff nelements, exint tuplesize=TSIZE)
GLint GLint GLsizei GLint GLenum GLenum type
Definition: glcorearb.h:107
T add(T val)
Atomically adds val to myValue, returning the new value of myValue.
static void hardenConstantPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
getOption("OpenEXR.storage") storage
Definition: HDK_Image.dox:276
Container class for all geometry.
Definition: GA_Detail.h:96
int64 getMemoryUsage(bool inclusive) const
static SYS_FORCE_INLINE UT_PageNum pageNum(IDX_T i)
UT_PageArray< DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > ThisType
Definition: UT_PageArray.h:243
static void hardenSharedPage(PageTable *pages, PageTableEntry *page)
void setCapacityIfNeeded(IDX_T newcapacity)
Definition: UT_PageArray.h:908
void forceHardenTable(IDX_T newcapacity)
SYS_FORCE_INLINE int64 getGuaranteedInt(IDX_T i) const
SYS_FORCE_INLINE void relaxedStore(T val)
Definition: SYS_AtomicInt.h:68
SYS_FORCE_INLINE NotVoidType * unshareConstantPage(UT_PageNum pagenum)
short int16
Definition: SYS_Types.h:32
#define SYSmin(a, b)
Definition: SYS_Math.h:1366
SYS_FORCE_INLINE bool isPageHard(UT_PageNum pagenum) const
UT_PageArray< DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > ParentType
SYS_FORCE_INLINE constexpr int UTstorageSize(UT_Storage storage)
Returns the number of bytes in the given storage type.
Definition: UT_Storage.h:54
SYS_FORCE_INLINE void initZero()
SYS_FORCE_INLINE const NotVoidType * getFirstPtr() const
Returns the data pointer, if not a constant page.
#define UT_ASSERT_MSG(ZZ, MM)
Definition: UT_Assert.h:105
float fpreal32
Definition: SYS_Types.h:190
SYS_FORCE_INLINE NotVoidType * getFirstPtr()
Returns the data pointer, if not a constant page.
void setPageConstantMismatchedType(UT_PageNum pagenum, const SRC_DATA_T *values)
void replace(const UT_PageArray< SRC_DATA_T, SRC_TSIZE, SRC_TABLEHARDENED, SRC_PAGESHARDENED, THEPAGEBITS, IDX_T > &that)
Definition: UT_PageArray.h:388
SYS_FORCE_INLINE NotVoidType & operator()(IDX_T i, exint component=0)
Definition: UT_PageArray.h:562
SYS_FORCE_INLINE T relaxedLoad() const
Definition: SYS_AtomicInt.h:61
NotVoidType * getPageData(UT_PageNum pagenum)
void setVectorRange(IDX_T deststart, IDX_T nelements, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE, SRC_INSTANTIATED > *src)
SYS_FORCE_INLINE void decRef()
static void replacePage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destpagesize, UT_PageOff destpagecapacity)
SYS_FORCE_INLINE UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castType()
GLenum src
Definition: glcorearb.h:1792