HDK
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
UT_PageArray.h
Go to the documentation of this file.
1 /*
2  * PROPRIETARY INFORMATION. This software is proprietary to
3  * Side Effects Software Inc., and is not to be reproduced,
4  * transmitted, or disclosed in any way without written permission.
5  *
6  * NAME: UT_PageArray.h (UT Library, C++)
7  *
8  * COMMENTS: An array class with special handling of constant pages and
9  * shared page data.
10  */
11 
12 #pragma once
13 
14 #ifndef __UT_PageArray__
15 #define __UT_PageArray__
16 
17 #include "UT_API.h"
18 #include "UT_Assert.h"
19 #include "UT_FixedVector.h"
20 #include "UT_SmallArray.h"
21 #include "UT_Storage.h"
22 #include <SYS/SYS_AtomicInt.h>
23 #include <SYS/SYS_BitUtil.h>
24 #include <SYS/SYS_Compiler.h>
25 #include <SYS/SYS_Inline.h>
26 #include <SYS/SYS_Math.h>
27 #include <SYS/SYS_Types.h>
28 #include <SYS/SYS_TypeTraits.h>
29 #include <stdint.h>
30 #include <stdlib.h>
31 #include <string.h>
32 
33 
34 typedef exint UT_PageNum;
35 typedef exint UT_PageOff;
36 
37 
38 class UT_Defaults;
39 class UT_MemoryCounter;
40 
41 
42 /// @internal
43 /// W E L C O M E
44 /// to scenic
45 /// Template Hell
46 ///
47 /// Population: You
48 ///
49 /// If you're just using this class, hopefully you'll have a short and
50 /// not-so-painful visit!
51 ///
52 /// If, however, you're attempting to modify this class, BEWARE!
53 /// Any missteps can and likely will result in application of the
54 /// Doughnut Rules, so be sure to review them before visiting.
55 /// It may take several days to escape, so it may be best to just
56 /// revert and make absolutely sure that it works on all platforms
57 /// before committing again.
58 ///
59 /// This class is the result of a wide variety of needs for a wide variety
60 /// of use cases, most of which need both good performance and flexibility.
61 /// You may want to skip reading this gigantic comment, but whether it's
62 /// immediately or in a few months, you'll realize that skipping reading
63 /// this was a huge mistake.
64 ///
65 /// This array consists of a pointer to a page table, optionally a
66 /// tuple size (if TSIZE==-1), and optionally a UT_Storage indicating
67 /// the storage type (if DATA_T is void). The page table, if not nullptr,
68 /// can be shared, and has the format: @code
69 /// +-------------------+
70 /// | ref count | (1 if unshared; >1 if shared)
71 /// +-------------------+
72 /// | array size | (number of real elements/tuples in the array)
73 /// +-------------------+
74 /// | array capacity | (number of allocated elements/tuples if all pages of the table were hardened)
75 /// +-------------------+
76 /// | page 0 pointer | (pointer to page if bit 0 is 0, else constant page representation)
77 /// +-------------------+
78 /// | page 1 pointer |
79 /// +-------------------+
80 /// | page 2 pointer |
81 /// +-------------------+
82 /// | ... |
83 /// +-------------------+
84 /// @endcode
85 ///
86 /// If a page pointer has bit 0 clear, (i.e. an address aligned to a 2-byte boundary),
87 /// it is a non-constant page, it can be shared, and has the format: @code
88 /// +-------------------+
89 /// | ref count | (1 if unshared; >1 if shared)
90 /// +-------------------+
91 /// | (unused) | (padding so that data starts on a 16-byte boundary)
92 /// +-------------------+
93 /// | tuple 0 |
94 /// +-------------------+
95 /// | tuple 1 |
96 /// +-------------------+
97 /// | tuple 2 |
98 /// +-------------------+
99 /// | ... |
100 /// +-------------------+
101 /// @endcode
102 ///
103 /// A normal capacity page contains 2^THEPAGEBITS tuples, e.g. the default
104 /// THEPAGEBITS==10 gives a page size of 1024 tuples. If there is exactly
105 /// one page, it may have a capacity that is any power of two up to
106 /// THEPAGEBITS, which can be checked by checking if the overall array has
107 /// capacity less than 2^THEPAGEBITS.
108 ///
109 /// If, instead, the page pointer has bit 0 set to 1, which would be an odd
110 /// address, all tuples in the page are the same. There are 3 ways this may
111 /// be represented:
112 ///
113 /// 1) If the size of a full tuple is strictly smaller than the size of a
114 /// pointer, (e.g. 7 bytes or fewer on a 64-bit platform), the constant value
115 /// will be stored inside the space that would have been a pointer, aligned
116 /// to the end of the pointer. That ensures that 4-byte tuples will be
117 /// aligned to 4 bytes, and 2- or 6-byte tuples will be aligned to 2 bytes.
118 /// This is most useful for fpreal32[1] and int32[1] (i.e. 4-byte scalars).
119 ///
120 /// 2) If the page pointer is equal to 1 (i.e. nullptr apart from bit 0 being 1),
121 /// the value is all zero. This covers the extremely
122 /// common case of a zero default value, especially for fpreal32[3] and
123 /// fpreal64[3].
124 ///
125 /// 3) Otherwise, the page pointer, ignoring bit zero, points to a page
126 /// that contains just a single tuple.
127 ///
128 /// That covers the basic structure. It sounds so nice and simple...
129 /// but then there are the templates.
130 ///
131 /// DATA_T:
132 ///
133 /// If DATA_T is something other than void, that type is the type of each
134 /// component of each tuple. This is the only way that non-numeric types
135 /// are possible; DATA_T can also be a numeric type. If DATA_T is void,
136 /// myImpl contains a UT_Storage variable indicating what the data type is;
137 /// it must be a numeric type, i.e. it cannot be UT_Storage::INVALID.
138 /// The ...UnknownType functions switch on the UT_Storage to determine the
139 /// data type and call the corresponding function. If DATA_T is non-void,
140 /// it saves a lot of extra work at runtime. Because the codepaths that
141 /// will never be called due to DATA_T checks still need to compile with
142 /// DATA_T being void, NotVoidType ensures that there is some arbitrary type
143 /// in place so that the code will compile, even though it will never be
144 /// called. The same issue prevents using UT_ASSERT_COMPILETIME to verify
145 /// DATA_T, so UT_ASSERT_P is used instead.
146 /// myImpl is always structured so that if the type becomes known, by checking
147 /// getStorage(), this can be cast to have the now-known DATA_T set.
148 /// For example: @code
149 /// if (array.getStorage() == UT_Storage::REAL32)
150 /// array.castType<fpreal32>().someFunctionCall();
151 /// @endcode
152 ///
153 /// TSIZE:
154 ///
155 /// If TSIZE >= 1, it is the tuple size (the number of components in each
156 /// tuple), and is known at compile time, avoiding extra work at compile-time
157 /// that depends on the tuple size, especially if TSIZE is 1. Odds are that
158 /// for non-numeric types, TSIZE will almost always be 1, since it rarely
159 /// makes sense to be something else. If TSIZE is 0, something is almost
160 /// certainly wrong. If the tuple size is not known at compile-time, TSIZE
161 /// must be -1. In that case, myImpl contains an integer indicating the tuple
162 /// size. Like with DATA_T, because the codepaths that will never be called
163 /// due to TSIZE checks still need to compile with other TSIZE values, so
164 /// UT_ASSERT_COMPILETIME can't be used to verify TSIZE; UT_ASSERT_P is used
165 /// instead. There are a number of functions that require TSIZE to be the
166 /// tuple size, in order to avoid having to send an extra parameter and
167 /// incurring the slowdowns of their counterparts that assume that the tuple
168 /// size is what's passed in. If you modify one, REMEMBER TO MODIFY THE OTHER!
169 /// myImpl is always structured so that if the tuple size becomes
170 /// known, by checking the getTupleSize(), this can be cast to have the
171 /// now-known TSIZE set.
172 /// For example: @code
173 /// if (array.getTupleSize() == 3)
174 /// array.castTupleSize<3>().someFunctionCall();
175 /// @endcode
176 ///
177 /// TABLEHARDENED:
178 ///
179 /// Since page tables can be shared if all of the data are identical, modifying
180 /// the array in any way requires copying (hardening) the page table and may
181 /// require copying one or more shared pages or hardening constant pages.
182 /// Since every modification requires hardening the page table, it's useful
183 /// to harden the page table once and be able to skip checking whether it needs
184 /// hardening for all modifications after that. Calling hardenTable()
185 /// will ensure that the table is hardened, and return this with TABLEHARDENED
186 /// set to true, so that further calls on it to modify the array won't need to
187 /// check whether the table needs hardening. The page table should be
188 /// quite light compared to the data, so it's not the end of the world if
189 /// the table is hardened pre-emptively and there end up being no actual
190 /// modifications.
191 ///
192 /// PAGESHARDENED:
193 ///
194 /// When writing data in parallel to random locations, all pages that could
195 /// be written-to must be hardened in advance, to avoid a race condition on
196 /// the hardening of any page. Also, if it's known that most pages will
197 /// likely be hardened anyway when writing data, it'd be advantageous to
198 /// be able to harden all pages in advance and skip checking when making
199 /// modifications. The code for hardening pages just-in-time can
200 /// dramatically increase code size and time overhead. To avoid this,
201 /// hardenAllPages() will ensure that the table and all pages are hardened,
202 /// and will return this with TABLEHARDENED and PAGESHARDENED both set to true.
203 /// Any calls to modify page data made on that returned pointer won't check
204 /// whether any pages need hardening, because it will know that all pages are
205 /// already hardened. If you use this, be sure not to use the pointer
206 /// with PAGESHARDENED set to true after any call to tryCompressAllPages()
207 /// on the original array pointer.
208 ///
209 /// THEPAGEBITS:
210 ///
211 /// Full pages will contain 2^THEPAGEBITS tuples, e.g. THEPAGEBITS==10
212 /// means there will be 1024 tuples per full page.
213 ///
214 /// IDX_T:
215 ///
216 /// This allows templating on the array index type so that strict types
217 /// can be used, e.g. GA_Offset.
218 ///
219 /// FIXME BEFORE USING:
220 /// - Should make generic UT_R[OW]Handle.+ classes for easy accessing
221 /// of runtime-determined data type and tuple size data as tuples
222 /// with type and tuple size determined at compile-time.
223 /// - Make UT_PageNum and UT_PageOff strict types when strict types are
224 /// enabled.
225 /// - Add more functions for page-level access and block (bulk sequential)
226 /// access, for easy reading/writing to/from flat arrays.
227 /// @endinternal
228 template<typename DATA_T,exint TSIZE=1,bool TABLEHARDENED=false,bool PAGESHARDENED=false,exint THEPAGEBITS=10,typename IDX_T=exint>
230 {
231 protected:
232  /// This class is just to get code to compile when DATA_T is void,
233  /// by picking any type other than void if it is void.
234  template<typename T>
235  struct NotVoid
237 
239 
240  // These friend declarations are needed for calling protected functions on hardened/cast types.
241  template<typename DATA_T2,exint TSIZE2,bool TABLEHARDENED2,bool PAGESHARDENED2,exint THEPAGEBITS2,typename IDX_T2>
242  friend class UT_PageArray;
243 public:
245  typedef DATA_T DataType;
247  static const exint theTupleSize = TSIZE;
248  static const exint theSafeTupleSize = (TSIZE==-1) ? 3 : TSIZE;
249  static const exint thePageBits = THEPAGEBITS;
250  static const exint thePageSize = (exint(1)<<thePageBits);
251  static const exint thePageMask = thePageSize-1;
252  typedef IDX_T IndexType;
253 
254  /// The default constructor can only be used if the tuple size
255  /// and storage type are known at compile time.
257  {
258  UT_ASSERT_P(TSIZE >= 1);
260  myImpl.getPages() = nullptr;
261  }
262 
263  /// This constructor can only be used if the storage type is
264  /// known at compile time, but the tuple size is not.
265  UT_PageArray(exint tuplesize)
266  {
267  UT_ASSERT_P(TSIZE == -1 || TSIZE == tuplesize);
269  // NOTE: This is one case where a tuple size of zero is okay.
270  UT_ASSERT_P(tuplesize >= 0);
271  myImpl.getPages() = nullptr;
272  myImpl.setTupleSize(tuplesize);
273  }
274 
275  /// This constructor can only be used if the storage type is
276  /// known at compile time, but the tuple size is not.
278  {
279  UT_ASSERT_P(TSIZE >= 1);
281  UT_ASSERT_P(storage != UT_Storage::INVALID);
282  myImpl.getPages() = nullptr;
283  myImpl.setStorage(storage);
284  }
285 
286  /// This constructor can only be used if the tuple size and
287  /// storage type are unknown at compile time.
289  {
290  UT_ASSERT_P(TSIZE == -1 || TSIZE == tuplesize);
292  // NOTE: This is one case where a tuple size of zero is okay.
293  UT_ASSERT_P(tuplesize >= 0);
294  UT_ASSERT_P(storage != UT_Storage::INVALID);
295  myImpl.getPages() = nullptr;
296  myImpl.setStorage(storage);
297  myImpl.setTupleSize(tuplesize);
298  }
299 
300  /// This just references that's page table, unless TABLEHARDENED
301  /// or PAGESHARDENED. If TABLEHARDENED and !PAGESHARDENED,
302  /// the pages are referenced, and if PAGESHARDENED, the pages
303  /// are copied.
304  UT_PageArray(const ThisType &that)
305  {
306  myImpl.getPages() = nullptr;
307  *this = that;
308  }
309 
310  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
312  {
313  myImpl.getPages() = nullptr;
314  *this = that;
315  }
316 
319  {
320  clearAndDestroy();
321  }
322 
324  void clear()
325  {
326  if (myImpl.getPages())
327  myImpl.getPages()->setSize(IDX_T(0));
328  }
329 
331  {
332  if (!myImpl.getPages())
333  return;
334 
335  const exint tuplesize = myImpl.getTupleSize();
336 
337  // Just decrement the reference count to free the table safely
338  if constexpr (!SYSisSame<DATA_T,void>())
339  myImpl.getPages()->decRef(tuplesize);
340  else
341  {
342  const UT_Storage storage = myImpl.getStorage();
343  switch (storage)
344  {
345  case UT_Storage::INVALID:
346  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
347  return;
348  case UT_Storage::INT8:
349  castType<int8>().myImpl.getPages()->decRef(tuplesize); return;
350  case UT_Storage::UINT8:
351  castType<uint8>().myImpl.getPages()->decRef(tuplesize); return;
352  case UT_Storage::INT16:
353  castType<int16>().myImpl.getPages()->decRef(tuplesize); return;
354  case UT_Storage::INT32:
355  castType<int32>().myImpl.getPages()->decRef(tuplesize); return;
356  case UT_Storage::INT64:
357  castType<int64>().myImpl.getPages()->decRef(tuplesize); return;
358  case UT_Storage::REAL16:
359  castType<fpreal16>().myImpl.getPages()->decRef(tuplesize); return;
360  case UT_Storage::REAL32:
361  castType<fpreal32>().myImpl.getPages()->decRef(tuplesize); return;
362  case UT_Storage::REAL64:
363  castType<fpreal64>().myImpl.getPages()->decRef(tuplesize); return;
364  }
365  }
366  myImpl.getPages() = nullptr;
367  }
368 
369  /// NOTE: Compilers seem not to use the operator= below when the template
370  /// arguments match, so this operator= is necessary in order to
371  /// avoid the compiler silently using its implicitly-defined
372  /// assignment operator.
375  {
376  replace(that);
377  return *this;
378  }
379 
380  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
383  {
384  replace(that);
385  return *this;
386  }
387 
388  /// Completely replace this UT_PageArray with that, referencing
389  /// the pages or table, if allowed by the template arguments.
390  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
392  {
393  // NOTE: Careful if you add template parameter checks on this,
394  // because, e.g., a UT_PageArray<void,-1> could refer to the
395  // same exact array as a UT_PageArray<float 3>.
396  if ((void *)this == (const void *)&that)
397  return;
398  // NOTE: The storage and tuple size checks are so that if both
399  // tables are nullptr, it still copies the storage and tuple
400  // size below.
401  if (!TABLEHARDENED &&
402  myImpl.getStorage() == that.myImpl.getStorage() &&
403  myImpl.getTupleSize() == that.myImpl.getTupleSize() &&
404  (void *)myImpl.getPages() == (const void *)that.myImpl.getPages())
405  return;
406 
407  clearAndDestroy();
408 
409  if constexpr (SYSisSame<DATA_T,void>())
410  myImpl.setStorage(that.myImpl.getStorage());
411  if constexpr (TSIZE == -1)
412  myImpl.setTupleSize(that.myImpl.getTupleSize());
413 
414  // May be able to share pages or table if matching tuple size and storage type
415  if constexpr ((SYSisSame<DATA_T,void>() || SYSisSame<DATA_T,SRC_DATA_T>()) && (TSIZE == -1 || TSIZE == SRC_TSIZE))
416  {
417  PageTable *pages = reinterpret_cast<PageTable *>(SYSconst_cast(that.myImpl.getPages()));
418  myImpl.getPages() = pages;
419  if (!pages)
420  return;
421  pages->incRef();
422  if constexpr (TABLEHARDENED || PAGESHARDENED || SRC_TABLEHARDENED || SRC_PAGESHARDENED)
423  {
424  forceHardenTable(pages->capacity());
425  if (PAGESHARDENED || SRC_PAGESHARDENED)
426  {
427  // If we don't cast to PAGESHARDENED==false, it won't do anything.
429  ->template hardenAllPages<false>();
430  }
431  }
432  }
433  else
434  {
435  // Can't share table or pages, so just use moveRange
436  IDX_T capacity = that.capacity();
437  IDX_T size = that.size();
438  forceHardenTable(capacity);
439  moveRange(that, IDX_T(0), IDX_T(0), size);
440  }
441  }
442 
443  /// Read element i's specified component (default component 0).
444  SYS_FORCE_INLINE NotVoidType operator()(IDX_T i,exint component=0) const
445  {
446  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
447  UT_ASSERT_P(TSIZE != 0);
448  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
449  UT_ASSERT_P(i >= IDX_T(0) && i < size());
450  UT_ASSERT_P(component >= 0 && component < myImpl.getTupleSize());
451 
452  UT_PageNum pagenum = pageNum(i);
453  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
454 
455  // If we know that the pages are hardened, we can omit
456  // the constant page case at compile time.
457  bool isconst = !PAGESHARDENED && page->isConstant();
458 
459  if (TSIZE >= 1)
460  {
461  if (!isconst)
462  {
463  UT_PageOff pageoff = pageOff(i);
464  return page->getFirstPtr()[TSIZE*pageoff + component];
465  }
466  return getConstant(page, component);
467  }
468  else
469  {
470  if (!isconst)
471  {
472  UT_PageOff pageoff = pageOff(i);
473  return page->getFirstPtr()[myImpl.getTupleSize()*pageoff + component];
474  }
475  return getConstant(page, component, myImpl.getTupleSize());
476  }
477  }
478 
479  /// This get function may seem redundant, given operator(), but it gives a
480  /// simple way to read from a non-const UT_PageArray without calling the
481  /// non-const operator() and potentially hardening pages. It just
482  /// explicitly adds const. It also allows data type conversion and
483  /// reading when the type is unknown at compile time.
484  /// It should hopefully be inlined in the no-conversion case;
485  /// it's less critical in the conversion case.
486  template<typename DEST_DATA_T=DATA_T>
487  SYS_FORCE_INLINE DEST_DATA_T get(IDX_T i,exint component=0) const
488  {
489  UT_ASSERT_P(i >= IDX_T(0) && i < size());
490  if constexpr (!(SYSisSame<DATA_T,void>()) && (SYSisSame<DEST_DATA_T,DATA_T>() || theStorage != UT_Storage::INVALID))
491  return UTconvertStorage<DEST_DATA_T>((*this)(i,component));
492  else
493  return getUnknownType<DEST_DATA_T>(i,component);
494  }
495 
496  template<typename DEST_DATA_T=NotVoidType,exint DEST_TSIZE=theSafeTupleSize>
498  {
499  if constexpr (SYSisSame<DATA_T,void>())
500  return getVectorUnknownType<DEST_DATA_T,DEST_TSIZE>(i);
501  else if constexpr (!SYSisSame<DEST_DATA_T,DATA_T>())
502  return convertVectorStorage<DEST_DATA_T>(getVector<NotVoidType,DEST_TSIZE>(i));
503 
504  if constexpr (TSIZE==-1)
505  return getVectorUnknownSize<DEST_DATA_T,DEST_TSIZE>(i);
506  else if constexpr (TSIZE!=DEST_TSIZE)
507  return getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
508 
509  UT_ASSERT_P(i >= IDX_T(0) && i < size());
510  UT_ASSERT_P(TSIZE==DEST_TSIZE);
511  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()));
512 
513  // Types and tuple sizes are known and match
514  UT_PageNum pagenum = pageNum(i);
515  // NOTE: We can dereference here because we're returning by value.
516  // getConstantPtr may return a pointer into this stack variable
517  // if the page is constant and inline.
518  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
519 
520  // If we know that the pages are hardened, we can omit
521  // the constant page case at compile time.
522  bool isconst = !PAGESHARDENED && page->isConstant();
523 
524  if (!isconst)
525  {
526  UT_PageOff pageoff = pageOff(i);
527  return ((const UT_FixedVector<DEST_DATA_T,DEST_TSIZE>*)page->getFirstPtr())[pageoff];
528  }
529  const NotVoidType *constdata = getConstantPtr(page);
530  if (!constdata)
531  {
533  }
534 
535  return *(const UT_FixedVector<DEST_DATA_T,DEST_TSIZE>*)constdata;
536  }
537 
538  /// Get a non-const reference to element i's specified component
539  /// (default component 0).
540  /// WARNING: DO NOT call this if DATA_T is void!
541  /// NOTE: Unlike set(), this must always harden, even if you're just
542  /// writing a value that is already there.
544  {
545  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
546  UT_ASSERT_P(TSIZE != 0);
547  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
548  UT_ASSERT_P(i >= IDX_T(0) && i < size());
549  UT_ASSERT_P(component >= 0 && component < myImpl.getTupleSize());
550 
551  // NOTE: This is a no-op if TABLEHARDENED is already true.
552  hardenTable();
553 
554  UT_PageNum pagenum = pageNum(i);
555  UT_PageOff pageoff = pageOff(i);
556  PageTable *pages = myImpl.getPages();
557  PageTableEntry *page = pages->getPPage(pagenum);
558 
559  if (!PAGESHARDENED)
560  {
561  if (page->isConstant())
562  {
563  if (TSIZE >= 1)
564  hardenConstantPage(pages, page);
565  else
566  hardenConstantPage(pages, page, myImpl.getTupleSize());
567  }
568  else if (page->isShared())
569  {
570  if (TSIZE >= 1)
571  hardenSharedPage(pages, page);
572  else
573  hardenSharedPage(pages, page, myImpl.getTupleSize());
574  }
575  }
576 
577  NotVoidType *data = page->getFirstPtr()
578  + pageoff*((TSIZE >= 1) ? TSIZE : myImpl.getTupleSize())
579  + component;
580 
581  return *data;
582  }
583 
584  /// component == 0 in this version
585  template<typename SRC_DATA_T>
586  SYS_FORCE_INLINE void set(IDX_T i,SRC_DATA_T v)
587  {
588  set(i, 0, v);
589  }
590  /// component == 0 in this version
591  template<typename SRC_DATA_T>
592  SYS_FORCE_INLINE void add(IDX_T i,SRC_DATA_T v)
593  {
594  add(i, 0, v);
595  }
596 
597  /// This looks gigantic, but it's mostly because of the different
598  /// behaviour for different template values. If
599  /// PAGESHARDENED is true, it'll skip checking for constant
600  /// or shared pages altogether.
601  template<typename SRC_DATA_T>
602  SYS_FORCE_INLINE void set(IDX_T i,exint component,SRC_DATA_T v)
603  {
604  op<0>(i, component, v);
605  }
606  template<typename SRC_DATA_T>
607  SYS_FORCE_INLINE void add(IDX_T i,exint component,SRC_DATA_T v)
608  {
609  op<1>(i, component, v);
610  }
611  template<int OP,typename SRC_DATA_T>
612  SYS_FORCE_INLINE void op(IDX_T i,exint component,SRC_DATA_T v)
613  {
614  UT_ASSERT_P(i >= IDX_T(0) && i < size());
615  UT_ASSERT_P(component >= 0 && component < getTupleSize());
616  UT_ASSERT_MSG_P(OP == 0 || OP == 1, "Only OP 0 (set) and 1 (add) are defined!");
617 
618  if constexpr (SYSisSame<DATA_T,void>())
619  {
620  opUnknownType<OP>(i, component, v);
621  return;
622  }
623 
624  // Make sure that v is the correct type before comparing.
625  NotVoidType vt(UTconvertStorage<NotVoidType>(v));
626 
627  UT_ASSERT_P(TSIZE != 0);
628  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
629 
630  hardenTable();
631 
632  UT_PageNum pagenum = pageNum(i);
633  UT_PageOff pageoff = pageOff(i);
634  PageTable *pages = myImpl.getPages();
635  PageTableEntry *page = pages->getPPage(pagenum);
636 
637  if constexpr (!PAGESHARDENED)
638  {
639  if (page->isConstant())
640  {
641  // If we're setting, check to see if it's a new value
642  if (OP == 0)
643  {
644  if (TSIZE >= 1)
645  {
646  if (getConstant(page, component) == vt)
647  return;
648  }
649  else
650  {
651  if (getConstant(page, component, myImpl.getTupleSize()) == vt)
652  return;
653  }
654  }
655  if (TSIZE >= 1)
656  hardenConstantPage(pages, page);
657  else
658  hardenConstantPage(pages, page, myImpl.getTupleSize());
659  }
660  else if (page->isShared())
661  {
662  // If we're setting, check to see if it's a new value
663  if (OP == 0)
664  {
665  NotVoidType *data = page->getFirstPtr()
666  + pageoff*((TSIZE >= 1) ? TSIZE : myImpl.getTupleSize())
667  + component;
668  if (*data == vt)
669  return;
670  }
671  if (TSIZE >= 1)
672  hardenSharedPage(pages, page);
673  else
674  hardenSharedPage(pages, page, myImpl.getTupleSize());
675  }
676  }
677 
678  NotVoidType *data = page->getFirstPtr()
679  + pageoff*((TSIZE >= 1) ? TSIZE : myImpl.getTupleSize())
680  + component;
681 
682  if (OP == 0)
683  *data = vt;
684  else if (OP == 1)
685  *data += vt;
686  else
687  {
688  UT_ASSERT_MSG_P(0,"Only OP 0 (set) and 1 (add) are defined!");
689  }
690  }
691 
692  template<typename TS>
693  SYS_FORCE_INLINE void setVector(IDX_T i,const TS& as)
694  {
695  opVector<0>(i, UTmakeFixedVector(as));
696  }
697 
698  template<typename TS>
699  SYS_FORCE_INLINE void addVector(IDX_T i,const TS& as)
700  {
701  opVector<1>(i, UTmakeFixedVector(as));
702  }
703 
704  /// This looks gigantic, but it's mostly because of the different
705  /// behaviour for different template values. If
706  /// PAGESHARDENED is true, it'll skip checking for constant
707  /// or shared pages altogether.
708  template<int OP,typename SRC_DATA_T,exint SRC_TSIZE>
710  {
711  UT_ASSERT_P(i >= IDX_T(0) && i < size());
712  UT_ASSERT_MSG_P(OP == 0 || OP == 1, "Only OP 0 (set) and 1 (add) are defined!");
713 
714  if constexpr (SYSisSame<DATA_T,void>())
715  {
716  opVectorUnknownType<OP>(i, v);
717  return;
718  }
719 
720  const exint tuplesize = myImpl.getTupleSize();
721  if constexpr (TSIZE == -1)
722  {
723  if (tuplesize == SRC_TSIZE)
724  {
725  castTupleSize<SRC_TSIZE>().template opVector<OP>(i, v);
726  return;
727  }
728  }
729 
730  const exint mintuplesize = SYSmin(tuplesize, SRC_TSIZE);
731 
732  // Make sure that v is the correct type before comparing.
733  // Hopefully this will avoid copying if SRC_DATA_T is NotVoidType.
734  constexpr bool TYPE_MATCH = SYSisSame<SRC_DATA_T,NotVoidType>();
736  if (!TYPE_MATCH)
737  converted_v = convertVectorStorage<NotVoidType>(v);
738 
739  UT_ASSERT_P(TSIZE != 0);
740  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
741 
742  hardenTable();
743 
744  UT_PageNum pagenum = pageNum(i);
745  UT_PageOff pageoff = pageOff(i);
746  PageTable *pages = myImpl.getPages();
747  PageTableEntry *page = pages->getPPage(pagenum);
748 
749  if constexpr (!PAGESHARDENED)
750  {
751  if (page->isConstant())
752  {
753  // If we're setting, check to see if it's a new value
754  if constexpr (OP == 0)
755  {
756  const NotVoidType *data;
757  if (PageTableEntry::typeFitsInline(tuplesize))
758  data = page->getInlinePtr(tuplesize);
759  else
760  data = page->getMaskedPtr();
761  if (data)
762  {
763  if constexpr (TYPE_MATCH)
764  {
765  if (isEqual(data, v.data(), mintuplesize))
766  return;
767  }
768  else
769  {
770  if (isEqual(data, converted_v.data(), mintuplesize))
771  return;
772  }
773  }
774  else
775  {
776  if constexpr (TYPE_MATCH)
777  {
778  if (isZero(v.data(), mintuplesize))
779  return;
780  }
781  else
782  {
783  if (isZero(converted_v.data(), mintuplesize))
784  return;
785  }
786  }
787  }
788  if (TSIZE >= 1)
789  hardenConstantPage(pages, page);
790  else
791  hardenConstantPage(pages, page, tuplesize);
792  }
793  else if (page->isShared())
794  {
795  // If we're setting, check to see if it's a new value
796  if constexpr (OP == 0)
797  {
798  const NotVoidType *data = page->getFirstPtr()
799  + pageoff*((TSIZE >= 0) ? TSIZE : tuplesize);
800  if constexpr (TYPE_MATCH)
801  {
802  if (isEqual(data, v.data(), mintuplesize))
803  return;
804  }
805  else
806  {
807  if (isEqual(data, converted_v.data(), mintuplesize))
808  return;
809  }
810  }
811  if constexpr (TSIZE >= 1)
812  hardenSharedPage(pages, page);
813  else
814  hardenSharedPage(pages, page, tuplesize);
815  }
816  }
817 
818  NotVoidType *data = page->getFirstPtr() + pageoff*((TSIZE >= 0) ? TSIZE : tuplesize);
819  for (exint component = 0; component < mintuplesize; ++component)
820  {
821  if constexpr (OP == 0)
822  {
823  if constexpr (TYPE_MATCH)
824  data[component] = v[component];
825  else
826  data[component] = converted_v[component];
827  }
828  else if constexpr (OP == 1)
829  {
830  if constexpr (TYPE_MATCH)
831  data[component] += v[component];
832  else
833  data[component] += converted_v[component];
834  }
835  else
836  {
837  UT_ASSERT_MSG_P(0,"Only OP 0 (set) and 1 (add) are defined!");
838  }
839  }
840  }
841 
842  SYS_FORCE_INLINE IDX_T size() const
843  {
844  const PageTable *pages = myImpl.getPages();
845  if (!pages)
846  return IDX_T(0);
847  return pages->size();
848  }
849 
851  {
852  const PageTable *pages = myImpl.getPages();
853  if (!pages)
854  return IDX_T(0);
855  return pages->capacity();
856  }
857 
858  /// This helper function forces a (non-negative) capacity to be an
859  /// acceptable value: either
860  /// A) a power of two less than a full page in size, or
861  /// B) a multiple of the page size.
863  static IDX_T roundUpCapacity(IDX_T capacity)
864  {
865  UT_ASSERT_P(capacity >= IDX_T(0));
866 
867  // Round up to page size if more than half of one page.
868  if (capacity > IDX_T(thePageSize>>1))
869  return IDX_T((capacity + thePageMask) & ~thePageMask);
870 
871  // Round up to power of two if half a page or less.
872  // (Zero stays zero.)
873  if (capacity <= IDX_T(2))
874  return capacity;
875  return IDX_T(SYSmakePow2(exint(capacity)));
876  }
877 
878  void setCapacity(IDX_T newcapacity)
879  {
880  UT_ASSERT_P(newcapacity >= IDX_T(0));
881  newcapacity = roundUpCapacity(newcapacity);
882  if (newcapacity == capacity())
883  return;
884  if (newcapacity < size())
885  {
886  setSize(newcapacity);
887  }
888  forceHardenTable(newcapacity);
889  }
890 
891  void setCapacityIfNeeded(IDX_T newcapacity)
892  {
893  UT_ASSERT_P(newcapacity >= IDX_T(0));
894  if (newcapacity <= capacity())
895  return;
896  newcapacity = roundUpCapacity(newcapacity);
897  forceHardenTable(newcapacity);
898  }
899 
900  /// Sets the size of the array without initializing any added
901  /// elements.
902  void setSize(IDX_T newsize)
903  {
904  UT_ASSERT_P(newsize >= IDX_T(0));
905 
906  setCapacityIfNeeded(newsize);
907  hardenTable();
908  PageTable *pages = myImpl.getPages();
909  UT_ASSERT_P(pages || newsize == IDX_T(0));
910  if (pages)
911  pages->setSize(newsize);
912  }
913 
914  /// Sets the size of the array and initializes any new elements to initval.
915  /// This version of setSize only works if DATA_T is a numeric type.
916  /// If the tuple size is > 1, it will write initval to all components.
917  /// Include UT_PageArrayImpl.h to call this.
918  void setSize(IDX_T newsize, NotVoidType initval);
919 
920  /// Sets the size of the array and initializes any new elements
921  /// to initval.
922  /// This version of setSize only works if DATA_T is a numeric type
923  /// and TSIZE >= 1.
924  /// Include UT_PageArrayImpl.h to call this.
925  void setSize(IDX_T newsize, const UT_FixedVector<NotVoidType,theSafeTupleSize> &initval);
926 
927  /// Sets the size of the array and initializes any new elements to initval.
928  /// Include UT_PageArrayImpl.h to call this.
929  void setSize(IDX_T newsize, const UT_Defaults &initval);
930 
931  /// Set all of the values in the range [start,end) to v.
932  /// This will constant-compress pages if allowed and the pages are
933  /// completely overwritten with the given value.
934  /// This version of setConstant only works if DATA_T is a numeric type.
935  /// If the tuple size is > 1, it will write v to all components.
936  void setConstant(IDX_T start, IDX_T end, NotVoidType v);
937 
938  /// Set all of the values in the range [start,end) to v.
939  /// This will constant-compress pages if allowed and the pages are
940  /// completely overwritten with the given value.
941  /// This version of setConstant only works if DATA_T is a numeric type
942  /// and TSIZE >= 1.
944 
945  /// Set all of the values in the range [start,end) to v.
946  /// This will constant-compress pages if allowed and the pages are
947  /// completely overwritten with the given value.
948  void setConstant(IDX_T start, IDX_T end, const UT_Defaults &v);
949 
950  /// Set all of the values in the range [start,end) to the tuple in values.
951  /// values must have length equal to getTupleSize().
952  /// This will constant-compress pages if allowed and the pages are
953  /// completely overwritten with the given value.
954  template<typename SRC_T>
955  void setConstant(IDX_T start, IDX_T end, const SRC_T *values);
956 
957  /// Changes the storage type and converts the data to the new type.
958  /// NOTE: This only makes sense if DATA_T is void and newstorage is
959  /// something other than INVALID.
960  /// Include UT_PageArrayImpl.h to call this.
961  void setStorage(const UT_Storage newstorage);
962 
963  /// Changes the tuple size, and if the new size is larger, pads with
964  /// values from the UT_Defaults.
965  /// NOTE: This only makes sense if TSIZE == -1.
966  /// Include UT_PageArrayImpl.h to call this.
967  void setTupleSize(exint newtuplesize, const UT_Defaults &v);
968 
969  /// Get the total of all memory referred-to by this.
970  /// NOTE: This does not handle shared memory correctly.
971  /// Include UT_PageArrayImpl.h to call this.
972  int64 getMemoryUsage(bool inclusive) const;
973 
974  /// Count memory usage using a UT_MemoryCounter in order to count
975  /// shared memory correctly.
976  /// If inclusive is true, the size of this object is counted,
977  /// else only memory owned by this object is counted.
978  /// If this is pointed to by the calling object, inclusive should be true.
979  /// If this is contained in the calling object, inclusive should be false.
980  /// (Its memory was already counted in the size of the calling object.)
981  /// Include UT_PageArrayImpl.h to call this.
982  void countMemory(UT_MemoryCounter &counter, bool inclusive) const;
983 
984  /// Copies the nelements from srcstart to deststart
985  /// The two ranges *are* allowed to overlap.
986  /// Include UT_PageArrayImpl.h to call this.
987  void moveRange(IDX_T srcstart, IDX_T deststart, IDX_T nelements);
988 
989  /// Copies the nelements from fromstart in src to tostart in this.
990  /// The two ranges *are* allowed to overlap, even if &src==this.
991  /// Include UT_PageArrayImpl.h to call this.
992  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
993  void moveRange(const UT_PageArray<SRC_DATA_T,SRC_TSIZE,SRC_TABLEHARDENED,SRC_PAGESHARDENED,THEPAGEBITS,IDX_T> &src, IDX_T srcstart, IDX_T deststart, IDX_T nelements);
994 
995  /// Swaps the nelements at astart with the nelements at bstart.
996  /// The two ranges *must* be non-overlapping, else swapping is ill-defined.
997  /// Include UT_PageArrayImpl.h to call this.
998  void swapRange(IDX_T astart, IDX_T bstart, IDX_T nelements);
999 
1000  /// Copies a single element from src to dest within this
1001  void copy(IDX_T dest, IDX_T src)
1002  {
1003  UT_ASSERT_P(dest >= IDX_T(0) && dest < size());
1004  UT_ASSERT_P(src >= IDX_T(0) && src < size());
1005 
1006  hardenTable();
1007 
1008  // Because no conversions are necessary, only the element size
1009  // in bytes is needed.
1010  const exint bytesize = (SYSisSame<DATA_T,void>() ? UTstorageSize(getStorage()) : sizeof(NotVoidType))
1011  * getTupleSize();
1012 
1013  UT_PageNum destpagenum = pageNum(dest);
1014  UT_PageNum srcpagenum = pageNum(src);
1015 
1016  PageTable *pages = myImpl.getPages();
1017  PageTableEntry *destpage = pages->getPPage(destpagenum);
1018  const PageTableEntry *srcpage = pages->getPPage(srcpagenum);
1019 
1020  const int8 *srcdata;
1021  if (srcpage->isConstant())
1022  {
1023  if (bytesize < sizeof(PageTableEntry))
1024  {
1025  // Copying within equal constant pages does nothing
1026  if (*srcpage == *destpage)
1027  return;
1028 
1029  srcdata = ((const int8 *)(srcpage+1)) - bytesize;
1030  }
1031  else if (srcpage->getUnmaskedPtrVoid() == destpage->getUnmaskedPtrVoid())
1032  {
1033  // Copying within equal constant pages does nothing
1034  return;
1035  }
1036  else
1037  {
1038  // NOTE: srcdata may be set to nullptr here.
1039  srcdata = (const int8 *)srcpage->getMaskedPtrVoid();
1040  }
1041  }
1042  else
1043  {
1044  srcdata = ((const int8*)srcpage->getFirstPtrVoid()) + pageOff(src)*bytesize;
1045  }
1046 
1047  int8 *destdata;
1048  if (destpage->isConstant())
1049  {
1050  if (bytesize < sizeof(PageTableEntry))
1051  destdata = ((int8 *)(destpage+1)) - bytesize;
1052  else
1053  {
1054  void *destmasked = destpage->getMaskedPtrVoid();
1055  if (destmasked == nullptr)
1056  {
1057  if (isZero(srcdata, bytesize))
1058  return;
1059  destdata = nullptr;
1060  }
1061  else
1062  destdata = (int8*)destmasked;
1063  }
1064  if (destdata != nullptr && isEqual(srcdata, destdata, bytesize))
1065  return;
1066 
1067  if constexpr (!SYSisSame<DATA_T,void>())
1068  hardenConstantPage(pages, destpage, getTupleSize());
1069  else
1070  {
1071  switch (getStorage())
1072  {
1073  case UT_Storage::INT8:
1074  case UT_Storage::UINT8:
1075  {
1076  auto &a = castType<int8>();
1077  auto a_pages = a.myImpl.getPages();
1078  auto a_destpage = a_pages->getPPage(destpagenum);
1079  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1080  break;
1081  }
1082  case UT_Storage::INT16:
1083  case UT_Storage::REAL16:
1084  {
1085  auto &a = castType<int16>();
1086  auto a_pages = a.myImpl.getPages();
1087  auto a_destpage = a_pages->getPPage(destpagenum);
1088  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1089  break;
1090  }
1091  case UT_Storage::INT32:
1092  case UT_Storage::REAL32:
1093  {
1094  auto &a = castType<int32>();
1095  auto a_pages = a.myImpl.getPages();
1096  auto a_destpage = a_pages->getPPage(destpagenum);
1097  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1098  break;
1099  }
1100  case UT_Storage::INT64:
1101  case UT_Storage::REAL64:
1102  {
1103  auto &a = castType<int64>();
1104  auto a_pages = a.myImpl.getPages();
1105  auto a_destpage = a_pages->getPPage(destpagenum);
1106  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1107  break;
1108  }
1109  case UT_Storage::INVALID:
1110  UT_ASSERT_MSG(0, "Storage must be valid if DATA_T is void.");
1111  break;
1112  }
1113  }
1114  }
1115  else if (!PAGESHARDENED && destpage->isShared())
1116  {
1117  destdata = ((int8*)destpage->getFirstPtrVoid()) + pageOff(dest)*bytesize;
1118  if (srcdata)
1119  {
1120  if (isEqual(srcdata, destdata, bytesize))
1121  return;
1122  }
1123  else
1124  {
1125  if (isZero(destdata, bytesize))
1126  return;
1127  }
1128 
1129  if constexpr (!SYSisSame<DATA_T,void>())
1130  hardenSharedPage(pages, destpage, getTupleSize());
1131  else
1132  {
1133  switch (getStorage())
1134  {
1135  case UT_Storage::INT8:
1136  case UT_Storage::UINT8:
1137  {
1138  auto &a = castType<int8>();
1139  auto a_pages = a.myImpl.getPages();
1140  auto a_destpage = a_pages->getPPage(destpagenum);
1141  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1142  break;
1143  }
1144  case UT_Storage::INT16:
1145  case UT_Storage::REAL16:
1146  {
1147  auto &a = castType<int16>();
1148  auto a_pages = a.myImpl.getPages();
1149  auto a_destpage = a_pages->getPPage(destpagenum);
1150  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1151  break;
1152  }
1153  case UT_Storage::INT32:
1154  case UT_Storage::REAL32:
1155  {
1156  auto &a = castType<int32>();
1157  auto a_pages = a.myImpl.getPages();
1158  auto a_destpage = a_pages->getPPage(destpagenum);
1159  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1160  break;
1161  }
1162  case UT_Storage::INT64:
1163  case UT_Storage::REAL64:
1164  {
1165  auto &a = castType<int64>();
1166  auto a_pages = a.myImpl.getPages();
1167  auto a_destpage = a_pages->getPPage(destpagenum);
1168  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1169  break;
1170  }
1171  case UT_Storage::INVALID:
1172  UT_ASSERT_MSG(0, "Storage must be valid if DATA_T is void.");
1173  break;
1174  }
1175  }
1176  }
1177  destdata = ((int8*)destpage->getFirstPtrVoid()) + pageOff(dest)*bytesize;
1178 
1179  for (const int8 *srcdataend = srcdata+bytesize; srcdata != srcdataend; ++srcdata, ++destdata)
1180  *destdata = *srcdata;
1181  }
1182 
1183  /// Copies nelements from srcstart in this to dest array
1184  /// Include UT_PageArrayImpl.h to call this.
1185  template<typename T>
1186  SYS_FORCE_INLINE void getRange(IDX_T srcstart, IDX_T nelements, T *dest) const
1187  {
1188  // Always cast to associated UT_FixedVector type, so that
1189  // we can just delegate to getVectorRange that always uses UT_FixedVector
1190  getVectorRange(srcstart, nelements, (typename UT_FixedVectorTraits<T>::FixedVectorType*)dest);
1191  }
1192 
1193  /// Copies nelements from src array to deststart in this
1194  /// Include UT_PageArrayImpl.h to call this.
1195  template<typename T>
1196  SYS_FORCE_INLINE void setRange(IDX_T deststart, IDX_T nelements, const T *src)
1197  {
1198  // Always cast to associated UT_FixedVector type, so that
1199  // we can just delegate to getVectorRange that always uses UT_FixedVector
1200  setVectorRange(deststart, nelements, (const typename UT_FixedVectorTraits<T>::FixedVectorType*)src);
1201  }
1202 
1203  /// Copies nelements from srcstart in this to dest array
1204  /// Include UT_PageArrayImpl.h to call this.
1205  template<typename DEST_DATA_T,exint DEST_TSIZE>
1206  void getVectorRange(IDX_T srcstart, IDX_T nelements, UT_FixedVector<DEST_DATA_T,DEST_TSIZE> *dest) const;
1207 
1208  /// Copies nelements from src array to deststart in this
1209  /// Include UT_PageArrayImpl.h to call this.
1210  template<typename SRC_DATA_T,exint SRC_TSIZE>
1211  void setVectorRange(IDX_T deststart, IDX_T nelements, const UT_FixedVector<SRC_DATA_T,SRC_TSIZE> *src);
1212 
1214  {
1215  return UT_PageNum(exint(i) >> thePageBits);
1216  }
1217  SYS_FORCE_INLINE static UT_PageNum numPages(IDX_T nelements)
1218  {
1219  return UT_PageNum((exint(nelements)+thePageMask) >> thePageBits);
1220  }
1222  {
1223  return UT_PageOff(exint(i) & thePageMask);
1224  }
1226  {
1227  return IDX_T(exint(page) << thePageBits);
1228  }
1229 
1230  /// Returns true iff the table isn't shared with any other UT_PageArray's.
1232  {
1233  // If TABLEHARDENED, assert that the page table hasn't become shared
1234  // after the fact! Nobody should share the page table after templating
1235  // on having hardened!
1236  UT_ASSERT_P(!TABLEHARDENED || !myImpl.getPages() || !myImpl.getPages()->isShared());
1237 
1238  return TABLEHARDENED || !myImpl.getPages() || !myImpl.getPages()->isShared();
1239  }
1240 
1241  /// This ensures that the table itself is not shared.
1242  /// It's a good idea to do this before writing, to avoid having to check
1243  /// every time. Remember to use the returned reference, since it
1244  /// records in the template that it doesn't need to be hardened again.
1245  /// It should be a no-op if TABLEHARDENED is already true.
1247  {
1248  if (!isTableHardened())
1249  {
1250  // Keep same capacity
1251  forceHardenTable(myImpl.getPages()->capacity());
1252  }
1253 
1255  }
1256 
1257  /// This forces all pages to be hardened, optionally including pages
1258  /// that are in the capacity but outside the size.
1259  template <bool including_capacity=false>
1261  {
1262  // Nothing to do if PAGESHARDENED
1263  PageTable *pages = myImpl.getPages();
1264  if (PAGESHARDENED || (pages == nullptr))
1266 
1267  auto &hard = hardenTable();
1268 
1269  if (end < IDX_T(0))
1270  end = including_capacity ? pages->capacity() : pages->size();
1271 
1272  UT_ASSERT_P(start >= IDX_T(0));
1274 
1275  if constexpr (!SYSisSame<DATA_T,void>())
1276  {
1277  if constexpr (TSIZE >= 1)
1278  pages->hardenAllPages(start, end);
1279  else
1280  pages->hardenAllPages(start, end, myImpl.getTupleSize());
1281  }
1282  else
1283  {
1284  // Don't want to have to re-check that the table is hardened,
1285  // so call hardenAllPagesUnknownType on hard.
1286  hard.hardenAllPagesUnknownType(start, end);
1287  }
1288 
1290  }
1291 
1292  template <bool including_capacity=false>
1293  SYS_FORCE_INLINE void tryCompressAllPages(IDX_T start = IDX_T(0), IDX_T end = IDX_T(-1))
1294  {
1295  // Can't have compressed pages if PAGESHARDENED
1296  PageTable *pages = myImpl.getPages();
1297  if (PAGESHARDENED || (pages == nullptr))
1298  return;
1299 
1300  auto &hard = hardenTable();
1301 
1302  if (end < IDX_T(0))
1303  end = including_capacity ? pages->capacity() : pages->size();
1304 
1305  UT_ASSERT_P(start >= IDX_T(0));
1307 
1308  if constexpr (!SYSisSame<DATA_T,void>())
1309  {
1310  if constexpr (TSIZE >= 1)
1311  pages->tryCompressAllPages(start, end);
1312  else
1313  pages->tryCompressAllPages(start, end, myImpl.getTupleSize());
1314  }
1315  else
1316  {
1317  // Don't want to have to re-check that the table is hardened,
1318  // so call hardenAllPagesUnknownType on hard.
1319  hard.tryCompressAllPagesUnknownType(start, end);
1320  }
1321  }
1322 
1323  /// This is for setting the DATA_T template after checking getStorage(),
1324  /// if DATA_T wasn't already known.
1325  /// @{
1326  template<typename DEST_DATA_T>
1328  castType() const
1329  {
1331  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()) || (getStorage() != UT_Storage::INVALID && getStorage() == UT_StorageNum<DEST_DATA_T>::theStorage));
1333  }
1334  template<typename DEST_DATA_T>
1337  {
1339  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()) || (getStorage() != UT_Storage::INVALID && getStorage() == UT_StorageNum<DEST_DATA_T>::theStorage));
1341  }
1342  /// @}
1343 
1344  /// This is for setting the TSIZE template after checking getTupleSize(),
1345  /// if TSIZE wasn't already known.
1346  /// @{
1347  template<exint DEST_TSIZE>
1350  {
1351  UT_ASSERT_P(TSIZE == -1 || TSIZE == DEST_TSIZE);
1352  UT_ASSERT_P(getTupleSize() == DEST_TSIZE);
1354  }
1355  template<exint DEST_TSIZE>
1358  {
1359  UT_ASSERT_P(TSIZE == -1 || TSIZE == DEST_TSIZE);
1360  UT_ASSERT_P(getTupleSize() == DEST_TSIZE);
1362  }
1363  /// @}
1364 
1366  {
1367  return myImpl.getTupleSize();
1368  }
1370  {
1371  return myImpl.getStorage();
1372  }
1373 
1374  /// Returns true iff the specified page is constant-compressed
1376  {
1377  UT_ASSERT_P(myImpl.getPages() != nullptr);
1378  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1379  return myImpl.getPages()->getPPage(pagenum)->isConstant();
1380  }
1381 
1382  /// Returns true iff the entire array is constant & zero
1384  {
1385  // Zero capacity array is zero:
1386  if (myImpl.getPages() == nullptr)
1387  return true;
1388 
1389  const PageTable *pages = myImpl.getPages();
1390 
1391  UT_PageNum npages = numPages(pages->capacity());
1392  for (UT_PageNum pagenum = 0; pagenum < npages; pagenum++)
1393  {
1394  const PageTableEntry *page = pages->getPPage(pagenum);
1395  if (!page->isConstantAndZeroSafe())
1396  return false;
1397  }
1398  return true;
1399  }
1400 
1401  /// Sets all components of all elements of the specified page to
1402  /// the given value.
1404  {
1405  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1406  UT_ASSERT_P(myImpl.getPages() != nullptr);
1407  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1408 
1409  hardenTable();
1410 
1411  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1412  if (TSIZE >= 1)
1413  makeConstant(page, val);
1414  else
1415  makeConstant(page, val, myImpl.getTupleSize());
1416  }
1417 
1418  /// Sets all components of all elements of the specified page to
1419  /// the given value.
1420  template<typename SRC_DATA_T>
1422  {
1423  if constexpr (SYSisSame<DATA_T,void>())
1424  {
1425  setPageConstantUnknownType(pagenum, val);
1426  return;
1427  }
1428 
1429  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1430 
1431  if constexpr (!SYSisSame<DATA_T,SRC_DATA_T>())
1432  {
1434  for (exint i = 0; i < theSafeTupleSize; ++i)
1435  new_val[i] = UTconvertStorage<NotVoidType>(val[i]);
1436  setPageConstant(pagenum, new_val);
1437 
1438  return;
1439  }
1440 
1441  UT_ASSERT_P((SYSisSame<DATA_T,SRC_DATA_T>()));
1442  UT_ASSERT_P(myImpl.getPages() != nullptr);
1443  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1444  UT_ASSERT_P(TSIZE >= 1);
1445 
1446  hardenTable();
1447 
1448  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1449  makeConstant(page, val);
1450  }
1451 
1452  /// Sets all components of all elements of the specified page to
1453  /// the given values.
1454  /// NOTE: The length of values must be equal to the tuple size.
1455  template<typename SRC_DATA_T>
1456  SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const SRC_DATA_T *values)
1457  {
1458  if constexpr (SYSisSame<DATA_T,void>())
1459  {
1460  setPageConstantUnknownType(pagenum, values);
1461  return;
1462  }
1463 
1464  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1465 
1466  if constexpr (!SYSisSame<DATA_T,SRC_DATA_T>())
1467  {
1468  setPageConstantMismatchedType(pagenum, values);
1469  return;
1470  }
1471 
1472  UT_ASSERT_P((SYSisSame<DATA_T,SRC_DATA_T>()));
1473  UT_ASSERT_P(myImpl.getPages() != nullptr);
1474  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1475 
1476  hardenTable();
1477 
1478  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1479  // NOTE: The cast will never actually be hit unless DATA_T is SRC_DATA_T,
1480  // as asserted above. It's just to get this to compile in other cases.
1481  makeConstant(page, (const NotVoidType *)values, myImpl.getTupleSize());
1482  }
1483 
1484  /// Returns true iff the specified page is "hard",
1485  /// i.e. NOT constant-compressed and NOT shared.
1486  /// NOTE: The *table* may or may not be hard.
1488  {
1489  UT_ASSERT_P(myImpl.getPages() != nullptr);
1490  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1491  if constexpr (PAGESHARDENED)
1492  return true;
1493  else
1494  {
1495  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1496  return !page->isConstant() && !page->isShared();
1497  }
1498  }
1499 
1500  /// Hardens the specified page if it is constant-compressed or shared.
1501  /// It returns a pointer to the data, but it will only be the correct
1502  /// type if DATA_T is not void.
1504  {
1505  UT_ASSERT_P(myImpl.getPages() != nullptr);
1506  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1507 
1508  hardenTable();
1509 
1510  PageTable *pages = myImpl.getPages();
1511  PageTableEntry *page = pages->getPPage(pagenum);
1512 
1513  if constexpr (PAGESHARDENED)
1514  return page->getFirstPtr();
1515 
1516  if (page->isConstant())
1517  hardenConstantPage(pages, page, myImpl.getTupleSize());
1518  else if (page->isShared())
1519  hardenSharedPage(pages, page, myImpl.getTupleSize());
1520  return page->getFirstPtr();
1521  }
1522 
1524  {
1525  UT_ASSERT_P(myImpl.getPages() != nullptr);
1526  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1527 
1528  hardenTable();
1529 
1530  PageTable *pages = myImpl.getPages();
1531  PageTableEntry *page = pages->getPPage(pagenum);
1532 
1533  if constexpr (PAGESHARDENED)
1534  return page->getFirstPtr();
1535 
1536  if (page->isConstant())
1537  hardenConstantPageNoInit(pages, page, myImpl.getTupleSize());
1538  else if (page->isShared())
1539  hardenSharedPageNoInit(pages, page, myImpl.getTupleSize());
1540  return page->getFirstPtr();
1541  }
1542 
1543  /// If the specified page is constant and shared, (only relevant for
1544  /// nonzero tuples that are at least the size of a pointer), this
1545  /// keeps the page constant, but unshares it, so that the constant value
1546  /// can be modified.
1547  /// NOTE: This should ONLY be called on constant pages! Check
1548  /// isPageConstant(pagenum).
1550  {
1551  UT_ASSERT_P(myImpl.getPages() != nullptr);
1552  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1553 
1554  hardenTable();
1555 
1556  PageTable *pages = myImpl.getPages();
1557  PageTableEntry *page = pages->getPPage(pagenum);
1558 
1559  UT_ASSERT_P(!PAGESHARDENED && page->isConstant());
1560 
1561  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1562  UT_ASSERT_P(TSIZE == -1 || TSIZE >= 0);
1563 
1564  const exint tuplesize = (TSIZE >= 0) ? TSIZE : myImpl.getTupleSize();
1565  if (PageTableEntry::typeFitsInline(tuplesize))
1566  return page->getInlinePtr(tuplesize);
1567 
1568  NotVoidType *src = page->getMaskedPtr();
1569  if (!page->isShared())
1570  return src;
1571 
1572  PageTableEntry newpage;
1573  newpage.alloc(UT_PageOff(1), tuplesize);
1574  NotVoidType *dest = newpage.getFirstPtr();
1575  memcpy(dest, src, tuplesize*sizeof(NotVoidType));
1576  newpage.setConstantBit();
1577  page->decRef();
1578  (*page) = newpage;
1579 
1580  return dest;
1581  }
1582 
1583  /// Checks if the specified page is all a single value,
1584  /// and if so, makes it a constant page.
1586  {
1587  // Can't have compressed pages if PAGESHARDENED
1588  PageTable *pages = myImpl.getPages();
1589  if (PAGESHARDENED || (pages == nullptr))
1590  return;
1591 
1592  auto &hard = hardenTable();
1593 
1594  UT_ASSERT_P(pagenum >= UT_PageNum(0));
1595  UT_ASSERT_P(pagenum < numPages(pages->capacity()));
1596 
1597  if constexpr (!SYSisSame<DATA_T,void>())
1598  {
1599  if constexpr (TSIZE >= 1)
1600  pages->tryCompressPage(pagenum);
1601  else
1602  pages->tryCompressPage(pagenum, myImpl.getTupleSize());
1603  }
1604  else
1605  {
1606  // Don't want to have to re-check that the table is hardened,
1607  // so call hardenAllPagesUnknownType on hard.
1608  hard.tryCompressPageUnknownType(pagenum);
1609  }
1610  }
1611 
1612  /// Returns a pointer to the data of a page.
1613  /// WARNING: DO NOT call this if DATA_T is void! The pointer returned
1614  /// can depend on the type.
1615  /// This will return nullptr if the page is constant compressed to zero.
1616  /// If it's non-nullptr and isPageConstant(pagenum) returns true, there
1617  /// will only be one element (tuple) at that address.
1618  const NotVoidType *getPageData(UT_PageNum pagenum) const
1619  {
1620  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1621  UT_ASSERT_P(myImpl.getPages() != nullptr);
1622  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1623  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1624  if (!page->isConstant())
1625  return page->getFirstPtr();
1626  else if constexpr (TSIZE >= 1)
1627  return getConstantPtr(page);
1628  else
1629  return getConstantPtr(page, 0, getTupleSize());
1630  }
1631 
1632  /// Returns a pointer to the data of a page.
1633  /// WARNING: DO NOT call this if DATA_T is void! The pointer returned
1634  /// can depend on the type.
1635  /// NOTE: DO NOT MODIFY THE DATA if there's any chance the page or the table
1636  /// might be shared. Check isTableHard() and isPageHard(pagenum).
1637  /// This will return nullptr if the page is constant compressed to zero.
1638  /// If it's non-nullptr and isPageConstant(pagenum) returns true, there
1639  /// will only be one element (tuple) at that address.
1641  {
1642  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1643  UT_ASSERT_P(myImpl.getPages() != nullptr);
1644  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1645  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1646  if (!page->isConstant())
1647  return page->getFirstPtr();
1648  else if constexpr (TSIZE >= 1)
1649  return getConstantPtr(page);
1650  else
1651  return getConstantPtr(page, 0, getTupleSize());
1652  }
1653 
1654  /// NOTE: Don't use this unless you know what you're doing!
1655  /// This is for the case where a non-POD type that requires destruction
1656  /// is masquerading as a POD type in here, e.g. GA_OffsetList as
1657  /// UT_FixedVector<int64,2>. When destroying a table, this first needs
1658  /// to be called. If it returns true, the table was shared,
1659  /// so it was just decRef'd, meaning that the data don't need to be
1660  /// destructed. If it returns false, the table was not shared,
1661  /// so the data need to be destructed before deleting the table.
1663  {
1664  if (TABLEHARDENED)
1665  return false;
1666 
1667  PageTable *table = myImpl.getPages();
1668  if (!table)
1669  return false;
1670 
1671  bool decremented = table->decRefIffShared();
1672  if (decremented)
1673  myImpl.getPages() = nullptr;
1674  return decremented;
1675  }
1676 
1677  /// NOTE: Don't use this unless you know what you're doing!
1678  /// This is for the case where a non-POD type that requires destruction
1679  /// is masquerading as a POD type in here, e.g. GA_OffsetList as
1680  /// UT_FixedVector<int64,2>. When destroying a page, this first needs
1681  /// to be called. If it returns true, the page was shared,
1682  /// so it was just decRef'd, meaning that the data don't need to be
1683  /// destructed. If it returns false, the page was not shared,
1684  /// so the data need to be destructed before deleting the page.
1686  {
1687  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1688  UT_ASSERT_P(TSIZE >= 1);
1689  UT_ASSERT_P(myImpl.getPages() != nullptr);
1690  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1691 
1692  if (PAGESHARDENED)
1693  return false;
1694 
1695  hardenTable();
1696 
1697  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1698  if (page->isConstant())
1699  {
1700  if (sizeof(NotVoidType)*TSIZE < sizeof(PageTableEntry))
1701  return false;
1702  }
1703  bool decremented = page->decRefIffShared();
1704  if (decremented)
1705  page->initZero();
1706  return decremented;
1707  }
1708 
1709  /// Only call this if DATA_T is void and you are absolutely, 100% CERTAIN
1710  /// that the storage type is int16, int32, or int64.
1711  /// NOTE: Really! Be careful!
1713  int64 getGuaranteedInt(IDX_T i) const
1714  {
1715  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1716  UT_ASSERT_P(TSIZE == 1);
1718 
1719  UT_PageNum pagenum = pageNum(i);
1720  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1721 
1722  // If we know that the pages are hardened, we can omit
1723  // the constant page case at compile time.
1724  bool isconst = !PAGESHARDENED && page->isConstant();
1725 
1726  const UT_Storage storage = getStorage();
1727  if (!isconst)
1728  {
1729  UT_PageOff pageoff = pageOff(i);
1730  const void *first = page->getFirstPtrVoid();
1731  switch (storage)
1732  {
1733  default:
1734  UT_ASSERT_MSG_P(0, "getGuaranteedInt() only supports int16, int32, and int64.");
1736  case UT_Storage::INT16:
1737  return (int64)((int16*)first)[pageoff];
1738  case UT_Storage::INT32:
1739  return (int64)((int32*)first)[pageoff];
1740  case UT_Storage::INT64:
1741  return ((int64*)first)[pageoff];
1742  }
1743  }
1744 
1745  // Special case for zero.
1746  if (!page->isConstantZero())
1747  return int64(0);
1748 
1749  // Special case for if the constant value is too big to fit in a PageTableEntry.
1750  // At the moment, PageTableEntry is 16 bytes on 64-bit platforms,
1751  // and 8 bytes on 32-bit platforms, so this case will not occur.
1752  if ((sizeof(PageTableEntry) < 9 && storage == UT_Storage::INT64) ||
1753  (sizeof(PageTableEntry) < 5 && storage == UT_Storage::INT32))
1754  {
1755  const void *v = page->getMaskedPtrVoid();
1756  if (sizeof(PageTableEntry) < 5 && storage == UT_Storage::INT32)
1757  {
1758  // This will not be run on 64-bit platforms
1759  return (int64)*(const int32 *)v;
1760  }
1761  return *(const int64 *)v;
1762  }
1763  else
1764  {
1765  // Any types that are strictly smaller than a pointer will be stored inline.
1766  // This is checked at compile time, so there'll be no op at runtime.
1767  if (sizeof(PageTableEntry) >= 9 && storage == UT_Storage::INT64)
1768  {
1769  return *page->template castType<int64>()->getInlinePtr(1);
1770  }
1771  if (sizeof(PageTableEntry) >= 5 && storage == UT_Storage::INT32)
1772  {
1773  return (int64)*page->template castType<int32>()->getInlinePtr(1);
1774  }
1775  return (int64)*page->template castType<int16>()->getInlinePtr(1);
1776  }
1777  }
1778 
1779  /// Returns true iff there is at least one NaN value in the given range.
1780  /// Include UT_PageArrayImpl.h to call this.
1781  bool hasNanInRange(IDX_T start, IDX_T end) const;
1782 
1783 protected:
1784  template<typename DEST_DATA_T=DATA_T>
1785  DEST_DATA_T getUnknownType(IDX_T i,exint component=0) const
1786  {
1787  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1788  UT_ASSERT_P(TSIZE != 0);
1789 
1790  // Make sure common work is extracted out where possible,
1791  // to avoid duplication.
1792 
1793  UT_PageNum pagenum = pageNum(i);
1794  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1795 
1796  // If we know that the pages are hardened, we can omit
1797  // the constant page case at compile time.
1798  bool isconst = !PAGESHARDENED && page->isConstant();
1799 
1800  const UT_Storage storage = getStorage();
1801  if (!isconst)
1802  {
1803  UT_PageOff pageoff = pageOff(i);
1804  const void *first = page->getFirstPtrVoid();
1805  exint i;
1806  if (TSIZE == 1)
1807  i = (exint)pageoff;
1808  else if (TSIZE > 1)
1809  i = TSIZE*pageoff + component;
1810  else
1811  i = myImpl.getTupleSize()*pageoff + component;
1812  switch (storage)
1813  {
1814  case UT_Storage::INVALID:
1815  UT_ASSERT_MSG_P(0, "Can't read a non-numeric type with a cast to a different type.");
1816  return DEST_DATA_T();
1817  case UT_Storage::INT8:
1818  return (DEST_DATA_T)((int8*)first)[i];
1819  case UT_Storage::UINT8:
1820  return (DEST_DATA_T)((uint8*)first)[i];
1821  case UT_Storage::INT16:
1822  return (DEST_DATA_T)((int16*)first)[i];
1823  case UT_Storage::INT32:
1824  return (DEST_DATA_T)((int32*)first)[i];
1825  case UT_Storage::INT64:
1826  return (DEST_DATA_T)((int64*)first)[i];
1827  case UT_Storage::REAL16:
1828  return (DEST_DATA_T)((fpreal16*)first)[i];
1829  case UT_Storage::REAL32:
1830  return (DEST_DATA_T)((fpreal32*)first)[i];
1831  case UT_Storage::REAL64:
1832  return (DEST_DATA_T)((fpreal64*)first)[i];
1833  }
1834  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1835  return DEST_DATA_T();
1836  }
1837 
1838  switch (storage)
1839  {
1840  case UT_Storage::INVALID:
1841  UT_ASSERT_MSG_P(0, "Can't read a non-numeric type with a cast to a different type.");
1842  return DEST_DATA_T();
1843  case UT_Storage::INT8:
1844  return (DEST_DATA_T)castType<int8>().getConstant(page->template castType<int8>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1845  case UT_Storage::UINT8:
1846  return (DEST_DATA_T)castType<uint8>().getConstant(page->template castType<uint8>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1847  case UT_Storage::INT16:
1848  return (DEST_DATA_T)castType<int16>().getConstant(page->template castType<int16>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1849  case UT_Storage::INT32:
1850  return (DEST_DATA_T)castType<int32>().getConstant(page->template castType<int32>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1851  case UT_Storage::INT64:
1852  return (DEST_DATA_T)castType<int64>().getConstant(page->template castType<int64>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1853  case UT_Storage::REAL16:
1854  return (DEST_DATA_T)castType<fpreal16>().getConstant(page->template castType<fpreal16>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1855  case UT_Storage::REAL32:
1856  return (DEST_DATA_T)castType<fpreal32>().getConstant(page->template castType<fpreal32>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1857  case UT_Storage::REAL64:
1858  return (DEST_DATA_T)castType<fpreal64>().getConstant(page->template castType<fpreal64>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1859  }
1860  return DEST_DATA_T();
1861  }
1862 
1863  template<int OP,typename SRC_DATA_T>
1864  void opUnknownType(IDX_T i,exint component,SRC_DATA_T v)
1865  {
1866  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1867  const UT_Storage storage = getStorage();
1868  switch (storage)
1869  {
1870  case UT_Storage::INVALID:
1871  UT_ASSERT_MSG_P(0, "Can't write a non-numeric type with opUnknownType.");
1872  return;
1873  case UT_Storage::INT8:
1874  castType<int8 >().template op<OP>(i,component, UTconvertStorage<int8>(v)); return;
1875  case UT_Storage::UINT8:
1876  castType<uint8 >().template op<OP>(i,component, UTconvertStorage<uint8>(v)); return;
1877  case UT_Storage::INT16:
1878  castType<int16 >().template op<OP>(i,component, UTconvertStorage<int16>(v)); return;
1879  case UT_Storage::INT32:
1880  castType<int32 >().template op<OP>(i,component, UTconvertStorage<int32>(v)); return;
1881  case UT_Storage::INT64:
1882  castType<int64 >().template op<OP>(i,component, UTconvertStorage<int64>(v)); return;
1883  case UT_Storage::REAL16:
1884  castType<fpreal16>().template op<OP>(i,component, (fpreal16)v); return;
1885  case UT_Storage::REAL32:
1886  castType<fpreal32>().template op<OP>(i,component, (fpreal32)v); return;
1887  case UT_Storage::REAL64:
1888  castType<fpreal64>().template op<OP>(i,component, (fpreal64)v); return;
1889  }
1890  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1891  }
1892  template<int OP,typename SRC_DATA_T,exint SRC_TSIZE>
1894  {
1895  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1896  const UT_Storage storage = getStorage();
1897  switch (storage)
1898  {
1899  case UT_Storage::INVALID:
1900  UT_ASSERT_MSG_P(0, "Can't write a non-numeric type with opVectorUnknownType.");
1901  return;
1902  case UT_Storage::INT8:
1903  castType<int8 >().template opVector<OP>(i,v); return;
1904  case UT_Storage::UINT8:
1905  castType<uint8 >().template opVector<OP>(i,v); return;
1906  case UT_Storage::INT16:
1907  castType<int16 >().template opVector<OP>(i,v); return;
1908  case UT_Storage::INT32:
1909  castType<int32 >().template opVector<OP>(i,v); return;
1910  case UT_Storage::INT64:
1911  castType<int64 >().template opVector<OP>(i,v); return;
1912  case UT_Storage::REAL16:
1913  castType<fpreal16>().template opVector<OP>(i,v); return;
1914  case UT_Storage::REAL32:
1915  castType<fpreal32>().template opVector<OP>(i,v); return;
1916  case UT_Storage::REAL64:
1917  castType<fpreal64>().template opVector<OP>(i,v); return;
1918  }
1919  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1920  }
1921  template<typename DEST_DATA_T,exint DEST_TSIZE>
1923  {
1924  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1925 
1926  const UT_Storage storage = getStorage();
1927 
1928  // First, check for match on destination type
1930  return castType<DEST_DATA_T>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1931 
1932  // Then, try second-guess type (e.g. reading double as float or vice versa)
1934  return castType<typename UT_StorageNum<DEST_DATA_T>::SecondGuess>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1935 
1936  switch (storage)
1937  {
1938  case UT_Storage::INVALID:
1939  UT_ASSERT_MSG_P(0, "Can't read a non-numeric type with getVectorUnknownType.");
1940  break;
1941  case UT_Storage::INT8:
1942  return castType<int8 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1943  case UT_Storage::UINT8:
1944  return castType<uint8 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1945  case UT_Storage::INT16:
1946  return castType<int16 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1947  case UT_Storage::INT32:
1948  return castType<int32 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1949  case UT_Storage::INT64:
1950  return castType<int64 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1951  case UT_Storage::REAL16:
1952  return castType<fpreal16>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1953  case UT_Storage::REAL32:
1954  return castType<fpreal32>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1955  case UT_Storage::REAL64:
1956  return castType<fpreal64>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1957  }
1958  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1960  }
1961  template<typename DEST_DATA_T,exint DEST_TSIZE>
1963  {
1964  UT_ASSERT_P(TSIZE==-1);
1965  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()));
1966 
1967  const exint tuplesize = getTupleSize();
1968 
1969  // First, check for match on destination type
1970  if (tuplesize == DEST_TSIZE)
1971  return castTupleSize<DEST_TSIZE>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1972 
1973  // Then try small sizes
1974  if (tuplesize == 1)
1975  return castTupleSize<1>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
1976  if (tuplesize == 2)
1977  return castTupleSize<2>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
1978  if (tuplesize == 3)
1979  return castTupleSize<3>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
1980  if (tuplesize == 4)
1981  return castTupleSize<4>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
1982 
1983  // Mismatched large, unknown tuplesize (uncommon case)
1985  const exint minsize = SYSmin(tuplesize,DEST_TSIZE);
1986  for (exint component = 0; component < minsize; ++component)
1987  v[component] = get<DEST_DATA_T>(i, component);
1988 
1989  return v;
1990  }
1991  template<typename DEST_DATA_T,exint DEST_TSIZE>
1993  {
1994  UT_ASSERT_P(TSIZE!=-1);
1995  UT_ASSERT_P(TSIZE!=DEST_TSIZE);
1996  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()));
1997 
1998  // Types and tuple sizes are known, but don't match
1999  UT_PageNum pagenum = pageNum(i);
2000  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
2001 
2002  // If we know that the pages are hardened, we can omit
2003  // the constant page case at compile time.
2004  bool isconst = !PAGESHARDENED && page->isConstant();
2005 
2006  const NotVoidType *srctuple;
2007  if (!isconst)
2008  {
2009  UT_PageOff pageoff = pageOff(i);
2010  srctuple = (const NotVoidType *)(((const UT_FixedVector<NotVoidType,theSafeTupleSize>*)page->getFirstPtr()) + pageoff);
2011  }
2012  else
2013  {
2014  srctuple = getConstantPtr(page);
2015  if (!srctuple)
2017  }
2018 
2019  if (DEST_TSIZE < TSIZE)
2020  return *(const UT_FixedVector<DEST_DATA_T,DEST_TSIZE>*)srctuple;
2021 
2023  for (exint component = 0; component < TSIZE; ++component)
2024  dsttuple[component] = srctuple[component];
2025  for (exint component = TSIZE; component < DEST_TSIZE; ++component)
2026  dsttuple[component] = DEST_DATA_T(0);
2027  return dsttuple;
2028  }
2030  {
2031  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2032  const UT_Storage storage = getStorage();
2033  switch (storage)
2034  {
2035  case UT_Storage::INVALID:
2036  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2037  return;
2038  case UT_Storage::INT8:
2039  castType<int8>().template hardenAllPages<false>(start, end); return;
2040  case UT_Storage::UINT8:
2041  castType<uint8>().template hardenAllPages<false>(start, end); return;
2042  case UT_Storage::INT16:
2043  castType<int16>().template hardenAllPages<false>(start, end); return;
2044  case UT_Storage::INT32:
2045  castType<int32>().template hardenAllPages<false>(start, end); return;
2046  case UT_Storage::INT64:
2047  castType<int64>().template hardenAllPages<false>(start, end); return;
2048  case UT_Storage::REAL16:
2049  castType<fpreal16>().template hardenAllPages<false>(start, end); return;
2050  case UT_Storage::REAL32:
2051  castType<fpreal32>().template hardenAllPages<false>(start, end); return;
2052  case UT_Storage::REAL64:
2053  castType<fpreal64>().template hardenAllPages<false>(start, end); return;
2054  }
2055  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2056  }
2057 
2059  {
2060  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2061  const UT_Storage storage = getStorage();
2062  switch (storage)
2063  {
2064  case UT_Storage::INVALID:
2065  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2066  return;
2067  case UT_Storage::INT8:
2068  castType<int8>().template tryCompressAllPages<false>(start, end); return;
2069  case UT_Storage::UINT8:
2070  castType<uint8>().template tryCompressAllPages<false>(start, end); return;
2071  case UT_Storage::INT16:
2072  castType<int16>().template tryCompressAllPages<false>(start, end); return;
2073  case UT_Storage::INT32:
2074  castType<int32>().template tryCompressAllPages<false>(start, end); return;
2075  case UT_Storage::INT64:
2076  castType<int64>().template tryCompressAllPages<false>(start, end); return;
2077  case UT_Storage::REAL16:
2078  castType<fpreal16>().template tryCompressAllPages<false>(start, end); return;
2079  case UT_Storage::REAL32:
2080  castType<fpreal32>().template tryCompressAllPages<false>(start, end); return;
2081  case UT_Storage::REAL64:
2082  castType<fpreal64>().template tryCompressAllPages<false>(start, end); return;
2083  }
2084  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2085  }
2086 
2088  {
2089  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2090  const UT_Storage storage = getStorage();
2091  switch (storage)
2092  {
2093  case UT_Storage::INVALID:
2094  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2095  return;
2096  case UT_Storage::INT8:
2097  castType<int8>().tryCompressPage(pagenum); return;
2098  case UT_Storage::UINT8:
2099  castType<uint8>().tryCompressPage(pagenum); return;
2100  case UT_Storage::INT16:
2101  castType<int16>().tryCompressPage(pagenum); return;
2102  case UT_Storage::INT32:
2103  castType<int32>().tryCompressPage(pagenum); return;
2104  case UT_Storage::INT64:
2105  castType<int64>().tryCompressPage(pagenum); return;
2106  case UT_Storage::REAL16:
2107  castType<fpreal16>().tryCompressPage(pagenum); return;
2108  case UT_Storage::REAL32:
2109  castType<fpreal32>().tryCompressPage(pagenum); return;
2110  case UT_Storage::REAL64:
2111  castType<fpreal64>().tryCompressPage(pagenum); return;
2112  }
2113  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2114  }
2115 
2116  template<typename SRC_DATA_T>
2118  {
2119  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2120  const UT_Storage storage = getStorage();
2121  switch (storage)
2122  {
2123  case UT_Storage::INVALID:
2124  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2125  return;
2126  case UT_Storage::INT8:
2127  castType<int8>().setPageConstant(pagenum, val); return;
2128  case UT_Storage::UINT8:
2129  castType<uint8>().setPageConstant(pagenum, val); return;
2130  case UT_Storage::INT16:
2131  castType<int16>().setPageConstant(pagenum, val); return;
2132  case UT_Storage::INT32:
2133  castType<int32>().setPageConstant(pagenum, val); return;
2134  case UT_Storage::INT64:
2135  castType<int64>().setPageConstant(pagenum, val); return;
2136  case UT_Storage::REAL16:
2137  castType<fpreal16>().setPageConstant(pagenum, val); return;
2138  case UT_Storage::REAL32:
2139  castType<fpreal32>().setPageConstant(pagenum, val); return;
2140  case UT_Storage::REAL64:
2141  castType<fpreal64>().setPageConstant(pagenum, val); return;
2142  }
2143  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2144  }
2145 
2146  template<typename SRC_DATA_T>
2147  void setPageConstantUnknownType(UT_PageNum pagenum, const SRC_DATA_T *values)
2148  {
2149  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2150  const UT_Storage storage = getStorage();
2151  switch (storage)
2152  {
2153  case UT_Storage::INVALID:
2154  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2155  return;
2156  case UT_Storage::INT8:
2157  castType<int8>().setPageConstant(pagenum, values); return;
2158  case UT_Storage::UINT8:
2159  castType<uint8>().setPageConstant(pagenum, values); return;
2160  case UT_Storage::INT16:
2161  castType<int16>().setPageConstant(pagenum, values); return;
2162  case UT_Storage::INT32:
2163  castType<int32>().setPageConstant(pagenum, values); return;
2164  case UT_Storage::INT64:
2165  castType<int64>().setPageConstant(pagenum, values); return;
2166  case UT_Storage::REAL16:
2167  castType<fpreal16>().setPageConstant(pagenum, values); return;
2168  case UT_Storage::REAL32:
2169  castType<fpreal32>().setPageConstant(pagenum, values); return;
2170  case UT_Storage::REAL64:
2171  castType<fpreal64>().setPageConstant(pagenum, values); return;
2172  }
2173  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2174  }
2175 
2176  template<typename SRC_DATA_T>
2177  void setPageConstantMismatchedType(UT_PageNum pagenum, const SRC_DATA_T *values)
2178  {
2179  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2180  UT_ASSERT_P(!(SYSisSame<DATA_T,SRC_DATA_T>()));
2181 
2182  // This could be done directly, instead of converting
2183  // to a UT_SmallArray<DATA_T>, but this approach is simpler for now.
2184 
2185  UT_SmallArray<NotVoidType, ((TSIZE >= 1) ? TSIZE : 16)*sizeof(NotVoidType)> dest_values;
2186  const exint tuplesize = myImpl.getTupleSize();
2187  dest_values.setSize(tuplesize);
2188 
2189  for (exint i = 0; i < tuplesize; ++i)
2190  dest_values(i) = UTconvertStorage<NotVoidType>(values[i]);
2191 
2192  setPageConstant(pagenum, dest_values.array());
2193  }
2194 
2196  {
2197  public:
2200  static const uintptr_t theConstantPageBit = 1;
2201 
2202  /// This is always valid to call.
2204  {
2205  return (uintptr_t(myData) & theConstantPageBit) != 0;
2206  }
2207  /// This is only valid to call if the type doesn't fit inline.
2209  {
2210  return (uintptr_t(myData) == theConstantPageBit);
2211  }
2212  /// This is always valid to call, and will return true
2213  /// iff this is either inline and all zero or
2214  /// using the constant zero optimization.
2216  {
2217  return (uintptr_t(myData) == theConstantPageBit)
2218  && (myPRefCount == nullptr);
2219  }
2220  /// This is only valid to call when DATA_T is non-void.
2221  SYS_FORCE_INLINE static bool typeFitsInline(exint tuplesize = TSIZE)
2222  {
2224  if (sizeof(PageTableEntry) <= sizeof(NotVoidType))
2225  return false;
2226  return (sizeof(NotVoidType)*((TSIZE >= 0) ? TSIZE : tuplesize) < sizeof(PageTableEntry));
2227  }
2228 
2229  /// This is always valid to call, but be aware that the parameter is the
2230  /// number of *bytes* in a tuple, not the number of components in a
2231  /// tuple.
2232  SYS_FORCE_INLINE bool isRefd(exint tuplebytes) const
2233  {
2234  if (!isConstant())
2235  return true;
2236 
2237  // This will be checked at compile time, when possible.
2238  if (sizeof(PageTableEntry) > ((TSIZE >= 0 && !SYSisSame<DATA_T,void>()) ? TSIZE*sizeof(NotVoidType) : tuplebytes))
2239  return false;
2240 
2241  return !isConstantZero();
2242  }
2243 
2244  /// NOTE: This should only be called when there's an actual allocated page,
2245  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2247  {
2248  UT_ASSERT_P(myPRefCount->relaxedLoad() > 0);
2249  return myPRefCount->relaxedLoad() != 1;
2250  }
2251 
2252  /// NOTE: This should only be called when there's an actual allocated page to decRef,
2253  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2255  {
2256  myPRefCount->add(1);
2257  }
2258 
2259  /// NOTE: This should only be called when there's an actual allocated page to decRef,
2260  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2262  {
2263  int64 new_count = myPRefCount->add(-1);
2264  UT_ASSERT_P(new_count >= 0);
2265  if (new_count == 0)
2266  {
2267  free((NotVoidType*)(uintptr_t(myData) & ~theConstantPageBit));
2268  free(myPRefCount);
2269 #if 0
2270  printf("Free page %d pages total\n", int(thePageTablePageCount.add(-1)));
2271  fflush(stdout);
2272 #endif
2273  }
2274  }
2275 
2276  /// NOTE: This should only be called when there's an actual allocated page,
2277  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2279  {
2280  UT_ASSERT_P(myPRefCount->relaxedLoad() > 0);
2281  return myPRefCount->relaxedLoad();
2282  }
2283 
2284  SYS_FORCE_INLINE void alloc(UT_PageOff nelements, exint tuplesize = TSIZE)
2285  {
2287  myData = (NotVoidType *)malloc(nelements*((TSIZE >= 0) ? TSIZE : tuplesize)*sizeof(NotVoidType));
2288  myPRefCount = (SYS_AtomicCounter *)malloc(sizeof(SYS_AtomicCounter));
2289  myPRefCount->relaxedStore(1);
2290 #if 0
2291  printf("Alloc page %d pages total\n", int(thePageTablePageCount.add(1)));
2292  fflush(stdout);
2293 #endif
2294  }
2295  void realloc(UT_PageOff sizetocopy, UT_PageOff newpagecapacity, exint tuplesize = TSIZE)
2296  {
2298  UT_ASSERT_P(sizetocopy >= 0 && sizetocopy <= newpagecapacity);
2299  UT_ASSERT_P(!isConstant());
2300 
2301  UT_ASSERT_P(tuplesize >= 0);
2302  // This is just to try to force the compiler to constant-value-optimize.
2303  if (TSIZE >= 0)
2304  tuplesize = TSIZE;
2305 
2306  UT_ASSERT_P(myPRefCount->relaxedLoad() > 0);
2307  if (myPRefCount->relaxedLoad() > 1)
2308  {
2309  // If shared, need to make a new page.
2310  PageTableEntry newpage;
2311  newpage.alloc(newpagecapacity, tuplesize);
2312  NotVoidType *__restrict newdata = newpage.myData;
2313  const NotVoidType *__restrict olddata = myData;
2314  // Yes, this loop is an inline memcpy, because memcpy won't inline,
2315  // but the compiler will optimize it if it's like this.
2316  sizetocopy *= tuplesize;
2317  while (sizetocopy)
2318  {
2319  *newdata = *olddata;
2320  ++newdata;
2321  ++olddata;
2322  --sizetocopy;
2323  }
2324  decRef();
2325  myData = newpage.myData;
2326  myPRefCount = newpage.myPRefCount;
2327  }
2328  else
2329  {
2330  // If not shared, we can potentially reuse the same memory.
2331  myData = (NotVoidType*)::realloc(myData, newpagecapacity*(tuplesize*sizeof(NotVoidType)));
2332  // NOTE: Reference count doesn't need to change, since it's 1.
2333  }
2334  }
2335 
2336  /// Atomically decrement the reference count iff the page is shared.
2337  /// See the description of decRefIffSharedPage() above for
2338  /// why this function exists.
2340  bool decRefIffShared(exint tuplesize = TSIZE)
2341  {
2342  if (isConstant())
2343  {
2344  UT_ASSERT_P(tuplesize >= 0);
2345  if (typeFitsInline(tuplesize) || isConstantZero())
2346  return false;
2347  }
2348  return decCounterIffAbove1(*myPRefCount);
2349  }
2350 
2351  /// Returns the data pointer, if not a constant page
2354  {
2355  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtr() is for non-constant pages");
2357 
2358  return myData;
2359  }
2360 
2361  /// Returns the data pointer, if not a constant page
2363  const NotVoidType *getFirstPtr() const
2364  {
2365  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtr() is for non-constant pages");
2367 
2368  return myData;
2369  }
2370 
2371  /// Returns the data pointer, if not a constant page
2374  {
2375  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtrVoid() is for non-constant pages");
2376 
2377  return myData;
2378  }
2379 
2380  /// Returns the data pointer, if not a constant page
2382  const void *getFirstPtrVoid() const
2383  {
2384  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtrVoid() is for non-constant pages");
2385 
2386  return myData;
2387  }
2388 
2389  /// Returns the data pointer, or whatever's in its place
2390  /// with bit 0 set to 1 if it's something constant.
2391  /// If it's constant, it might not be a flagged pointer.
2394  {
2395  return myData;
2396  }
2397 
2398  /// Returns the data pointer, or whatever's in its place
2399  /// with bit 0 set to 1 if it's something constant.
2400  /// If it's constant, it might not be a flagged pointer.
2402  const void *getFirstPtrVoidUnsafe() const
2403  {
2404  return myData;
2405  }
2406 
2407  /// Returns the data pointer, if an inline constant page
2410  {
2411  UT_ASSERT_MSG_P(isConstant() && typeFitsInline(tuplesize), "getInlinePtr() is for inline constant pages");
2413  return (NotVoidType *)(((int8 *)(this+1)) - (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize)));
2414  }
2415 
2416  /// Returns the data pointer, if an inline constant page
2418  const NotVoidType *getInlinePtr(exint tuplesize) const
2419  {
2420  UT_ASSERT_MSG_P(isConstant() && typeFitsInline(tuplesize), "getInlinePtr() is for inline constant pages");
2422  return (const NotVoidType *)(((const int8 *)(this+1)) - (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize)));
2423  }
2424 
2425  /// Returns the data pointer, if a non-inline constant page.
2426  /// Returns nullptr for a zero constant page.
2429  {
2430  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtr() is for constant pages");
2432 
2433  return (NotVoidType *)(uintptr_t(myData) & ~theConstantPageBit);
2434  }
2435 
2436  /// Returns the data pointer, if a non-inline constant page.
2437  /// Returns nullptr for a zero constant page.
2439  const NotVoidType *getMaskedPtr() const
2440  {
2441  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtr() is for constant pages");
2443 
2444  return (const NotVoidType *)(uintptr_t(myData) & ~theConstantPageBit);
2445  }
2446 
2447  /// Returns the data pointer, if a non-inline constant page.
2448  /// Returns nullptr for a zero constant page.
2451  {
2452  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtrVoid() is for constant pages");
2453 
2454  return (void *)(uintptr_t(myData) & ~theConstantPageBit);
2455  }
2456 
2457  /// Returns the data pointer, if a non-inline constant page.
2458  /// Returns nullptr for a zero constant page.
2460  const void *getMaskedPtrVoid() const
2461  {
2462  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtrVoid() is for constant pages");
2463 
2464  return (const void *)(uintptr_t(myData) & ~theConstantPageBit);
2465  }
2466 
2467  /// Initializes a page to constant zero, assuming that it hasn't been
2468  /// initialized yet.
2470  void initZero()
2471  {
2472  myData = (NotVoidType*)theConstantPageBit;
2473  myPRefCount = nullptr;
2474  }
2475 
2478  {
2479  myData = (NotVoidType *)(uintptr_t(myData) | theConstantPageBit);
2480  }
2481 
2482  void tryCompressPage(UT_PageOff pagesize, exint tuplesize = TSIZE)
2483  {
2484  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2485  UT_ASSERT_P(!isConstant());
2486 
2487  tuplesize = (TSIZE >= 0 ? TSIZE : tuplesize);
2488 
2489  const NotVoidType *const first = myData;
2490  const NotVoidType *const end = first + (TSIZE >= 0 ? TSIZE : tuplesize)*exint(pagesize);
2491  for (const NotVoidType *other = first+tuplesize; other < end; other += tuplesize)
2492  {
2493  for (exint component = 0; component < (TSIZE >= 0 ? TSIZE : tuplesize); ++component)
2494  {
2495  if (other[component] != first[component])
2496  return;
2497  }
2498  }
2499  // We have a constant page! Compress it!
2500 
2501  PageTableEntry newpage;
2502  newpage.initZero();
2503  const exint bytesize = sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize);
2504  // Value stored inline if it's strictly smaller than a pointer
2505  // The first check can be done at compile time, so it's only logically redundant,
2506  // not functionally redundant.
2507  if (sizeof(PageTableEntry) > sizeof(NotVoidType) && sizeof(PageTableEntry) > bytesize)
2508  {
2509  memcpy(((int8*)&newpage) + (sizeof(PageTableEntry) - bytesize),
2510  first, bytesize);
2511  }
2512  else
2513  {
2514  if (!isZero(first, (TSIZE >= 0 ? TSIZE : tuplesize)))
2515  {
2516  newpage.alloc(UT_PageOff(1), tuplesize);
2517  NotVoidType *newfirst = newpage.getFirstPtr();
2518  memcpy(newfirst, first, bytesize);
2519  newpage.setConstantBit();
2520  }
2521  }
2522  decRef();
2523  *this = newpage;
2524  }
2525 
2526  /// NOTE: This always excludes sizeof(*this), and is only valid for
2527  /// non-small pages, (okay if constant).
2529  {
2530  if (!isConstant())
2531  return sizeof(SYS_AtomicCounter) + tuplebytes*thePageSize;
2532  if (tuplebytes < sizeof(PageTableEntry) || isConstantZero())
2533  return 0;
2534  return sizeof(SYS_AtomicCounter) + tuplebytes;
2535  }
2536 
2537  template<typename DEST_DATA_T>
2540  castType() const
2541  {
2544  }
2545  template<typename DEST_DATA_T>
2549  {
2552  }
2553 
2554  /// NOTE: This just does a shallow comparison.
2556  bool operator==(const ThisType &that) const
2557  {
2558  return (myData == that.myData) && (myPRefCount == that.myPRefCount);
2559  }
2560  /// NOTE: This just does a shallow comparison.
2562  bool operator!=(const ThisType &that) const
2563  {
2564  return (myData != that.myData) || (myPRefCount != that.myPRefCount);
2565  }
2566 
2567  private:
2568  NotVoidType *myData;
2569  SYS_AtomicCounter *myPRefCount;
2570  };
2571 
2574  {
2575  exint cur_count = counter.relaxedLoad();
2576 
2577  // It should be rare for this to ever loop, because looping
2578  // means that something else referring to the same page has
2579  // done an incRef, decRef, or decRefIffShared on this same page,
2580  // between the relaxedLoad and the compare_swap calls, or on
2581  // subsequent loops, between the compare_swap calls.
2582  while (true)
2583  {
2584  if (cur_count == 1)
2585  return false; // Not shared, so don't decrement
2586  exint new_count = cur_count-1;
2587  exint prev_count = counter.compare_swap(cur_count, new_count);
2588  if (prev_count == cur_count)
2589  return true; // Successfully decremented
2590  cur_count = prev_count;
2591  }
2592  // Execution never gets here, because the only escapes from
2593  // the loops return.
2594  }
2595 
2596  SYS_FORCE_INLINE static NotVoidType getConstant(const PageTableEntry *const page,exint component,exint tuplesize = TSIZE)
2597  {
2598  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2599  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2600  UT_ASSERT_P(component >= 0 && component < tuplesize);
2601  UT_ASSERT_P(page->isConstant());
2602 
2603  // Any types whose tuples are strictly smaller than a pointer will be stored inline.
2604  // If a single one is at least as large as a pointer, it's
2605  // rejected at compile time, so there'll be no op at runtime in that case.
2606  if (PageTableEntry::typeFitsInline(tuplesize))
2607  {
2608  return page->getInlinePtr(tuplesize)[component];
2609  }
2610 
2611  const NotVoidType *masked = page->getMaskedPtr();
2612  if (!masked)
2613  {
2614  NotVoidType zero;
2615  memset(&zero, 0, sizeof(NotVoidType));
2616  return zero;
2617  }
2618 
2619  // Other types have a 1-element page.
2620  return masked[component];
2621  }
2622  /// NOTE: It may seem like overkill to take a PageTableEntry *, but the
2623  /// returned pointer may be *inside* the pointer that the caller owns,
2624  /// so we need the caller's pointer, not a copy. getConstant doesn't
2625  /// have this issue, since it can use the local pointer and read the
2626  /// value.
2627  SYS_FORCE_INLINE static const NotVoidType *getConstantPtr(const PageTableEntry *page,exint component = 0,exint tuplesize = TSIZE)
2628  {
2629  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2630  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2631  UT_ASSERT_P(component >= 0 && component < tuplesize);
2632  UT_ASSERT_P(page->isConstant());
2633 
2634  // Any types whose tuples are strictly smaller than PageTableEntry will be stored inline.
2635  if (PageTableEntry::typeFitsInline(tuplesize))
2636  {
2637  return page->getInlinePtr(tuplesize) + component;
2638  }
2639 
2640  const NotVoidType *masked = page->getMaskedPtr();
2641  if (!masked)
2642  return nullptr;
2643 
2644  // Other types have a 1-element page.
2645  return masked + component;
2646  }
2647  /// NOTE: It may seem like overkill to take a PageTableEntry *, but the
2648  /// returned pointer may be *inside* the pointer that the caller owns,
2649  /// so we need the caller's pointer, not a copy. getConstant doesn't
2650  /// have this issue, since it can use the local pointer and read the
2651  /// value.
2652  SYS_FORCE_INLINE static NotVoidType *getConstantPtr(PageTableEntry *page,exint component = 0,exint tuplesize = TSIZE)
2653  {
2654  return SYSconst_cast(getConstantPtr((const PageTableEntry *)page, component, tuplesize));
2655  }
2657  {
2658  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2659  UT_ASSERT_P(TSIZE >= 1);
2660  UT_ASSERT_P(!PAGESHARDENED);
2661 
2662  if (PAGESHARDENED)
2663  return;
2664 
2666 
2667  // Unref existing non-constant page
2668  bool wasconst = page->isConstant();
2669  if (!wasconst)
2670  page->decRef();
2671 
2672  // Easy for inline case, checked at compile-time.
2673  if (sizeof(PageTableEntry) > sizeof(Tuple))
2674  {
2675  page->initZero();
2676  NotVoidType *tuple = page->getInlinePtr(theSafeTupleSize);
2677  *(Tuple *)tuple = val;
2678  return;
2679  }
2680 
2681  bool iszero = isZero(val);
2682 
2683  // In other cases, we may or may not have to unref constant page
2684  if (wasconst)
2685  {
2686  NotVoidType *masked = page->getMaskedPtr();
2687  if (!masked)
2688  {
2689  // Fairly common case: already zero, making zero.
2690  if (iszero)
2691  return;
2692  }
2693  else if (iszero || (page->isShared() && (*(const Tuple*)masked != val)))
2694  {
2695  // No longer need this old constant page
2696  page->decRef();
2697  }
2698  else if (!page->isShared())
2699  {
2700  // Reuse the unshared constant page
2701  *(Tuple*)masked = val;
2702  return;
2703  }
2704  else
2705  {
2706  // Already equal; nothing to do
2707  UT_ASSERT_P(*(const Tuple*)masked == val);
2708  return;
2709  }
2710  }
2711 
2712  if (iszero)
2713  {
2714  page->initZero();
2715  return;
2716  }
2717 
2718  // Need to allocate new constant page
2719  PageTableEntry newpage;
2720  newpage.alloc(UT_PageOff(1), theSafeTupleSize);
2721  NotVoidType *tuple = newpage.getFirstPtr();
2722  (*(Tuple*)tuple) = val;
2723  newpage.setConstantBit();
2724  (*page) = newpage;
2725  }
2726 
2727  static void makeConstant(PageTableEntry *page, const NotVoidType &val, exint tuplesize = TSIZE)
2728  {
2729  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2730  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2731  UT_ASSERT_P(!PAGESHARDENED);
2732  UT_ASSERT_P(tuplesize >= 1);
2733 
2734  if (PAGESHARDENED)
2735  return;
2736 
2737  // Unref existing non-constant page
2738  bool wasconst = page->isConstant();
2739  if (!wasconst)
2740  page->decRef();
2741 
2742  // Easy for inline case, partly checked at compile-time.
2743  if (PageTableEntry::typeFitsInline(tuplesize))
2744  {
2745  page->initZero();
2746  NotVoidType *tuple = page->getInlinePtr(tuplesize);
2747  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2748  tuple[i] = val;
2749  return;
2750  }
2751 
2752  bool iszero = isZero(val);
2753 
2754  // In other cases, we may or may not have to unref constant page
2755  if (wasconst)
2756  {
2757  NotVoidType *masked = page->getMaskedPtr();
2758  if (!masked)
2759  {
2760  // Fairly common case: already zero, making zero.
2761  if (iszero)
2762  return;
2763  }
2764  else if (iszero)
2765  {
2766  // No longer need this old constant page
2767  page->decRef();
2768  }
2769  else if (page->isShared())
2770  {
2771  bool equal = true;
2772  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2773  {
2774  if (masked[i] != val)
2775  {
2776  equal = false;
2777  break;
2778  }
2779  }
2780 
2781  if (equal)
2782  {
2783  // Already equal; nothing to do
2784  return;
2785  }
2786 
2787  // No longer need this old constant page
2788  page->decRef();
2789  }
2790  else
2791  {
2792  // Reuse the unshared constant page
2793  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2794  masked[i] = val;
2795  return;
2796  }
2797  }
2798 
2799  if (iszero)
2800  {
2801  page->initZero();
2802  return;
2803  }
2804 
2805  // Need to allocate new constant page
2806  PageTableEntry newpage;
2807  newpage.alloc(UT_PageOff(1), tuplesize);
2808  NotVoidType *tuple = newpage.getFirstPtr();
2809  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2810  tuple[i] = val;
2811  newpage.setConstantBit();
2812  (*page) = newpage;
2813  }
2814 
2815  static void makeConstant(PageTableEntry *page, const NotVoidType *values, exint tuplesize)
2816  {
2817  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2818  UT_ASSERT_P(TSIZE == -1 || TSIZE == tuplesize);
2819  UT_ASSERT_P(!PAGESHARDENED);
2820  UT_ASSERT_P(tuplesize >= 1);
2821 
2822  if (PAGESHARDENED)
2823  return;
2824 
2825  // Unref existing non-constant page
2826  bool wasconst = page->isConstant();
2827  if (!wasconst)
2828  page->decRef();
2829 
2830  // Easy for inline case, partly checked at compile-time.
2831  if (PageTableEntry::typeFitsInline(tuplesize))
2832  {
2833  page->initZero();
2834  NotVoidType *tuple = page->getInlinePtr(tuplesize);
2835  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2836  tuple[i] = values[i];
2837  return;
2838  }
2839 
2840  bool iszero = isZero(values, (TSIZE >= 0 ? TSIZE : tuplesize));
2841 
2842  // In other cases, we may or may not have to unref constant page
2843  if (wasconst)
2844  {
2845  NotVoidType *masked = page->getMaskedPtr();
2846  if (!masked)
2847  {
2848  // Fairly common case: already zero, making zero.
2849  if (iszero)
2850  return;
2851  }
2852  else if (iszero)
2853  {
2854  // No longer need this old constant page
2855  page->decRef();
2856  }
2857  else if (page->isShared())
2858  {
2859  bool equal = true;
2860  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2861  {
2862  if (masked[i] != values[i])
2863  {
2864  equal = false;
2865  break;
2866  }
2867  }
2868 
2869  if (equal)
2870  {
2871  // Already equal; nothing to do
2872  return;
2873  }
2874 
2875  // No longer need this old constant page
2876  page->decRef();
2877  }
2878  else
2879  {
2880  // Reuse the unshared constant page
2881  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2882  masked[i] = values[i];
2883  return;
2884  }
2885  }
2886 
2887  if (iszero)
2888  {
2889  page->initZero();
2890  return;
2891  }
2892 
2893  // Need to allocate new constant page
2894  PageTableEntry newpage;
2895  newpage.alloc(UT_PageOff(1), tuplesize);
2896  NotVoidType *tuple = newpage.getFirstPtr();
2897  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2898  tuple[i] = values[i];
2899  newpage.setConstantBit();
2900  (*page) = newpage;
2901  }
2902 
2903  /// This replaces dest with src. It references if possible and keeps
2904  /// dest constant if easy. If desttuplesize > srctuplesize, it only
2905  /// replaces the first srctuplesize components.
2906  template<typename SrcType>
2907  static void replacePage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destpagesize, UT_PageOff destpagecapacity);
2908 
2909  /// This replaces part of dest with part of src. It keeps
2910  /// dest constant if easy. If desttuplesize > srctuplesize, it only
2911  /// replaces the first srctuplesize components.
2912  /// This will *not* check to see if the whole page is being replaced,
2913  /// so if you want the src page to be referenced, or overwrite a
2914  /// constant value, when writing the whole page, use replacePage.
2915  template<typename SrcType>
2916  static void copyPartialPage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destoff, UT_PageOff srcoff, UT_PageOff ntuples, UT_PageOff destpagecapacity);
2917 
2918  /// This version of makeConstant copies a const page onto this page,
2919  /// converting type if necessary, and supporting a different tuple size,
2920  /// so long as the destination size either isn't larger or is constant.
2921  /// NOTE: This would be templated on the actual templates of UT_PageArray,
2922  /// but somehow, Visual C++ can't deduce the type, so the template type
2923  /// has to be specified by the caller, and one argument is much easier
2924  /// to specify on every call than 4.
2925  template<typename SrcType>
2926  static void makeConstantFrom(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize);
2927 
2928  /// This is just a helper function for writing a single source tuple (or nullptr) to a block of tuple data, with conversion.
2929  /// NOTE: destpagedata must point to an *UNSHARED* page's data.
2930  template<typename SrcNotVoidType>
2931  static void fillNonConstWithConst(NotVoidType *destpagedata, NotVoidType *destpageend, const SrcNotVoidType *stuple, exint mintuplesize, exint desttupleextra);
2932 
2933  /// This is just a helper function for copying a non-constant source page to a non-constant destination page, with conversion.
2934  /// NOTE: destpagedata must point to an *UNSHARED* page's data.
2935  /// NOTE: The two data ranges must be *NON-OVERLAPPING*.
2936  template<typename SrcNotVoidType>
2937  static void copyNonConst(NotVoidType *destpagedata, const SrcNotVoidType *srcpagedata, exint desttuplesize, exint srctuplesize, UT_PageOff ntuples);
2938 
2939  /// This is just a helper function for checking whether two constant pages are equal, up to
2940  /// the smaller of the two, to see if writing is necessary.
2941  template<typename SrcNotVoidType>
2942  static bool isEqualConst(const NotVoidType *tuple, const SrcNotVoidType *stuple, exint mintuplesize);
2943 
2944  static void hardenConstantPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
2945  {
2946  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2947  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2948  UT_ASSERT_P(page->isConstant());
2949 
2950  PageTableEntry newpage;
2951  newpage.alloc(pagecapacity, tuplesize);
2952  NotVoidType *dest = newpage.getFirstPtr();
2953  bool isinline = PageTableEntry::typeFitsInline(tuplesize);
2954  if (isinline ? page->isConstantAndZeroSafe() : page->isConstantZero())
2955  {
2956  memset(dest, 0, (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize))*pagecapacity);
2957  }
2958  else
2959  {
2960  const NotVoidType *src;
2961  if (isinline)
2962  src = page->getInlinePtr(tuplesize);
2963  else
2964  src = page->getMaskedPtr();
2965 
2966  for (UT_PageOff i = 0; i < pagecapacity; ++i)
2967  {
2968  for (exint component = 0; component < tuplesize; ++component, ++dest)
2969  *dest = src[component];
2970  }
2971 
2972  if (!isinline)
2973  page->decRef();
2974  }
2975  (*page) = newpage;
2976  }
2977 
2978  static void hardenConstantPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
2979  {
2980  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2981  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2982  UT_ASSERT_P(page->isConstant());
2983 
2984  PageTableEntry newpage;
2985  newpage.alloc(pagecapacity, tuplesize);
2986  // Compile-time check, to speed up case where even one element doesn't fit inline.
2987  bool isinline = PageTableEntry::typeFitsInline(tuplesize);
2988  if (!isinline && !page->isConstantZero())
2989  page->decRef();
2990  (*page) = newpage;
2991  }
2992 
2993  static void hardenSharedPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
2994  {
2995  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2996  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2997  UT_ASSERT_P(!page->isConstant());
2998  // This may technically no longer be shared between checking and calling
2999  // due to other threads decRef'ing the page. It's okay to harden anyway.
3000  //UT_ASSERT_P(page->isShared());
3001 
3002  PageTableEntry newpage;
3003  newpage.alloc(pagecapacity, tuplesize);
3004  NotVoidType *dest = newpage.getFirstPtr();
3005  const NotVoidType *src = page->getFirstPtr();
3006  memcpy(dest, src, (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize))*pagecapacity);
3007  page->decRef();
3008  (*page) = newpage;
3009  }
3010 
3011  static void hardenSharedPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
3012  {
3013  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
3014  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3015  UT_ASSERT_P(!page->isConstant());
3016  // This may technically no longer be shared between checking and calling
3017  // due to other threads decRef'ing the page. It's okay to harden anyway.
3018  //UT_ASSERT_P(page->isShared());
3019 
3020  PageTableEntry newpage;
3021  newpage.alloc(pagecapacity, tuplesize);
3022  page->decRef();
3023  (*page) = newpage;
3024  }
3025 
3027  {
3028  public:
3031 
3033  {
3034  UT_ASSERT_P(myCapacity > IDX_T(0));
3035  return reinterpret_cast<PageTableEntry*>(this+1);
3036  }
3038  {
3039  UT_ASSERT_P(myCapacity > IDX_T(0));
3040  return reinterpret_cast<const PageTableEntry *>(this+1);
3041  }
3042  // NOTE: Clang sometimes decides to ignore const references and pass by
3043  // value, so we have to return by pointer, in case the page
3044  // is constant and inlined, so that the address of the constant
3045  // value is accessible through this function.
3046  // Yes, many cases don't rely on this, but to reduce the risk of
3047  // bugs, the only way to access page pointers in a const PageTable
3048  // is to call this to get the page pointer address and explicitly
3049  // dereference, if valid in the given circumstance.
3051  {
3052  UT_ASSERT_P(myCapacity > IDX_T(i << THEPAGEBITS));
3053  return reinterpret_cast<const PageTableEntry *>(this+1) + i;
3054  }
3056  {
3057  UT_ASSERT_P(myCapacity > IDX_T(i << THEPAGEBITS));
3058  return reinterpret_cast<PageTableEntry*>(this+1) + i;
3059  }
3061  {
3062  UT_ASSERT_P(myRefCount.relaxedLoad() > 0);
3063  return myRefCount.relaxedLoad() != 1;
3064  }
3066  {
3067  return myRefCount.relaxedLoad();
3068  }
3069  /// NOTE: This is the size of the full array, not the number of pages.
3070  SYS_FORCE_INLINE IDX_T size() const
3071  {
3072  return mySize;
3073  }
3074  /// NOTE: This is the capacity of the full array, not the capacity of pages.
3076  {
3077  return myCapacity;
3078  }
3079  /// NOTE: This will *assume* the capacity is large enough! The caller *must* check!
3080  /// NOTE: This is the size of the full array, not the number of pages.
3081  SYS_FORCE_INLINE void setSize(IDX_T newsize)
3082  {
3083  UT_ASSERT_P(newsize <= capacity());
3084  UT_ASSERT_MSG_P(myRefCount.relaxedLoad() == 1, "The table must already be hardened before we modify it!");
3085  if (PAGESHARDENED && newsize > mySize)
3086  {
3087  UT_ASSERT_P(TSIZE >= 0);
3088  // Need to ensure pages between mySize and newsize are hardened.
3089  // Pages containing offsets less than or equal to mySize are
3090  // already guaranteed to be hardened.
3091  UT_PageNum startpage = numPages(mySize);
3092  UT_PageNum endpage = numPages(newsize);
3093  UT_PageOff pagecapacity(thePageSize);
3094  if (endpage == UT_PageNum(1) && exint(capacity()) < thePageSize)
3095  pagecapacity = capacity();
3096  PageTableEntry *page = getPPage(startpage);
3097  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3098  {
3099  if (page->isConstant())
3100  hardenConstantPage(page, pagecapacity);
3101  else if (page->isShared())
3102  hardenSharedPage(page, pagecapacity);
3103  }
3104  }
3105  mySize = newsize;
3106  }
3107  /// NOTE: This will *assume* the capacity is large enough! The caller *must* check!
3108  /// NOTE: This is the size of the full array, not the number of pages.
3109  SYS_FORCE_INLINE void setSize(IDX_T newsize, exint tuplesize)
3110  {
3111  UT_ASSERT_P(newsize <= capacity());
3112  UT_ASSERT_MSG_P(myRefCount.relaxedLoad() == 1, "The table must already be hardened before we modify it!");
3113  if (PAGESHARDENED && newsize > mySize)
3114  {
3115  // Need to ensure pages between mySize and newsize are hardened.
3116  // Pages containing offsets less than or equal to mySize are
3117  // already guaranteed to be hardened.
3118  UT_PageNum startpage = numPages(mySize);
3119  UT_PageNum endpage = numPages(newsize);
3120  UT_PageOff pagecapacity(thePageSize);
3121  if (endpage == UT_PageNum(1) && exint(capacity()) < thePageSize)
3122  pagecapacity = capacity();
3123  PageTableEntry *page = getPPage(startpage);
3124  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3125  {
3126  if (page->isConstant())
3127  hardenConstantPage(page, pagecapacity, tuplesize);
3128  else if (page->isShared())
3129  hardenSharedPage(page, pagecapacity, tuplesize);
3130  }
3131  }
3132  mySize = newsize;
3133  }
3134  void fill(IDX_T start, IDX_T end, const NotVoidType &val);
3135  void fill(IDX_T start, IDX_T end, NotVoidType val, exint tuplesize);
3136  void fill(IDX_T start, IDX_T end, const NotVoidType *values, exint tuplesize);
3137  void fill(IDX_T start, IDX_T end, const UT_FixedVector<NotVoidType,theSafeTupleSize> &val);
3138 
3139  void incRef()
3140  {
3141  myRefCount.add(1);
3142  }
3143  void decRef(exint tuplesize)
3144  {
3145  const int64 new_count = myRefCount.add(-1);
3146  if (new_count != 0)
3147  {
3148  UT_ASSERT_P(new_count > 0);
3149  return;
3150  }
3151 
3152  UT_ASSERT_P(!(SYSisSame<DATA_T, void>()));
3153 
3154  const size_t bytesize = sizeof(NotVoidType)*((TSIZE >= 0) ? TSIZE : tuplesize);
3155 
3156  // If we're deleting the table, first, decRef all pages.
3157  UT_PageNum npages = numPages(myCapacity);
3158  PageTableEntry *page = getFirstPage();
3159  for (UT_PageNum p = 0; p < npages; ++p, ++page)
3160  {
3161  if (page->isRefd(bytesize))
3162  page->decRef();
3163  }
3164  free(this);
3165 #if 0
3166  printf("Free table %d pages (case 0); %d tables total\n",int(npages), int(thePageTableCount.add(-1)));
3167  fflush(stdout);
3168 #endif
3169  }
3170 
3171  void hardenAllPages(IDX_T start, IDX_T end, exint tuplesize = TSIZE)
3172  {
3173  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3174  UT_ASSERT_P(tuplesize >= 0);
3175  UT_PageNum startpage = pageNum(start);
3176  UT_PageNum endpage = numPages(end);
3177  UT_PageOff pagecapacity(thePageSize);
3178  if (endpage == UT_PageNum(1) && exint(capacity()) < thePageSize)
3179  pagecapacity = capacity();
3180  PageTableEntry *page = getPPage(startpage);
3181  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3182  {
3183  if (page->isConstant())
3184  hardenConstantPage(page, pagecapacity, tuplesize);
3185  else if (page->isShared())
3186  hardenSharedPage(page, pagecapacity, tuplesize);
3187  }
3188  }
3189 
3190  void tryCompressAllPages(IDX_T start, IDX_T end, exint tuplesize = TSIZE)
3191  {
3192  UT_ASSERT_P(tuplesize >= 0);
3193  UT_PageNum startpage = pageNum(start);
3194  UT_PageNum endpage = numPages(end);
3195  UT_PageNum npages = numPages(mySize);
3196  PageTableEntry *page = getPPage(startpage);
3197  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3198  {
3199  if (page->isConstant())
3200  continue;
3201 
3202  UT_PageOff pagesize = thePageSize;
3203  if (p+1 >= npages)
3204  {
3205  if (p+1 == npages)
3206  {
3207  UT_PageOff pagesizetemp = pageOff(mySize);
3208  if (pagesizetemp != UT_PageOff(0))
3209  pagesize = pagesizetemp;
3210  }
3211  else
3212  pagesize = UT_PageOff(0);
3213  }
3214  page->tryCompressPage(pagesize, tuplesize);
3215  }
3216  }
3217 
3218  void tryCompressPage(UT_PageNum pagenum, exint tuplesize = TSIZE)
3219  {
3220  UT_ASSERT_P(tuplesize >= 0);
3221  PageTableEntry *page = getPPage(pagenum);
3222  if (page->isConstant())
3223  return;
3224 
3225  UT_PageOff pagesize = thePageSize;
3226  UT_PageNum npages = numPages(mySize);
3227  if (pagenum+1 >= npages)
3228  {
3229  if (pagenum+1 == npages)
3230  {
3231  UT_PageOff pagesizetemp = pageOff(mySize);
3232  if (pagesizetemp != UT_PageOff(0))
3233  pagesize = pagesizetemp;
3234  }
3235  else
3236  pagesize = UT_PageOff(0);
3237  }
3238  page->tryCompressPage(pagesize, tuplesize);
3239  }
3240 
3241  static ThisType *alloc(UT_PageNum npages)
3242  {
3243  UT_ASSERT_MSG_P(npages > UT_PageNum(0), "Shouldn't have a zero-page page table; nullptr should be used instead.");
3244  ThisType *p = (ThisType *)malloc(sizeof(ThisType) + npages*sizeof(PageTableEntry));
3245  //UT_ASSERT_P(_CrtIsValidHeapPointer(p));
3246  p->myRefCount.relaxedStore(1);
3247 #if 0
3248  printf("Alloc table %d pages; %d tables total\n",int(npages), int(thePageTableCount.add(1)));
3249  fflush(stdout);
3250 #endif
3251  return p;
3252  }
3253  static ThisType *harden(ThisType *pages, IDX_T newcapacity, exint tuplesize=TSIZE)
3254  {
3255  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3256 
3257  // If we're setting capacity to zero, free the table.
3258  if (newcapacity == IDX_T(0))
3259  {
3260  if (pages)
3261  pages->decRef(tuplesize);
3262  return nullptr;
3263  }
3264 
3265  UT_ASSERT_P(tuplesize >= 0);
3266  UT_ASSERT_MSG_P(newcapacity == roundUpCapacity(newcapacity), "Caller should already have adjusted capacity");
3267 
3268  bool wasshared = pages && pages->isShared();
3269  UT_PageNum npages = numPages(newcapacity);
3270  IDX_T oldcapacity = pages ? pages->capacity() : IDX_T(0);
3271  UT_PageNum oldnpages = numPages(oldcapacity);
3272 
3273  // Caller must force capacity to be on a page boundary if more than one page, else power of 2.
3274  UT_ASSERT_P((newcapacity < IDX_T(thePageSize) && SYSisPow2(exint(newcapacity))) || newcapacity == IDX_T(exint(npages)<<thePageBits));
3275 
3276  const bool nonconstfirst = pages && !pages->getFirstPage()->isConstant();
3277 
3278  if (npages == UT_PageNum(1) && oldnpages == UT_PageNum(1) && !wasshared)
3279  {
3280  // Can reuse existing page table if we're staying at one page, but changing its capacity.
3281 
3282  // NOTE: The "newcapacity != oldcapacity" will almost always be true here, but
3283  // we need to make absolutely sure that we don't realloc a shared page when just
3284  // calling hardenTable()! If we didn't have this check, it could happen, because
3285  // between the reference count check in hardenTable() could return 2, and then
3286  // before the pages->isShared() call above, the reference count drops to 1,
3287  // so we have an already hardened table, but we can identify that we don't need
3288  // to do anything because the capacity won't be changing.
3289  // The reason we can't have hardenTable() realloc a shared page is that in
3290  // GA_PrimitiveList, it needs to incDataRef() on all GA_OffsetLists in a shared
3291  // page before ever hardening one.
3292  if (nonconstfirst && newcapacity != oldcapacity)
3293  {
3294  // Just need to realloc the page.
3295  UT_PageOff newpagecapacity = UT_PageOff(exint(newcapacity));
3296  UT_PageOff sizetocopy = SYSmin(UT_PageOff(exint(pages->mySize)), newpagecapacity);
3297  PageTableEntry *page = pages->getFirstPage();
3298  page->realloc(sizetocopy, newpagecapacity, tuplesize);
3299  }
3300  // Don't need to change anything but capacity (and possibly size)
3301  // if constant page, even if it's an allocated constant page.
3302 
3303  pages->myCapacity = newcapacity;
3304  if (pages->mySize > newcapacity)
3305  pages->mySize = newcapacity;
3306  return pages;
3307  }
3308 
3309  PageTable *table = PageTable::alloc(npages);
3310  table->myCapacity = newcapacity;
3311  if (pages)
3312  {
3313  UT_ASSERT_P(pages->getRefCount() > 0 && table->getRefCount() > 0);
3314  UT_ASSERT_P(pages->capacity() > IDX_T(0));
3315  UT_ASSERT_P(oldnpages > UT_PageNum(0));
3316  const bool wassmallfirstpage = pages->capacity() < IDX_T(thePageSize);
3317  const bool willbesmallfirstpage = newcapacity < IDX_T(thePageSize);
3318 
3319  UT_PageNum minnpages = SYSmin(npages, oldnpages);
3320  table->mySize = pages->mySize;
3321  memcpy(table->getFirstPage(), pages->getFirstPage(), (sizeof(PageTableEntry))*minnpages);
3322 
3323  UT_ASSERT_P(!(SYSisSame<DATA_T, void>()));
3324 
3325  const size_t bytesize = sizeof(NotVoidType)*((TSIZE >= 0) ? TSIZE : tuplesize);
3326 
3327  if (wasshared)
3328  {
3329  // Increment all of the reference counts of reused pages,
3330  // if we're unsharing a table.
3331  PageTableEntry *page = table->getFirstPage();
3332  for (UT_PageNum p = 0; p < minnpages; ++p, ++page)
3333  {
3334  if (page->isRefd(bytesize))
3335  page->incRef();
3336  }
3337  pages->decRef(tuplesize);
3338  }
3339  else
3340  {
3341  // pages isn't shared, so we can just decRef() any pages no
3342  // longer referenced and free pages. This avoids calling
3343  // incRef on all kept pages and decRef on all pages like the
3344  // above would do if pages had its reference count go to
3345  // zero.
3346  PageTableEntry *page = pages->getFirstPage() + npages;
3347  for (UT_PageNum p = npages; p < oldnpages; ++p, ++page)
3348  {
3349  if (page->isRefd(bytesize))
3350  page->decRef();
3351  }
3352  UT_ASSERT_MSG_P(!pages->isShared(), "This should still be unshared.");
3353  free(pages);
3354 #if 0
3355  printf("Free table %d pages (case 1); %d tables total\n",int(oldnpages), int(thePageTableCount.add(-1)));
3356  fflush(stdout);
3357 #endif
3358  }
3359  // Can't refer to pages beyond this point!
3360  UT_IF_ASSERT_P(pages = nullptr;)
3361 
3362  if (table->mySize > newcapacity)
3363  table->mySize = newcapacity;
3364 
3365  // NOTE: The "newcapacity != oldcapacity" is to make absolutely
3366  // sure that we don't realloc a shared page when just
3367  // calling hardenTable()! This is because in
3368  // GA_PrimitiveList, it needs to incDataRef() on all
3369  // GA_OffsetLists in a shared page before ever hardening one.
3370  if (nonconstfirst && (wassmallfirstpage || willbesmallfirstpage) && newcapacity != oldcapacity)
3371  {
3372  UT_PageOff newpagecapacity = SYSmin(UT_PageOff(exint(newcapacity)), UT_PageOff(thePageSize));
3373  UT_PageOff sizetocopy = SYSmin(UT_PageOff(exint(table->mySize)), newpagecapacity);
3374  PageTableEntry *page = table->getFirstPage();
3375  page->realloc(sizetocopy, newpagecapacity, tuplesize);
3376  }
3377  }
3378  else
3379  {
3380  oldnpages = 0;
3381  table->mySize = IDX_T(0);
3382  }
3383 
3384  // Set any new pages to be constant zero.
3385  PageTableEntry *page = table->getFirstPage() + oldnpages;
3386  for (UT_PageNum p = oldnpages; p < npages; ++p, ++page)
3387  {
3388  page->initZero();
3389  }
3390 
3391  if (PAGESHARDENED && wasshared)
3392  {
3393  if (TSIZE >= 1)
3394  table->hardenAllPages(IDX_T(0), table->mySize);
3395  else
3396  table->hardenAllPages(IDX_T(0), table->mySize, tuplesize);
3397  }
3398 
3399  return table;
3400  }
3401 
3402  /// Atomically decrement the reference count iff the page is shared.
3403  /// See the description of decRefIffSharedPage() above for
3404  /// why this function exists.
3406  {
3407  return decCounterIffAbove1(myRefCount);
3408  }
3409 
3410  private:
3411  // The constructor/destructor should never be called, since the class
3412  // can only be allocated with malloc.
3413  PageTable() { UT_ASSERT_P(0); }
3414  PageTable(const PageTable &) { UT_ASSERT_P(0); }
3415  ~PageTable() { UT_ASSERT_P(0); }
3416 
3417  SYS_AtomicCounter myRefCount;
3418  IDX_T mySize;
3419  IDX_T myCapacity;
3420  };
3421 
3422  /// NOTE: These have to be below the PageTable class definition, because
3423  /// for no apparent reason, even though you can have a regular local
3424  /// variable of type PageTable before, you can't have a function
3425  /// parameter of type PageTable before. *sigh*
3426  /// @{
3427  static void hardenConstantPage(PageTable *pages, PageTableEntry *page)
3428  {
3429  const exint arraycapacity(exint(pages->capacity()));
3430  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3431  hardenConstantPage(page, pagecapacity);
3432  }
3433  static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page)
3434  {
3435  const exint arraycapacity(exint(pages->capacity()));
3436  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3437  hardenConstantPageNoInit(page, pagecapacity);
3438  }
3439  static void hardenConstantPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
3440  {
3441  const exint arraycapacity(exint(pages->capacity()));
3442  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3443  hardenConstantPage(page, pagecapacity, tuplesize);
3444  }
3445  static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
3446  {
3447  const exint arraycapacity(exint(pages->capacity()));
3448  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3449  hardenConstantPageNoInit(page, pagecapacity, tuplesize);
3450  }
3451  static void hardenSharedPage(PageTable *pages, PageTableEntry *page)
3452  {
3453  const exint arraycapacity(exint(pages->capacity()));
3454  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3455  hardenSharedPage(page, pagecapacity);
3456  }
3457  static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page)
3458  {
3459  const exint arraycapacity(exint(pages->capacity()));
3460  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3461  hardenSharedPageNoInit(page, pagecapacity);
3462  }
3463  static void hardenSharedPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
3464  {
3465  const exint arraycapacity(exint(pages->capacity()));
3466  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3467  hardenSharedPage(page, pagecapacity, tuplesize);
3468  }
3469  static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
3470  {
3471  const exint arraycapacity(exint(pages->capacity()));
3472  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3473  hardenSharedPageNoInit(page, pagecapacity, tuplesize);
3474  }
3475  /// @}
3476 
3477  void forceHardenTable(IDX_T newcapacity)
3478  {
3479  if constexpr (SYSisSame<DATA_T,void>())
3480  {
3481  const UT_Storage storage = myImpl.getStorage();
3482  switch (storage)
3483  {
3484  case UT_Storage::INVALID:
3485  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
3486  return;
3487  case UT_Storage::INT8:
3488  castType<int8>().forceHardenTable(newcapacity); return;
3489  case UT_Storage::UINT8:
3490  castType<uint8>().forceHardenTable(newcapacity); return;
3491  case UT_Storage::INT16:
3492  castType<int16>().forceHardenTable(newcapacity); return;
3493  case UT_Storage::INT32:
3494  castType<int32>().forceHardenTable(newcapacity); return;
3495  case UT_Storage::INT64:
3496  castType<int64>().forceHardenTable(newcapacity); return;
3497  case UT_Storage::REAL16:
3498  castType<fpreal16>().forceHardenTable(newcapacity); return;
3499  case UT_Storage::REAL32:
3500  castType<fpreal32>().forceHardenTable(newcapacity); return;
3501  case UT_Storage::REAL64:
3502  castType<fpreal64>().forceHardenTable(newcapacity); return;
3503  }
3504  return;
3505  }
3506 
3507  PageTable *&pages = myImpl.getPages();
3508  pages = PageTable::harden(pages, newcapacity, myImpl.getTupleSize());
3509  }
3510 
3511 protected:
3512  template<typename T>
3514  static bool isZero(const T &val)
3515  {
3516  // TODO: Find a way to check for zero that will still compile, but
3517  // that avoids the mess of dealing with memset for POD types.
3518  T zero;
3519  memset(&zero, 0, sizeof(T));
3520  return (val == zero);
3521  }
3522 
3523  template<typename T>
3525  static bool isZero(const T *val,exint tuplesize)
3526  {
3527  UT_ASSERT_P(tuplesize >= 1);
3528  if (tuplesize == 0)
3529  return true;
3530 
3531  // TODO: Find a way to check for zero that will still compile, but
3532  // that avoids the mess of dealing with memset for POD types.
3533  T zero;
3534  memset(&zero, 0, sizeof(T));
3535 
3536  bool iszero = (val[0] == zero);
3537  for (exint i = 1; i < tuplesize; ++i)
3538  iszero &= (val[i] == zero);
3539  return iszero;
3540  }
3541 
3542  template<typename T0,typename T1>
3544  static bool isEqual(const T0 *a,const T1 *b,exint tuplesize)
3545  {
3546  UT_ASSERT_P(tuplesize >= 1);
3547  UT_ASSERT_P(a != nullptr && b != nullptr);
3548 
3549  for (exint i = 0; i < tuplesize; ++i)
3550  {
3551  if (a[i] != b[i])
3552  return false;
3553  }
3554  return true;
3555  }
3556 
3557  template<typename DEST_TYPE,typename SRC_TYPE,exint SRC_TSIZE>
3560  {
3561  if constexpr (SYSisSame<DEST_TYPE,SRC_TYPE>())
3563  else
3564  {
3566  for (exint i = 0; i < SRC_TSIZE; ++i)
3567  dest[i] = UTconvertStorage<DEST_TYPE>(src[i]);
3568  return UT_FixedVector<DEST_TYPE, SRC_TSIZE>{ dest };
3569  }
3570  }
3571 
3572 private:
3573  /// This class wraps the data of UT_PageArray, just so that if
3574  /// the tuple size and data type are known at compile-time, it doesn't
3575  /// need to store that information as an int and a UT_Storage.
3576  /// myPages is the same in all of them, but can't be extracted out,
3577  /// because the C++ standard, unfortunately, forbids 0-byte classes.
3578  /// @{
3579  struct ImplPageData
3580  {
3581  PageTable *myPages;
3582  };
3583 
3584  template<exint ITSIZE, typename IDATA_T>
3585  class Impl :
3586  private ImplPageData,
3587  private UT_TupleType<ITSIZE, IDATA_T>
3588  {
3589  public:
3590  SYS_FORCE_INLINE const PageTable *getPages() const { return ImplPageData::myPages; }
3591  SYS_FORCE_INLINE PageTable *&getPages() { return ImplPageData::myPages; }
3596  };
3597 
3598  Impl<TSIZE,DATA_T> myImpl;
3599  /// @}
3600 };
3601 
3602 #endif
void hardenAllPages(IDX_T start, IDX_T end, exint tuplesize=TSIZE)
SYS_FORCE_INLINE NotVoidType operator()(IDX_T i, exint component=0) const
Read element i's specified component (default component 0).
Definition: UT_PageArray.h:444
type
Definition: core.h:556
static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page)
GLint first
Definition: glcorearb.h:405
static void copyNonConst(NotVoidType *destpagedata, const SrcNotVoidType *srcpagedata, exint desttuplesize, exint srctuplesize, UT_PageOff ntuples)
SYS_FORCE_INLINE bool isPageConstant(UT_PageNum pagenum) const
Returns true iff the specified page is constant-compressed.
static void hardenConstantPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
SYS_FORCE_INLINE void opVector(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE > &v)
Definition: UT_PageArray.h:709
static SYS_FORCE_INLINE const NotVoidType * getConstantPtr(const PageTableEntry *page, exint component=0, exint tuplesize=TSIZE)
void SecondGuess
Definition: UT_Storage.h:91
SYS_FORCE_INLINE void setSize(IDX_T newsize)
static SYS_FORCE_INLINE bool decCounterIffAbove1(SYS_AtomicCounter &counter)
int int32
Definition: SYS_Types.h:39
SYS_FORCE_INLINE const NotVoidType * getInlinePtr(exint tuplesize) const
Returns the data pointer, if an inline constant page.
SYS_FORCE_INLINE void * getFirstPtrVoid()
Returns the data pointer, if not a constant page.
UT_Storage
Definition: UT_Storage.h:28
static void hardenSharedPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
void hardenAllPagesUnknownType(IDX_T start, IDX_T end)
void tryCompressPage(UT_PageOff pagesize, exint tuplesize=TSIZE)
static ThisType * harden(ThisType *pages, IDX_T newcapacity, exint tuplesize=TSIZE)
static void hardenSharedPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
SYS_FORCE_INLINE bool isArrayZero() const
Returns true iff the entire array is constant & zero.
static const UT_Storage theStorage
Definition: UT_PageArray.h:246
static SYS_FORCE_INLINE NotVoidType getConstant(const PageTableEntry *const page, exint component, exint tuplesize=TSIZE)
NotVoid< DATA_T >::type NotVoidType
Definition: UT_PageArray.h:238
UT_PageArray(const ThisType &that)
Definition: UT_PageArray.h:304
void setTupleSize(exint newtuplesize, const UT_Defaults &v)
static ThisType * alloc(UT_PageNum npages)
void tryCompressPage(UT_PageNum pagenum, exint tuplesize=TSIZE)
SYS_FORCE_INLINE const NotVoidType * getMaskedPtr() const
getFileOption("OpenEXR:storage") storage
Definition: HDK_Image.dox:276
GLboolean * data
Definition: glcorearb.h:131
void copy(IDX_T dest, IDX_T src)
Copies a single element from src to dest within this.
void tryCompressAllPagesUnknownType(IDX_T start, IDX_T end)
const GLdouble * v
Definition: glcorearb.h:837
auto printf(const S &fmt, const T &...args) -> int
Definition: printf.h:670
uint128_t uintptr_t
Definition: format.h:479
SYS_FORCE_INLINE ~UT_PageArray()
Definition: UT_PageArray.h:318
SYS_FORCE_INLINE bool operator!=(const ThisType &that) const
NOTE: This just does a shallow comparison.
void tryCompressAllPages(IDX_T start, IDX_T end, exint tuplesize=TSIZE)
GLuint start
Definition: glcorearb.h:475
constexpr const T * data() const noexcept
SYS_FORCE_INLINE void clear()
Definition: UT_PageArray.h:324
SYS_FORCE_INLINE ThisType & operator=(const UT_PageArray< SRC_DATA_T, SRC_TSIZE, SRC_TABLEHARDENED, SRC_PAGESHARDENED, THEPAGEBITS, IDX_T > &that)
Definition: UT_PageArray.h:382
static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
void opVectorUnknownType(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE > &v)
SYS_FORCE_INLINE IDX_T size() const
Definition: UT_PageArray.h:842
SYS_FORCE_INLINE T * SYSconst_cast(const T *foo)
Definition: SYS_Types.h:136
constexpr UT_FixedVector< SYS_FixedArrayElement_t< TS >, SYS_FixedArraySize_v< TS > > UTmakeFixedVector(const TS &as) noexcept
SYS_FORCE_INLINE void add(IDX_T i, SRC_DATA_T v)
component == 0 in this version
Definition: UT_PageArray.h:592
UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVectorUnknownSize(IDX_T i) const
SYS_FORCE_INLINE void relaxedStore(T val)
SYS_FORCE_INLINE bool isShared() const
void countMemory(UT_MemoryCounter &counter, bool inclusive) const
int64 exint
Definition: SYS_Types.h:125
SYS_FORCE_INLINE PageTableEntry * getFirstPage()
GLboolean GLboolean GLboolean GLboolean a
Definition: glcorearb.h:1222
static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page)
void setConstant(IDX_T start, IDX_T end, NotVoidType v)
void opUnknownType(IDX_T i, exint component, SRC_DATA_T v)
exint UT_PageNum
Definition: UT_PageArray.h:34
SYS_FORCE_INLINE const PageTableEntry * getPPage(UT_PageNum i) const
static void hardenConstantPage(PageTable *pages, PageTableEntry *page)
IMATH_HOSTDEVICE constexpr bool equal(T1 a, T2 b, T3 t) IMATH_NOEXCEPT
Definition: ImathFun.h:105
SYS_FORCE_INLINE UT_PageArray< DATA_T, TSIZE, true, true, THEPAGEBITS, IDX_T > & hardenAllPages(IDX_T start=IDX_T(0), IDX_T end=IDX_T(-1))
static const exint theTupleSize
Definition: UT_PageArray.h:247
SYS_FORCE_INLINE UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T >::PageTableEntry * castType()
SYS_FORCE_INLINE exint getTupleSize() const
DEST_DATA_T getUnknownType(IDX_T i, exint component=0) const
SYS_SelectType< T, float, SYS_IsSame< T, void >::value >::type type
Definition: UT_PageArray.h:236
#define UT_ASSERT_MSG_P(ZZ,...)
Definition: UT_Assert.h:158
SYS_FORCE_INLINE IDX_T size() const
NOTE: This is the size of the full array, not the number of pages.
SYS_FORCE_INLINE UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVector(IDX_T i) const
Definition: UT_PageArray.h:497
#define UT_IF_ASSERT_P(ZZ)
Definition: UT_Assert.h:182
SYS_FORCE_INLINE void set(IDX_T i, SRC_DATA_T v)
component == 0 in this version
Definition: UT_PageArray.h:586
SYS_FORCE_INLINE UT_Storage getStorage() const
static SYS_FORCE_INLINE IDX_T roundUpCapacity(IDX_T capacity)
Definition: UT_PageArray.h:863
SYS_FORCE_INLINE bool isTableHardened() const
Returns true iff the table isn't shared with any other UT_PageArray's.
float fpreal32
Definition: SYS_Types.h:200
SYS_FORCE_INLINE exint getRefCount() const
SYS_FORCE_INLINE UT_PageArray< DATA_T, DEST_TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castTupleSize()
bool hasNanInRange(IDX_T start, IDX_T end) const
void setVectorRange(IDX_T deststart, IDX_T nelements, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE > *src)
UT_PageArray(const UT_PageArray< SRC_DATA_T, SRC_TSIZE, SRC_TABLEHARDENED, SRC_PAGESHARDENED, THEPAGEBITS, IDX_T > &that)
Definition: UT_PageArray.h:311
UT_PageArray(exint tuplesize, UT_Storage storage)
Definition: UT_PageArray.h:288
void tryCompressPageUnknownType(UT_PageNum pagenum)
void decRef(exint tuplesize)
SYS_FORCE_INLINE bool isShared() const
void fill(IDX_T start, IDX_T end, const NotVoidType &val)
static bool isEqualConst(const NotVoidType *tuple, const SrcNotVoidType *stuple, exint mintuplesize)
SYS_FORCE_INLINE void setConstantBit()
double fpreal64
Definition: SYS_Types.h:201
#define UT_ASSERT_MSG(ZZ,...)
Definition: UT_Assert.h:159
unsigned char uint8
Definition: SYS_Types.h:36
UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVectorMismatchedSize(IDX_T i) const
UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVectorUnknownType(IDX_T i) const
SYS_FORCE_INLINE exint getRefCount() const
SYS_FORCE_INLINE bool isConstantZero() const
This is only valid to call if the type doesn't fit inline.
SYS_FORCE_INLINE void setSize(IDX_T newsize, exint tuplesize)
static SYS_FORCE_INLINE UT_PageOff pageOff(IDX_T i)
UT_PageArray(UT_Storage storage)
Definition: UT_PageArray.h:277
SYS_FORCE_INLINE void tryCompressPage(UT_PageNum pagenum)
SYS_FORCE_INLINE ThisType & operator=(const ThisType &that)
Definition: UT_PageArray.h:374
static SYS_FORCE_INLINE bool typeFitsInline(exint tuplesize=TSIZE)
This is only valid to call when DATA_T is non-void.
SYS_FORCE_INLINE const PageTableEntry * getFirstPage() const
SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const UT_FixedVector< SRC_DATA_T, theSafeTupleSize > &val)
static SYS_FORCE_INLINE UT_PageNum numPages(IDX_T nelements)
static SYS_FORCE_INLINE bool isEqual(const T0 *a, const T1 *b, exint tuplesize)
SYS_FORCE_INLINE void getRange(IDX_T srcstart, IDX_T nelements, T *dest) const
SYS_FORCE_INLINE bool isRefd(exint tuplebytes) const
#define UT_ASSERT_P(ZZ)
Definition: UT_Assert.h:155
SYS_FORCE_INLINE bool operator==(const ThisType &that) const
NOTE: This just does a shallow comparison.
SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const NotVoidType &val)
SYS_FORCE_INLINE IDX_T capacity() const
NOTE: This is the capacity of the full array, not the capacity of pages.
static SYS_FORCE_INLINE bool isZero(const T &val)
SYS_FORCE_INLINE IDX_T capacity() const
Definition: UT_PageArray.h:850
#define SYS_FALLTHROUGH
Definition: SYS_Compiler.h:68
GLuint GLuint end
Definition: glcorearb.h:475
#define SYS_FORCE_INLINE
Definition: SYS_Inline.h:45
T compare_swap(T expected, T desired)
SYS_FORCE_INLINE NotVoidType * hardenPage(UT_PageNum pagenum)
UT_FixedVector< NotVoidType, theSafeTupleSize > Tuple
DATA_T DataType
Definition: UT_PageArray.h:245
SYS_FORCE_INLINE void op(IDX_T i, exint component, SRC_DATA_T v)
Definition: UT_PageArray.h:612
SYS_FORCE_INLINE NotVoidType * getInlinePtr(exint tuplesize)
Returns the data pointer, if an inline constant page.
void setCapacity(IDX_T newcapacity)
Definition: UT_PageArray.h:878
long long int64
Definition: SYS_Types.h:116
void clearAndDestroy()
Definition: UT_PageArray.h:330
SYS_FORCE_INLINE void * getMaskedPtrVoid()
static SYS_FORCE_INLINE UT_PageOff pageStart(IDX_T page)
UT_PageArray(exint tuplesize)
Definition: UT_PageArray.h:265
SYS_FORCE_INLINE const void * getFirstPtrVoidUnsafe() const
static void copyPartialPage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destoff, UT_PageOff srcoff, UT_PageOff ntuples, UT_PageOff destpagecapacity)
SYS_FORCE_INLINE bool isConstantAndZeroSafe() const
signed char int8
Definition: SYS_Types.h:35
void setPageConstantUnknownType(UT_PageNum pagenum, const UT_FixedVector< SRC_DATA_T, theSafeTupleSize > &val)
SYS_FORCE_INLINE bool decRefIffShared(exint tuplesize=TSIZE)
static void makeConstant(PageTableEntry *page, const NotVoidType *values, exint tuplesize)
SYS_FORCE_INLINE NotVoidType * hardenPageNoInit(UT_PageNum pagenum)
GLboolean GLboolean GLboolean b
Definition: glcorearb.h:1222
SYS_FORCE_INLINE void tryCompressAllPages(IDX_T start=IDX_T(0), IDX_T end=IDX_T(-1))
static const exint thePageBits
Definition: UT_PageArray.h:249
SYS_FORCE_INLINE void add(IDX_T i, exint component, SRC_DATA_T v)
Definition: UT_PageArray.h:607
const NotVoidType * getPageData(UT_PageNum pagenum) const
exint UT_PageOff
Definition: UT_PageArray.h:35
static const exint theSafeTupleSize
Definition: UT_PageArray.h:248
SYS_FORCE_INLINE void incRef()
GLenum GLenum GLsizei void * table
Definition: glad.h:5129
void realloc(UT_PageOff sizetocopy, UT_PageOff newpagecapacity, exint tuplesize=TSIZE)
static void makeConstantFrom(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize)
PageTableEntry ThisType
SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const SRC_DATA_T *values)
SYS_FORCE_INLINE void addVector(IDX_T i, const TS &as)
Definition: UT_PageArray.h:699
SYS_FORCE_INLINE T relaxedLoad() const
void moveRange(IDX_T srcstart, IDX_T deststart, IDX_T nelements)
static const exint thePageMask
Definition: UT_PageArray.h:251
static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
GLsizeiptr size
Definition: glcorearb.h:664
static const uintptr_t theConstantPageBit
SYS_FORCE_INLINE int64 getMemoryUsage(exint tuplebytes) const
SYS_AtomicInt< int32 > SYS_AtomicCounter
SYS_FORCE_INLINE const UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castType() const
void setSize(IDX_T newsize)
Definition: UT_PageArray.h:902
SYS_FORCE_INLINE const void * getMaskedPtrVoid() const
SYS_FORCE_INLINE const UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T >::PageTableEntry * castType() const
GLenum GLsizei GLsizei GLint * values
Definition: glcorearb.h:1602
void swapRange(IDX_T astart, IDX_T bstart, IDX_T nelements)
SYS_FORCE_INLINE void * getFirstPtrVoidUnsafe()
short int16
Definition: SYS_Types.h:37
bool decRefIffSharedPage(UT_PageNum pagenum)
void setPageConstantUnknownType(UT_PageNum pagenum, const SRC_DATA_T *values)
SYS_FORCE_INLINE PageTableEntry * getPPage(UT_PageNum i)
void setStorage(const UT_Storage newstorage)
IMATH_HOSTDEVICE constexpr bool iszero(T a, T t) IMATH_NOEXCEPT
Definition: ImathFun.h:98
SYS_FORCE_INLINE void set(IDX_T i, exint component, SRC_DATA_T v)
Definition: UT_PageArray.h:602
static void hardenConstantPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
static void fillNonConstWithConst(NotVoidType *destpagedata, NotVoidType *destpageend, const SrcNotVoidType *stuple, exint mintuplesize, exint desttupleextra)
SYS_FORCE_INLINE NotVoidType * getMaskedPtr()
bool decRefIffSharedTable()
SYS_FORCE_INLINE const UT_PageArray< DATA_T, DEST_TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castTupleSize() const
SYS_FORCE_INLINE bool isConstant() const
This is always valid to call.
SYS_FORCE_INLINE void setRange(IDX_T deststart, IDX_T nelements, const T *src)
static void hardenSharedPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
SYS_FORCE_INLINE const void * getFirstPtrVoid() const
Returns the data pointer, if not a constant page.
GLuint GLfloat * val
Definition: glcorearb.h:1608
static SYS_FORCE_INLINE NotVoidType * getConstantPtr(PageTableEntry *page, exint component=0, exint tuplesize=TSIZE)
static SYS_FORCE_INLINE void makeConstant(PageTableEntry *page, const UT_FixedVector< NotVoidType, theSafeTupleSize > &val)
static SYS_FORCE_INLINE UT_FixedVector< DEST_TYPE, SRC_TSIZE > convertVectorStorage(const UT_FixedVector< SRC_TYPE, SRC_TSIZE > &src)
static void makeConstant(PageTableEntry *page, const NotVoidType &val, exint tuplesize=TSIZE)
SYS_FORCE_INLINE UT_PageArray< DATA_T, TSIZE, true, PAGESHARDENED, THEPAGEBITS, IDX_T > & hardenTable()
static const exint thePageSize
Definition: UT_PageArray.h:250
static SYS_FORCE_INLINE bool isZero(const T *val, exint tuplesize)
SYS_FORCE_INLINE void alloc(UT_PageOff nelements, exint tuplesize=TSIZE)
T add(T val)
Atomically adds val to myValue, returning the new value of myValue.
static void hardenConstantPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
int64 getMemoryUsage(bool inclusive) const
static SYS_FORCE_INLINE UT_PageNum pageNum(IDX_T i)
UT_PageArray< DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > ThisType
Definition: UT_PageArray.h:244
static void hardenSharedPage(PageTable *pages, PageTableEntry *page)
void setCapacityIfNeeded(IDX_T newcapacity)
Definition: UT_PageArray.h:891
SYS_FORCE_INLINE void setVector(IDX_T i, const TS &as)
Definition: UT_PageArray.h:693
void getVectorRange(IDX_T srcstart, IDX_T nelements, UT_FixedVector< DEST_DATA_T, DEST_TSIZE > *dest) const
void forceHardenTable(IDX_T newcapacity)
SYS_FORCE_INLINE int64 getGuaranteedInt(IDX_T i) const
SYS_FORCE_INLINE NotVoidType * unshareConstantPage(UT_PageNum pagenum)
#define SYSmin(a, b)
Definition: SYS_Math.h:1583
SYS_FORCE_INLINE bool isPageHard(UT_PageNum pagenum) const
UT_PageArray< DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > ParentType
SYS_FORCE_INLINE constexpr int UTstorageSize(UT_Storage storage)
Returns the number of bytes in the given storage type.
Definition: UT_Storage.h:57
SYS_FORCE_INLINE void initZero()
SYS_FORCE_INLINE const NotVoidType * getFirstPtr() const
Returns the data pointer, if not a constant page.
SYS_FORCE_INLINE NotVoidType * getFirstPtr()
Returns the data pointer, if not a constant page.
void setPageConstantMismatchedType(UT_PageNum pagenum, const SRC_DATA_T *values)
void replace(const UT_PageArray< SRC_DATA_T, SRC_TSIZE, SRC_TABLEHARDENED, SRC_PAGESHARDENED, THEPAGEBITS, IDX_T > &that)
Definition: UT_PageArray.h:391
Definition: format.h:1821
SYS_FORCE_INLINE NotVoidType & operator()(IDX_T i, exint component=0)
Definition: UT_PageArray.h:543
NotVoidType * getPageData(UT_PageNum pagenum)
SYS_FORCE_INLINE void decRef()
static void replacePage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destpagesize, UT_PageOff destpagecapacity)
SYS_FORCE_INLINE UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castType()
GLenum src
Definition: glcorearb.h:1793