HDK
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
UT_PageArray.h
Go to the documentation of this file.
1 /*
2  * PROPRIETARY INFORMATION. This software is proprietary to
3  * Side Effects Software Inc., and is not to be reproduced,
4  * transmitted, or disclosed in any way without written permission.
5  *
6  * NAME: UT_PageArray.h (UT Library, C++)
7  *
8  * COMMENTS: An array class with special handling of constant pages and
9  * shared page data.
10  */
11 
12 #pragma once
13 
14 #ifndef __UT_PageArray__
15 #define __UT_PageArray__
16 
17 #include "UT_API.h"
18 #include "UT_Assert.h"
19 #include "UT_FixedVector.h"
20 #include "UT_SmallArray.h"
21 #include "UT_Storage.h"
22 #include <SYS/SYS_AtomicInt.h>
23 #include <SYS/SYS_BitUtil.h>
24 #include <SYS/SYS_CallIf.h>
25 #include <SYS/SYS_Compiler.h>
26 #include <SYS/SYS_Inline.h>
27 #include <SYS/SYS_Math.h>
28 #include <SYS/SYS_Types.h>
29 #include <SYS/SYS_TypeTraits.h>
30 #include <stdint.h>
31 #include <stdlib.h>
32 #include <string.h>
33 
34 #ifdef WIN32
35 // Default template arguments on functions are legal. Don't error on them!
36 #pragma warning(disable: 4519)
37 #endif
38 
39 
40 typedef exint UT_PageNum;
41 typedef exint UT_PageOff;
42 
43 
44 class UT_Defaults;
45 class UT_MemoryCounter;
46 
47 
48 /// @internal
49 /// W E L C O M E
50 /// to scenic
51 /// Template Hell
52 ///
53 /// Population: You
54 ///
55 /// If you're just using this class, hopefully you'll have a short and
56 /// not-so-painful visit!
57 ///
58 /// If, however, you're attempting to modify this class, BEWARE!
59 /// Any missteps can and likely will result in application of the
60 /// Doughnut Rules, so be sure to review them before visiting.
61 /// It may take several days to escape, so it may be best to just
62 /// revert and make absolutely sure that it works on all platforms
63 /// before committing again.
64 ///
65 /// This class is the result of a wide variety of needs for a wide variety
66 /// of use cases, most of which need both good performance and flexibility.
67 /// You may want to skip reading this gigantic comment, but whether it's
68 /// immediately or in a few months, you'll realize that skipping reading
69 /// this was a huge mistake.
70 ///
71 /// This array consists of a pointer to a page table, optionally a
72 /// tuple size (if TSIZE==-1), and optionally a UT_Storage indicating
73 /// the storage type (if DATA_T is void). The page table, if not nullptr,
74 /// can be shared, and has the format: @code
75 /// +-------------------+
76 /// | ref count | (1 if unshared; >1 if shared)
77 /// +-------------------+
78 /// | array size | (number of real elements/tuples in the array)
79 /// +-------------------+
80 /// | array capacity | (number of allocated elements/tuples if all pages of the table were hardened)
81 /// +-------------------+
82 /// | page 0 pointer | (pointer to page if bit 0 is 0, else constant page representation)
83 /// +-------------------+
84 /// | page 1 pointer |
85 /// +-------------------+
86 /// | page 2 pointer |
87 /// +-------------------+
88 /// | ... |
89 /// +-------------------+
90 /// @endcode
91 ///
92 /// If a page pointer has bit 0 clear, (i.e. an address aligned to a 2-byte boundary),
93 /// it is a non-constant page, it can be shared, and has the format: @code
94 /// +-------------------+
95 /// | ref count | (1 if unshared; >1 if shared)
96 /// +-------------------+
97 /// | (unused) | (padding so that data starts on a 16-byte boundary)
98 /// +-------------------+
99 /// | tuple 0 |
100 /// +-------------------+
101 /// | tuple 1 |
102 /// +-------------------+
103 /// | tuple 2 |
104 /// +-------------------+
105 /// | ... |
106 /// +-------------------+
107 /// @endcode
108 ///
109 /// A normal capacity page contains 2^THEPAGEBITS tuples, e.g. the default
110 /// THEPAGEBITS==10 gives a page size of 1024 tuples. If there is exactly
111 /// one page, it may have a capacity that is any power of two up to
112 /// THEPAGEBITS, which can be checked by checking if the overall array has
113 /// capacity less than 2^THEPAGEBITS.
114 ///
115 /// If, instead, the page pointer has bit 0 set to 1, which would be an odd
116 /// address, all tuples in the page are the same. There are 3 ways this may
117 /// be represented:
118 ///
119 /// 1) If the size of a full tuple is strictly smaller than the size of a
120 /// pointer, (e.g. 7 bytes or fewer on a 64-bit platform), the constant value
121 /// will be stored inside the space that would have been a pointer, aligned
122 /// to the end of the pointer. That ensures that 4-byte tuples will be
123 /// aligned to 4 bytes, and 2- or 6-byte tuples will be aligned to 2 bytes.
124 /// This is most useful for fpreal32[1] and int32[1] (i.e. 4-byte scalars).
125 ///
126 /// 2) If the page pointer is equal to 1 (i.e. nullptr apart from bit 0 being 1),
127 /// the value is all zero. This covers the extremely
128 /// common case of a zero default value, especially for fpreal32[3] and
129 /// fpreal64[3].
130 ///
131 /// 3) Otherwise, the page pointer, ignoring bit zero, points to a page
132 /// that contains just a single tuple.
133 ///
134 /// That covers the basic structure. It sounds so nice and simple...
135 /// but then there are the templates.
136 ///
137 /// DATA_T:
138 ///
139 /// If DATA_T is something other than void, that type is the type of each
140 /// component of each tuple. This is the only way that non-numeric types
141 /// are possible; DATA_T can also be a numeric type. If DATA_T is void,
142 /// myImpl contains a UT_Storage variable indicating what the data type is;
143 /// it must be a numeric type, i.e. it cannot be UT_Storage::INVALID.
144 /// The ...UnknownType functions switch on the UT_Storage to determine the
145 /// data type and call the corresponding function. If DATA_T is non-void,
146 /// it saves a lot of extra work at runtime. Because the codepaths that
147 /// will never be called due to DATA_T checks still need to compile with
148 /// DATA_T being void, NotVoidType ensures that there is some arbitrary type
149 /// in place so that the code will compile, even though it will never be
150 /// called. The same issue prevents using UT_ASSERT_COMPILETIME to verify
151 /// DATA_T, so UT_ASSERT_P is used instead.
152 /// myImpl is always structured so that if the type becomes known, by checking
153 /// getStorage(), this can be cast to have the now-known DATA_T set.
154 /// For example: @code
155 /// if (array.getStorage() == UT_Storage::REAL32)
156 /// array.castType<fpreal32>().someFunctionCall();
157 /// @endcode
158 ///
159 /// TSIZE:
160 ///
161 /// If TSIZE >= 1, it is the tuple size (the number of components in each
162 /// tuple), and is known at compile time, avoiding extra work at compile-time
163 /// that depends on the tuple size, especially if TSIZE is 1. Odds are that
164 /// for non-numeric types, TSIZE will almost always be 1, since it rarely
165 /// makes sense to be something else. If TSIZE is 0, something is almost
166 /// certainly wrong. If the tuple size is not known at compile-time, TSIZE
167 /// must be -1. In that case, myImpl contains an integer indicating the tuple
168 /// size. Like with DATA_T, because the codepaths that will never be called
169 /// due to TSIZE checks still need to compile with other TSIZE values, so
170 /// UT_ASSERT_COMPILETIME can't be used to verify TSIZE; UT_ASSERT_P is used
171 /// instead. There are a number of functions that require TSIZE to be the
172 /// tuple size, in order to avoid having to send an extra parameter and
173 /// incurring the slowdowns of their counterparts that assume that the tuple
174 /// size is what's passed in. If you modify one, REMEMBER TO MODIFY THE OTHER!
175 /// myImpl is always structured so that if the tuple size becomes
176 /// known, by checking the getTupleSize(), this can be cast to have the
177 /// now-known TSIZE set.
178 /// For example: @code
179 /// if (array.getTupleSize() == 3)
180 /// array.castTupleSize<3>().someFunctionCall();
181 /// @endcode
182 ///
183 /// TABLEHARDENED:
184 ///
185 /// Since page tables can be shared if all of the data are identical, modifying
186 /// the array in any way requires copying (hardening) the page table and may
187 /// require copying one or more shared pages or hardening constant pages.
188 /// Since every modification requires hardening the page table, it's useful
189 /// to harden the page table once and be able to skip checking whether it needs
190 /// hardening for all modifications after that. Calling hardenTable()
191 /// will ensure that the table is hardened, and return this with TABLEHARDENED
192 /// set to true, so that further calls on it to modify the array won't need to
193 /// check whether the table needs hardening. The page table should be
194 /// quite light compared to the data, so it's not the end of the world if
195 /// the table is hardened pre-emptively and there end up being no actual
196 /// modifications.
197 ///
198 /// PAGESHARDENED:
199 ///
200 /// When writing data in parallel to random locations, all pages that could
201 /// be written-to must be hardened in advance, to avoid a race condition on
202 /// the hardening of any page. Also, if it's known that most pages will
203 /// likely be hardened anyway when writing data, it'd be advantageous to
204 /// be able to harden all pages in advance and skip checking when making
205 /// modifications. The code for hardening pages just-in-time can
206 /// dramatically increase code size and time overhead. To avoid this,
207 /// hardenAllPages() will ensure that the table and all pages are hardened,
208 /// and will return this with TABLEHARDENED and PAGESHARDENED both set to true.
209 /// Any calls to modify page data made on that returned pointer won't check
210 /// whether any pages need hardening, because it will know that all pages are
211 /// already hardened. If you use this, be sure not to use the pointer
212 /// with PAGESHARDENED set to true after any call to tryCompressAllPages()
213 /// on the original array pointer.
214 ///
215 /// THEPAGEBITS:
216 ///
217 /// Full pages will contain 2^THEPAGEBITS tuples, e.g. THEPAGEBITS==10
218 /// means there will be 1024 tuples per full page.
219 ///
220 /// IDX_T:
221 ///
222 /// This allows templating on the array index type so that strict types
223 /// can be used, e.g. GA_Offset.
224 ///
225 /// FIXME BEFORE USING:
226 /// - Should make generic UT_R[OW]Handle.+ classes for easy accessing
227 /// of runtime-determined data type and tuple size data as tuples
228 /// with type and tuple size determined at compile-time.
229 /// - Make UT_PageNum and UT_PageOff strict types when strict types are
230 /// enabled.
231 /// - Add more functions for page-level access and block (bulk sequential)
232 /// access, for easy reading/writing to/from flat arrays.
233 /// @endinternal
234 template<typename DATA_T,exint TSIZE=1,bool TABLEHARDENED=false,bool PAGESHARDENED=false,exint THEPAGEBITS=10,typename IDX_T=exint>
236 {
237 protected:
238  /// This class is just to get code to compile when DATA_T is void,
239  /// by picking any type other than void if it is void.
240  template<typename T>
241  struct NotVoid
243 
245 
246  // These friend declarations are needed for calling protected functions on hardened/cast types.
247  template<typename DATA_T2,exint TSIZE2,bool TABLEHARDENED2,bool PAGESHARDENED2,exint THEPAGEBITS2,typename IDX_T2>
248  friend class UT_PageArray;
249 public:
251  typedef DATA_T DataType;
253  static const exint theTupleSize = TSIZE;
254  static const exint theSafeTupleSize = (TSIZE==-1) ? 3 : TSIZE;
255  static const exint thePageBits = THEPAGEBITS;
256  static const exint thePageSize = (exint(1)<<thePageBits);
257  static const exint thePageMask = thePageSize-1;
258  typedef IDX_T IndexType;
259 
260  /// The default constructor can only be used if the tuple size
261  /// and storage type are known at compile time.
263  {
264  UT_ASSERT_P(TSIZE >= 1);
266  myImpl.getPages() = nullptr;
267  }
268 
269  /// This constructor can only be used if the storage type is
270  /// known at compile time, but the tuple size is not.
271  UT_PageArray(exint tuplesize)
272  {
273  UT_ASSERT_P(TSIZE == -1 || TSIZE == tuplesize);
275  // NOTE: This is one case where a tuple size of zero is okay.
276  UT_ASSERT_P(tuplesize >= 0);
277  myImpl.getPages() = nullptr;
278  myImpl.setTupleSize(tuplesize);
279  }
280 
281  /// This constructor can only be used if the storage type is
282  /// known at compile time, but the tuple size is not.
284  {
285  UT_ASSERT_P(TSIZE >= 1);
287  UT_ASSERT_P(storage != UT_Storage::INVALID);
288  myImpl.getPages() = nullptr;
289  myImpl.setStorage(storage);
290  }
291 
292  /// This constructor can only be used if the tuple size and
293  /// storage type are unknown at compile time.
295  {
296  UT_ASSERT_P(TSIZE == -1 || TSIZE == tuplesize);
298  // NOTE: This is one case where a tuple size of zero is okay.
299  UT_ASSERT_P(tuplesize >= 0);
300  UT_ASSERT_P(storage != UT_Storage::INVALID);
301  myImpl.getPages() = nullptr;
302  myImpl.setStorage(storage);
303  myImpl.setTupleSize(tuplesize);
304  }
305 
306  /// This just references that's page table, unless TABLEHARDENED
307  /// or PAGESHARDENED. If TABLEHARDENED and !PAGESHARDENED,
308  /// the pages are referenced, and if PAGESHARDENED, the pages
309  /// are copied.
310  UT_PageArray(const ThisType &that)
311  {
312  myImpl.getPages() = nullptr;
313  *this = that;
314  }
315 
316  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
318  {
319  myImpl.getPages() = nullptr;
320  *this = that;
321  }
322 
325  {
326  clearAndDestroy();
327  }
328 
330  void clear()
331  {
332  if (myImpl.getPages())
333  myImpl.getPages()->setSize(IDX_T(0));
334  }
335 
337  {
338  if (!myImpl.getPages())
339  return;
340 
341  const exint tuplesize = myImpl.getTupleSize();
342 
343  // Just decrement the reference count to free the table safely
344  if (!SYSisSame<DATA_T,void>())
345  myImpl.getPages()->decRef(tuplesize);
346  else
347  {
348  const UT_Storage storage = myImpl.getStorage();
349  switch (storage)
350  {
351  case UT_Storage::INVALID:
352  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
353  return;
354  case UT_Storage::INT8:
355  castType<int8>().myImpl.getPages()->decRef(tuplesize); return;
356  case UT_Storage::INT16:
357  castType<int16>().myImpl.getPages()->decRef(tuplesize); return;
358  case UT_Storage::INT32:
359  castType<int32>().myImpl.getPages()->decRef(tuplesize); return;
360  case UT_Storage::INT64:
361  castType<int64>().myImpl.getPages()->decRef(tuplesize); return;
362  case UT_Storage::REAL16:
363  castType<fpreal16>().myImpl.getPages()->decRef(tuplesize); return;
364  case UT_Storage::REAL32:
365  castType<fpreal32>().myImpl.getPages()->decRef(tuplesize); return;
366  case UT_Storage::REAL64:
367  castType<fpreal64>().myImpl.getPages()->decRef(tuplesize); return;
368  }
369  }
370  myImpl.getPages() = nullptr;
371  }
372 
373  /// NOTE: Compilers seem not to use the operator= below when the template
374  /// arguments match, so this operator= is necessary in order to
375  /// avoid the compiler silently using its implicitly-defined
376  /// assignment operator.
379  {
380  replace(that);
381  return *this;
382  }
383 
384  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
387  {
388  replace(that);
389  return *this;
390  }
391 
392  /// Completely replace this UT_PageArray with that, referencing
393  /// the pages or table, if allowed by the template arguments.
394  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
396  {
397  // NOTE: Careful if you add template parameter checks on this,
398  // because, e.g., a UT_PageArray<void,-1> could refer to the
399  // same exact array as a UT_PageArray<float 3>.
400  if ((void *)this == (const void *)&that)
401  return;
402  // NOTE: The storage and tuple size checks are so that if both
403  // tables are nullptr, it still copies the storage and tuple
404  // size below.
405  if (!TABLEHARDENED &&
406  myImpl.getStorage() == that.myImpl.getStorage() &&
407  myImpl.getTupleSize() == that.myImpl.getTupleSize() &&
408  (void *)myImpl.getPages() == (const void *)that.myImpl.getPages())
409  return;
410 
411  clearAndDestroy();
412 
413  if (SYSisSame<DATA_T,void>())
414  myImpl.setStorage(that.myImpl.getStorage());
415  if (TSIZE == -1)
416  myImpl.setTupleSize(that.myImpl.getTupleSize());
417 
418  // May be able to share pages or table if matching tuple size and storage type
419  if ((SYSisSame<DATA_T,void>() || SYSisSame<DATA_T,SRC_DATA_T>()) && (TSIZE == -1 || TSIZE == SRC_TSIZE))
420  {
421  PageTable *pages = reinterpret_cast<PageTable *>(SYSconst_cast(that.myImpl.getPages()));
422  myImpl.getPages() = pages;
423  if (!pages)
424  return;
425  pages->incRef();
426  if (TABLEHARDENED || PAGESHARDENED || SRC_TABLEHARDENED || SRC_PAGESHARDENED)
427  {
428  forceHardenTable(pages->capacity());
429  if (PAGESHARDENED || SRC_PAGESHARDENED)
430  {
431  // If we don't cast to PAGESHARDENED==false, it won't do anything.
433  ->template hardenAllPages<false>();
434  }
435  }
436  }
437  else
438  {
439  // Can't share table or pages, so just use moveRange
440  IDX_T capacity = that.capacity();
441  IDX_T size = that.size();
442  forceHardenTable(capacity);
443  moveRange(that, IDX_T(0), IDX_T(0), size);
444  }
445  }
446 
447  /// Read element i's specified component (default component 0).
448  SYS_FORCE_INLINE NotVoidType operator()(IDX_T i,exint component=0) const
449  {
450  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
451  UT_ASSERT_P(TSIZE != 0);
452  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
453  UT_ASSERT_P(i >= IDX_T(0) && i < size());
454  UT_ASSERT_P(component >= 0 && component < myImpl.getTupleSize());
455 
456  UT_PageNum pagenum = pageNum(i);
457  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
458 
459  // If we know that the pages are hardened, we can omit
460  // the constant page case at compile time.
461  bool isconst = !PAGESHARDENED && page->isConstant();
462 
463  if (TSIZE >= 1)
464  {
465  if (!isconst)
466  {
467  UT_PageOff pageoff = pageOff(i);
468  return page->getFirstPtr()[TSIZE*pageoff + component];
469  }
470  return getConstant(page, component);
471  }
472  else
473  {
474  if (!isconst)
475  {
476  UT_PageOff pageoff = pageOff(i);
477  return page->getFirstPtr()[myImpl.getTupleSize()*pageoff + component];
478  }
479  return getConstant(page, component, myImpl.getTupleSize());
480  }
481  }
482 
483  /// This get function may seem redundant, given operator(), but it gives a
484  /// simple way to read from a non-const UT_PageArray without calling the
485  /// non-const operator() and potentially hardening pages. It just
486  /// explicitly adds const. It also allows data type conversion and
487  /// reading when the type is unknown at compile time.
488  /// It should hopefully be inlined in the no-conversion case;
489  /// it's less critical in the conversion case.
490  template<typename DEST_DATA_T=DATA_T>
491  SYS_FORCE_INLINE DEST_DATA_T get(IDX_T i,exint component=0) const
492  {
493  UT_ASSERT_P(i >= IDX_T(0) && i < size());
494  if (!(SYSisSame<DATA_T,void>()) && (SYSisSame<DEST_DATA_T,DATA_T>() || theStorage != UT_Storage::INVALID))
495  return UTconvertStorage<DEST_DATA_T>((*this)(i,component));
496 
497  return getUnknownType<DEST_DATA_T>(i,component);
498  }
499 
500  template<typename DEST_DATA_T=NotVoidType,exint DEST_TSIZE=theSafeTupleSize>
502  {
503  if (SYSisSame<DATA_T,void>())
504  {
506  SYS_CallIf<SYSisSame<DATA_T,void>()>::call([this,&dest,i](SYS_CALLIF_AUTO){
507  dest = this->getVectorUnknownType<DEST_DATA_T,DEST_TSIZE>(i);
508  });
509  return dest;
510  }
511  if (!SYSisSame<DEST_DATA_T,DATA_T>())
512  {
515  dest = this->convertVectorStorage<DEST_DATA_T>(this->getVector<NotVoidType,DEST_TSIZE>(i));
516  });
517  return dest;
518  }
519 
520  if (TSIZE==-1)
521  {
524  dest = this->getVectorUnknownSize<DEST_DATA_T,DEST_TSIZE>(i);
525  });
526  return dest;
527  }
528  if (TSIZE!=DEST_TSIZE)
529  {
532  dest = this->getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
533  });
534  return dest;
535  }
536 
537  UT_ASSERT_P(i >= IDX_T(0) && i < size());
538  UT_ASSERT_P(TSIZE==DEST_TSIZE);
539  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()));
540 
541  // Types and tuple sizes are known and match
542  UT_PageNum pagenum = pageNum(i);
543  // NOTE: We can dereference here because we're returning by value.
544  // getConstantPtr may return a pointer into this stack variable
545  // if the page is constant and inline.
546  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
547 
548  // If we know that the pages are hardened, we can omit
549  // the constant page case at compile time.
550  bool isconst = !PAGESHARDENED && page->isConstant();
551 
552  if (!isconst)
553  {
554  UT_PageOff pageoff = pageOff(i);
555  return ((const UT_FixedVector<DEST_DATA_T,DEST_TSIZE>*)page->getFirstPtr())[pageoff];
556  }
557  const NotVoidType *constdata = getConstantPtr(page);
558  if (!constdata)
559  return UT_FixedVector<DEST_DATA_T,DEST_TSIZE>(DEST_DATA_T(0));
560 
561  return *(const UT_FixedVector<DEST_DATA_T,DEST_TSIZE>*)constdata;
562  }
563 
564  /// Get a non-const reference to element i's specified component
565  /// (default component 0).
566  /// WARNING: DO NOT call this if DATA_T is void!
567  /// NOTE: Unlike set(), this must always harden, even if you're just
568  /// writing a value that is already there.
570  {
571  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
572  UT_ASSERT_P(TSIZE != 0);
573  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
574  UT_ASSERT_P(i >= IDX_T(0) && i < size());
575  UT_ASSERT_P(component >= 0 && component < myImpl.getTupleSize());
576 
577  // NOTE: This is a no-op if TABLEHARDENED is already true.
578  hardenTable();
579 
580  UT_PageNum pagenum = pageNum(i);
581  UT_PageOff pageoff = pageOff(i);
582  PageTable *pages = myImpl.getPages();
583  PageTableEntry *page = pages->getPPage(pagenum);
584 
585  if (!PAGESHARDENED)
586  {
587  if (page->isConstant())
588  {
589  if (TSIZE >= 1)
590  hardenConstantPage(pages, page);
591  else
592  hardenConstantPage(pages, page, myImpl.getTupleSize());
593  }
594  else if (page->isShared())
595  {
596  if (TSIZE >= 1)
597  hardenSharedPage(pages, page);
598  else
599  hardenSharedPage(pages, page, myImpl.getTupleSize());
600  }
601  }
602 
603  NotVoidType *data = page->getFirstPtr()
604  + pageoff*((TSIZE >= 1) ? TSIZE : myImpl.getTupleSize())
605  + component;
606 
607  return *data;
608  }
609 
610  /// component == 0 in this version
611  template<typename SRC_DATA_T>
612  SYS_FORCE_INLINE void set(IDX_T i,SRC_DATA_T v)
613  {
614  set(i, 0, v);
615  }
616  /// component == 0 in this version
617  template<typename SRC_DATA_T>
618  SYS_FORCE_INLINE void add(IDX_T i,SRC_DATA_T v)
619  {
620  add(i, 0, v);
621  }
622 
623  /// This looks gigantic, but it's mostly because of the different
624  /// behaviour for different template values. If
625  /// PAGESHARDENED is true, it'll skip checking for constant
626  /// or shared pages altogether.
627  template<typename SRC_DATA_T>
628  SYS_FORCE_INLINE void set(IDX_T i,exint component,SRC_DATA_T v)
629  {
630  op<0>(i, component, v);
631  }
632  template<typename SRC_DATA_T>
633  SYS_FORCE_INLINE void add(IDX_T i,exint component,SRC_DATA_T v)
634  {
635  op<1>(i, component, v);
636  }
637  template<int OP,typename SRC_DATA_T>
638  SYS_FORCE_INLINE void op(IDX_T i,exint component,SRC_DATA_T v)
639  {
640  UT_ASSERT_P(i >= IDX_T(0) && i < size());
641  UT_ASSERT_P(component >= 0 && component < getTupleSize());
642  UT_ASSERT_MSG_P(OP == 0 || OP == 1, "Only OP 0 (set) and 1 (add) are defined!");
643 
644  if (SYSisSame<DATA_T,void>())
645  {
646  opUnknownType<OP>(i, component, v);
647  return;
648  }
649 
650  // Make sure that v is the correct type before comparing.
651  NotVoidType vt(UTconvertStorage<NotVoidType>(v));
652 
653  UT_ASSERT_P(TSIZE != 0);
654  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
655 
656  hardenTable();
657 
658  UT_PageNum pagenum = pageNum(i);
659  UT_PageOff pageoff = pageOff(i);
660  PageTable *pages = myImpl.getPages();
661  PageTableEntry *page = pages->getPPage(pagenum);
662 
663  if (!PAGESHARDENED)
664  {
665  if (page->isConstant())
666  {
667  // If we're setting, check to see if it's a new value
668  if (OP == 0)
669  {
670  if (TSIZE >= 1)
671  {
672  if (getConstant(page, component) == vt)
673  return;
674  }
675  else
676  {
677  if (getConstant(page, component, myImpl.getTupleSize()) == vt)
678  return;
679  }
680  }
681  if (TSIZE >= 1)
682  hardenConstantPage(pages, page);
683  else
684  hardenConstantPage(pages, page, myImpl.getTupleSize());
685  }
686  else if (page->isShared())
687  {
688  // If we're setting, check to see if it's a new value
689  if (OP == 0)
690  {
691  NotVoidType *data = page->getFirstPtr()
692  + pageoff*((TSIZE >= 1) ? TSIZE : myImpl.getTupleSize())
693  + component;
694  if (*data == vt)
695  return;
696  }
697  if (TSIZE >= 1)
698  hardenSharedPage(pages, page);
699  else
700  hardenSharedPage(pages, page, myImpl.getTupleSize());
701  }
702  }
703 
704  NotVoidType *data = page->getFirstPtr()
705  + pageoff*((TSIZE >= 1) ? TSIZE : myImpl.getTupleSize())
706  + component;
707 
708  if (OP == 0)
709  *data = vt;
710  else if (OP == 1)
711  *data += vt;
712  else
713  {
714  UT_ASSERT_MSG_P(0,"Only OP 0 (set) and 1 (add) are defined!");
715  }
716  }
717 
718  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_INSTANTIATED>
720  {
721  opVector<0>(i, reinterpret_cast<const UT_FixedVector<SRC_DATA_T,SRC_TSIZE> &>(v));
722  }
723  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_INSTANTIATED>
725  {
726  opVector<1>(i, reinterpret_cast<const UT_FixedVector<SRC_DATA_T,SRC_TSIZE> &>(v));
727  }
728  /// This looks gigantic, but it's mostly because of the different
729  /// behaviour for different template values. If
730  /// PAGESHARDENED is true, it'll skip checking for constant
731  /// or shared pages altogether.
732  template<int OP,typename SRC_DATA_T,exint SRC_TSIZE>
734  {
735  UT_ASSERT_P(i >= IDX_T(0) && i < size());
736  UT_ASSERT_MSG_P(OP == 0 || OP == 1, "Only OP 0 (set) and 1 (add) are defined!");
737 
738  if (SYSisSame<DATA_T,void>())
739  {
741  this->opVectorUnknownType<OP>(i, v);
742  });
743  return;
744  }
745 
746  const exint tuplesize = myImpl.getTupleSize();
747  if (TSIZE == -1 && tuplesize == SRC_TSIZE)
748  {
750  this->castTupleSize<SRC_TSIZE>().template opVector<OP>(i, v);
751  });
752  return;
753  }
754  const exint mintuplesize = SYSmin(tuplesize, SRC_TSIZE);
755 
756  // Make sure that v is the correct type before comparing.
757  // Hopefully this will avoid copying if SRC_DATA_T is NotVoidType.
758  constexpr bool TYPE_MATCH = SYSisSame<SRC_DATA_T,NotVoidType>();
760  if (!TYPE_MATCH)
761  converted_v = convertVectorStorage<NotVoidType>(v);
762 
763  UT_ASSERT_P(TSIZE != 0);
764  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
765 
766  hardenTable();
767 
768  UT_PageNum pagenum = pageNum(i);
769  UT_PageOff pageoff = pageOff(i);
770  PageTable *pages = myImpl.getPages();
771  PageTableEntry *page = pages->getPPage(pagenum);
772 
773  if (!PAGESHARDENED)
774  {
775  if (page->isConstant())
776  {
777  // If we're setting, check to see if it's a new value
778  if (OP == 0)
779  {
780  const NotVoidType *data;
781  if (PageTableEntry::typeFitsInline(tuplesize))
782  data = page->getInlinePtr(tuplesize);
783  else
784  data = page->getMaskedPtr();
785  if (data)
786  {
787  if (TYPE_MATCH)
788  {
789  if (isEqual(data, v.data(), mintuplesize))
790  return;
791  }
792  else
793  {
794  if (isEqual(data, converted_v.data(), mintuplesize))
795  return;
796  }
797  }
798  else
799  {
800  if (TYPE_MATCH)
801  {
802  if (isZero(v.data(), mintuplesize))
803  return;
804  }
805  else
806  {
807  if (isZero(converted_v.data(), mintuplesize))
808  return;
809  }
810  }
811  }
812  if (TSIZE >= 1)
813  hardenConstantPage(pages, page);
814  else
815  hardenConstantPage(pages, page, tuplesize);
816  }
817  else if (page->isShared())
818  {
819  // If we're setting, check to see if it's a new value
820  if (OP == 0)
821  {
822  const NotVoidType *data = page->getFirstPtr()
823  + pageoff*((TSIZE >= 0) ? TSIZE : tuplesize);
824  if (TYPE_MATCH)
825  {
826  if (isEqual(data, v.data(), mintuplesize))
827  return;
828  }
829  else
830  {
831  if (isEqual(data, converted_v.data(), mintuplesize))
832  return;
833  }
834  }
835  if (TSIZE >= 1)
836  hardenSharedPage(pages, page);
837  else
838  hardenSharedPage(pages, page, tuplesize);
839  }
840  }
841 
842  NotVoidType *data = page->getFirstPtr() + pageoff*((TSIZE >= 0) ? TSIZE : tuplesize);
843  for (exint component = 0; component < mintuplesize; ++component)
844  {
845  if (OP == 0)
846  {
847  if (TYPE_MATCH)
848  data[component] = v[component];
849  else
850  data[component] = converted_v[component];
851  }
852  else if (OP == 1)
853  {
854  if (TYPE_MATCH)
855  data[component] += v[component];
856  else
857  data[component] += converted_v[component];
858  }
859  else
860  {
861  UT_ASSERT_MSG_P(0,"Only OP 0 (set) and 1 (add) are defined!");
862  }
863  }
864  }
865 
866  SYS_FORCE_INLINE IDX_T size() const
867  {
868  const PageTable *pages = myImpl.getPages();
869  if (!pages)
870  return IDX_T(0);
871  return pages->size();
872  }
873 
875  {
876  const PageTable *pages = myImpl.getPages();
877  if (!pages)
878  return IDX_T(0);
879  return pages->capacity();
880  }
881 
882  /// This helper function forces a (non-negative) capacity to be an
883  /// acceptable value: either
884  /// A) a power of two less than a full page in size, or
885  /// B) a multiple of the page size.
887  static IDX_T roundUpCapacity(IDX_T capacity)
888  {
889  UT_ASSERT_P(capacity >= IDX_T(0));
890 
891  // Round up to page size if more than half of one page.
892  if (capacity > IDX_T(thePageSize>>1))
893  return IDX_T((capacity + thePageMask) & ~thePageMask);
894 
895  // Round up to power of two if half a page or less.
896  // (Zero stays zero.)
897  if (capacity <= IDX_T(2))
898  return capacity;
899  return IDX_T(SYSmakePow2(exint(capacity)));
900  }
901 
902  void setCapacity(IDX_T newcapacity)
903  {
904  UT_ASSERT_P(newcapacity >= IDX_T(0));
905  newcapacity = roundUpCapacity(newcapacity);
906  if (newcapacity == capacity())
907  return;
908  if (newcapacity < size())
909  {
910  setSize(newcapacity);
911  }
912  forceHardenTable(newcapacity);
913  }
914 
915  void setCapacityIfNeeded(IDX_T newcapacity)
916  {
917  UT_ASSERT_P(newcapacity >= IDX_T(0));
918  if (newcapacity <= capacity())
919  return;
920  newcapacity = roundUpCapacity(newcapacity);
921  forceHardenTable(newcapacity);
922  }
923 
924  /// Sets the size of the array without initializing any added
925  /// elements.
926  void setSize(IDX_T newsize)
927  {
928  UT_ASSERT_P(newsize >= IDX_T(0));
929 
930  setCapacityIfNeeded(newsize);
931  hardenTable();
932  PageTable *pages = myImpl.getPages();
933  UT_ASSERT_P(pages || newsize == IDX_T(0));
934  if (pages)
935  pages->setSize(newsize);
936  }
937 
938  /// Sets the size of the array and initializes any new elements to initval.
939  /// This version of setSize only works if DATA_T is a numeric type.
940  /// If the tuple size is > 1, it will write initval to all components.
941  /// Include UT_PageArrayImpl.h to call this.
942  void setSize(IDX_T newsize, NotVoidType initval);
943 
944  /// Sets the size of the array and initializes any new elements
945  /// to initval.
946  /// This version of setSize only works if DATA_T is a numeric type
947  /// and TSIZE >= 1.
948  /// Include UT_PageArrayImpl.h to call this.
949  void setSize(IDX_T newsize, const UT_FixedVector<NotVoidType,theSafeTupleSize> &initval);
950 
951  /// Sets the size of the array and initializes any new elements to initval.
952  /// Include UT_PageArrayImpl.h to call this.
953  void setSize(IDX_T newsize, const UT_Defaults &initval);
954 
955  /// Set all of the values in the range [start,end) to v.
956  /// This will constant-compress pages if allowed and the pages are
957  /// completely overwritten with the given value.
958  /// This version of setConstant only works if DATA_T is a numeric type.
959  /// If the tuple size is > 1, it will write v to all components.
960  void setConstant(IDX_T start, IDX_T end, NotVoidType v);
961 
962  /// Set all of the values in the range [start,end) to v.
963  /// This will constant-compress pages if allowed and the pages are
964  /// completely overwritten with the given value.
965  /// This version of setConstant only works if DATA_T is a numeric type
966  /// and TSIZE >= 1.
968 
969  /// Set all of the values in the range [start,end) to v.
970  /// This will constant-compress pages if allowed and the pages are
971  /// completely overwritten with the given value.
972  void setConstant(IDX_T start, IDX_T end, const UT_Defaults &v);
973 
974  /// Changes the storage type and converts the data to the new type.
975  /// NOTE: This only makes sense if DATA_T is void and newstorage is
976  /// something other than INVALID.
977  /// Include UT_PageArrayImpl.h to call this.
978  void setStorage(const UT_Storage newstorage);
979 
980  /// Changes the tuple size, and if the new size is larger, pads with
981  /// values from the UT_Defaults.
982  /// NOTE: This only makes sense if TSIZE == -1.
983  /// Include UT_PageArrayImpl.h to call this.
984  void setTupleSize(exint newtuplesize, const UT_Defaults &v);
985 
986  /// Get the total of all memory referred-to by this.
987  /// NOTE: This does not handle shared memory correctly.
988  /// Include UT_PageArrayImpl.h to call this.
989  int64 getMemoryUsage(bool inclusive) const;
990 
991  /// Count memory usage using a UT_MemoryCounter in order to count
992  /// shared memory correctly.
993  /// If inclusive is true, the size of this object is counted,
994  /// else only memory owned by this object is counted.
995  /// If this is pointed to by the calling object, inclusive should be true.
996  /// If this is contained in the calling object, inclusive should be false.
997  /// (Its memory was already counted in the size of the calling object.)
998  /// Include UT_PageArrayImpl.h to call this.
999  void countMemory(UT_MemoryCounter &counter, bool inclusive) const;
1000 
1001  /// Copies the nelements from srcstart to deststart
1002  /// The two ranges *are* allowed to overlap.
1003  /// Include UT_PageArrayImpl.h to call this.
1004  void moveRange(IDX_T srcstart, IDX_T deststart, IDX_T nelements);
1005 
1006  /// Copies the nelements from fromstart in src to tostart in this.
1007  /// The two ranges *are* allowed to overlap, even if &src==this.
1008  /// Include UT_PageArrayImpl.h to call this.
1009  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
1010  void moveRange(const UT_PageArray<SRC_DATA_T,SRC_TSIZE,SRC_TABLEHARDENED,SRC_PAGESHARDENED,THEPAGEBITS,IDX_T> &src, IDX_T srcstart, IDX_T deststart, IDX_T nelements);
1011 
1012  /// Swaps the nelements at astart with the nelements at bstart.
1013  /// The two ranges *must* be non-overlapping, else swapping is ill-defined.
1014  /// Include UT_PageArrayImpl.h to call this.
1015  void swapRange(IDX_T astart, IDX_T bstart, IDX_T nelements);
1016 
1017  /// Copies a single element from src to dest within this
1018  void copy(IDX_T dest, IDX_T src)
1019  {
1020  UT_ASSERT_P(dest >= IDX_T(0) && dest < size());
1021  UT_ASSERT_P(src >= IDX_T(0) && src < size());
1022 
1023  hardenTable();
1024 
1025  // Because no conversions are necessary, only the element size
1026  // in bytes is needed.
1027  const exint bytesize = (SYSisSame<DATA_T,void>() ? UTstorageSize(getStorage()) : sizeof(NotVoidType))
1028  * getTupleSize();
1029 
1030  UT_PageNum destpagenum = pageNum(dest);
1031  UT_PageNum srcpagenum = pageNum(src);
1032 
1033  PageTable *pages = myImpl.getPages();
1034  PageTableEntry *destpage = pages->getPPage(destpagenum);
1035  const PageTableEntry *srcpage = pages->getPPage(srcpagenum);
1036 
1037  const int8 *srcdata;
1038  if (srcpage->isConstant())
1039  {
1040  if (bytesize < sizeof(PageTableEntry))
1041  {
1042  // Copying within equal constant pages does nothing
1043  if (*srcpage == *destpage)
1044  return;
1045 
1046  srcdata = ((const int8 *)(srcpage+1)) - bytesize;
1047  }
1048  else if (srcpage->getUnmaskedPtrVoid() == destpage->getUnmaskedPtrVoid())
1049  {
1050  // Copying within equal constant pages does nothing
1051  return;
1052  }
1053  else
1054  {
1055  // NOTE: srcdata may be set to nullptr here.
1056  srcdata = (const int8 *)srcpage->getMaskedPtrVoid();
1057  }
1058  }
1059  else
1060  {
1061  srcdata = ((const int8*)srcpage->getFirstPtrVoid()) + pageOff(src)*bytesize;
1062  }
1063 
1064  int8 *destdata;
1065  if (destpage->isConstant())
1066  {
1067  if (bytesize < sizeof(PageTableEntry))
1068  destdata = ((int8 *)(destpage+1)) - bytesize;
1069  else
1070  {
1071  void *destmasked = destpage->getMaskedPtrVoid();
1072  if (destmasked == nullptr)
1073  {
1074  if (isZero(srcdata, bytesize))
1075  return;
1076  destdata = nullptr;
1077  }
1078  else
1079  destdata = (int8*)destmasked;
1080  }
1081  if (destdata != nullptr && isEqual(srcdata, destdata, bytesize))
1082  return;
1083 
1084  if (!SYSisSame<DATA_T,void>())
1085  hardenConstantPage(pages, destpage, getTupleSize());
1086  else
1087  {
1088  switch (getStorage())
1089  {
1090  case UT_Storage::INT8:
1091  {
1092  auto &a = castType<int8>();
1093  auto a_pages = a.myImpl.getPages();
1094  auto a_destpage = a_pages->getPPage(destpagenum);
1095  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1096  break;
1097  }
1098  case UT_Storage::INT16:
1099  case UT_Storage::REAL16:
1100  {
1101  auto &a = castType<int16>();
1102  auto a_pages = a.myImpl.getPages();
1103  auto a_destpage = a_pages->getPPage(destpagenum);
1104  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1105  break;
1106  }
1107  case UT_Storage::INT32:
1108  case UT_Storage::REAL32:
1109  {
1110  auto &a = castType<int32>();
1111  auto a_pages = a.myImpl.getPages();
1112  auto a_destpage = a_pages->getPPage(destpagenum);
1113  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1114  break;
1115  }
1116  case UT_Storage::INT64:
1117  case UT_Storage::REAL64:
1118  {
1119  auto &a = castType<int64>();
1120  auto a_pages = a.myImpl.getPages();
1121  auto a_destpage = a_pages->getPPage(destpagenum);
1122  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1123  break;
1124  }
1125  case UT_Storage::INVALID:
1126  UT_ASSERT_MSG(0, "Storage must be valid if DATA_T is void.");
1127  break;
1128  }
1129  }
1130  }
1131  else if (!PAGESHARDENED && destpage->isShared())
1132  {
1133  destdata = ((int8*)destpage->getFirstPtrVoid()) + pageOff(dest)*bytesize;
1134  if (srcdata)
1135  {
1136  if (isEqual(srcdata, destdata, bytesize))
1137  return;
1138  }
1139  else
1140  {
1141  if (isZero(destdata, bytesize))
1142  return;
1143  }
1144 
1145  if (!SYSisSame<DATA_T,void>())
1146  hardenSharedPage(pages, destpage, getTupleSize());
1147  else
1148  {
1149  switch (getStorage())
1150  {
1151  case UT_Storage::INT8:
1152  {
1153  auto &a = castType<int8>();
1154  auto a_pages = a.myImpl.getPages();
1155  auto a_destpage = a_pages->getPPage(destpagenum);
1156  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1157  break;
1158  }
1159  case UT_Storage::INT16:
1160  case UT_Storage::REAL16:
1161  {
1162  auto &a = castType<int16>();
1163  auto a_pages = a.myImpl.getPages();
1164  auto a_destpage = a_pages->getPPage(destpagenum);
1165  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1166  break;
1167  }
1168  case UT_Storage::INT32:
1169  case UT_Storage::REAL32:
1170  {
1171  auto &a = castType<int32>();
1172  auto a_pages = a.myImpl.getPages();
1173  auto a_destpage = a_pages->getPPage(destpagenum);
1174  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1175  break;
1176  }
1177  case UT_Storage::INT64:
1178  case UT_Storage::REAL64:
1179  {
1180  auto &a = castType<int64>();
1181  auto a_pages = a.myImpl.getPages();
1182  auto a_destpage = a_pages->getPPage(destpagenum);
1183  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1184  break;
1185  }
1186  case UT_Storage::INVALID:
1187  UT_ASSERT_MSG(0, "Storage must be valid if DATA_T is void.");
1188  break;
1189  }
1190  }
1191  }
1192  destdata = ((int8*)destpage->getFirstPtrVoid()) + pageOff(dest)*bytesize;
1193 
1194  for (const int8 *srcdataend = srcdata+bytesize; srcdata != srcdataend; ++srcdata, ++destdata)
1195  *destdata = *srcdata;
1196  }
1197 
1198  /// Copies nelements from srcstart in this to dest array
1199  /// Include UT_PageArrayImpl.h to call this.
1200  template<typename T>
1201  SYS_FORCE_INLINE void getRange(IDX_T srcstart, IDX_T nelements, T *dest) const
1202  {
1203  // Always cast to associated UT_FixedVector type, so that
1204  // we can just delegate to getVectorRange that always uses UT_FixedVector
1205  getVectorRange(srcstart, nelements, (typename UT_FixedVectorTraits<T>::FixedVectorType*)dest);
1206  }
1207 
1208  /// Copies nelements from src array to deststart in this
1209  /// Include UT_PageArrayImpl.h to call this.
1210  template<typename T>
1211  SYS_FORCE_INLINE void setRange(IDX_T deststart, IDX_T nelements, const T *src)
1212  {
1213  // Always cast to associated UT_FixedVector type, so that
1214  // we can just delegate to getVectorRange that always uses UT_FixedVector
1215  setVectorRange(deststart, nelements, (const typename UT_FixedVectorTraits<T>::FixedVectorType*)src);
1216  }
1217 
1218  /// Copies nelements from srcstart in this to dest array
1219  /// Include UT_PageArrayImpl.h to call this.
1220  template<typename DEST_DATA_T,exint DEST_TSIZE,bool DEST_INSTANTIATED>
1221  void getVectorRange(IDX_T srcstart, IDX_T nelements, UT_FixedVector<DEST_DATA_T,DEST_TSIZE,DEST_INSTANTIATED> *dest) const;
1222 
1223  /// Copies nelements from src array to deststart in this
1224  /// Include UT_PageArrayImpl.h to call this.
1225  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_INSTANTIATED>
1226  void setVectorRange(IDX_T deststart, IDX_T nelements, const UT_FixedVector<SRC_DATA_T,SRC_TSIZE,SRC_INSTANTIATED> *src);
1227 
1229  {
1230  return UT_PageNum(exint(i) >> thePageBits);
1231  }
1232  SYS_FORCE_INLINE static UT_PageNum numPages(IDX_T nelements)
1233  {
1234  return UT_PageNum((exint(nelements)+thePageMask) >> thePageBits);
1235  }
1237  {
1238  return UT_PageOff(exint(i) & thePageMask);
1239  }
1241  {
1242  return IDX_T(exint(page) << thePageBits);
1243  }
1244 
1245  /// Returns true iff the table isn't shared with any other UT_PageArray's.
1247  {
1248  // If TABLEHARDENED, assert that the page table hasn't become shared
1249  // after the fact! Nobody should share the page table after templating
1250  // on having hardened!
1251  UT_ASSERT_P(!TABLEHARDENED || !myImpl.getPages() || !myImpl.getPages()->isShared());
1252 
1253  return TABLEHARDENED || !myImpl.getPages() || !myImpl.getPages()->isShared();
1254  }
1255 
1256  /// This ensures that the table itself is not shared.
1257  /// It's a good idea to do this before writing, to avoid having to check
1258  /// every time. Remember to use the returned reference, since it
1259  /// records in the template that it doesn't need to be hardened again.
1260  /// It should be a no-op if TABLEHARDENED is already true.
1262  {
1263  if (!isTableHardened())
1264  {
1265  // Keep same capacity
1266  forceHardenTable(myImpl.getPages()->capacity());
1267  }
1268 
1270  }
1271 
1272  /// This forces all pages to be hardened, optionally including pages
1273  /// that are in the capacity but outside the size.
1274  template <bool including_capacity=false>
1276  {
1277  // Nothing to do if PAGESHARDENED
1278  PageTable *pages = myImpl.getPages();
1279  if (PAGESHARDENED || (pages == nullptr))
1281 
1282  auto &hard = hardenTable();
1283 
1284  if (end < IDX_T(0))
1285  end = including_capacity ? pages->capacity() : pages->size();
1286 
1287  UT_ASSERT_P(start >= IDX_T(0));
1289 
1290  if (!SYSisSame<DATA_T,void>())
1291  {
1292  if (TSIZE >= 1)
1293  pages->hardenAllPages(start, end);
1294  else
1295  pages->hardenAllPages(start, end, myImpl.getTupleSize());
1296  }
1297  else
1298  {
1299  // Don't want to have to re-check that the table is hardened,
1300  // so call hardenAllPagesUnknownType on hard.
1301  hard.hardenAllPagesUnknownType(start, end);
1302  }
1303 
1305  }
1306 
1307  template <bool including_capacity=false>
1308  SYS_FORCE_INLINE void tryCompressAllPages(IDX_T start = IDX_T(0), IDX_T end = IDX_T(-1))
1309  {
1310  // Can't have compressed pages if PAGESHARDENED
1311  PageTable *pages = myImpl.getPages();
1312  if (PAGESHARDENED || (pages == nullptr))
1313  return;
1314 
1315  auto &hard = hardenTable();
1316 
1317  if (end < IDX_T(0))
1318  end = including_capacity ? pages->capacity() : pages->size();
1319 
1320  UT_ASSERT_P(start >= IDX_T(0));
1322 
1323  if (!SYSisSame<DATA_T,void>())
1324  {
1325  if (TSIZE >= 1)
1326  pages->tryCompressAllPages(start, end);
1327  else
1328  pages->tryCompressAllPages(start, end, myImpl.getTupleSize());
1329  }
1330  else
1331  {
1332  // Don't want to have to re-check that the table is hardened,
1333  // so call hardenAllPagesUnknownType on hard.
1334  hard.tryCompressAllPagesUnknownType(start, end);
1335  }
1336  }
1337 
1338  /// This is for setting the DATA_T template after checking getStorage(),
1339  /// if DATA_T wasn't already known.
1340  /// @{
1341  template<typename DEST_DATA_T>
1343  castType() const
1344  {
1346  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()) || (getStorage() != UT_Storage::INVALID && getStorage() == UT_StorageNum<DEST_DATA_T>::theStorage));
1348  }
1349  template<typename DEST_DATA_T>
1352  {
1354  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()) || (getStorage() != UT_Storage::INVALID && getStorage() == UT_StorageNum<DEST_DATA_T>::theStorage));
1356  }
1357  /// @}
1358 
1359  /// This is for setting the TSIZE template after checking getTupleSize(),
1360  /// if TSIZE wasn't already known.
1361  /// @{
1362  template<exint DEST_TSIZE>
1365  {
1366  UT_ASSERT_P(TSIZE == -1 || TSIZE == DEST_TSIZE);
1367  UT_ASSERT_P(getTupleSize() == DEST_TSIZE);
1369  }
1370  template<exint DEST_TSIZE>
1373  {
1374  UT_ASSERT_P(TSIZE == -1 || TSIZE == DEST_TSIZE);
1375  UT_ASSERT_P(getTupleSize() == DEST_TSIZE);
1377  }
1378  /// @}
1379 
1381  {
1382  return myImpl.getTupleSize();
1383  }
1385  {
1386  return myImpl.getStorage();
1387  }
1388 
1389  /// Returns true iff the specified page is constant-compressed
1391  {
1392  UT_ASSERT_P(myImpl.getPages() != nullptr);
1393  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1394  return myImpl.getPages()->getPPage(pagenum)->isConstant();
1395  }
1396 
1397  /// Returns true iff the entire array is constant & zero
1399  {
1400  // Zero capacity array is zero:
1401  if (myImpl.getPages() == nullptr)
1402  return true;
1403 
1404  const PageTable *pages = myImpl.getPages();
1405 
1406  UT_PageNum npages = numPages(pages->capacity());
1407  for (UT_PageNum pagenum = 0; pagenum < npages; pagenum++)
1408  {
1409  const PageTableEntry *page = pages->getPPage(pagenum);
1410  if (!page->isConstantAndZeroSafe())
1411  return false;
1412  }
1413  return true;
1414  }
1415 
1416  /// Sets all components of all elements of the specified page to
1417  /// the given value.
1419  {
1420  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1421  UT_ASSERT_P(myImpl.getPages() != nullptr);
1422  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1423 
1424  hardenTable();
1425 
1426  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1427  if (TSIZE >= 1)
1428  makeConstant(page, val);
1429  else
1430  makeConstant(page, val, myImpl.getTupleSize());
1431  }
1432 
1433  /// Sets all components of all elements of the specified page to
1434  /// the given value.
1435  template<typename SRC_DATA_T>
1437  {
1438  if (SYSisSame<DATA_T,void>())
1439  {
1440  SYS_CallIf<SYSisSame<DATA_T,void>()>::call([this,pagenum,&val](SYS_CALLIF_AUTO){
1441  setPageConstantUnknownType(pagenum, val);
1442  });
1443  return;
1444  }
1445 
1446  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1447 
1448  if (!SYSisSame<DATA_T,SRC_DATA_T>())
1449  {
1450  SYS_CallIf<!SYSisSame<DATA_T,SRC_DATA_T>()>::call([this,pagenum,&val](SYS_CALLIF_AUTO){
1452  for (exint i = 0; i < theSafeTupleSize; ++i)
1453  new_val[i] = UTconvertStorage<NotVoidType>(val[i]);
1454  this->setPageConstant(pagenum, new_val);
1455  });
1456  return;
1457  }
1458 
1459  UT_ASSERT_P((SYSisSame<DATA_T,SRC_DATA_T>()));
1460  UT_ASSERT_P(myImpl.getPages() != nullptr);
1461  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1462  UT_ASSERT_P(TSIZE >= 1);
1463 
1464  hardenTable();
1465 
1466  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1467  makeConstant(page, val);
1468  }
1469 
1470  /// Sets all components of all elements of the specified page to
1471  /// the given values.
1472  /// NOTE: The length of values must be equal to the tuple size.
1473  template<typename SRC_DATA_T>
1474  SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const SRC_DATA_T *values)
1475  {
1476  if (SYSisSame<DATA_T,void>())
1477  {
1478  SYS_CallIf<SYSisSame<DATA_T,void>()>::call([this,pagenum,values](SYS_CALLIF_AUTO){
1479  this->setPageConstantUnknownType(pagenum, values);
1480  });
1481  return;
1482  }
1483 
1484  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1485 
1486  if (!SYSisSame<DATA_T,SRC_DATA_T>())
1487  {
1488  SYS_CallIf<!SYSisSame<DATA_T,SRC_DATA_T>()>::call([this,pagenum,values](SYS_CALLIF_AUTO){
1489  this->setPageConstantMismatchedType(pagenum, values);
1490  });
1491  return;
1492  }
1493 
1494  UT_ASSERT_P((SYSisSame<DATA_T,SRC_DATA_T>()));
1495  UT_ASSERT_P(myImpl.getPages() != nullptr);
1496  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1497 
1498  hardenTable();
1499 
1500  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1501  // NOTE: The cast will never actually be hit unless DATA_T is SRC_DATA_T,
1502  // as asserted above. It's just to get this to compile in other cases.
1503  makeConstant(page, (const NotVoidType *)values, myImpl.getTupleSize());
1504  }
1505 
1506  /// Returns true iff the specified page is "hard",
1507  /// i.e. NOT constant-compressed and NOT shared.
1508  /// NOTE: The *table* may or may not be hard.
1510  {
1511  UT_ASSERT_P(myImpl.getPages() != nullptr);
1512  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1513  if (PAGESHARDENED)
1514  return true;
1515  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1516  return !page->isConstant() && !page->isShared();
1517  }
1518 
1519  /// Hardens the specified page if it is constant-compressed or shared.
1520  /// It returns a pointer to the data, but it will only be the correct
1521  /// type if DATA_T is not void.
1523  {
1524  UT_ASSERT_P(myImpl.getPages() != nullptr);
1525  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1526 
1527  hardenTable();
1528 
1529  PageTable *pages = myImpl.getPages();
1530  PageTableEntry *page = pages->getPPage(pagenum);
1531 
1532  if (PAGESHARDENED)
1533  return page->getFirstPtr();
1534 
1535  if (page->isConstant())
1536  hardenConstantPage(pages, page, myImpl.getTupleSize());
1537  else if (page->isShared())
1538  hardenSharedPage(pages, page, myImpl.getTupleSize());
1539  return page->getFirstPtr();
1540  }
1541 
1543  {
1544  UT_ASSERT_P(myImpl.getPages() != nullptr);
1545  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1546 
1547  hardenTable();
1548 
1549  PageTable *pages = myImpl.getPages();
1550  PageTableEntry *page = pages->getPPage(pagenum);
1551 
1552  if (PAGESHARDENED)
1553  return page->getFirstPtr();
1554 
1555  if (page->isConstant())
1556  hardenConstantPageNoInit(pages, page, myImpl.getTupleSize());
1557  else if (page->isShared())
1558  hardenSharedPageNoInit(pages, page, myImpl.getTupleSize());
1559  return page->getFirstPtr();
1560  }
1561 
1562  /// If the specified page is constant and shared, (only relevant for
1563  /// nonzero tuples that are at least the size of a pointer), this
1564  /// keeps the page constant, but unshares it, so that the constant value
1565  /// can be modified.
1566  /// NOTE: This should ONLY be called on constant pages! Check
1567  /// isPageConstant(pagenum).
1569  {
1570  UT_ASSERT_P(myImpl.getPages() != nullptr);
1571  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1572 
1573  hardenTable();
1574 
1575  PageTable *pages = myImpl.getPages();
1576  PageTableEntry *page = pages->getPPage(pagenum);
1577 
1578  UT_ASSERT_P(!PAGESHARDENED && page->isConstant());
1579 
1580  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1581  UT_ASSERT_P(TSIZE == -1 || TSIZE >= 0);
1582 
1583  const exint tuplesize = (TSIZE >= 0) ? TSIZE : myImpl.getTupleSize();
1584  if (PageTableEntry::typeFitsInline(tuplesize))
1585  return page->getInlinePtr(tuplesize);
1586 
1587  NotVoidType *src = page->getMaskedPtr();
1588  if (!page->isShared())
1589  return src;
1590 
1591  PageTableEntry newpage;
1592  newpage.alloc(UT_PageOff(1), tuplesize);
1593  NotVoidType *dest = newpage.getFirstPtr();
1594  memcpy(dest, src, tuplesize*sizeof(NotVoidType));
1595  newpage.setConstantBit();
1596  page->decRef();
1597  (*page) = newpage;
1598 
1599  return dest;
1600  }
1601 
1602  /// Checks if the specified page is all a single value,
1603  /// and if so, makes it a constant page.
1605  {
1606  // Can't have compressed pages if PAGESHARDENED
1607  PageTable *pages = myImpl.getPages();
1608  if (PAGESHARDENED || (pages == nullptr))
1609  return;
1610 
1611  auto &hard = hardenTable();
1612 
1613  UT_ASSERT_P(pagenum >= UT_PageNum(0));
1614  UT_ASSERT_P(pagenum < numPages(pages->capacity()));
1615 
1616  if (!SYSisSame<DATA_T,void>())
1617  {
1618  if (TSIZE >= 1)
1619  pages->tryCompressPage(pagenum);
1620  else
1621  pages->tryCompressPage(pagenum, myImpl.getTupleSize());
1622  }
1623  else
1624  {
1625  // Don't want to have to re-check that the table is hardened,
1626  // so call hardenAllPagesUnknownType on hard.
1627  hard.tryCompressPageUnknownType(pagenum);
1628  }
1629  }
1630 
1631  /// Returns a pointer to the data of a page.
1632  /// WARNING: DO NOT call this if DATA_T is void! The pointer returned
1633  /// can depend on the type.
1634  /// This will return nullptr if the page is constant compressed to zero.
1635  /// If it's non-nullptr and isPageConstant(pagenum) returns true, there
1636  /// will only be one element (tuple) at that address.
1637  const NotVoidType *getPageData(UT_PageNum pagenum) const
1638  {
1639  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1640  UT_ASSERT_P(myImpl.getPages() != nullptr);
1641  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1642  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1643  if (!page->isConstant())
1644  return page->getFirstPtr();
1645  if (TSIZE >= 1)
1646  return getConstantPtr(page);
1647  return getConstantPtr(page, 0, getTupleSize());
1648  }
1649 
1650  /// Returns a pointer to the data of a page.
1651  /// WARNING: DO NOT call this if DATA_T is void! The pointer returned
1652  /// can depend on the type.
1653  /// NOTE: DO NOT MODIFY THE DATA if there's any chance the page or the table
1654  /// might be shared. Check isTableHard() and isPageHard(pagenum).
1655  /// This will return nullptr if the page is constant compressed to zero.
1656  /// If it's non-nullptr and isPageConstant(pagenum) returns true, there
1657  /// will only be one element (tuple) at that address.
1659  {
1660  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1661  UT_ASSERT_P(myImpl.getPages() != nullptr);
1662  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1663  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1664  if (!page->isConstant())
1665  return page->getFirstPtr();
1666  if (TSIZE >= 1)
1667  return getConstantPtr(page);
1668  return getConstantPtr(page, 0, getTupleSize());
1669  }
1670 
1671  /// NOTE: Don't use this unless you know what you're doing!
1672  /// This is for the case where a non-POD type that requires destruction
1673  /// is masquerading as a POD type in here, e.g. GA_OffsetList as
1674  /// UT_FixedVector<int64,2>. When destroying a table, this first needs
1675  /// to be called. If it returns true, the table was shared,
1676  /// so it was just decRef'd, meaning that the data don't need to be
1677  /// destructed. If it returns false, the table was not shared,
1678  /// so the data need to be destructed before deleting the table.
1680  {
1681  if (TABLEHARDENED)
1682  return false;
1683 
1684  PageTable *table = myImpl.getPages();
1685  if (!table)
1686  return false;
1687 
1688  bool decremented = table->decRefIffShared();
1689  if (decremented)
1690  myImpl.getPages() = nullptr;
1691  return decremented;
1692  }
1693 
1694  /// NOTE: Don't use this unless you know what you're doing!
1695  /// This is for the case where a non-POD type that requires destruction
1696  /// is masquerading as a POD type in here, e.g. GA_OffsetList as
1697  /// UT_FixedVector<int64,2>. When destroying a page, this first needs
1698  /// to be called. If it returns true, the page was shared,
1699  /// so it was just decRef'd, meaning that the data don't need to be
1700  /// destructed. If it returns false, the page was not shared,
1701  /// so the data need to be destructed before deleting the page.
1703  {
1704  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1705  UT_ASSERT_P(TSIZE >= 1);
1706  UT_ASSERT_P(myImpl.getPages() != nullptr);
1707  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1708 
1709  if (PAGESHARDENED)
1710  return false;
1711 
1712  hardenTable();
1713 
1714  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1715  if (page->isConstant())
1716  {
1717  if (sizeof(NotVoidType)*TSIZE < sizeof(PageTableEntry))
1718  return false;
1719  }
1720  bool decremented = page->decRefIffShared();
1721  if (decremented)
1722  page->initZero();
1723  return decremented;
1724  }
1725 
1726  /// Only call this if DATA_T is void and you are absolutely, 100% CERTAIN
1727  /// that the storage type is int16, int32, or int64.
1728  /// NOTE: Really! Be careful!
1730  int64 getGuaranteedInt(IDX_T i) const
1731  {
1732  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1733  UT_ASSERT_P(TSIZE == 1);
1735 
1736  UT_PageNum pagenum = pageNum(i);
1737  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1738 
1739  // If we know that the pages are hardened, we can omit
1740  // the constant page case at compile time.
1741  bool isconst = !PAGESHARDENED && page->isConstant();
1742 
1743  const UT_Storage storage = getStorage();
1744  if (!isconst)
1745  {
1746  UT_PageOff pageoff = pageOff(i);
1747  const void *first = page->getFirstPtrVoid();
1748  switch (storage)
1749  {
1750  default:
1751  UT_ASSERT_MSG_P(0, "getGuaranteedInt() only supports int16, int32, and int64.");
1753  case UT_Storage::INT16:
1754  return (int64)((int16*)first)[pageoff];
1755  case UT_Storage::INT32:
1756  return (int64)((int32*)first)[pageoff];
1757  case UT_Storage::INT64:
1758  return ((int64*)first)[pageoff];
1759  }
1760  }
1761 
1762  // Special case for zero.
1763  if (!page->isConstantZero())
1764  return int64(0);
1765 
1766  // Special case for if the constant value is too big to fit in a PageTableEntry.
1767  // At the moment, PageTableEntry is 16 bytes on 64-bit platforms,
1768  // and 8 bytes on 32-bit platforms, so this case will not occur.
1769  if ((sizeof(PageTableEntry) < 9 && storage == UT_Storage::INT64) ||
1770  (sizeof(PageTableEntry) < 5 && storage == UT_Storage::INT32))
1771  {
1772  const void *v = page->getMaskedPtrVoid();
1773  if (sizeof(PageTableEntry) < 5 && storage == UT_Storage::INT32)
1774  {
1775  // This will not be run on 64-bit platforms
1776  return (int64)*(const int32 *)v;
1777  }
1778  return *(const int64 *)v;
1779  }
1780  else
1781  {
1782  // Any types that are strictly smaller than a pointer will be stored inline.
1783  // This is checked at compile time, so there'll be no op at runtime.
1784  if (sizeof(PageTableEntry) >= 9 && storage == UT_Storage::INT64)
1785  {
1786  return *page->template castType<int64>()->getInlinePtr(1);
1787  }
1788  if (sizeof(PageTableEntry) >= 5 && storage == UT_Storage::INT32)
1789  {
1790  return (int64)*page->template castType<int32>()->getInlinePtr(1);
1791  }
1792  return (int64)*page->template castType<int16>()->getInlinePtr(1);
1793  }
1794  }
1795 
1796  /// Returns true iff there is at least one NaN value in the given range.
1797  /// Include UT_PageArrayImpl.h to call this.
1798  bool hasNanInRange(IDX_T start, IDX_T end) const;
1799 
1800 protected:
1801  template<typename DEST_DATA_T=DATA_T>
1802  DEST_DATA_T getUnknownType(IDX_T i,exint component=0) const
1803  {
1804  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1805  UT_ASSERT_P(TSIZE != 0);
1806 
1807  // Make sure common work is extracted out where possible,
1808  // to avoid duplication.
1809 
1810  UT_PageNum pagenum = pageNum(i);
1811  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1812 
1813  // If we know that the pages are hardened, we can omit
1814  // the constant page case at compile time.
1815  bool isconst = !PAGESHARDENED && page->isConstant();
1816 
1817  const UT_Storage storage = getStorage();
1818  if (!isconst)
1819  {
1820  UT_PageOff pageoff = pageOff(i);
1821  const void *first = page->getFirstPtrVoid();
1822  exint i;
1823  if (TSIZE == 1)
1824  i = (exint)pageoff;
1825  else if (TSIZE > 1)
1826  i = TSIZE*pageoff + component;
1827  else
1828  i = myImpl.getTupleSize()*pageoff + component;
1829  switch (storage)
1830  {
1831  case UT_Storage::INVALID:
1832  UT_ASSERT_MSG_P(0, "Can't read a non-numeric type with a cast to a different type.");
1833  return DEST_DATA_T();
1834  case UT_Storage::INT8:
1835  return (DEST_DATA_T)((int8*)first)[i];
1836  case UT_Storage::INT16:
1837  return (DEST_DATA_T)((int16*)first)[i];
1838  case UT_Storage::INT32:
1839  return (DEST_DATA_T)((int32*)first)[i];
1840  case UT_Storage::INT64:
1841  return (DEST_DATA_T)((int64*)first)[i];
1842  case UT_Storage::REAL16:
1843  return (DEST_DATA_T)((fpreal16*)first)[i];
1844  case UT_Storage::REAL32:
1845  return (DEST_DATA_T)((fpreal32*)first)[i];
1846  case UT_Storage::REAL64:
1847  return (DEST_DATA_T)((fpreal64*)first)[i];
1848  }
1849  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1850  return DEST_DATA_T();
1851  }
1852 
1853  switch (storage)
1854  {
1855  case UT_Storage::INVALID:
1856  UT_ASSERT_MSG_P(0, "Can't read a non-numeric type with a cast to a different type.");
1857  return DEST_DATA_T();
1858  case UT_Storage::INT8:
1859  return (DEST_DATA_T)castType<int8>().getConstant(page->template castType<int8>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1860  case UT_Storage::INT16:
1861  return (DEST_DATA_T)castType<int16>().getConstant(page->template castType<int16>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1862  case UT_Storage::INT32:
1863  return (DEST_DATA_T)castType<int32>().getConstant(page->template castType<int32>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1864  case UT_Storage::INT64:
1865  return (DEST_DATA_T)castType<int64>().getConstant(page->template castType<int64>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1866  case UT_Storage::REAL16:
1867  return (DEST_DATA_T)castType<fpreal16>().getConstant(page->template castType<fpreal16>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1868  case UT_Storage::REAL32:
1869  return (DEST_DATA_T)castType<fpreal32>().getConstant(page->template castType<fpreal32>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1870  case UT_Storage::REAL64:
1871  return (DEST_DATA_T)castType<fpreal64>().getConstant(page->template castType<fpreal64>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1872  }
1873  return DEST_DATA_T();
1874  }
1875 
1876  template<int OP,typename SRC_DATA_T>
1877  void opUnknownType(IDX_T i,exint component,SRC_DATA_T v)
1878  {
1879  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1880  const UT_Storage storage = getStorage();
1881  switch (storage)
1882  {
1883  case UT_Storage::INVALID:
1884  UT_ASSERT_MSG_P(0, "Can't write a non-numeric type with opUnknownType.");
1885  return;
1886  case UT_Storage::INT8:
1887  castType<int8 >().template op<OP>(i,component, UTconvertStorage<int8>(v)); return;
1888  case UT_Storage::INT16:
1889  castType<int16 >().template op<OP>(i,component, UTconvertStorage<int16>(v)); return;
1890  case UT_Storage::INT32:
1891  castType<int32 >().template op<OP>(i,component, UTconvertStorage<int32>(v)); return;
1892  case UT_Storage::INT64:
1893  castType<int64 >().template op<OP>(i,component, UTconvertStorage<int64>(v)); return;
1894  case UT_Storage::REAL16:
1895  castType<fpreal16>().template op<OP>(i,component, (fpreal16)v); return;
1896  case UT_Storage::REAL32:
1897  castType<fpreal32>().template op<OP>(i,component, (fpreal32)v); return;
1898  case UT_Storage::REAL64:
1899  castType<fpreal64>().template op<OP>(i,component, (fpreal64)v); return;
1900  }
1901  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1902  }
1903  template<int OP,typename SRC_DATA_T,exint SRC_TSIZE>
1905  {
1906  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1907  const UT_Storage storage = getStorage();
1908  switch (storage)
1909  {
1910  case UT_Storage::INVALID:
1911  UT_ASSERT_MSG_P(0, "Can't write a non-numeric type with opVectorUnknownType.");
1912  return;
1913  case UT_Storage::INT8:
1914  castType<int8 >().template opVector<OP>(i,v); return;
1915  case UT_Storage::INT16:
1916  castType<int16 >().template opVector<OP>(i,v); return;
1917  case UT_Storage::INT32:
1918  castType<int32 >().template opVector<OP>(i,v); return;
1919  case UT_Storage::INT64:
1920  castType<int64 >().template opVector<OP>(i,v); return;
1921  case UT_Storage::REAL16:
1922  castType<fpreal16>().template opVector<OP>(i,v); return;
1923  case UT_Storage::REAL32:
1924  castType<fpreal32>().template opVector<OP>(i,v); return;
1925  case UT_Storage::REAL64:
1926  castType<fpreal64>().template opVector<OP>(i,v); return;
1927  }
1928  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1929  }
1930  template<typename DEST_DATA_T,exint DEST_TSIZE>
1932  {
1933  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1934 
1935  const UT_Storage storage = getStorage();
1936 
1937  // First, check for match on destination type
1939  return castType<DEST_DATA_T>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1940 
1941  // Then, try second-guess type (e.g. reading double as float or vice versa)
1943  return castType<typename UT_StorageNum<DEST_DATA_T>::SecondGuess>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1944 
1945  switch (storage)
1946  {
1947  case UT_Storage::INVALID:
1948  UT_ASSERT_MSG_P(0, "Can't read a non-numeric type with getVectorUnknownType.");
1949  break;
1950  case UT_Storage::INT8:
1951  return castType<int8 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1952  case UT_Storage::INT16:
1953  return castType<int16 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1954  case UT_Storage::INT32:
1955  return castType<int32 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1956  case UT_Storage::INT64:
1957  return castType<int64 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1958  case UT_Storage::REAL16:
1959  return castType<fpreal16>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1960  case UT_Storage::REAL32:
1961  return castType<fpreal32>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1962  case UT_Storage::REAL64:
1963  return castType<fpreal64>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1964  }
1965  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1966  return UT_FixedVector<DEST_DATA_T,DEST_TSIZE>(DEST_DATA_T(0));
1967  }
1968  template<typename DEST_DATA_T,exint DEST_TSIZE>
1970  {
1971  UT_ASSERT_P(TSIZE==-1);
1972  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()));
1973 
1974  const exint tuplesize = getTupleSize();
1975 
1976  // First, check for match on destination type
1977  if (tuplesize == DEST_TSIZE)
1978  return castTupleSize<DEST_TSIZE>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1979 
1980  // Then try small sizes
1981  if (tuplesize == 1)
1982  return castTupleSize<1>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
1983  if (tuplesize == 2)
1984  return castTupleSize<2>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
1985  if (tuplesize == 3)
1986  return castTupleSize<3>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
1987  if (tuplesize == 4)
1988  return castTupleSize<4>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
1989 
1990  // Mismatched large, unknown tuplesize (uncommon case)
1991  UT_FixedVector<DEST_DATA_T,DEST_TSIZE> v(DEST_DATA_T(0));
1992  const exint minsize = SYSmin(tuplesize,DEST_TSIZE);
1993  for (exint component = 0; component < minsize; ++component)
1994  v[component] = get<DEST_DATA_T>(i, component);
1995 
1996  return v;
1997  }
1998  template<typename DEST_DATA_T,exint DEST_TSIZE>
2000  {
2001  UT_ASSERT_P(TSIZE!=-1);
2002  UT_ASSERT_P(TSIZE!=DEST_TSIZE);
2003  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()));
2004 
2005  // Types and tuple sizes are known, but don't match
2006  UT_PageNum pagenum = pageNum(i);
2007  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
2008 
2009  // If we know that the pages are hardened, we can omit
2010  // the constant page case at compile time.
2011  bool isconst = !PAGESHARDENED && page->isConstant();
2012 
2013  const NotVoidType *srctuple;
2014  if (!isconst)
2015  {
2016  UT_PageOff pageoff = pageOff(i);
2017  srctuple = (const NotVoidType *)(((const UT_FixedVector<NotVoidType,theSafeTupleSize>*)page->getFirstPtr()) + pageoff);
2018  }
2019  else
2020  {
2021  srctuple = getConstantPtr(page);
2022  if (!srctuple)
2023  return UT_FixedVector<DEST_DATA_T,DEST_TSIZE>(DEST_DATA_T(0));
2024  }
2025 
2026  if (DEST_TSIZE < TSIZE)
2027  return *(const UT_FixedVector<DEST_DATA_T,DEST_TSIZE>*)srctuple;
2028 
2030  for (exint component = 0; component < TSIZE; ++component)
2031  dsttuple[component] = srctuple[component];
2032  for (exint component = TSIZE; component < DEST_TSIZE; ++component)
2033  dsttuple[component] = DEST_DATA_T(0);
2034  return dsttuple;
2035  }
2037  {
2038  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2039  const UT_Storage storage = getStorage();
2040  switch (storage)
2041  {
2042  case UT_Storage::INVALID:
2043  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2044  return;
2045  case UT_Storage::INT8:
2046  castType<int8>().template hardenAllPages<false>(start, end); return;
2047  case UT_Storage::INT16:
2048  castType<int16>().template hardenAllPages<false>(start, end); return;
2049  case UT_Storage::INT32:
2050  castType<int32>().template hardenAllPages<false>(start, end); return;
2051  case UT_Storage::INT64:
2052  castType<int64>().template hardenAllPages<false>(start, end); return;
2053  case UT_Storage::REAL16:
2054  castType<fpreal16>().template hardenAllPages<false>(start, end); return;
2055  case UT_Storage::REAL32:
2056  castType<fpreal32>().template hardenAllPages<false>(start, end); return;
2057  case UT_Storage::REAL64:
2058  castType<fpreal64>().template hardenAllPages<false>(start, end); return;
2059  }
2060  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2061  }
2062 
2064  {
2065  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2066  const UT_Storage storage = getStorage();
2067  switch (storage)
2068  {
2069  case UT_Storage::INVALID:
2070  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2071  return;
2072  case UT_Storage::INT8:
2073  castType<int8>().template tryCompressAllPages<false>(start, end); return;
2074  case UT_Storage::INT16:
2075  castType<int16>().template tryCompressAllPages<false>(start, end); return;
2076  case UT_Storage::INT32:
2077  castType<int32>().template tryCompressAllPages<false>(start, end); return;
2078  case UT_Storage::INT64:
2079  castType<int64>().template tryCompressAllPages<false>(start, end); return;
2080  case UT_Storage::REAL16:
2081  castType<fpreal16>().template tryCompressAllPages<false>(start, end); return;
2082  case UT_Storage::REAL32:
2083  castType<fpreal32>().template tryCompressAllPages<false>(start, end); return;
2084  case UT_Storage::REAL64:
2085  castType<fpreal64>().template tryCompressAllPages<false>(start, end); return;
2086  }
2087  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2088  }
2089 
2091  {
2092  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2093  const UT_Storage storage = getStorage();
2094  switch (storage)
2095  {
2096  case UT_Storage::INVALID:
2097  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2098  return;
2099  case UT_Storage::INT8:
2100  castType<int8>().tryCompressPage(pagenum); return;
2101  case UT_Storage::INT16:
2102  castType<int16>().tryCompressPage(pagenum); return;
2103  case UT_Storage::INT32:
2104  castType<int32>().tryCompressPage(pagenum); return;
2105  case UT_Storage::INT64:
2106  castType<int64>().tryCompressPage(pagenum); return;
2107  case UT_Storage::REAL16:
2108  castType<fpreal16>().tryCompressPage(pagenum); return;
2109  case UT_Storage::REAL32:
2110  castType<fpreal32>().tryCompressPage(pagenum); return;
2111  case UT_Storage::REAL64:
2112  castType<fpreal64>().tryCompressPage(pagenum); return;
2113  }
2114  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2115  }
2116 
2117  template<typename SRC_DATA_T>
2119  {
2120  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2121  const UT_Storage storage = getStorage();
2122  switch (storage)
2123  {
2124  case UT_Storage::INVALID:
2125  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2126  return;
2127  case UT_Storage::INT8:
2128  castType<int8>().setPageConstant(pagenum, val); return;
2129  case UT_Storage::INT16:
2130  castType<int16>().setPageConstant(pagenum, val); return;
2131  case UT_Storage::INT32:
2132  castType<int32>().setPageConstant(pagenum, val); return;
2133  case UT_Storage::INT64:
2134  castType<int64>().setPageConstant(pagenum, val); return;
2135  case UT_Storage::REAL16:
2136  castType<fpreal16>().setPageConstant(pagenum, val); return;
2137  case UT_Storage::REAL32:
2138  castType<fpreal32>().setPageConstant(pagenum, val); return;
2139  case UT_Storage::REAL64:
2140  castType<fpreal64>().setPageConstant(pagenum, val); return;
2141  }
2142  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2143  }
2144 
2145  template<typename SRC_DATA_T>
2146  void setPageConstantUnknownType(UT_PageNum pagenum, const SRC_DATA_T *values)
2147  {
2148  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2149  const UT_Storage storage = getStorage();
2150  switch (storage)
2151  {
2152  case UT_Storage::INVALID:
2153  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2154  return;
2155  case UT_Storage::INT8:
2156  castType<int8>().setPageConstant(pagenum, values); return;
2157  case UT_Storage::INT16:
2158  castType<int16>().setPageConstant(pagenum, values); return;
2159  case UT_Storage::INT32:
2160  castType<int32>().setPageConstant(pagenum, values); return;
2161  case UT_Storage::INT64:
2162  castType<int64>().setPageConstant(pagenum, values); return;
2163  case UT_Storage::REAL16:
2164  castType<fpreal16>().setPageConstant(pagenum, values); return;
2165  case UT_Storage::REAL32:
2166  castType<fpreal32>().setPageConstant(pagenum, values); return;
2167  case UT_Storage::REAL64:
2168  castType<fpreal64>().setPageConstant(pagenum, values); return;
2169  }
2170  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2171  }
2172 
2173  template<typename SRC_DATA_T>
2174  void setPageConstantMismatchedType(UT_PageNum pagenum, const SRC_DATA_T *values)
2175  {
2176  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2177  UT_ASSERT_P(!(SYSisSame<DATA_T,SRC_DATA_T>()));
2178 
2179  // This could be done directly, instead of converting
2180  // to a UT_SmallArray<DATA_T>, but this approach is simpler for now.
2181 
2182  UT_SmallArray<NotVoidType, ((TSIZE >= 1) ? TSIZE : 16)*sizeof(NotVoidType)> dest_values;
2183  const exint tuplesize = myImpl.getTupleSize();
2184  dest_values.setSize(tuplesize);
2185 
2186  for (exint i = 0; i < tuplesize; ++i)
2187  dest_values(i) = UTconvertStorage<NotVoidType>(values[i]);
2188 
2189  setPageConstant(pagenum, dest_values.array());
2190  }
2191 
2193  {
2194  public:
2197  static const uintptr_t theConstantPageBit = 1;
2198 
2199  /// This is always valid to call.
2201  {
2202  return (uintptr_t(myData) & theConstantPageBit) != 0;
2203  }
2204  /// This is only valid to call if the type doesn't fit inline.
2206  {
2207  return (uintptr_t(myData) == theConstantPageBit);
2208  }
2209  /// This is always valid to call, and will return true
2210  /// iff this is either inline and all zero or
2211  /// using the constant zero optimization.
2213  {
2214  return (uintptr_t(myData) == theConstantPageBit)
2215  && (myPRefCount == nullptr);
2216  }
2217  /// This is only valid to call when DATA_T is non-void.
2218  SYS_FORCE_INLINE static bool typeFitsInline(exint tuplesize = TSIZE)
2219  {
2221  if (sizeof(PageTableEntry) <= sizeof(NotVoidType))
2222  return false;
2223  return (sizeof(NotVoidType)*((TSIZE >= 0) ? TSIZE : tuplesize) < sizeof(PageTableEntry));
2224  }
2225 
2226  /// This is always valid to call, but be aware that the parameter is the
2227  /// number of *bytes* in a tuple, not the number of components in a
2228  /// tuple.
2229  SYS_FORCE_INLINE bool isRefd(exint tuplebytes) const
2230  {
2231  if (!isConstant())
2232  return true;
2233 
2234  // This will be checked at compile time, when possible.
2235  if (sizeof(PageTableEntry) > ((TSIZE >= 0 && !SYSisSame<DATA_T,void>()) ? TSIZE*sizeof(NotVoidType) : tuplebytes))
2236  return false;
2237 
2238  return !isConstantZero();
2239  }
2240 
2241  /// NOTE: This should only be called when there's an actual allocated page,
2242  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2244  {
2245  UT_ASSERT_P(myPRefCount->relaxedLoad() > 0);
2246  return myPRefCount->relaxedLoad() != 1;
2247  }
2248 
2249  /// NOTE: This should only be called when there's an actual allocated page to decRef,
2250  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2252  {
2253  myPRefCount->add(1);
2254  }
2255 
2256  /// NOTE: This should only be called when there's an actual allocated page to decRef,
2257  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2259  {
2260  int64 new_count = myPRefCount->add(-1);
2261  UT_ASSERT_P(new_count >= 0);
2262  if (new_count == 0)
2263  {
2264  free((NotVoidType*)(uintptr_t(myData) & ~theConstantPageBit));
2265  free(myPRefCount);
2266 #if 0
2267  printf("Free page %d pages total\n", int(thePageTablePageCount.add(-1)));
2268  fflush(stdout);
2269 #endif
2270  }
2271  }
2272 
2273  /// NOTE: This should only be called when there's an actual allocated page,
2274  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2276  {
2277  UT_ASSERT_P(myPRefCount->relaxedLoad() > 0);
2278  return myPRefCount->relaxedLoad();
2279  }
2280 
2281  SYS_FORCE_INLINE void alloc(UT_PageOff nelements, exint tuplesize = TSIZE)
2282  {
2284  myData = (NotVoidType *)malloc(nelements*((TSIZE >= 0) ? TSIZE : tuplesize)*sizeof(NotVoidType));
2285  myPRefCount = (SYS_AtomicCounter *)malloc(sizeof(SYS_AtomicCounter));
2286  myPRefCount->relaxedStore(1);
2287 #if 0
2288  printf("Alloc page %d pages total\n", int(thePageTablePageCount.add(1)));
2289  fflush(stdout);
2290 #endif
2291  }
2292  void realloc(UT_PageOff sizetocopy, UT_PageOff newpagecapacity, exint tuplesize = TSIZE)
2293  {
2295  UT_ASSERT_P(sizetocopy >= 0 && sizetocopy <= newpagecapacity);
2296  UT_ASSERT_P(!isConstant());
2297 
2298  UT_ASSERT_P(tuplesize >= 0);
2299  // This is just to try to force the compiler to constant-value-optimize.
2300  if (TSIZE >= 0)
2301  tuplesize = TSIZE;
2302 
2303  UT_ASSERT_P(myPRefCount->relaxedLoad() > 0);
2304  if (myPRefCount->relaxedLoad() > 1)
2305  {
2306  // If shared, need to make a new page.
2307  PageTableEntry newpage;
2308  newpage.alloc(newpagecapacity, tuplesize);
2309  NotVoidType *__restrict newdata = newpage.myData;
2310  const NotVoidType *__restrict olddata = myData;
2311  // Yes, this loop is an inline memcpy, because memcpy won't inline,
2312  // but the compiler will optimize it if it's like this.
2313  sizetocopy *= tuplesize;
2314  while (sizetocopy)
2315  {
2316  *newdata = *olddata;
2317  ++newdata;
2318  ++olddata;
2319  --sizetocopy;
2320  }
2321  decRef();
2322  myData = newpage.myData;
2323  myPRefCount = newpage.myPRefCount;
2324  }
2325  else
2326  {
2327  // If not shared, we can potentially reuse the same memory.
2328  myData = (NotVoidType*)::realloc(myData, newpagecapacity*(tuplesize*sizeof(NotVoidType)));
2329  // NOTE: Reference count doesn't need to change, since it's 1.
2330  }
2331  }
2332 
2333  /// Atomically decrement the reference count iff the page is shared.
2334  /// See the description of decRefIffSharedPage() above for
2335  /// why this function exists.
2337  bool decRefIffShared(exint tuplesize = TSIZE)
2338  {
2339  if (isConstant())
2340  {
2341  UT_ASSERT_P(tuplesize >= 0);
2342  if (typeFitsInline(tuplesize) || isConstantZero())
2343  return false;
2344  }
2345  return decCounterIffAbove1(*myPRefCount);
2346  }
2347 
2348  /// Returns the data pointer, if not a constant page
2351  {
2352  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtr() is for non-constant pages");
2354 
2355  return myData;
2356  }
2357 
2358  /// Returns the data pointer, if not a constant page
2360  const NotVoidType *getFirstPtr() const
2361  {
2362  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtr() is for non-constant pages");
2364 
2365  return myData;
2366  }
2367 
2368  /// Returns the data pointer, if not a constant page
2371  {
2372  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtrVoid() is for non-constant pages");
2373 
2374  return myData;
2375  }
2376 
2377  /// Returns the data pointer, if not a constant page
2379  const void *getFirstPtrVoid() const
2380  {
2381  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtrVoid() is for non-constant pages");
2382 
2383  return myData;
2384  }
2385 
2386  /// Returns the data pointer, or whatever's in its place
2387  /// with bit 0 set to 1 if it's something constant.
2388  /// If it's constant, it might not be a flagged pointer.
2391  {
2392  return myData;
2393  }
2394 
2395  /// Returns the data pointer, or whatever's in its place
2396  /// with bit 0 set to 1 if it's something constant.
2397  /// If it's constant, it might not be a flagged pointer.
2399  const void *getFirstPtrVoidUnsafe() const
2400  {
2401  return myData;
2402  }
2403 
2404  /// Returns the data pointer, if an inline constant page
2407  {
2408  UT_ASSERT_MSG_P(isConstant() && typeFitsInline(tuplesize), "getInlinePtr() is for inline constant pages");
2410  return (NotVoidType *)(((int8 *)(this+1)) - (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize)));
2411  }
2412 
2413  /// Returns the data pointer, if an inline constant page
2415  const NotVoidType *getInlinePtr(exint tuplesize) const
2416  {
2417  UT_ASSERT_MSG_P(isConstant() && typeFitsInline(tuplesize), "getInlinePtr() is for inline constant pages");
2419  return (const NotVoidType *)(((const int8 *)(this+1)) - (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize)));
2420  }
2421 
2422  /// Returns the data pointer, if a non-inline constant page.
2423  /// Returns nullptr for a zero constant page.
2426  {
2427  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtr() is for constant pages");
2429 
2430  return (NotVoidType *)(uintptr_t(myData) & ~theConstantPageBit);
2431  }
2432 
2433  /// Returns the data pointer, if a non-inline constant page.
2434  /// Returns nullptr for a zero constant page.
2436  const NotVoidType *getMaskedPtr() const
2437  {
2438  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtr() is for constant pages");
2440 
2441  return (const NotVoidType *)(uintptr_t(myData) & ~theConstantPageBit);
2442  }
2443 
2444  /// Returns the data pointer, if a non-inline constant page.
2445  /// Returns nullptr for a zero constant page.
2448  {
2449  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtrVoid() is for constant pages");
2450 
2451  return (void *)(uintptr_t(myData) & ~theConstantPageBit);
2452  }
2453 
2454  /// Returns the data pointer, if a non-inline constant page.
2455  /// Returns nullptr for a zero constant page.
2457  const void *getMaskedPtrVoid() const
2458  {
2459  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtrVoid() is for constant pages");
2460 
2461  return (const void *)(uintptr_t(myData) & ~theConstantPageBit);
2462  }
2463 
2464  /// Initializes a page to constant zero, assuming that it hasn't been
2465  /// initialized yet.
2467  void initZero()
2468  {
2469  myData = (NotVoidType*)theConstantPageBit;
2470  myPRefCount = nullptr;
2471  }
2472 
2475  {
2476  myData = (NotVoidType *)(uintptr_t(myData) | theConstantPageBit);
2477  }
2478 
2479  void tryCompressPage(UT_PageOff pagesize, exint tuplesize = TSIZE)
2480  {
2481  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2482  UT_ASSERT_P(!isConstant());
2483 
2484  tuplesize = (TSIZE >= 0 ? TSIZE : tuplesize);
2485 
2486  const NotVoidType *const first = myData;
2487  const NotVoidType *const end = first + (TSIZE >= 0 ? TSIZE : tuplesize)*exint(pagesize);
2488  for (const NotVoidType *other = first+tuplesize; other < end; other += tuplesize)
2489  {
2490  for (exint component = 0; component < (TSIZE >= 0 ? TSIZE : tuplesize); ++component)
2491  {
2492  if (other[component] != first[component])
2493  return;
2494  }
2495  }
2496  // We have a constant page! Compress it!
2497 
2498  PageTableEntry newpage;
2499  newpage.initZero();
2500  const exint bytesize = sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize);
2501  // Value stored inline if it's strictly smaller than a pointer
2502  // The first check can be done at compile time, so it's only logically redundant,
2503  // not functionally redundant.
2504  if (sizeof(PageTableEntry) > sizeof(NotVoidType) && sizeof(PageTableEntry) > bytesize)
2505  {
2506  memcpy(((int8*)&newpage) + (sizeof(PageTableEntry) - bytesize),
2507  first, bytesize);
2508  }
2509  else
2510  {
2511  if (!isZero(first, (TSIZE >= 0 ? TSIZE : tuplesize)))
2512  {
2513  newpage.alloc(UT_PageOff(1), tuplesize);
2514  NotVoidType *newfirst = newpage.getFirstPtr();
2515  memcpy(newfirst, first, bytesize);
2516  newpage.setConstantBit();
2517  }
2518  }
2519  decRef();
2520  *this = newpage;
2521  }
2522 
2523  /// NOTE: This always excludes sizeof(*this), and is only valid for
2524  /// non-small pages, (okay if constant).
2526  {
2527  if (!isConstant())
2528  return sizeof(SYS_AtomicCounter) + tuplebytes*thePageSize;
2529  if (tuplebytes < sizeof(PageTableEntry) || isConstantZero())
2530  return 0;
2531  return sizeof(SYS_AtomicCounter) + tuplebytes;
2532  }
2533 
2534  template<typename DEST_DATA_T>
2537  castType() const
2538  {
2541  }
2542  template<typename DEST_DATA_T>
2546  {
2549  }
2550 
2551  /// NOTE: This just does a shallow comparison.
2553  bool operator==(const ThisType &that) const
2554  {
2555  return (myData == that.myData) && (myPRefCount == that.myPRefCount);
2556  }
2557  /// NOTE: This just does a shallow comparison.
2559  bool operator!=(const ThisType &that) const
2560  {
2561  return (myData != that.myData) || (myPRefCount != that.myPRefCount);
2562  }
2563 
2564  private:
2565  NotVoidType *myData;
2566  SYS_AtomicCounter *myPRefCount;
2567  };
2568 
2571  {
2572  exint cur_count = counter.relaxedLoad();
2573 
2574  // It should be rare for this to ever loop, because looping
2575  // means that something else referring to the same page has
2576  // done an incRef, decRef, or decRefIffShared on this same page,
2577  // between the relaxedLoad and the compare_swap calls, or on
2578  // subsequent loops, between the compare_swap calls.
2579  while (true)
2580  {
2581  if (cur_count == 1)
2582  return false; // Not shared, so don't decrement
2583  exint new_count = cur_count-1;
2584  exint prev_count = counter.compare_swap(cur_count, new_count);
2585  if (prev_count == cur_count)
2586  return true; // Successfully decremented
2587  cur_count = prev_count;
2588  }
2589  // Execution never gets here, because the only escapes from
2590  // the loops return.
2591  }
2592 
2593  SYS_FORCE_INLINE static NotVoidType getConstant(const PageTableEntry *const page,exint component,exint tuplesize = TSIZE)
2594  {
2595  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2596  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2597  UT_ASSERT_P(component >= 0 && component < tuplesize);
2598  UT_ASSERT_P(page->isConstant());
2599 
2600  // Any types whose tuples are strictly smaller than a pointer will be stored inline.
2601  // If a single one is at least as large as a pointer, it's
2602  // rejected at compile time, so there'll be no op at runtime in that case.
2603  if (PageTableEntry::typeFitsInline(tuplesize))
2604  {
2605  return page->getInlinePtr(tuplesize)[component];
2606  }
2607 
2608  const NotVoidType *masked = page->getMaskedPtr();
2609  if (!masked)
2610  {
2611  NotVoidType zero;
2612  memset(&zero, 0, sizeof(NotVoidType));
2613  return zero;
2614  }
2615 
2616  // Other types have a 1-element page.
2617  return masked[component];
2618  }
2619  /// NOTE: It may seem like overkill to take a PageTableEntry *, but the
2620  /// returned pointer may be *inside* the pointer that the caller owns,
2621  /// so we need the caller's pointer, not a copy. getConstant doesn't
2622  /// have this issue, since it can use the local pointer and read the
2623  /// value.
2624  SYS_FORCE_INLINE static const NotVoidType *getConstantPtr(const PageTableEntry *page,exint component = 0,exint tuplesize = TSIZE)
2625  {
2626  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2627  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2628  UT_ASSERT_P(component >= 0 && component < tuplesize);
2629  UT_ASSERT_P(page->isConstant());
2630 
2631  // Any types whose tuples are strictly smaller than PageTableEntry will be stored inline.
2632  if (PageTableEntry::typeFitsInline(tuplesize))
2633  {
2634  return page->getInlinePtr(tuplesize) + component;
2635  }
2636 
2637  const NotVoidType *masked = page->getMaskedPtr();
2638  if (!masked)
2639  return nullptr;
2640 
2641  // Other types have a 1-element page.
2642  return masked + component;
2643  }
2644  /// NOTE: It may seem like overkill to take a PageTableEntry *, but the
2645  /// returned pointer may be *inside* the pointer that the caller owns,
2646  /// so we need the caller's pointer, not a copy. getConstant doesn't
2647  /// have this issue, since it can use the local pointer and read the
2648  /// value.
2649  SYS_FORCE_INLINE static NotVoidType *getConstantPtr(PageTableEntry *page,exint component = 0,exint tuplesize = TSIZE)
2650  {
2651  return SYSconst_cast(getConstantPtr((const PageTableEntry *)page, component, tuplesize));
2652  }
2654  {
2655  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2656  UT_ASSERT_P(TSIZE >= 1);
2657  UT_ASSERT_P(!PAGESHARDENED);
2658 
2659  if (PAGESHARDENED)
2660  return;
2661 
2663 
2664  // Unref existing non-constant page
2665  bool wasconst = page->isConstant();
2666  if (!wasconst)
2667  page->decRef();
2668 
2669  // Easy for inline case, checked at compile-time.
2670  if (sizeof(PageTableEntry) > sizeof(Tuple))
2671  {
2672  page->initZero();
2673  NotVoidType *tuple = page->getInlinePtr(theSafeTupleSize);
2674  *(Tuple *)tuple = val;
2675  return;
2676  }
2677 
2678  bool iszero = val.isZero();
2679 
2680  // In other cases, we may or may not have to unref constant page
2681  if (wasconst)
2682  {
2683  NotVoidType *masked = page->getMaskedPtr();
2684  if (!masked)
2685  {
2686  // Fairly common case: already zero, making zero.
2687  if (iszero)
2688  return;
2689  }
2690  else if (iszero || (page->isShared() && (*(const Tuple*)masked != val)))
2691  {
2692  // No longer need this old constant page
2693  page->decRef();
2694  }
2695  else if (!page->isShared())
2696  {
2697  // Reuse the unshared constant page
2698  *(Tuple*)masked = val;
2699  return;
2700  }
2701  else
2702  {
2703  // Already equal; nothing to do
2704  UT_ASSERT_P(*(const Tuple*)masked == val);
2705  return;
2706  }
2707  }
2708 
2709  if (iszero)
2710  {
2711  page->initZero();
2712  return;
2713  }
2714 
2715  // Need to allocate new constant page
2716  PageTableEntry newpage;
2717  newpage.alloc(UT_PageOff(1), theSafeTupleSize);
2718  NotVoidType *tuple = newpage.getFirstPtr();
2719  (*(Tuple*)tuple) = val;
2720  newpage.setConstantBit();
2721  (*page) = newpage;
2722  }
2723 
2724  static void makeConstant(PageTableEntry *page, const NotVoidType &val, exint tuplesize = TSIZE)
2725  {
2726  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2727  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2728  UT_ASSERT_P(!PAGESHARDENED);
2729  UT_ASSERT_P(tuplesize >= 1);
2730 
2731  if (PAGESHARDENED)
2732  return;
2733 
2734  // Unref existing non-constant page
2735  bool wasconst = page->isConstant();
2736  if (!wasconst)
2737  page->decRef();
2738 
2739  // Easy for inline case, partly checked at compile-time.
2740  if (PageTableEntry::typeFitsInline(tuplesize))
2741  {
2742  page->initZero();
2743  NotVoidType *tuple = page->getInlinePtr(tuplesize);
2744  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2745  tuple[i] = val;
2746  return;
2747  }
2748 
2749  bool iszero = isZero(val);
2750 
2751  // In other cases, we may or may not have to unref constant page
2752  if (wasconst)
2753  {
2754  NotVoidType *masked = page->getMaskedPtr();
2755  if (!masked)
2756  {
2757  // Fairly common case: already zero, making zero.
2758  if (iszero)
2759  return;
2760  }
2761  else if (iszero)
2762  {
2763  // No longer need this old constant page
2764  page->decRef();
2765  }
2766  else if (page->isShared())
2767  {
2768  bool equal = true;
2769  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2770  {
2771  if (masked[i] != val)
2772  {
2773  equal = false;
2774  break;
2775  }
2776  }
2777 
2778  if (equal)
2779  {
2780  // Already equal; nothing to do
2781  return;
2782  }
2783 
2784  // No longer need this old constant page
2785  page->decRef();
2786  }
2787  else
2788  {
2789  // Reuse the unshared constant page
2790  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2791  masked[i] = val;
2792  return;
2793  }
2794  }
2795 
2796  if (iszero)
2797  {
2798  page->initZero();
2799  return;
2800  }
2801 
2802  // Need to allocate new constant page
2803  PageTableEntry newpage;
2804  newpage.alloc(UT_PageOff(1), tuplesize);
2805  NotVoidType *tuple = newpage.getFirstPtr();
2806  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2807  tuple[i] = val;
2808  newpage.setConstantBit();
2809  (*page) = newpage;
2810  }
2811 
2812  static void makeConstant(PageTableEntry *page, const NotVoidType *values, exint tuplesize)
2813  {
2814  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2815  UT_ASSERT_P(TSIZE == -1 || TSIZE == tuplesize);
2816  UT_ASSERT_P(!PAGESHARDENED);
2817  UT_ASSERT_P(tuplesize >= 1);
2818 
2819  if (PAGESHARDENED)
2820  return;
2821 
2822  // Unref existing non-constant page
2823  bool wasconst = page->isConstant();
2824  if (!wasconst)
2825  page->decRef();
2826 
2827  // Easy for inline case, partly checked at compile-time.
2828  if (PageTableEntry::typeFitsInline(tuplesize))
2829  {
2830  page->initZero();
2831  NotVoidType *tuple = page->getInlinePtr(tuplesize);
2832  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2833  tuple[i] = values[i];
2834  return;
2835  }
2836 
2837  bool iszero = isZero(values, (TSIZE >= 0 ? TSIZE : tuplesize));
2838 
2839  // In other cases, we may or may not have to unref constant page
2840  if (wasconst)
2841  {
2842  NotVoidType *masked = page->getMaskedPtr();
2843  if (!masked)
2844  {
2845  // Fairly common case: already zero, making zero.
2846  if (iszero)
2847  return;
2848  }
2849  else if (iszero)
2850  {
2851  // No longer need this old constant page
2852  page->decRef();
2853  }
2854  else if (page->isShared())
2855  {
2856  bool equal = true;
2857  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2858  {
2859  if (masked[i] != values[i])
2860  {
2861  equal = false;
2862  break;
2863  }
2864  }
2865 
2866  if (equal)
2867  {
2868  // Already equal; nothing to do
2869  return;
2870  }
2871 
2872  // No longer need this old constant page
2873  page->decRef();
2874  }
2875  else
2876  {
2877  // Reuse the unshared constant page
2878  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2879  masked[i] = values[i];
2880  return;
2881  }
2882  }
2883 
2884  if (iszero)
2885  {
2886  page->initZero();
2887  return;
2888  }
2889 
2890  // Need to allocate new constant page
2891  PageTableEntry newpage;
2892  newpage.alloc(UT_PageOff(1), tuplesize);
2893  NotVoidType *tuple = newpage.getFirstPtr();
2894  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2895  tuple[i] = values[i];
2896  newpage.setConstantBit();
2897  (*page) = newpage;
2898  }
2899 
2900  /// This replaces dest with src. It references if possible and keeps
2901  /// dest constant if easy. If desttuplesize > srctuplesize, it only
2902  /// replaces the first srctuplesize components.
2903  template<typename SrcType>
2904  static void replacePage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destpagesize, UT_PageOff destpagecapacity);
2905 
2906  /// This replaces part of dest with part of src. It keeps
2907  /// dest constant if easy. If desttuplesize > srctuplesize, it only
2908  /// replaces the first srctuplesize components.
2909  /// This will *not* check to see if the whole page is being replaced,
2910  /// so if you want the src page to be referenced, or overwrite a
2911  /// constant value, when writing the whole page, use replacePage.
2912  template<typename SrcType>
2913  static void copyPartialPage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destoff, UT_PageOff srcoff, UT_PageOff ntuples, UT_PageOff destpagecapacity);
2914 
2915  /// This version of makeConstant copies a const page onto this page,
2916  /// converting type if necessary, and supporting a different tuple size,
2917  /// so long as the destination size either isn't larger or is constant.
2918  /// NOTE: This would be templated on the actual templates of UT_PageArray,
2919  /// but somehow, Visual C++ can't deduce the type, so the template type
2920  /// has to be specified by the caller, and one argument is much easier
2921  /// to specify on every call than 4.
2922  template<typename SrcType>
2923  static void makeConstantFrom(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize);
2924 
2925  /// This is just a helper function for writing a single source tuple (or nullptr) to a block of tuple data, with conversion.
2926  /// NOTE: destpagedata must point to an *UNSHARED* page's data.
2927  template<typename SrcNotVoidType>
2928  static void fillNonConstWithConst(NotVoidType *destpagedata, NotVoidType *destpageend, const SrcNotVoidType *stuple, exint mintuplesize, exint desttupleextra);
2929 
2930  /// This is just a helper function for copying a non-constant source page to a non-constant destination page, with conversion.
2931  /// NOTE: destpagedata must point to an *UNSHARED* page's data.
2932  /// NOTE: The two data ranges must be *NON-OVERLAPPING*.
2933  template<typename SrcNotVoidType>
2934  static void copyNonConst(NotVoidType *destpagedata, const SrcNotVoidType *srcpagedata, exint desttuplesize, exint srctuplesize, UT_PageOff ntuples);
2935 
2936  /// This is just a helper function for checking whether two constant pages are equal, up to
2937  /// the smaller of the two, to see if writing is necessary.
2938  template<typename SrcNotVoidType>
2939  static bool isEqualConst(const NotVoidType *tuple, const SrcNotVoidType *stuple, exint mintuplesize);
2940 
2941  static void hardenConstantPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
2942  {
2943  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2944  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2945  UT_ASSERT_P(page->isConstant());
2946 
2947  PageTableEntry newpage;
2948  newpage.alloc(pagecapacity, tuplesize);
2949  NotVoidType *dest = newpage.getFirstPtr();
2950  bool isinline = PageTableEntry::typeFitsInline(tuplesize);
2951  if (isinline ? page->isConstantAndZeroSafe() : page->isConstantZero())
2952  {
2953  memset(dest, 0, (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize))*pagecapacity);
2954  }
2955  else
2956  {
2957  const NotVoidType *src;
2958  if (isinline)
2959  src = page->getInlinePtr(tuplesize);
2960  else
2961  src = page->getMaskedPtr();
2962 
2963  for (UT_PageOff i = 0; i < pagecapacity; ++i)
2964  {
2965  for (exint component = 0; component < tuplesize; ++component, ++dest)
2966  *dest = src[component];
2967  }
2968 
2969  if (!isinline)
2970  page->decRef();
2971  }
2972  (*page) = newpage;
2973  }
2974 
2975  static void hardenConstantPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
2976  {
2977  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2978  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2979  UT_ASSERT_P(page->isConstant());
2980 
2981  PageTableEntry newpage;
2982  newpage.alloc(pagecapacity, tuplesize);
2983  // Compile-time check, to speed up case where even one element doesn't fit inline.
2984  bool isinline = PageTableEntry::typeFitsInline(tuplesize);
2985  if (!isinline && !page->isConstantZero())
2986  page->decRef();
2987  (*page) = newpage;
2988  }
2989 
2990  static void hardenSharedPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
2991  {
2992  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2993  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2994  UT_ASSERT_P(!page->isConstant());
2995  // This may technically no longer be shared between checking and calling
2996  // due to other threads decRef'ing the page. It's okay to harden anyway.
2997  //UT_ASSERT_P(page->isShared());
2998 
2999  PageTableEntry newpage;
3000  newpage.alloc(pagecapacity, tuplesize);
3001  NotVoidType *dest = newpage.getFirstPtr();
3002  const NotVoidType *src = page->getFirstPtr();
3003  memcpy(dest, src, (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize))*pagecapacity);
3004  page->decRef();
3005  (*page) = newpage;
3006  }
3007 
3008  static void hardenSharedPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
3009  {
3010  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
3011  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3012  UT_ASSERT_P(!page->isConstant());
3013  // This may technically no longer be shared between checking and calling
3014  // due to other threads decRef'ing the page. It's okay to harden anyway.
3015  //UT_ASSERT_P(page->isShared());
3016 
3017  PageTableEntry newpage;
3018  newpage.alloc(pagecapacity, tuplesize);
3019  page->decRef();
3020  (*page) = newpage;
3021  }
3022 
3024  {
3025  public:
3028 
3030  {
3031  UT_ASSERT_P(myCapacity > IDX_T(0));
3032  return reinterpret_cast<PageTableEntry*>(this+1);
3033  }
3035  {
3036  UT_ASSERT_P(myCapacity > IDX_T(0));
3037  return reinterpret_cast<const PageTableEntry *>(this+1);
3038  }
3039  // NOTE: Clang sometimes decides to ignore const references and pass by
3040  // value, so we have to return by pointer, in case the page
3041  // is constant and inlined, so that the address of the constant
3042  // value is accessible through this function.
3043  // Yes, many cases don't rely on this, but to reduce the risk of
3044  // bugs, the only way to access page pointers in a const PageTable
3045  // is to call this to get the page pointer address and explicitly
3046  // dereference, if valid in the given circumstance.
3048  {
3049  UT_ASSERT_P(myCapacity > IDX_T(i << THEPAGEBITS));
3050  return reinterpret_cast<const PageTableEntry *>(this+1) + i;
3051  }
3053  {
3054  UT_ASSERT_P(myCapacity > IDX_T(i << THEPAGEBITS));
3055  return reinterpret_cast<PageTableEntry*>(this+1) + i;
3056  }
3058  {
3059  UT_ASSERT_P(myRefCount.relaxedLoad() > 0);
3060  return myRefCount.relaxedLoad() != 1;
3061  }
3063  {
3064  return myRefCount.relaxedLoad();
3065  }
3066  /// NOTE: This is the size of the full array, not the number of pages.
3067  SYS_FORCE_INLINE IDX_T size() const
3068  {
3069  return mySize;
3070  }
3071  /// NOTE: This is the capacity of the full array, not the capacity of pages.
3073  {
3074  return myCapacity;
3075  }
3076  /// NOTE: This will *assume* the capacity is large enough! The caller *must* check!
3077  /// NOTE: This is the size of the full array, not the number of pages.
3078  SYS_FORCE_INLINE void setSize(IDX_T newsize)
3079  {
3080  UT_ASSERT_P(newsize <= capacity());
3081  UT_ASSERT_MSG_P(myRefCount.relaxedLoad() == 1, "The table must already be hardened before we modify it!");
3082  if (PAGESHARDENED && newsize > mySize)
3083  {
3084  UT_ASSERT_P(TSIZE >= 0);
3085  // Need to ensure pages between mySize and newsize are hardened.
3086  // Pages containing offsets less than or equal to mySize are
3087  // already guaranteed to be hardened.
3088  UT_PageNum startpage = numPages(mySize);
3089  UT_PageNum endpage = numPages(newsize);
3090  UT_PageOff pagecapacity(thePageSize);
3091  if (endpage == UT_PageNum(1) && exint(capacity()) < thePageSize)
3092  pagecapacity = capacity();
3093  PageTableEntry *page = getPPage(startpage);
3094  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3095  {
3096  if (page->isConstant())
3097  hardenConstantPage(page, pagecapacity);
3098  else if (page->isShared())
3099  hardenSharedPage(page, pagecapacity);
3100  }
3101  }
3102  mySize = newsize;
3103  }
3104  /// NOTE: This will *assume* the capacity is large enough! The caller *must* check!
3105  /// NOTE: This is the size of the full array, not the number of pages.
3106  SYS_FORCE_INLINE void setSize(IDX_T newsize, exint tuplesize)
3107  {
3108  UT_ASSERT_P(newsize <= capacity());
3109  UT_ASSERT_MSG_P(myRefCount.relaxedLoad() == 1, "The table must already be hardened before we modify it!");
3110  if (PAGESHARDENED && newsize > mySize)
3111  {
3112  // Need to ensure pages between mySize and newsize are hardened.
3113  // Pages containing offsets less than or equal to mySize are
3114  // already guaranteed to be hardened.
3115  UT_PageNum startpage = numPages(mySize);
3116  UT_PageNum endpage = numPages(newsize);
3117  UT_PageOff pagecapacity(thePageSize);
3118  if (endpage == UT_PageNum(1) && exint(capacity()) < thePageSize)
3119  pagecapacity = capacity();
3120  PageTableEntry *page = getPPage(startpage);
3121  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3122  {
3123  if (page->isConstant())
3124  hardenConstantPage(page, pagecapacity, tuplesize);
3125  else if (page->isShared())
3126  hardenSharedPage(page, pagecapacity, tuplesize);
3127  }
3128  }
3129  mySize = newsize;
3130  }
3131  void fill(IDX_T start, IDX_T end, const NotVoidType &val);
3132  void fill(IDX_T start, IDX_T end, NotVoidType val, exint tuplesize);
3133  void fill(IDX_T start, IDX_T end, const NotVoidType *values, exint tuplesize);
3134  void fill(IDX_T start, IDX_T end, const UT_FixedVector<NotVoidType,theSafeTupleSize> &val);
3135 
3136  void incRef()
3137  {
3138  myRefCount.add(1);
3139  }
3140  void decRef(exint tuplesize)
3141  {
3142  const int64 new_count = myRefCount.add(-1);
3143  if (new_count != 0)
3144  {
3145  UT_ASSERT_P(new_count > 0);
3146  return;
3147  }
3148 
3149  UT_ASSERT_P(!(SYSisSame<DATA_T, void>()));
3150 
3151  const size_t bytesize = sizeof(NotVoidType)*((TSIZE >= 0) ? TSIZE : tuplesize);
3152 
3153  // If we're deleting the table, first, decRef all pages.
3154  UT_PageNum npages = numPages(myCapacity);
3155  PageTableEntry *page = getFirstPage();
3156  for (UT_PageNum p = 0; p < npages; ++p, ++page)
3157  {
3158  if (page->isRefd(bytesize))
3159  page->decRef();
3160  }
3161  free(this);
3162 #if 0
3163  printf("Free table %d pages (case 0); %d tables total\n",int(npages), int(thePageTableCount.add(-1)));
3164  fflush(stdout);
3165 #endif
3166  }
3167 
3168  void hardenAllPages(IDX_T start, IDX_T end, exint tuplesize = TSIZE)
3169  {
3170  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3171  UT_ASSERT_P(tuplesize >= 0);
3172  UT_PageNum startpage = pageNum(start);
3173  UT_PageNum endpage = numPages(end);
3174  UT_PageOff pagecapacity(thePageSize);
3175  if (endpage == UT_PageNum(1) && exint(capacity()) < thePageSize)
3176  pagecapacity = capacity();
3177  PageTableEntry *page = getPPage(startpage);
3178  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3179  {
3180  if (page->isConstant())
3181  hardenConstantPage(page, pagecapacity, tuplesize);
3182  else if (page->isShared())
3183  hardenSharedPage(page, pagecapacity, tuplesize);
3184  }
3185  }
3186 
3187  void tryCompressAllPages(IDX_T start, IDX_T end, exint tuplesize = TSIZE)
3188  {
3189  UT_ASSERT_P(tuplesize >= 0);
3190  UT_PageNum startpage = pageNum(start);
3191  UT_PageNum endpage = numPages(end);
3192  UT_PageNum npages = numPages(mySize);
3193  PageTableEntry *page = getPPage(startpage);
3194  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3195  {
3196  if (page->isConstant())
3197  continue;
3198 
3199  UT_PageOff pagesize = thePageSize;
3200  if (p+1 >= npages)
3201  {
3202  if (p+1 == npages)
3203  {
3204  UT_PageOff pagesizetemp = pageOff(mySize);
3205  if (pagesizetemp != UT_PageOff(0))
3206  pagesize = pagesizetemp;
3207  }
3208  else
3209  pagesize = UT_PageOff(0);
3210  }
3211  page->tryCompressPage(pagesize, tuplesize);
3212  }
3213  }
3214 
3215  void tryCompressPage(UT_PageNum pagenum, exint tuplesize = TSIZE)
3216  {
3217  UT_ASSERT_P(tuplesize >= 0);
3218  PageTableEntry *page = getPPage(pagenum);
3219  if (page->isConstant())
3220  return;
3221 
3222  UT_PageOff pagesize = thePageSize;
3223  UT_PageNum npages = numPages(mySize);
3224  if (pagenum+1 >= npages)
3225  {
3226  if (pagenum+1 == npages)
3227  {
3228  UT_PageOff pagesizetemp = pageOff(mySize);
3229  if (pagesizetemp != UT_PageOff(0))
3230  pagesize = pagesizetemp;
3231  }
3232  else
3233  pagesize = UT_PageOff(0);
3234  }
3235  page->tryCompressPage(pagesize, tuplesize);
3236  }
3237 
3238  static ThisType *alloc(UT_PageNum npages)
3239  {
3240  UT_ASSERT_MSG_P(npages > UT_PageNum(0), "Shouldn't have a zero-page page table; nullptr should be used instead.");
3241  ThisType *p = (ThisType *)malloc(sizeof(ThisType) + npages*sizeof(PageTableEntry));
3242  //UT_ASSERT_P(_CrtIsValidHeapPointer(p));
3243  p->myRefCount.relaxedStore(1);
3244 #if 0
3245  printf("Alloc table %d pages; %d tables total\n",int(npages), int(thePageTableCount.add(1)));
3246  fflush(stdout);
3247 #endif
3248  return p;
3249  }
3250  static ThisType *harden(ThisType *pages, IDX_T newcapacity, exint tuplesize=TSIZE)
3251  {
3252  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3253 
3254  // If we're setting capacity to zero, free the table.
3255  if (newcapacity == IDX_T(0))
3256  {
3257  if (pages)
3258  pages->decRef(tuplesize);
3259  return nullptr;
3260  }
3261 
3262  UT_ASSERT_P(tuplesize >= 0);
3263  UT_ASSERT_MSG_P(newcapacity == roundUpCapacity(newcapacity), "Caller should already have adjusted capacity");
3264 
3265  bool wasshared = pages && pages->isShared();
3266  UT_PageNum npages = numPages(newcapacity);
3267  IDX_T oldcapacity = pages ? pages->capacity() : IDX_T(0);
3268  UT_PageNum oldnpages = numPages(oldcapacity);
3269 
3270  // Caller must force capacity to be on a page boundary if more than one page, else power of 2.
3271  UT_ASSERT_P((newcapacity < IDX_T(thePageSize) && SYSisPow2(exint(newcapacity))) || newcapacity == IDX_T(exint(npages)<<thePageBits));
3272 
3273  const bool nonconstfirst = pages && !pages->getFirstPage()->isConstant();
3274 
3275  if (npages == UT_PageNum(1) && oldnpages == UT_PageNum(1) && !wasshared)
3276  {
3277  // Can reuse existing page table if we're staying at one page, but changing its capacity.
3278 
3279  // NOTE: The "newcapacity != oldcapacity" will almost always be true here, but
3280  // we need to make absolutely sure that we don't realloc a shared page when just
3281  // calling hardenTable()! If we didn't have this check, it could happen, because
3282  // between the reference count check in hardenTable() could return 2, and then
3283  // before the pages->isShared() call above, the reference count drops to 1,
3284  // so we have an already hardened table, but we can identify that we don't need
3285  // to do anything because the capacity won't be changing.
3286  // The reason we can't have hardenTable() realloc a shared page is that in
3287  // GA_PrimitiveList, it needs to incDataRef() on all GA_OffsetLists in a shared
3288  // page before ever hardening one.
3289  if (nonconstfirst && newcapacity != oldcapacity)
3290  {
3291  // Just need to realloc the page.
3292  UT_PageOff newpagecapacity = UT_PageOff(exint(newcapacity));
3293  UT_PageOff sizetocopy = SYSmin(UT_PageOff(exint(pages->mySize)), newpagecapacity);
3294  PageTableEntry *page = pages->getFirstPage();
3295  page->realloc(sizetocopy, newpagecapacity, tuplesize);
3296  }
3297  // Don't need to change anything but capacity (and possibly size)
3298  // if constant page, even if it's an allocated constant page.
3299 
3300  pages->myCapacity = newcapacity;
3301  if (pages->mySize > newcapacity)
3302  pages->mySize = newcapacity;
3303  return pages;
3304  }
3305 
3306  PageTable *table = PageTable::alloc(npages);
3307  table->myCapacity = newcapacity;
3308  if (pages)
3309  {
3310  UT_ASSERT_P(pages->getRefCount() > 0 && table->getRefCount() > 0);
3311  UT_ASSERT_P(pages->capacity() > IDX_T(0));
3312  UT_ASSERT_P(oldnpages > UT_PageNum(0));
3313  const bool wassmallfirstpage = pages->capacity() < IDX_T(thePageSize);
3314  const bool willbesmallfirstpage = newcapacity < IDX_T(thePageSize);
3315 
3316  UT_PageNum minnpages = SYSmin(npages, oldnpages);
3317  table->mySize = pages->mySize;
3318  memcpy(table->getFirstPage(), pages->getFirstPage(), (sizeof(PageTableEntry))*minnpages);
3319 
3320  UT_ASSERT_P(!(SYSisSame<DATA_T, void>()));
3321 
3322  const size_t bytesize = sizeof(NotVoidType)*((TSIZE >= 0) ? TSIZE : tuplesize);
3323 
3324  if (wasshared)
3325  {
3326  // Increment all of the reference counts of reused pages,
3327  // if we're unsharing a table.
3328  PageTableEntry *page = table->getFirstPage();
3329  for (UT_PageNum p = 0; p < minnpages; ++p, ++page)
3330  {
3331  if (page->isRefd(bytesize))
3332  page->incRef();
3333  }
3334  pages->decRef(tuplesize);
3335  }
3336  else
3337  {
3338  // pages isn't shared, so we can just decRef() any pages no
3339  // longer referenced and free pages. This avoids calling
3340  // incRef on all kept pages and decRef on all pages like the
3341  // above would do if pages had its reference count go to
3342  // zero.
3343  PageTableEntry *page = pages->getFirstPage() + npages;
3344  for (UT_PageNum p = npages; p < oldnpages; ++p, ++page)
3345  {
3346  if (page->isRefd(bytesize))
3347  page->decRef();
3348  }
3349  UT_ASSERT_MSG_P(!pages->isShared(), "This should still be unshared.");
3350  free(pages);
3351 #if 0
3352  printf("Free table %d pages (case 1); %d tables total\n",int(oldnpages), int(thePageTableCount.add(-1)));
3353  fflush(stdout);
3354 #endif
3355  }
3356  // Can't refer to pages beyond this point!
3357  UT_IF_ASSERT_P(pages = nullptr;)
3358 
3359  if (table->mySize > newcapacity)
3360  table->mySize = newcapacity;
3361 
3362  // NOTE: The "newcapacity != oldcapacity" is to make absolutely
3363  // sure that we don't realloc a shared page when just
3364  // calling hardenTable()! This is because in
3365  // GA_PrimitiveList, it needs to incDataRef() on all
3366  // GA_OffsetLists in a shared page before ever hardening one.
3367  if (nonconstfirst && (wassmallfirstpage || willbesmallfirstpage) && newcapacity != oldcapacity)
3368  {
3369  UT_PageOff newpagecapacity = SYSmin(UT_PageOff(exint(newcapacity)), UT_PageOff(thePageSize));
3370  UT_PageOff sizetocopy = SYSmin(UT_PageOff(exint(table->mySize)), newpagecapacity);
3371  PageTableEntry *page = table->getFirstPage();
3372  page->realloc(sizetocopy, newpagecapacity, tuplesize);
3373  }
3374  }
3375  else
3376  {
3377  oldnpages = 0;
3378  table->mySize = IDX_T(0);
3379  }
3380 
3381  // Set any new pages to be constant zero.
3382  PageTableEntry *page = table->getFirstPage() + oldnpages;
3383  for (UT_PageNum p = oldnpages; p < npages; ++p, ++page)
3384  {
3385  page->initZero();
3386  }
3387 
3388  if (PAGESHARDENED && wasshared)
3389  {
3390  if (TSIZE >= 1)
3391  table->hardenAllPages(IDX_T(0), table->mySize);
3392  else
3393  table->hardenAllPages(IDX_T(0), table->mySize, tuplesize);
3394  }
3395 
3396  return table;
3397  }
3398 
3399  /// Atomically decrement the reference count iff the page is shared.
3400  /// See the description of decRefIffSharedPage() above for
3401  /// why this function exists.
3403  {
3404  return decCounterIffAbove1(myRefCount);
3405  }
3406 
3407  private:
3408  // The constructor/destructor should never be called, since the class
3409  // can only be allocated with malloc.
3410  PageTable() { UT_ASSERT_P(0); }
3411  PageTable(const PageTable &) { UT_ASSERT_P(0); }
3412  ~PageTable() { UT_ASSERT_P(0); }
3413 
3414  SYS_AtomicCounter myRefCount;
3415  IDX_T mySize;
3416  IDX_T myCapacity;
3417  };
3418 
3419  /// NOTE: These have to be below the PageTable class definition, because
3420  /// for no apparent reason, even though you can have a regular local
3421  /// variable of type PageTable before, you can't have a function
3422  /// parameter of type PageTable before. *sigh*
3423  /// @{
3424  static void hardenConstantPage(PageTable *pages, PageTableEntry *page)
3425  {
3426  const exint arraycapacity(exint(pages->capacity()));
3427  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3428  hardenConstantPage(page, pagecapacity);
3429  }
3430  static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page)
3431  {
3432  const exint arraycapacity(exint(pages->capacity()));
3433  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3434  hardenConstantPageNoInit(page, pagecapacity);
3435  }
3436  static void hardenConstantPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
3437  {
3438  const exint arraycapacity(exint(pages->capacity()));
3439  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3440  hardenConstantPage(page, pagecapacity, tuplesize);
3441  }
3442  static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
3443  {
3444  const exint arraycapacity(exint(pages->capacity()));
3445  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3446  hardenConstantPageNoInit(page, pagecapacity, tuplesize);
3447  }
3448  static void hardenSharedPage(PageTable *pages, PageTableEntry *page)
3449  {
3450  const exint arraycapacity(exint(pages->capacity()));
3451  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3452  hardenSharedPage(page, pagecapacity);
3453  }
3454  static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page)
3455  {
3456  const exint arraycapacity(exint(pages->capacity()));
3457  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3458  hardenSharedPageNoInit(page, pagecapacity);
3459  }
3460  static void hardenSharedPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
3461  {
3462  const exint arraycapacity(exint(pages->capacity()));
3463  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3464  hardenSharedPage(page, pagecapacity, tuplesize);
3465  }
3466  static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
3467  {
3468  const exint arraycapacity(exint(pages->capacity()));
3469  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3470  hardenSharedPageNoInit(page, pagecapacity, tuplesize);
3471  }
3472  /// @}
3473 
3474  void forceHardenTable(IDX_T newcapacity)
3475  {
3476  if (SYSisSame<DATA_T,void>())
3477  {
3478  const UT_Storage storage = myImpl.getStorage();
3479  switch (storage)
3480  {
3481  case UT_Storage::INVALID:
3482  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
3483  return;
3484  case UT_Storage::INT8:
3485  castType<int8>().forceHardenTable(newcapacity); return;
3486  case UT_Storage::INT16:
3487  castType<int16>().forceHardenTable(newcapacity); return;
3488  case UT_Storage::INT32:
3489  castType<int32>().forceHardenTable(newcapacity); return;
3490  case UT_Storage::INT64:
3491  castType<int64>().forceHardenTable(newcapacity); return;
3492  case UT_Storage::REAL16:
3493  castType<fpreal16>().forceHardenTable(newcapacity); return;
3494  case UT_Storage::REAL32:
3495  castType<fpreal32>().forceHardenTable(newcapacity); return;
3496  case UT_Storage::REAL64:
3497  castType<fpreal64>().forceHardenTable(newcapacity); return;
3498  }
3499  return;
3500  }
3501 
3502  PageTable *&pages = myImpl.getPages();
3503  pages = PageTable::harden(pages, newcapacity, myImpl.getTupleSize());
3504  }
3505 
3506 protected:
3507  template<typename T>
3509  static bool isZero(const T &val)
3510  {
3511  // TODO: Find a way to check for zero that will still compile, but
3512  // that avoids the mess of dealing with memset for POD types.
3513  T zero;
3514  memset(&zero, 0, sizeof(T));
3515  return (val == zero);
3516  }
3517 
3518  template<typename T>
3520  static bool isZero(const T *val,exint tuplesize)
3521  {
3522  UT_ASSERT_P(tuplesize >= 1);
3523  if (tuplesize == 0)
3524  return true;
3525 
3526  // TODO: Find a way to check for zero that will still compile, but
3527  // that avoids the mess of dealing with memset for POD types.
3528  T zero;
3529  memset(&zero, 0, sizeof(T));
3530 
3531  bool iszero = (val[0] == zero);
3532  for (exint i = 1; i < tuplesize; ++i)
3533  iszero &= (val[i] == zero);
3534  return iszero;
3535  }
3536 
3537  template<typename T0,typename T1>
3539  static bool isEqual(const T0 *a,const T1 *b,exint tuplesize)
3540  {
3541  UT_ASSERT_P(tuplesize >= 1);
3542  UT_ASSERT_P(a != nullptr && b != nullptr);
3543 
3544  for (exint i = 0; i < tuplesize; ++i)
3545  {
3546  if (a[i] != b[i])
3547  return false;
3548  }
3549  return true;
3550  }
3551 
3552  template<typename DEST_TYPE,typename SRC_TYPE,exint SRC_TSIZE,bool SRC_INSTANTIATED>
3555  {
3556  if (SYSisSame<DEST_TYPE,SRC_TYPE>())
3557  return src;
3558 
3560  for (exint i = 0; i < SRC_TSIZE; ++i)
3561  dest[i] = UTconvertStorage<DEST_TYPE>(src[i]);
3562  return dest;
3563  }
3564 
3565 private:
3566  /// This class wraps the data of UT_PageArray, just so that if
3567  /// the tuple size and data type are known at compile-time, it doesn't
3568  /// need to store that information as an int and a UT_Storage.
3569  /// myPages is the same in all of them, but can't be extracted out,
3570  /// because the C++ standard, unfortunately, forbids 0-byte classes.
3571  /// @{
3572  struct ImplPageData
3573  {
3574  PageTable *myPages;
3575  };
3576 
3577  template<exint ITSIZE, typename IDATA_T>
3578  class Impl :
3579  private ImplPageData,
3580  private UT_TupleType<ITSIZE, IDATA_T>
3581  {
3582  public:
3583  SYS_FORCE_INLINE const PageTable *getPages() const { return ImplPageData::myPages; }
3584  SYS_FORCE_INLINE PageTable *&getPages() { return ImplPageData::myPages; }
3589  };
3590 
3591  Impl<TSIZE,DATA_T> myImpl;
3592  /// @}
3593 };
3594 
3595 #endif
void hardenAllPages(IDX_T start, IDX_T end, exint tuplesize=TSIZE)
SYS_FORCE_INLINE NotVoidType operator()(IDX_T i, exint component=0) const
Read element i's specified component (default component 0).
Definition: UT_PageArray.h:448
SYS_FORCE_INLINE constexpr const T * data() const noexcept
static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page)
GLint first
Definition: glcorearb.h:404
static void copyNonConst(NotVoidType *destpagedata, const SrcNotVoidType *srcpagedata, exint desttuplesize, exint srctuplesize, UT_PageOff ntuples)
SYS_FORCE_INLINE bool isPageConstant(UT_PageNum pagenum) const
Returns true iff the specified page is constant-compressed.
static void hardenConstantPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
SYS_FORCE_INLINE void opVector(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE > &v)
Definition: UT_PageArray.h:733
static SYS_FORCE_INLINE const NotVoidType * getConstantPtr(const PageTableEntry *page, exint component=0, exint tuplesize=TSIZE)
void SecondGuess
Definition: UT_Storage.h:74
SYS_FORCE_INLINE void setSize(IDX_T newsize)
static SYS_FORCE_INLINE bool decCounterIffAbove1(SYS_AtomicCounter &counter)
SYS_FORCE_INLINE const NotVoidType * getInlinePtr(exint tuplesize) const
Returns the data pointer, if an inline constant page.
SYS_FORCE_INLINE void * getFirstPtrVoid()
Returns the data pointer, if not a constant page.
UT_Storage
Definition: UT_Storage.h:26
static void hardenSharedPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
void hardenAllPagesUnknownType(IDX_T start, IDX_T end)
void tryCompressPage(UT_PageOff pagesize, exint tuplesize=TSIZE)
static ThisType * harden(ThisType *pages, IDX_T newcapacity, exint tuplesize=TSIZE)
static void hardenSharedPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
SYS_FORCE_INLINE bool isArrayZero() const
Returns true iff the entire array is constant & zero.
static const UT_Storage theStorage
Definition: UT_PageArray.h:252
static SYS_FORCE_INLINE NotVoidType getConstant(const PageTableEntry *const page, exint component, exint tuplesize=TSIZE)
NotVoid< DATA_T >::type NotVoidType
Definition: UT_PageArray.h:244
UT_PageArray(const ThisType &that)
Definition: UT_PageArray.h:310
void setTupleSize(exint newtuplesize, const UT_Defaults &v)
static ThisType * alloc(UT_PageNum npages)
void tryCompressPage(UT_PageNum pagenum, exint tuplesize=TSIZE)
SYS_FORCE_INLINE const NotVoidType * getMaskedPtr() const
void copy(IDX_T dest, IDX_T src)
Copies a single element from src to dest within this.
void tryCompressAllPagesUnknownType(IDX_T start, IDX_T end)
const GLdouble * v
Definition: glcorearb.h:836
SYS_FORCE_INLINE ~UT_PageArray()
Definition: UT_PageArray.h:324
SYS_FORCE_INLINE bool operator!=(const ThisType &that) const
NOTE: This just does a shallow comparison.
SYS_FORCE_INLINE bool isZero() const noexcept
void tryCompressAllPages(IDX_T start, IDX_T end, exint tuplesize=TSIZE)
GLuint start
Definition: glcorearb.h:474
SYS_FORCE_INLINE void clear()
Definition: UT_PageArray.h:330
SYS_FORCE_INLINE ThisType & operator=(const UT_PageArray< SRC_DATA_T, SRC_TSIZE, SRC_TABLEHARDENED, SRC_PAGESHARDENED, THEPAGEBITS, IDX_T > &that)
Definition: UT_PageArray.h:386
static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
void opVectorUnknownType(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE > &v)
SYS_FORCE_INLINE IDX_T size() const
Definition: UT_PageArray.h:866
SYS_FORCE_INLINE T * SYSconst_cast(const T *foo)
Definition: SYS_Types.h:127
SYS_FORCE_INLINE void add(IDX_T i, SRC_DATA_T v)
component == 0 in this version
Definition: UT_PageArray.h:618
UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVectorUnknownSize(IDX_T i) const
SYS_FORCE_INLINE bool isShared() const
void countMemory(UT_MemoryCounter &counter, bool inclusive) const
bool iszero(T a, T t)
Definition: ImathFun.h:135
SYS_FORCE_INLINE PageTableEntry * getFirstPage()
GLboolean GLboolean GLboolean GLboolean a
Definition: glcorearb.h:1221
static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page)
void setConstant(IDX_T start, IDX_T end, NotVoidType v)
void opUnknownType(IDX_T i, exint component, SRC_DATA_T v)
exint UT_PageNum
Definition: UT_PageArray.h:40
SYS_FORCE_INLINE const PageTableEntry * getPPage(UT_PageNum i) const
static void hardenConstantPage(PageTable *pages, PageTableEntry *page)
SYS_FORCE_INLINE UT_PageArray< DATA_T, TSIZE, true, true, THEPAGEBITS, IDX_T > & hardenAllPages(IDX_T start=IDX_T(0), IDX_T end=IDX_T(-1))
static const exint theTupleSize
Definition: UT_PageArray.h:253
SYS_FORCE_INLINE UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T >::PageTableEntry * castType()
SYS_FORCE_INLINE exint getTupleSize() const
DEST_DATA_T getUnknownType(IDX_T i, exint component=0) const
SYS_SelectType< T, float, SYS_IsSame< T, void >::value >::type type
Definition: UT_PageArray.h:242
SYS_FORCE_INLINE IDX_T size() const
NOTE: This is the size of the full array, not the number of pages.
SYS_FORCE_INLINE UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVector(IDX_T i) const
Definition: UT_PageArray.h:501
#define UT_IF_ASSERT_P(ZZ)
Definition: UT_Assert.h:152
SYS_FORCE_INLINE void set(IDX_T i, SRC_DATA_T v)
component == 0 in this version
Definition: UT_PageArray.h:612
SYS_FORCE_INLINE UT_Storage getStorage() const
static SYS_FORCE_INLINE IDX_T roundUpCapacity(IDX_T capacity)
Definition: UT_PageArray.h:887
SYS_FORCE_INLINE bool isTableHardened() const
Returns true iff the table isn't shared with any other UT_PageArray's.
SYS_FORCE_INLINE exint getRefCount() const
png_uint_32 i
Definition: png.h:2877
SYS_FORCE_INLINE UT_PageArray< DATA_T, DEST_TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castTupleSize()
bool hasNanInRange(IDX_T start, IDX_T end) const
static SYS_FORCE_INLINE UT_FixedVector< DEST_TYPE, SRC_TSIZE > convertVectorStorage(const UT_FixedVector< SRC_TYPE, SRC_TSIZE, SRC_INSTANTIATED > &src)
UT_PageArray(const UT_PageArray< SRC_DATA_T, SRC_TSIZE, SRC_TABLEHARDENED, SRC_PAGESHARDENED, THEPAGEBITS, IDX_T > &that)
Definition: UT_PageArray.h:317
GLsizeiptr size
Definition: glcorearb.h:663
UT_PageArray(exint tuplesize, UT_Storage storage)
Definition: UT_PageArray.h:294
SYS_FORCE_INLINE void addVector(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE, SRC_INSTANTIATED > &v)
Definition: UT_PageArray.h:724
void tryCompressPageUnknownType(UT_PageNum pagenum)
void decRef(exint tuplesize)
SYS_FORCE_INLINE bool isShared() const
void fill(IDX_T start, IDX_T end, const NotVoidType &val)
static bool isEqualConst(const NotVoidType *tuple, const SrcNotVoidType *stuple, exint mintuplesize)
signed char int8
Definition: SYS_Types.h:31
SYS_FORCE_INLINE void setConstantBit()
UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVectorMismatchedSize(IDX_T i) const
static SYS_FORCE_INLINE void call(FUNCTOR functor)
Definition: SYS_CallIf.h:37
long long int64
Definition: SYS_Types.h:107
UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVectorUnknownType(IDX_T i) const
SYS_FORCE_INLINE exint getRefCount() const
SYS_FORCE_INLINE bool isConstantZero() const
This is only valid to call if the type doesn't fit inline.
SYS_FORCE_INLINE void setSize(IDX_T newsize, exint tuplesize)
static SYS_FORCE_INLINE UT_PageOff pageOff(IDX_T i)
UT_PageArray(UT_Storage storage)
Definition: UT_PageArray.h:283
SYS_FORCE_INLINE void tryCompressPage(UT_PageNum pagenum)
SYS_FORCE_INLINE ThisType & operator=(const ThisType &that)
Definition: UT_PageArray.h:378
static SYS_FORCE_INLINE bool typeFitsInline(exint tuplesize=TSIZE)
This is only valid to call when DATA_T is non-void.
SYS_FORCE_INLINE const PageTableEntry * getFirstPage() const
SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const UT_FixedVector< SRC_DATA_T, theSafeTupleSize > &val)
static SYS_FORCE_INLINE UT_PageNum numPages(IDX_T nelements)
static SYS_FORCE_INLINE bool isEqual(const T0 *a, const T1 *b, exint tuplesize)
int64 exint
Definition: SYS_Types.h:116
SYS_FORCE_INLINE void getRange(IDX_T srcstart, IDX_T nelements, T *dest) const
SYS_FORCE_INLINE bool isRefd(exint tuplebytes) const
#define UT_ASSERT_P(ZZ)
Definition: UT_Assert.h:125
SYS_FORCE_INLINE bool operator==(const ThisType &that) const
NOTE: This just does a shallow comparison.
SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const NotVoidType &val)
double fpreal64
Definition: SYS_Types.h:192
SYS_FORCE_INLINE IDX_T capacity() const
NOTE: This is the capacity of the full array, not the capacity of pages.
static SYS_FORCE_INLINE bool isZero(const T &val)
SYS_FORCE_INLINE IDX_T capacity() const
Definition: UT_PageArray.h:874
#define SYS_FALLTHROUGH
Definition: SYS_Compiler.h:75
GLuint GLuint end
Definition: glcorearb.h:474
#define SYS_FORCE_INLINE
Definition: SYS_Inline.h:45
T compare_swap(T expected, T desired)
SYS_FORCE_INLINE NotVoidType * hardenPage(UT_PageNum pagenum)
UT_FixedVector< NotVoidType, theSafeTupleSize > Tuple
DATA_T DataType
Definition: UT_PageArray.h:251
SYS_FORCE_INLINE void op(IDX_T i, exint component, SRC_DATA_T v)
Definition: UT_PageArray.h:638
SYS_FORCE_INLINE NotVoidType * getInlinePtr(exint tuplesize)
Returns the data pointer, if an inline constant page.
void setCapacity(IDX_T newcapacity)
Definition: UT_PageArray.h:902
void clearAndDestroy()
Definition: UT_PageArray.h:336
SYS_FORCE_INLINE void setVector(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE, SRC_INSTANTIATED > &v)
Definition: UT_PageArray.h:719
GLenum GLsizei GLsizei GLint * values
Definition: glcorearb.h:1601
SYS_FORCE_INLINE void * getMaskedPtrVoid()
static SYS_FORCE_INLINE UT_PageOff pageStart(IDX_T page)
UT_PageArray(exint tuplesize)
Definition: UT_PageArray.h:271
SYS_FORCE_INLINE const void * getFirstPtrVoidUnsafe() const
GLboolean * data
Definition: glcorearb.h:130
static void copyPartialPage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destoff, UT_PageOff srcoff, UT_PageOff ntuples, UT_PageOff destpagecapacity)
SYS_FORCE_INLINE bool isConstantAndZeroSafe() const
void setPageConstantUnknownType(UT_PageNum pagenum, const UT_FixedVector< SRC_DATA_T, theSafeTupleSize > &val)
int int32
Definition: SYS_Types.h:35
SYS_FORCE_INLINE bool decRefIffShared(exint tuplesize=TSIZE)
static void makeConstant(PageTableEntry *page, const NotVoidType *values, exint tuplesize)
SYS_FORCE_INLINE NotVoidType * hardenPageNoInit(UT_PageNum pagenum)
GLboolean GLboolean GLboolean b
Definition: glcorearb.h:1221
SYS_FORCE_INLINE void tryCompressAllPages(IDX_T start=IDX_T(0), IDX_T end=IDX_T(-1))
static const exint thePageBits
Definition: UT_PageArray.h:255
SYS_FORCE_INLINE void add(IDX_T i, exint component, SRC_DATA_T v)
Definition: UT_PageArray.h:633
const NotVoidType * getPageData(UT_PageNum pagenum) const
exint UT_PageOff
Definition: UT_PageArray.h:41
static const exint theSafeTupleSize
Definition: UT_PageArray.h:254
SYS_FORCE_INLINE void incRef()
void realloc(UT_PageOff sizetocopy, UT_PageOff newpagecapacity, exint tuplesize=TSIZE)
static void makeConstantFrom(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize)
PageTableEntry ThisType
SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const SRC_DATA_T *values)
void moveRange(IDX_T srcstart, IDX_T deststart, IDX_T nelements)
static const exint thePageMask
Definition: UT_PageArray.h:257
void getVectorRange(IDX_T srcstart, IDX_T nelements, UT_FixedVector< DEST_DATA_T, DEST_TSIZE, DEST_INSTANTIATED > *dest) const
static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
static const uintptr_t theConstantPageBit
SYS_FORCE_INLINE int64 getMemoryUsage(exint tuplebytes) const
SYS_AtomicInt< int32 > SYS_AtomicCounter
Definition: SYS_AtomicInt.h:85
SYS_FORCE_INLINE const UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castType() const
void setSize(IDX_T newsize)
Definition: UT_PageArray.h:926
SYS_FORCE_INLINE const void * getMaskedPtrVoid() const
SYS_FORCE_INLINE const UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T >::PageTableEntry * castType() const
#define UT_ASSERT_MSG(ZZ, MM)
Definition: UT_Assert.h:129
void swapRange(IDX_T astart, IDX_T bstart, IDX_T nelements)
SYS_FORCE_INLINE void * getFirstPtrVoidUnsafe()
bool decRefIffSharedPage(UT_PageNum pagenum)
void setPageConstantUnknownType(UT_PageNum pagenum, const SRC_DATA_T *values)
SYS_FORCE_INLINE PageTableEntry * getPPage(UT_PageNum i)
void setStorage(const UT_Storage newstorage)
SYS_FORCE_INLINE void set(IDX_T i, exint component, SRC_DATA_T v)
Definition: UT_PageArray.h:628
static void hardenConstantPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
static void fillNonConstWithConst(NotVoidType *destpagedata, NotVoidType *destpageend, const SrcNotVoidType *stuple, exint mintuplesize, exint desttupleextra)
SYS_FORCE_INLINE NotVoidType * getMaskedPtr()
bool decRefIffSharedTable()
SYS_FORCE_INLINE const UT_PageArray< DATA_T, DEST_TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castTupleSize() const
SYS_FORCE_INLINE bool isConstant() const
This is always valid to call.
SYS_FORCE_INLINE void setRange(IDX_T deststart, IDX_T nelements, const T *src)
static void hardenSharedPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
SYS_FORCE_INLINE const void * getFirstPtrVoid() const
Returns the data pointer, if not a constant page.
GLuint GLfloat * val
Definition: glcorearb.h:1607
static SYS_FORCE_INLINE NotVoidType * getConstantPtr(PageTableEntry *page, exint component=0, exint tuplesize=TSIZE)
static SYS_FORCE_INLINE void makeConstant(PageTableEntry *page, const UT_FixedVector< NotVoidType, theSafeTupleSize > &val)
bool equal(T1 a, T2 b, T3 t)
Definition: ImathFun.h:143
static void makeConstant(PageTableEntry *page, const NotVoidType &val, exint tuplesize=TSIZE)
SYS_FORCE_INLINE UT_PageArray< DATA_T, TSIZE, true, PAGESHARDENED, THEPAGEBITS, IDX_T > & hardenTable()
static const exint thePageSize
Definition: UT_PageArray.h:256
static SYS_FORCE_INLINE bool isZero(const T *val, exint tuplesize)
SYS_FORCE_INLINE void alloc(UT_PageOff nelements, exint tuplesize=TSIZE)
GLint GLint GLsizei GLint GLenum GLenum type
Definition: glcorearb.h:107
T add(T val)
Atomically adds val to myValue, returning the new value of myValue.
static void hardenConstantPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
getOption("OpenEXR.storage") storage
Definition: HDK_Image.dox:276
Container class for all geometry.
Definition: GA_Detail.h:95
int64 getMemoryUsage(bool inclusive) const
static SYS_FORCE_INLINE UT_PageNum pageNum(IDX_T i)
UT_PageArray< DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > ThisType
Definition: UT_PageArray.h:250
static void hardenSharedPage(PageTable *pages, PageTableEntry *page)
void setCapacityIfNeeded(IDX_T newcapacity)
Definition: UT_PageArray.h:915
void forceHardenTable(IDX_T newcapacity)
SYS_FORCE_INLINE int64 getGuaranteedInt(IDX_T i) const
SYS_FORCE_INLINE void relaxedStore(T val)
Definition: SYS_AtomicInt.h:67
SYS_FORCE_INLINE NotVoidType * unshareConstantPage(UT_PageNum pagenum)
short int16
Definition: SYS_Types.h:33
#define SYSmin(a, b)
Definition: SYS_Math.h:1368
SYS_FORCE_INLINE bool isPageHard(UT_PageNum pagenum) const
UT_PageArray< DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > ParentType
SYS_FORCE_INLINE constexpr int UTstorageSize(UT_Storage storage)
Returns the number of bytes in the given storage type.
Definition: UT_Storage.h:54
SYS_FORCE_INLINE void initZero()
SYS_FORCE_INLINE const NotVoidType * getFirstPtr() const
Returns the data pointer, if not a constant page.
float fpreal32
Definition: SYS_Types.h:191
SYS_FORCE_INLINE NotVoidType * getFirstPtr()
Returns the data pointer, if not a constant page.
void setPageConstantMismatchedType(UT_PageNum pagenum, const SRC_DATA_T *values)
void replace(const UT_PageArray< SRC_DATA_T, SRC_TSIZE, SRC_TABLEHARDENED, SRC_PAGESHARDENED, THEPAGEBITS, IDX_T > &that)
Definition: UT_PageArray.h:395
#define SYS_CALLIF_AUTO
Definition: SYS_CallIf.h:22
SYS_FORCE_INLINE NotVoidType & operator()(IDX_T i, exint component=0)
Definition: UT_PageArray.h:569
SYS_FORCE_INLINE T relaxedLoad() const
Definition: SYS_AtomicInt.h:60
NotVoidType * getPageData(UT_PageNum pagenum)
void setVectorRange(IDX_T deststart, IDX_T nelements, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE, SRC_INSTANTIATED > *src)
SYS_FORCE_INLINE void decRef()
#define UT_ASSERT_MSG_P(ZZ, MM)
Definition: UT_Assert.h:128
static void replacePage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destpagesize, UT_PageOff destpagecapacity)
SYS_FORCE_INLINE UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castType()
GLenum src
Definition: glcorearb.h:1792