HDK
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
UT_PageArray.h
Go to the documentation of this file.
1 /*
2  * PROPRIETARY INFORMATION. This software is proprietary to
3  * Side Effects Software Inc., and is not to be reproduced,
4  * transmitted, or disclosed in any way without written permission.
5  *
6  * NAME: UT_PageArray.h (UT Library, C++)
7  *
8  * COMMENTS: An array class with special handling of constant pages and
9  * shared page data.
10  */
11 
12 #pragma once
13 
14 #ifndef __UT_PageArray__
15 #define __UT_PageArray__
16 
17 #include "UT_API.h"
18 #include "UT_Assert.h"
19 #include "UT_FixedVector.h"
20 #include "UT_SmallArray.h"
21 #include "UT_Storage.h"
22 #include <SYS/SYS_AtomicInt.h>
23 #include <SYS/SYS_BitUtil.h>
24 #include <SYS/SYS_CallIf.h>
25 #include <SYS/SYS_Compiler.h>
26 #include <SYS/SYS_Inline.h>
27 #include <SYS/SYS_Math.h>
28 #include <SYS/SYS_Types.h>
29 #include <SYS/SYS_TypeTraits.h>
30 #include <stdint.h>
31 #include <stdlib.h>
32 #include <string.h>
33 
34 #ifdef WIN32
35 // Default template arguments on functions are legal. Don't error on them!
36 #pragma warning(disable: 4519)
37 #endif
38 
39 
40 typedef exint UT_PageNum;
41 typedef exint UT_PageOff;
42 
43 
44 class UT_Defaults;
45 class UT_MemoryCounter;
46 
47 
48 /// @internal
49 /// W E L C O M E
50 /// to scenic
51 /// Template Hell
52 ///
53 /// Population: You
54 ///
55 /// If you're just using this class, hopefully you'll have a short and
56 /// not-so-painful visit!
57 ///
58 /// If, however, you're attempting to modify this class, BEWARE!
59 /// Any missteps can and likely will result in application of the
60 /// Doughnut Rules, so be sure to review them before visiting.
61 /// It may take several days to escape, so it may be best to just
62 /// revert and make absolutely sure that it works on all platforms
63 /// before committing again.
64 ///
65 /// This class is the result of a wide variety of needs for a wide variety
66 /// of use cases, most of which need both good performance and flexibility.
67 /// You may want to skip reading this gigantic comment, but whether it's
68 /// immediately or in a few months, you'll realize that skipping reading
69 /// this was a huge mistake.
70 ///
71 /// This array consists of a pointer to a page table, optionally a
72 /// tuple size (if TSIZE==-1), and optionally a UT_Storage indicating
73 /// the storage type (if DATA_T is void). The page table, if not nullptr,
74 /// can be shared, and has the format: @code
75 /// +-------------------+
76 /// | ref count | (1 if unshared; >1 if shared)
77 /// +-------------------+
78 /// | array size | (number of real elements/tuples in the array)
79 /// +-------------------+
80 /// | array capacity | (number of allocated elements/tuples if all pages of the table were hardened)
81 /// +-------------------+
82 /// | page 0 pointer | (pointer to page if bit 0 is 0, else constant page representation)
83 /// +-------------------+
84 /// | page 1 pointer |
85 /// +-------------------+
86 /// | page 2 pointer |
87 /// +-------------------+
88 /// | ... |
89 /// +-------------------+
90 /// @endcode
91 ///
92 /// If a page pointer has bit 0 clear, (i.e. an address aligned to a 2-byte boundary),
93 /// it is a non-constant page, it can be shared, and has the format: @code
94 /// +-------------------+
95 /// | ref count | (1 if unshared; >1 if shared)
96 /// +-------------------+
97 /// | (unused) | (padding so that data starts on a 16-byte boundary)
98 /// +-------------------+
99 /// | tuple 0 |
100 /// +-------------------+
101 /// | tuple 1 |
102 /// +-------------------+
103 /// | tuple 2 |
104 /// +-------------------+
105 /// | ... |
106 /// +-------------------+
107 /// @endcode
108 ///
109 /// A normal capacity page contains 2^THEPAGEBITS tuples, e.g. the default
110 /// THEPAGEBITS==10 gives a page size of 1024 tuples. If there is exactly
111 /// one page, it may have a capacity that is any power of two up to
112 /// THEPAGEBITS, which can be checked by checking if the overall array has
113 /// capacity less than 2^THEPAGEBITS.
114 ///
115 /// If, instead, the page pointer has bit 0 set to 1, which would be an odd
116 /// address, all tuples in the page are the same. There are 3 ways this may
117 /// be represented:
118 ///
119 /// 1) If the size of a full tuple is strictly smaller than the size of a
120 /// pointer, (e.g. 7 bytes or fewer on a 64-bit platform), the constant value
121 /// will be stored inside the space that would have been a pointer, aligned
122 /// to the end of the pointer. That ensures that 4-byte tuples will be
123 /// aligned to 4 bytes, and 2- or 6-byte tuples will be aligned to 2 bytes.
124 /// This is most useful for fpreal32[1] and int32[1] (i.e. 4-byte scalars).
125 ///
126 /// 2) If the page pointer is equal to 1 (i.e. nullptr apart from bit 0 being 1),
127 /// the value is all zero. This covers the extremely
128 /// common case of a zero default value, especially for fpreal32[3] and
129 /// fpreal64[3].
130 ///
131 /// 3) Otherwise, the page pointer, ignoring bit zero, points to a page
132 /// that contains just a single tuple.
133 ///
134 /// That covers the basic structure. It sounds so nice and simple...
135 /// but then there are the templates.
136 ///
137 /// DATA_T:
138 ///
139 /// If DATA_T is something other than void, that type is the type of each
140 /// component of each tuple. This is the only way that non-numeric types
141 /// are possible; DATA_T can also be a numeric type. If DATA_T is void,
142 /// myImpl contains a UT_Storage variable indicating what the data type is;
143 /// it must be a numeric type, i.e. it cannot be UT_Storage::INVALID.
144 /// The ...UnknownType functions switch on the UT_Storage to determine the
145 /// data type and call the corresponding function. If DATA_T is non-void,
146 /// it saves a lot of extra work at runtime. Because the codepaths that
147 /// will never be called due to DATA_T checks still need to compile with
148 /// DATA_T being void, NotVoidType ensures that there is some arbitrary type
149 /// in place so that the code will compile, even though it will never be
150 /// called. The same issue prevents using UT_ASSERT_COMPILETIME to verify
151 /// DATA_T, so UT_ASSERT_P is used instead.
152 /// myImpl is always structured so that if the type becomes known, by checking
153 /// getStorage(), this can be cast to have the now-known DATA_T set.
154 /// For example: @code
155 /// if (array.getStorage() == UT_Storage::REAL32)
156 /// array.castType<fpreal32>().someFunctionCall();
157 /// @endcode
158 ///
159 /// TSIZE:
160 ///
161 /// If TSIZE >= 1, it is the tuple size (the number of components in each
162 /// tuple), and is known at compile time, avoiding extra work at compile-time
163 /// that depends on the tuple size, especially if TSIZE is 1. Odds are that
164 /// for non-numeric types, TSIZE will almost always be 1, since it rarely
165 /// makes sense to be something else. If TSIZE is 0, something is almost
166 /// certainly wrong. If the tuple size is not known at compile-time, TSIZE
167 /// must be -1. In that case, myImpl contains an integer indicating the tuple
168 /// size. Like with DATA_T, because the codepaths that will never be called
169 /// due to TSIZE checks still need to compile with other TSIZE values, so
170 /// UT_ASSERT_COMPILETIME can't be used to verify TSIZE; UT_ASSERT_P is used
171 /// instead. There are a number of functions that require TSIZE to be the
172 /// tuple size, in order to avoid having to send an extra parameter and
173 /// incurring the slowdowns of their counterparts that assume that the tuple
174 /// size is what's passed in. If you modify one, REMEMBER TO MODIFY THE OTHER!
175 /// myImpl is always structured so that if the tuple size becomes
176 /// known, by checking the getTupleSize(), this can be cast to have the
177 /// now-known TSIZE set.
178 /// For example: @code
179 /// if (array.getTupleSize() == 3)
180 /// array.castTupleSize<3>().someFunctionCall();
181 /// @endcode
182 ///
183 /// TABLEHARDENED:
184 ///
185 /// Since page tables can be shared if all of the data are identical, modifying
186 /// the array in any way requires copying (hardening) the page table and may
187 /// require copying one or more shared pages or hardening constant pages.
188 /// Since every modification requires hardening the page table, it's useful
189 /// to harden the page table once and be able to skip checking whether it needs
190 /// hardening for all modifications after that. Calling hardenTable()
191 /// will ensure that the table is hardened, and return this with TABLEHARDENED
192 /// set to true, so that further calls on it to modify the array won't need to
193 /// check whether the table needs hardening. The page table should be
194 /// quite light compared to the data, so it's not the end of the world if
195 /// the table is hardened pre-emptively and there end up being no actual
196 /// modifications.
197 ///
198 /// PAGESHARDENED:
199 ///
200 /// When writing data in parallel to random locations, all pages that could
201 /// be written-to must be hardened in advance, to avoid a race condition on
202 /// the hardening of any page. Also, if it's known that most pages will
203 /// likely be hardened anyway when writing data, it'd be advantageous to
204 /// be able to harden all pages in advance and skip checking when making
205 /// modifications. The code for hardening pages just-in-time can
206 /// dramatically increase code size and time overhead. To avoid this,
207 /// hardenAllPages() will ensure that the table and all pages are hardened,
208 /// and will return this with TABLEHARDENED and PAGESHARDENED both set to true.
209 /// Any calls to modify page data made on that returned pointer won't check
210 /// whether any pages need hardening, because it will know that all pages are
211 /// already hardened. If you use this, be sure not to use the pointer
212 /// with PAGESHARDENED set to true after any call to tryCompressAllPages()
213 /// on the original array pointer.
214 ///
215 /// THEPAGEBITS:
216 ///
217 /// Full pages will contain 2^THEPAGEBITS tuples, e.g. THEPAGEBITS==10
218 /// means there will be 1024 tuples per full page.
219 ///
220 /// IDX_T:
221 ///
222 /// This allows templating on the array index type so that strict types
223 /// can be used, e.g. GA_Offset.
224 ///
225 /// FIXME BEFORE USING:
226 /// - Should make generic UT_R[OW]Handle.+ classes for easy accessing
227 /// of runtime-determined data type and tuple size data as tuples
228 /// with type and tuple size determined at compile-time.
229 /// - Make UT_PageNum and UT_PageOff strict types when strict types are
230 /// enabled.
231 /// - Add more functions for page-level access and block (bulk sequential)
232 /// access, for easy reading/writing to/from flat arrays.
233 /// @endinternal
234 template<typename DATA_T,exint TSIZE=1,bool TABLEHARDENED=false,bool PAGESHARDENED=false,exint THEPAGEBITS=10,typename IDX_T=exint>
236 {
237 protected:
238  /// This class is just to get code to compile when DATA_T is void,
239  /// by picking any type other than void if it is void.
240  template<typename T>
241  struct NotVoid
243 
245 
246  // These friend declarations are needed for calling protected functions on hardened/cast types.
247  template<typename DATA_T2,exint TSIZE2,bool TABLEHARDENED2,bool PAGESHARDENED2,exint THEPAGEBITS2,typename IDX_T2>
248  friend class UT_PageArray;
249 public:
251  typedef DATA_T DataType;
253  static const exint theTupleSize = TSIZE;
254  static const exint theSafeTupleSize = (TSIZE==-1) ? 3 : TSIZE;
255  static const exint thePageBits = THEPAGEBITS;
256  static const exint thePageSize = (exint(1)<<thePageBits);
257  static const exint thePageMask = thePageSize-1;
258  typedef IDX_T IndexType;
259 
260  /// The default constructor can only be used if the tuple size
261  /// and storage type are known at compile time.
263  {
264  UT_ASSERT_P(TSIZE >= 1);
266  myImpl.getPages() = nullptr;
267  }
268 
269  /// This constructor can only be used if the storage type is
270  /// known at compile time, but the tuple size is not.
271  UT_PageArray(exint tuplesize)
272  {
273  UT_ASSERT_P(TSIZE == -1 || TSIZE == tuplesize);
275  // NOTE: This is one case where a tuple size of zero is okay.
276  UT_ASSERT_P(tuplesize >= 0);
277  myImpl.getPages() = nullptr;
278  myImpl.setTupleSize(tuplesize);
279  }
280 
281  /// This constructor can only be used if the storage type is
282  /// known at compile time, but the tuple size is not.
284  {
285  UT_ASSERT_P(TSIZE >= 1);
287  UT_ASSERT_P(storage != UT_Storage::INVALID);
288  myImpl.getPages() = nullptr;
289  myImpl.setStorage(storage);
290  }
291 
292  /// This constructor can only be used if the tuple size and
293  /// storage type are unknown at compile time.
295  {
296  UT_ASSERT_P(TSIZE == -1 || TSIZE == tuplesize);
298  // NOTE: This is one case where a tuple size of zero is okay.
299  UT_ASSERT_P(tuplesize >= 0);
300  UT_ASSERT_P(storage != UT_Storage::INVALID);
301  myImpl.getPages() = nullptr;
302  myImpl.setStorage(storage);
303  myImpl.setTupleSize(tuplesize);
304  }
305 
306  /// This just references that's page table, unless TABLEHARDENED
307  /// or PAGESHARDENED. If TABLEHARDENED and !PAGESHARDENED,
308  /// the pages are referenced, and if PAGESHARDENED, the pages
309  /// are copied.
310  UT_PageArray(const ThisType &that)
311  {
312  myImpl.getPages() = nullptr;
313  *this = that;
314  }
315 
316  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
318  {
319  myImpl.getPages() = nullptr;
320  *this = that;
321  }
322 
325  {
326  clearAndDestroy();
327  }
328 
330  void clear()
331  {
332  if (myImpl.getPages())
333  myImpl.getPages()->setSize(IDX_T(0));
334  }
335 
337  {
338  if (!myImpl.getPages())
339  return;
340 
341  const exint tuplesize = myImpl.getTupleSize();
342 
343  // Just decrement the reference count to free the table safely
344  if (!SYSisSame<DATA_T,void>())
345  myImpl.getPages()->decRef(tuplesize);
346  else
347  {
348  const UT_Storage storage = myImpl.getStorage();
349  switch (storage)
350  {
351  case UT_Storage::INVALID:
352  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
353  return;
354  case UT_Storage::INT8:
355  castType<int8>().myImpl.getPages()->decRef(tuplesize); return;
356  case UT_Storage::UINT8:
357  castType<uint8>().myImpl.getPages()->decRef(tuplesize); return;
358  case UT_Storage::INT16:
359  castType<int16>().myImpl.getPages()->decRef(tuplesize); return;
360  case UT_Storage::INT32:
361  castType<int32>().myImpl.getPages()->decRef(tuplesize); return;
362  case UT_Storage::INT64:
363  castType<int64>().myImpl.getPages()->decRef(tuplesize); return;
364  case UT_Storage::REAL16:
365  castType<fpreal16>().myImpl.getPages()->decRef(tuplesize); return;
366  case UT_Storage::REAL32:
367  castType<fpreal32>().myImpl.getPages()->decRef(tuplesize); return;
368  case UT_Storage::REAL64:
369  castType<fpreal64>().myImpl.getPages()->decRef(tuplesize); return;
370  }
371  }
372  myImpl.getPages() = nullptr;
373  }
374 
375  /// NOTE: Compilers seem not to use the operator= below when the template
376  /// arguments match, so this operator= is necessary in order to
377  /// avoid the compiler silently using its implicitly-defined
378  /// assignment operator.
381  {
382  replace(that);
383  return *this;
384  }
385 
386  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
389  {
390  replace(that);
391  return *this;
392  }
393 
394  /// Completely replace this UT_PageArray with that, referencing
395  /// the pages or table, if allowed by the template arguments.
396  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
398  {
399  // NOTE: Careful if you add template parameter checks on this,
400  // because, e.g., a UT_PageArray<void,-1> could refer to the
401  // same exact array as a UT_PageArray<float 3>.
402  if ((void *)this == (const void *)&that)
403  return;
404  // NOTE: The storage and tuple size checks are so that if both
405  // tables are nullptr, it still copies the storage and tuple
406  // size below.
407  if (!TABLEHARDENED &&
408  myImpl.getStorage() == that.myImpl.getStorage() &&
409  myImpl.getTupleSize() == that.myImpl.getTupleSize() &&
410  (void *)myImpl.getPages() == (const void *)that.myImpl.getPages())
411  return;
412 
413  clearAndDestroy();
414 
415  if (SYSisSame<DATA_T,void>())
416  myImpl.setStorage(that.myImpl.getStorage());
417  if (TSIZE == -1)
418  myImpl.setTupleSize(that.myImpl.getTupleSize());
419 
420  // May be able to share pages or table if matching tuple size and storage type
421  if ((SYSisSame<DATA_T,void>() || SYSisSame<DATA_T,SRC_DATA_T>()) && (TSIZE == -1 || TSIZE == SRC_TSIZE))
422  {
423  PageTable *pages = reinterpret_cast<PageTable *>(SYSconst_cast(that.myImpl.getPages()));
424  myImpl.getPages() = pages;
425  if (!pages)
426  return;
427  pages->incRef();
428  if (TABLEHARDENED || PAGESHARDENED || SRC_TABLEHARDENED || SRC_PAGESHARDENED)
429  {
430  forceHardenTable(pages->capacity());
431  if (PAGESHARDENED || SRC_PAGESHARDENED)
432  {
433  // If we don't cast to PAGESHARDENED==false, it won't do anything.
435  ->template hardenAllPages<false>();
436  }
437  }
438  }
439  else
440  {
441  // Can't share table or pages, so just use moveRange
442  IDX_T capacity = that.capacity();
443  IDX_T size = that.size();
444  forceHardenTable(capacity);
445  moveRange(that, IDX_T(0), IDX_T(0), size);
446  }
447  }
448 
449  /// Read element i's specified component (default component 0).
450  SYS_FORCE_INLINE NotVoidType operator()(IDX_T i,exint component=0) const
451  {
452  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
453  UT_ASSERT_P(TSIZE != 0);
454  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
455  UT_ASSERT_P(i >= IDX_T(0) && i < size());
456  UT_ASSERT_P(component >= 0 && component < myImpl.getTupleSize());
457 
458  UT_PageNum pagenum = pageNum(i);
459  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
460 
461  // If we know that the pages are hardened, we can omit
462  // the constant page case at compile time.
463  bool isconst = !PAGESHARDENED && page->isConstant();
464 
465  if (TSIZE >= 1)
466  {
467  if (!isconst)
468  {
469  UT_PageOff pageoff = pageOff(i);
470  return page->getFirstPtr()[TSIZE*pageoff + component];
471  }
472  return getConstant(page, component);
473  }
474  else
475  {
476  if (!isconst)
477  {
478  UT_PageOff pageoff = pageOff(i);
479  return page->getFirstPtr()[myImpl.getTupleSize()*pageoff + component];
480  }
481  return getConstant(page, component, myImpl.getTupleSize());
482  }
483  }
484 
485  /// This get function may seem redundant, given operator(), but it gives a
486  /// simple way to read from a non-const UT_PageArray without calling the
487  /// non-const operator() and potentially hardening pages. It just
488  /// explicitly adds const. It also allows data type conversion and
489  /// reading when the type is unknown at compile time.
490  /// It should hopefully be inlined in the no-conversion case;
491  /// it's less critical in the conversion case.
492  template<typename DEST_DATA_T=DATA_T>
493  SYS_FORCE_INLINE DEST_DATA_T get(IDX_T i,exint component=0) const
494  {
495  UT_ASSERT_P(i >= IDX_T(0) && i < size());
496  if (!(SYSisSame<DATA_T,void>()) && (SYSisSame<DEST_DATA_T,DATA_T>() || theStorage != UT_Storage::INVALID))
497  return UTconvertStorage<DEST_DATA_T>((*this)(i,component));
498 
499  return getUnknownType<DEST_DATA_T>(i,component);
500  }
501 
502  template<typename DEST_DATA_T=NotVoidType,exint DEST_TSIZE=theSafeTupleSize>
504  {
505  if (SYSisSame<DATA_T,void>())
506  {
508  SYS_CallIf<SYSisSame<DATA_T,void>()>::call([this,&dest,i](SYS_CALLIF_AUTO){
509  dest = this->getVectorUnknownType<DEST_DATA_T,DEST_TSIZE>(i);
510  });
511  return dest;
512  }
513  if (!SYSisSame<DEST_DATA_T,DATA_T>())
514  {
517  dest = this->convertVectorStorage<DEST_DATA_T>(this->getVector<NotVoidType,DEST_TSIZE>(i));
518  });
519  return dest;
520  }
521 
522  if (TSIZE==-1)
523  {
526  dest = this->getVectorUnknownSize<DEST_DATA_T,DEST_TSIZE>(i);
527  });
528  return dest;
529  }
530  if (TSIZE!=DEST_TSIZE)
531  {
534  dest = this->getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
535  });
536  return dest;
537  }
538 
539  UT_ASSERT_P(i >= IDX_T(0) && i < size());
540  UT_ASSERT_P(TSIZE==DEST_TSIZE);
541  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()));
542 
543  // Types and tuple sizes are known and match
544  UT_PageNum pagenum = pageNum(i);
545  // NOTE: We can dereference here because we're returning by value.
546  // getConstantPtr may return a pointer into this stack variable
547  // if the page is constant and inline.
548  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
549 
550  // If we know that the pages are hardened, we can omit
551  // the constant page case at compile time.
552  bool isconst = !PAGESHARDENED && page->isConstant();
553 
554  if (!isconst)
555  {
556  UT_PageOff pageoff = pageOff(i);
557  return ((const UT_FixedVector<DEST_DATA_T,DEST_TSIZE>*)page->getFirstPtr())[pageoff];
558  }
559  const NotVoidType *constdata = getConstantPtr(page);
560  if (!constdata)
561  return UT_FixedVector<DEST_DATA_T,DEST_TSIZE>(DEST_DATA_T(0));
562 
563  return *(const UT_FixedVector<DEST_DATA_T,DEST_TSIZE>*)constdata;
564  }
565 
566  /// Get a non-const reference to element i's specified component
567  /// (default component 0).
568  /// WARNING: DO NOT call this if DATA_T is void!
569  /// NOTE: Unlike set(), this must always harden, even if you're just
570  /// writing a value that is already there.
572  {
573  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
574  UT_ASSERT_P(TSIZE != 0);
575  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
576  UT_ASSERT_P(i >= IDX_T(0) && i < size());
577  UT_ASSERT_P(component >= 0 && component < myImpl.getTupleSize());
578 
579  // NOTE: This is a no-op if TABLEHARDENED is already true.
580  hardenTable();
581 
582  UT_PageNum pagenum = pageNum(i);
583  UT_PageOff pageoff = pageOff(i);
584  PageTable *pages = myImpl.getPages();
585  PageTableEntry *page = pages->getPPage(pagenum);
586 
587  if (!PAGESHARDENED)
588  {
589  if (page->isConstant())
590  {
591  if (TSIZE >= 1)
592  hardenConstantPage(pages, page);
593  else
594  hardenConstantPage(pages, page, myImpl.getTupleSize());
595  }
596  else if (page->isShared())
597  {
598  if (TSIZE >= 1)
599  hardenSharedPage(pages, page);
600  else
601  hardenSharedPage(pages, page, myImpl.getTupleSize());
602  }
603  }
604 
605  NotVoidType *data = page->getFirstPtr()
606  + pageoff*((TSIZE >= 1) ? TSIZE : myImpl.getTupleSize())
607  + component;
608 
609  return *data;
610  }
611 
612  /// component == 0 in this version
613  template<typename SRC_DATA_T>
614  SYS_FORCE_INLINE void set(IDX_T i,SRC_DATA_T v)
615  {
616  set(i, 0, v);
617  }
618  /// component == 0 in this version
619  template<typename SRC_DATA_T>
620  SYS_FORCE_INLINE void add(IDX_T i,SRC_DATA_T v)
621  {
622  add(i, 0, v);
623  }
624 
625  /// This looks gigantic, but it's mostly because of the different
626  /// behaviour for different template values. If
627  /// PAGESHARDENED is true, it'll skip checking for constant
628  /// or shared pages altogether.
629  template<typename SRC_DATA_T>
630  SYS_FORCE_INLINE void set(IDX_T i,exint component,SRC_DATA_T v)
631  {
632  op<0>(i, component, v);
633  }
634  template<typename SRC_DATA_T>
635  SYS_FORCE_INLINE void add(IDX_T i,exint component,SRC_DATA_T v)
636  {
637  op<1>(i, component, v);
638  }
639  template<int OP,typename SRC_DATA_T>
640  SYS_FORCE_INLINE void op(IDX_T i,exint component,SRC_DATA_T v)
641  {
642  UT_ASSERT_P(i >= IDX_T(0) && i < size());
643  UT_ASSERT_P(component >= 0 && component < getTupleSize());
644  UT_ASSERT_MSG_P(OP == 0 || OP == 1, "Only OP 0 (set) and 1 (add) are defined!");
645 
646  if (SYSisSame<DATA_T,void>())
647  {
648  opUnknownType<OP>(i, component, v);
649  return;
650  }
651 
652  // Make sure that v is the correct type before comparing.
653  NotVoidType vt(UTconvertStorage<NotVoidType>(v));
654 
655  UT_ASSERT_P(TSIZE != 0);
656  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
657 
658  hardenTable();
659 
660  UT_PageNum pagenum = pageNum(i);
661  UT_PageOff pageoff = pageOff(i);
662  PageTable *pages = myImpl.getPages();
663  PageTableEntry *page = pages->getPPage(pagenum);
664 
665  if (!PAGESHARDENED)
666  {
667  if (page->isConstant())
668  {
669  // If we're setting, check to see if it's a new value
670  if (OP == 0)
671  {
672  if (TSIZE >= 1)
673  {
674  if (getConstant(page, component) == vt)
675  return;
676  }
677  else
678  {
679  if (getConstant(page, component, myImpl.getTupleSize()) == vt)
680  return;
681  }
682  }
683  if (TSIZE >= 1)
684  hardenConstantPage(pages, page);
685  else
686  hardenConstantPage(pages, page, myImpl.getTupleSize());
687  }
688  else if (page->isShared())
689  {
690  // If we're setting, check to see if it's a new value
691  if (OP == 0)
692  {
693  NotVoidType *data = page->getFirstPtr()
694  + pageoff*((TSIZE >= 1) ? TSIZE : myImpl.getTupleSize())
695  + component;
696  if (*data == vt)
697  return;
698  }
699  if (TSIZE >= 1)
700  hardenSharedPage(pages, page);
701  else
702  hardenSharedPage(pages, page, myImpl.getTupleSize());
703  }
704  }
705 
706  NotVoidType *data = page->getFirstPtr()
707  + pageoff*((TSIZE >= 1) ? TSIZE : myImpl.getTupleSize())
708  + component;
709 
710  if (OP == 0)
711  *data = vt;
712  else if (OP == 1)
713  *data += vt;
714  else
715  {
716  UT_ASSERT_MSG_P(0,"Only OP 0 (set) and 1 (add) are defined!");
717  }
718  }
719 
720  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_INSTANTIATED>
722  {
723  opVector<0>(i, reinterpret_cast<const UT_FixedVector<SRC_DATA_T,SRC_TSIZE> &>(v));
724  }
725  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_INSTANTIATED>
727  {
728  opVector<1>(i, reinterpret_cast<const UT_FixedVector<SRC_DATA_T,SRC_TSIZE> &>(v));
729  }
730  /// This looks gigantic, but it's mostly because of the different
731  /// behaviour for different template values. If
732  /// PAGESHARDENED is true, it'll skip checking for constant
733  /// or shared pages altogether.
734  template<int OP,typename SRC_DATA_T,exint SRC_TSIZE>
736  {
737  UT_ASSERT_P(i >= IDX_T(0) && i < size());
738  UT_ASSERT_MSG_P(OP == 0 || OP == 1, "Only OP 0 (set) and 1 (add) are defined!");
739 
740  if (SYSisSame<DATA_T,void>())
741  {
743  this->opVectorUnknownType<OP>(i, v);
744  });
745  return;
746  }
747 
748  const exint tuplesize = myImpl.getTupleSize();
749  if (TSIZE == -1 && tuplesize == SRC_TSIZE)
750  {
752  this->castTupleSize<SRC_TSIZE>().template opVector<OP>(i, v);
753  });
754  return;
755  }
756  const exint mintuplesize = SYSmin(tuplesize, SRC_TSIZE);
757 
758  // Make sure that v is the correct type before comparing.
759  // Hopefully this will avoid copying if SRC_DATA_T is NotVoidType.
760  constexpr bool TYPE_MATCH = SYSisSame<SRC_DATA_T,NotVoidType>();
762  if (!TYPE_MATCH)
763  converted_v = convertVectorStorage<NotVoidType>(v);
764 
765  UT_ASSERT_P(TSIZE != 0);
766  UT_ASSERT_P(myImpl.getTupleSize() >= 1);
767 
768  hardenTable();
769 
770  UT_PageNum pagenum = pageNum(i);
771  UT_PageOff pageoff = pageOff(i);
772  PageTable *pages = myImpl.getPages();
773  PageTableEntry *page = pages->getPPage(pagenum);
774 
775  if (!PAGESHARDENED)
776  {
777  if (page->isConstant())
778  {
779  // If we're setting, check to see if it's a new value
780  if (OP == 0)
781  {
782  const NotVoidType *data;
783  if (PageTableEntry::typeFitsInline(tuplesize))
784  data = page->getInlinePtr(tuplesize);
785  else
786  data = page->getMaskedPtr();
787  if (data)
788  {
789  if (TYPE_MATCH)
790  {
791  if (isEqual(data, v.data(), mintuplesize))
792  return;
793  }
794  else
795  {
796  if (isEqual(data, converted_v.data(), mintuplesize))
797  return;
798  }
799  }
800  else
801  {
802  if (TYPE_MATCH)
803  {
804  if (isZero(v.data(), mintuplesize))
805  return;
806  }
807  else
808  {
809  if (isZero(converted_v.data(), mintuplesize))
810  return;
811  }
812  }
813  }
814  if (TSIZE >= 1)
815  hardenConstantPage(pages, page);
816  else
817  hardenConstantPage(pages, page, tuplesize);
818  }
819  else if (page->isShared())
820  {
821  // If we're setting, check to see if it's a new value
822  if (OP == 0)
823  {
824  const NotVoidType *data = page->getFirstPtr()
825  + pageoff*((TSIZE >= 0) ? TSIZE : tuplesize);
826  if (TYPE_MATCH)
827  {
828  if (isEqual(data, v.data(), mintuplesize))
829  return;
830  }
831  else
832  {
833  if (isEqual(data, converted_v.data(), mintuplesize))
834  return;
835  }
836  }
837  if (TSIZE >= 1)
838  hardenSharedPage(pages, page);
839  else
840  hardenSharedPage(pages, page, tuplesize);
841  }
842  }
843 
844  NotVoidType *data = page->getFirstPtr() + pageoff*((TSIZE >= 0) ? TSIZE : tuplesize);
845  for (exint component = 0; component < mintuplesize; ++component)
846  {
847  if (OP == 0)
848  {
849  if (TYPE_MATCH)
850  data[component] = v[component];
851  else
852  data[component] = converted_v[component];
853  }
854  else if (OP == 1)
855  {
856  if (TYPE_MATCH)
857  data[component] += v[component];
858  else
859  data[component] += converted_v[component];
860  }
861  else
862  {
863  UT_ASSERT_MSG_P(0,"Only OP 0 (set) and 1 (add) are defined!");
864  }
865  }
866  }
867 
868  SYS_FORCE_INLINE IDX_T size() const
869  {
870  const PageTable *pages = myImpl.getPages();
871  if (!pages)
872  return IDX_T(0);
873  return pages->size();
874  }
875 
877  {
878  const PageTable *pages = myImpl.getPages();
879  if (!pages)
880  return IDX_T(0);
881  return pages->capacity();
882  }
883 
884  /// This helper function forces a (non-negative) capacity to be an
885  /// acceptable value: either
886  /// A) a power of two less than a full page in size, or
887  /// B) a multiple of the page size.
889  static IDX_T roundUpCapacity(IDX_T capacity)
890  {
891  UT_ASSERT_P(capacity >= IDX_T(0));
892 
893  // Round up to page size if more than half of one page.
894  if (capacity > IDX_T(thePageSize>>1))
895  return IDX_T((capacity + thePageMask) & ~thePageMask);
896 
897  // Round up to power of two if half a page or less.
898  // (Zero stays zero.)
899  if (capacity <= IDX_T(2))
900  return capacity;
901  return IDX_T(SYSmakePow2(exint(capacity)));
902  }
903 
904  void setCapacity(IDX_T newcapacity)
905  {
906  UT_ASSERT_P(newcapacity >= IDX_T(0));
907  newcapacity = roundUpCapacity(newcapacity);
908  if (newcapacity == capacity())
909  return;
910  if (newcapacity < size())
911  {
912  setSize(newcapacity);
913  }
914  forceHardenTable(newcapacity);
915  }
916 
917  void setCapacityIfNeeded(IDX_T newcapacity)
918  {
919  UT_ASSERT_P(newcapacity >= IDX_T(0));
920  if (newcapacity <= capacity())
921  return;
922  newcapacity = roundUpCapacity(newcapacity);
923  forceHardenTable(newcapacity);
924  }
925 
926  /// Sets the size of the array without initializing any added
927  /// elements.
928  void setSize(IDX_T newsize)
929  {
930  UT_ASSERT_P(newsize >= IDX_T(0));
931 
932  setCapacityIfNeeded(newsize);
933  hardenTable();
934  PageTable *pages = myImpl.getPages();
935  UT_ASSERT_P(pages || newsize == IDX_T(0));
936  if (pages)
937  pages->setSize(newsize);
938  }
939 
940  /// Sets the size of the array and initializes any new elements to initval.
941  /// This version of setSize only works if DATA_T is a numeric type.
942  /// If the tuple size is > 1, it will write initval to all components.
943  /// Include UT_PageArrayImpl.h to call this.
944  void setSize(IDX_T newsize, NotVoidType initval);
945 
946  /// Sets the size of the array and initializes any new elements
947  /// to initval.
948  /// This version of setSize only works if DATA_T is a numeric type
949  /// and TSIZE >= 1.
950  /// Include UT_PageArrayImpl.h to call this.
951  void setSize(IDX_T newsize, const UT_FixedVector<NotVoidType,theSafeTupleSize> &initval);
952 
953  /// Sets the size of the array and initializes any new elements to initval.
954  /// Include UT_PageArrayImpl.h to call this.
955  void setSize(IDX_T newsize, const UT_Defaults &initval);
956 
957  /// Set all of the values in the range [start,end) to v.
958  /// This will constant-compress pages if allowed and the pages are
959  /// completely overwritten with the given value.
960  /// This version of setConstant only works if DATA_T is a numeric type.
961  /// If the tuple size is > 1, it will write v to all components.
962  void setConstant(IDX_T start, IDX_T end, NotVoidType v);
963 
964  /// Set all of the values in the range [start,end) to v.
965  /// This will constant-compress pages if allowed and the pages are
966  /// completely overwritten with the given value.
967  /// This version of setConstant only works if DATA_T is a numeric type
968  /// and TSIZE >= 1.
970 
971  /// Set all of the values in the range [start,end) to v.
972  /// This will constant-compress pages if allowed and the pages are
973  /// completely overwritten with the given value.
974  void setConstant(IDX_T start, IDX_T end, const UT_Defaults &v);
975 
976  /// Set all of the values in the range [start,end) to the tuple in values.
977  /// values must have length equal to getTupleSize().
978  /// This will constant-compress pages if allowed and the pages are
979  /// completely overwritten with the given value.
980  template<typename SRC_T>
981  void setConstant(IDX_T start, IDX_T end, const SRC_T *values);
982 
983  /// Changes the storage type and converts the data to the new type.
984  /// NOTE: This only makes sense if DATA_T is void and newstorage is
985  /// something other than INVALID.
986  /// Include UT_PageArrayImpl.h to call this.
987  void setStorage(const UT_Storage newstorage);
988 
989  /// Changes the tuple size, and if the new size is larger, pads with
990  /// values from the UT_Defaults.
991  /// NOTE: This only makes sense if TSIZE == -1.
992  /// Include UT_PageArrayImpl.h to call this.
993  void setTupleSize(exint newtuplesize, const UT_Defaults &v);
994 
995  /// Get the total of all memory referred-to by this.
996  /// NOTE: This does not handle shared memory correctly.
997  /// Include UT_PageArrayImpl.h to call this.
998  int64 getMemoryUsage(bool inclusive) const;
999 
1000  /// Count memory usage using a UT_MemoryCounter in order to count
1001  /// shared memory correctly.
1002  /// If inclusive is true, the size of this object is counted,
1003  /// else only memory owned by this object is counted.
1004  /// If this is pointed to by the calling object, inclusive should be true.
1005  /// If this is contained in the calling object, inclusive should be false.
1006  /// (Its memory was already counted in the size of the calling object.)
1007  /// Include UT_PageArrayImpl.h to call this.
1008  void countMemory(UT_MemoryCounter &counter, bool inclusive) const;
1009 
1010  /// Copies the nelements from srcstart to deststart
1011  /// The two ranges *are* allowed to overlap.
1012  /// Include UT_PageArrayImpl.h to call this.
1013  void moveRange(IDX_T srcstart, IDX_T deststart, IDX_T nelements);
1014 
1015  /// Copies the nelements from fromstart in src to tostart in this.
1016  /// The two ranges *are* allowed to overlap, even if &src==this.
1017  /// Include UT_PageArrayImpl.h to call this.
1018  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_TABLEHARDENED,bool SRC_PAGESHARDENED>
1019  void moveRange(const UT_PageArray<SRC_DATA_T,SRC_TSIZE,SRC_TABLEHARDENED,SRC_PAGESHARDENED,THEPAGEBITS,IDX_T> &src, IDX_T srcstart, IDX_T deststart, IDX_T nelements);
1020 
1021  /// Swaps the nelements at astart with the nelements at bstart.
1022  /// The two ranges *must* be non-overlapping, else swapping is ill-defined.
1023  /// Include UT_PageArrayImpl.h to call this.
1024  void swapRange(IDX_T astart, IDX_T bstart, IDX_T nelements);
1025 
1026  /// Copies a single element from src to dest within this
1027  void copy(IDX_T dest, IDX_T src)
1028  {
1029  UT_ASSERT_P(dest >= IDX_T(0) && dest < size());
1030  UT_ASSERT_P(src >= IDX_T(0) && src < size());
1031 
1032  hardenTable();
1033 
1034  // Because no conversions are necessary, only the element size
1035  // in bytes is needed.
1036  const exint bytesize = (SYSisSame<DATA_T,void>() ? UTstorageSize(getStorage()) : sizeof(NotVoidType))
1037  * getTupleSize();
1038 
1039  UT_PageNum destpagenum = pageNum(dest);
1040  UT_PageNum srcpagenum = pageNum(src);
1041 
1042  PageTable *pages = myImpl.getPages();
1043  PageTableEntry *destpage = pages->getPPage(destpagenum);
1044  const PageTableEntry *srcpage = pages->getPPage(srcpagenum);
1045 
1046  const int8 *srcdata;
1047  if (srcpage->isConstant())
1048  {
1049  if (bytesize < sizeof(PageTableEntry))
1050  {
1051  // Copying within equal constant pages does nothing
1052  if (*srcpage == *destpage)
1053  return;
1054 
1055  srcdata = ((const int8 *)(srcpage+1)) - bytesize;
1056  }
1057  else if (srcpage->getUnmaskedPtrVoid() == destpage->getUnmaskedPtrVoid())
1058  {
1059  // Copying within equal constant pages does nothing
1060  return;
1061  }
1062  else
1063  {
1064  // NOTE: srcdata may be set to nullptr here.
1065  srcdata = (const int8 *)srcpage->getMaskedPtrVoid();
1066  }
1067  }
1068  else
1069  {
1070  srcdata = ((const int8*)srcpage->getFirstPtrVoid()) + pageOff(src)*bytesize;
1071  }
1072 
1073  int8 *destdata;
1074  if (destpage->isConstant())
1075  {
1076  if (bytesize < sizeof(PageTableEntry))
1077  destdata = ((int8 *)(destpage+1)) - bytesize;
1078  else
1079  {
1080  void *destmasked = destpage->getMaskedPtrVoid();
1081  if (destmasked == nullptr)
1082  {
1083  if (isZero(srcdata, bytesize))
1084  return;
1085  destdata = nullptr;
1086  }
1087  else
1088  destdata = (int8*)destmasked;
1089  }
1090  if (destdata != nullptr && isEqual(srcdata, destdata, bytesize))
1091  return;
1092 
1093  if (!SYSisSame<DATA_T,void>())
1094  hardenConstantPage(pages, destpage, getTupleSize());
1095  else
1096  {
1097  switch (getStorage())
1098  {
1099  case UT_Storage::INT8:
1100  case UT_Storage::UINT8:
1101  {
1102  auto &a = castType<int8>();
1103  auto a_pages = a.myImpl.getPages();
1104  auto a_destpage = a_pages->getPPage(destpagenum);
1105  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1106  break;
1107  }
1108  case UT_Storage::INT16:
1109  case UT_Storage::REAL16:
1110  {
1111  auto &a = castType<int16>();
1112  auto a_pages = a.myImpl.getPages();
1113  auto a_destpage = a_pages->getPPage(destpagenum);
1114  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1115  break;
1116  }
1117  case UT_Storage::INT32:
1118  case UT_Storage::REAL32:
1119  {
1120  auto &a = castType<int32>();
1121  auto a_pages = a.myImpl.getPages();
1122  auto a_destpage = a_pages->getPPage(destpagenum);
1123  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1124  break;
1125  }
1126  case UT_Storage::INT64:
1127  case UT_Storage::REAL64:
1128  {
1129  auto &a = castType<int64>();
1130  auto a_pages = a.myImpl.getPages();
1131  auto a_destpage = a_pages->getPPage(destpagenum);
1132  a.hardenConstantPage(a_pages, a_destpage, getTupleSize());
1133  break;
1134  }
1135  case UT_Storage::INVALID:
1136  UT_ASSERT_MSG(0, "Storage must be valid if DATA_T is void.");
1137  break;
1138  }
1139  }
1140  }
1141  else if (!PAGESHARDENED && destpage->isShared())
1142  {
1143  destdata = ((int8*)destpage->getFirstPtrVoid()) + pageOff(dest)*bytesize;
1144  if (srcdata)
1145  {
1146  if (isEqual(srcdata, destdata, bytesize))
1147  return;
1148  }
1149  else
1150  {
1151  if (isZero(destdata, bytesize))
1152  return;
1153  }
1154 
1155  if (!SYSisSame<DATA_T,void>())
1156  hardenSharedPage(pages, destpage, getTupleSize());
1157  else
1158  {
1159  switch (getStorage())
1160  {
1161  case UT_Storage::INT8:
1162  case UT_Storage::UINT8:
1163  {
1164  auto &a = castType<int8>();
1165  auto a_pages = a.myImpl.getPages();
1166  auto a_destpage = a_pages->getPPage(destpagenum);
1167  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1168  break;
1169  }
1170  case UT_Storage::INT16:
1171  case UT_Storage::REAL16:
1172  {
1173  auto &a = castType<int16>();
1174  auto a_pages = a.myImpl.getPages();
1175  auto a_destpage = a_pages->getPPage(destpagenum);
1176  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1177  break;
1178  }
1179  case UT_Storage::INT32:
1180  case UT_Storage::REAL32:
1181  {
1182  auto &a = castType<int32>();
1183  auto a_pages = a.myImpl.getPages();
1184  auto a_destpage = a_pages->getPPage(destpagenum);
1185  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1186  break;
1187  }
1188  case UT_Storage::INT64:
1189  case UT_Storage::REAL64:
1190  {
1191  auto &a = castType<int64>();
1192  auto a_pages = a.myImpl.getPages();
1193  auto a_destpage = a_pages->getPPage(destpagenum);
1194  a.hardenSharedPage(a_pages, a_destpage, getTupleSize());
1195  break;
1196  }
1197  case UT_Storage::INVALID:
1198  UT_ASSERT_MSG(0, "Storage must be valid if DATA_T is void.");
1199  break;
1200  }
1201  }
1202  }
1203  destdata = ((int8*)destpage->getFirstPtrVoid()) + pageOff(dest)*bytesize;
1204 
1205  for (const int8 *srcdataend = srcdata+bytesize; srcdata != srcdataend; ++srcdata, ++destdata)
1206  *destdata = *srcdata;
1207  }
1208 
1209  /// Copies nelements from srcstart in this to dest array
1210  /// Include UT_PageArrayImpl.h to call this.
1211  template<typename T>
1212  SYS_FORCE_INLINE void getRange(IDX_T srcstart, IDX_T nelements, T *dest) const
1213  {
1214  // Always cast to associated UT_FixedVector type, so that
1215  // we can just delegate to getVectorRange that always uses UT_FixedVector
1216  getVectorRange(srcstart, nelements, (typename UT_FixedVectorTraits<T>::FixedVectorType*)dest);
1217  }
1218 
1219  /// Copies nelements from src array to deststart in this
1220  /// Include UT_PageArrayImpl.h to call this.
1221  template<typename T>
1222  SYS_FORCE_INLINE void setRange(IDX_T deststart, IDX_T nelements, const T *src)
1223  {
1224  // Always cast to associated UT_FixedVector type, so that
1225  // we can just delegate to getVectorRange that always uses UT_FixedVector
1226  setVectorRange(deststart, nelements, (const typename UT_FixedVectorTraits<T>::FixedVectorType*)src);
1227  }
1228 
1229  /// Copies nelements from srcstart in this to dest array
1230  /// Include UT_PageArrayImpl.h to call this.
1231  template<typename DEST_DATA_T,exint DEST_TSIZE,bool DEST_INSTANTIATED>
1232  void getVectorRange(IDX_T srcstart, IDX_T nelements, UT_FixedVector<DEST_DATA_T,DEST_TSIZE,DEST_INSTANTIATED> *dest) const;
1233 
1234  /// Copies nelements from src array to deststart in this
1235  /// Include UT_PageArrayImpl.h to call this.
1236  template<typename SRC_DATA_T,exint SRC_TSIZE,bool SRC_INSTANTIATED>
1237  void setVectorRange(IDX_T deststart, IDX_T nelements, const UT_FixedVector<SRC_DATA_T,SRC_TSIZE,SRC_INSTANTIATED> *src);
1238 
1240  {
1241  return UT_PageNum(exint(i) >> thePageBits);
1242  }
1243  SYS_FORCE_INLINE static UT_PageNum numPages(IDX_T nelements)
1244  {
1245  return UT_PageNum((exint(nelements)+thePageMask) >> thePageBits);
1246  }
1248  {
1249  return UT_PageOff(exint(i) & thePageMask);
1250  }
1252  {
1253  return IDX_T(exint(page) << thePageBits);
1254  }
1255 
1256  /// Returns true iff the table isn't shared with any other UT_PageArray's.
1258  {
1259  // If TABLEHARDENED, assert that the page table hasn't become shared
1260  // after the fact! Nobody should share the page table after templating
1261  // on having hardened!
1262  UT_ASSERT_P(!TABLEHARDENED || !myImpl.getPages() || !myImpl.getPages()->isShared());
1263 
1264  return TABLEHARDENED || !myImpl.getPages() || !myImpl.getPages()->isShared();
1265  }
1266 
1267  /// This ensures that the table itself is not shared.
1268  /// It's a good idea to do this before writing, to avoid having to check
1269  /// every time. Remember to use the returned reference, since it
1270  /// records in the template that it doesn't need to be hardened again.
1271  /// It should be a no-op if TABLEHARDENED is already true.
1273  {
1274  if (!isTableHardened())
1275  {
1276  // Keep same capacity
1277  forceHardenTable(myImpl.getPages()->capacity());
1278  }
1279 
1281  }
1282 
1283  /// This forces all pages to be hardened, optionally including pages
1284  /// that are in the capacity but outside the size.
1285  template <bool including_capacity=false>
1287  {
1288  // Nothing to do if PAGESHARDENED
1289  PageTable *pages = myImpl.getPages();
1290  if (PAGESHARDENED || (pages == nullptr))
1292 
1293  auto &hard = hardenTable();
1294 
1295  if (end < IDX_T(0))
1296  end = including_capacity ? pages->capacity() : pages->size();
1297 
1298  UT_ASSERT_P(start >= IDX_T(0));
1300 
1301  if (!SYSisSame<DATA_T,void>())
1302  {
1303  if (TSIZE >= 1)
1304  pages->hardenAllPages(start, end);
1305  else
1306  pages->hardenAllPages(start, end, myImpl.getTupleSize());
1307  }
1308  else
1309  {
1310  // Don't want to have to re-check that the table is hardened,
1311  // so call hardenAllPagesUnknownType on hard.
1312  hard.hardenAllPagesUnknownType(start, end);
1313  }
1314 
1316  }
1317 
1318  template <bool including_capacity=false>
1319  SYS_FORCE_INLINE void tryCompressAllPages(IDX_T start = IDX_T(0), IDX_T end = IDX_T(-1))
1320  {
1321  // Can't have compressed pages if PAGESHARDENED
1322  PageTable *pages = myImpl.getPages();
1323  if (PAGESHARDENED || (pages == nullptr))
1324  return;
1325 
1326  auto &hard = hardenTable();
1327 
1328  if (end < IDX_T(0))
1329  end = including_capacity ? pages->capacity() : pages->size();
1330 
1331  UT_ASSERT_P(start >= IDX_T(0));
1333 
1334  if (!SYSisSame<DATA_T,void>())
1335  {
1336  if (TSIZE >= 1)
1337  pages->tryCompressAllPages(start, end);
1338  else
1339  pages->tryCompressAllPages(start, end, myImpl.getTupleSize());
1340  }
1341  else
1342  {
1343  // Don't want to have to re-check that the table is hardened,
1344  // so call hardenAllPagesUnknownType on hard.
1345  hard.tryCompressAllPagesUnknownType(start, end);
1346  }
1347  }
1348 
1349  /// This is for setting the DATA_T template after checking getStorage(),
1350  /// if DATA_T wasn't already known.
1351  /// @{
1352  template<typename DEST_DATA_T>
1354  castType() const
1355  {
1357  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()) || (getStorage() != UT_Storage::INVALID && getStorage() == UT_StorageNum<DEST_DATA_T>::theStorage));
1359  }
1360  template<typename DEST_DATA_T>
1363  {
1365  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()) || (getStorage() != UT_Storage::INVALID && getStorage() == UT_StorageNum<DEST_DATA_T>::theStorage));
1367  }
1368  /// @}
1369 
1370  /// This is for setting the TSIZE template after checking getTupleSize(),
1371  /// if TSIZE wasn't already known.
1372  /// @{
1373  template<exint DEST_TSIZE>
1376  {
1377  UT_ASSERT_P(TSIZE == -1 || TSIZE == DEST_TSIZE);
1378  UT_ASSERT_P(getTupleSize() == DEST_TSIZE);
1380  }
1381  template<exint DEST_TSIZE>
1384  {
1385  UT_ASSERT_P(TSIZE == -1 || TSIZE == DEST_TSIZE);
1386  UT_ASSERT_P(getTupleSize() == DEST_TSIZE);
1388  }
1389  /// @}
1390 
1392  {
1393  return myImpl.getTupleSize();
1394  }
1396  {
1397  return myImpl.getStorage();
1398  }
1399 
1400  /// Returns true iff the specified page is constant-compressed
1402  {
1403  UT_ASSERT_P(myImpl.getPages() != nullptr);
1404  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1405  return myImpl.getPages()->getPPage(pagenum)->isConstant();
1406  }
1407 
1408  /// Returns true iff the entire array is constant & zero
1410  {
1411  // Zero capacity array is zero:
1412  if (myImpl.getPages() == nullptr)
1413  return true;
1414 
1415  const PageTable *pages = myImpl.getPages();
1416 
1417  UT_PageNum npages = numPages(pages->capacity());
1418  for (UT_PageNum pagenum = 0; pagenum < npages; pagenum++)
1419  {
1420  const PageTableEntry *page = pages->getPPage(pagenum);
1421  if (!page->isConstantAndZeroSafe())
1422  return false;
1423  }
1424  return true;
1425  }
1426 
1427  /// Sets all components of all elements of the specified page to
1428  /// the given value.
1430  {
1431  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1432  UT_ASSERT_P(myImpl.getPages() != nullptr);
1433  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1434 
1435  hardenTable();
1436 
1437  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1438  if (TSIZE >= 1)
1439  makeConstant(page, val);
1440  else
1441  makeConstant(page, val, myImpl.getTupleSize());
1442  }
1443 
1444  /// Sets all components of all elements of the specified page to
1445  /// the given value.
1446  template<typename SRC_DATA_T>
1448  {
1449  if (SYSisSame<DATA_T,void>())
1450  {
1451  SYS_CallIf<SYSisSame<DATA_T,void>()>::call([this,pagenum,&val](SYS_CALLIF_AUTO){
1452  setPageConstantUnknownType(pagenum, val);
1453  });
1454  return;
1455  }
1456 
1457  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1458 
1459  if (!SYSisSame<DATA_T,SRC_DATA_T>())
1460  {
1461  SYS_CallIf<!SYSisSame<DATA_T,SRC_DATA_T>()>::call([this,pagenum,&val](SYS_CALLIF_AUTO){
1463  for (exint i = 0; i < theSafeTupleSize; ++i)
1464  new_val[i] = UTconvertStorage<NotVoidType>(val[i]);
1465  this->setPageConstant(pagenum, new_val);
1466  });
1467  return;
1468  }
1469 
1470  UT_ASSERT_P((SYSisSame<DATA_T,SRC_DATA_T>()));
1471  UT_ASSERT_P(myImpl.getPages() != nullptr);
1472  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1473  UT_ASSERT_P(TSIZE >= 1);
1474 
1475  hardenTable();
1476 
1477  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1478  makeConstant(page, val);
1479  }
1480 
1481  /// Sets all components of all elements of the specified page to
1482  /// the given values.
1483  /// NOTE: The length of values must be equal to the tuple size.
1484  template<typename SRC_DATA_T>
1485  SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const SRC_DATA_T *values)
1486  {
1487  if (SYSisSame<DATA_T,void>())
1488  {
1489  SYS_CallIf<SYSisSame<DATA_T,void>()>::call([this,pagenum,values](SYS_CALLIF_AUTO){
1490  this->setPageConstantUnknownType(pagenum, values);
1491  });
1492  return;
1493  }
1494 
1495  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1496 
1497  if (!SYSisSame<DATA_T,SRC_DATA_T>())
1498  {
1499  SYS_CallIf<!SYSisSame<DATA_T,SRC_DATA_T>()>::call([this,pagenum,values](SYS_CALLIF_AUTO){
1500  this->setPageConstantMismatchedType(pagenum, values);
1501  });
1502  return;
1503  }
1504 
1505  UT_ASSERT_P((SYSisSame<DATA_T,SRC_DATA_T>()));
1506  UT_ASSERT_P(myImpl.getPages() != nullptr);
1507  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1508 
1509  hardenTable();
1510 
1511  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1512  // NOTE: The cast will never actually be hit unless DATA_T is SRC_DATA_T,
1513  // as asserted above. It's just to get this to compile in other cases.
1514  makeConstant(page, (const NotVoidType *)values, myImpl.getTupleSize());
1515  }
1516 
1517  /// Returns true iff the specified page is "hard",
1518  /// i.e. NOT constant-compressed and NOT shared.
1519  /// NOTE: The *table* may or may not be hard.
1521  {
1522  UT_ASSERT_P(myImpl.getPages() != nullptr);
1523  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1524  if (PAGESHARDENED)
1525  return true;
1526  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1527  return !page->isConstant() && !page->isShared();
1528  }
1529 
1530  /// Hardens the specified page if it is constant-compressed or shared.
1531  /// It returns a pointer to the data, but it will only be the correct
1532  /// type if DATA_T is not void.
1534  {
1535  UT_ASSERT_P(myImpl.getPages() != nullptr);
1536  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1537 
1538  hardenTable();
1539 
1540  PageTable *pages = myImpl.getPages();
1541  PageTableEntry *page = pages->getPPage(pagenum);
1542 
1543  if (PAGESHARDENED)
1544  return page->getFirstPtr();
1545 
1546  if (page->isConstant())
1547  hardenConstantPage(pages, page, myImpl.getTupleSize());
1548  else if (page->isShared())
1549  hardenSharedPage(pages, page, myImpl.getTupleSize());
1550  return page->getFirstPtr();
1551  }
1552 
1554  {
1555  UT_ASSERT_P(myImpl.getPages() != nullptr);
1556  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1557 
1558  hardenTable();
1559 
1560  PageTable *pages = myImpl.getPages();
1561  PageTableEntry *page = pages->getPPage(pagenum);
1562 
1563  if (PAGESHARDENED)
1564  return page->getFirstPtr();
1565 
1566  if (page->isConstant())
1567  hardenConstantPageNoInit(pages, page, myImpl.getTupleSize());
1568  else if (page->isShared())
1569  hardenSharedPageNoInit(pages, page, myImpl.getTupleSize());
1570  return page->getFirstPtr();
1571  }
1572 
1573  /// If the specified page is constant and shared, (only relevant for
1574  /// nonzero tuples that are at least the size of a pointer), this
1575  /// keeps the page constant, but unshares it, so that the constant value
1576  /// can be modified.
1577  /// NOTE: This should ONLY be called on constant pages! Check
1578  /// isPageConstant(pagenum).
1580  {
1581  UT_ASSERT_P(myImpl.getPages() != nullptr);
1582  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1583 
1584  hardenTable();
1585 
1586  PageTable *pages = myImpl.getPages();
1587  PageTableEntry *page = pages->getPPage(pagenum);
1588 
1589  UT_ASSERT_P(!PAGESHARDENED && page->isConstant());
1590 
1591  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1592  UT_ASSERT_P(TSIZE == -1 || TSIZE >= 0);
1593 
1594  const exint tuplesize = (TSIZE >= 0) ? TSIZE : myImpl.getTupleSize();
1595  if (PageTableEntry::typeFitsInline(tuplesize))
1596  return page->getInlinePtr(tuplesize);
1597 
1598  NotVoidType *src = page->getMaskedPtr();
1599  if (!page->isShared())
1600  return src;
1601 
1602  PageTableEntry newpage;
1603  newpage.alloc(UT_PageOff(1), tuplesize);
1604  NotVoidType *dest = newpage.getFirstPtr();
1605  memcpy(dest, src, tuplesize*sizeof(NotVoidType));
1606  newpage.setConstantBit();
1607  page->decRef();
1608  (*page) = newpage;
1609 
1610  return dest;
1611  }
1612 
1613  /// Checks if the specified page is all a single value,
1614  /// and if so, makes it a constant page.
1616  {
1617  // Can't have compressed pages if PAGESHARDENED
1618  PageTable *pages = myImpl.getPages();
1619  if (PAGESHARDENED || (pages == nullptr))
1620  return;
1621 
1622  auto &hard = hardenTable();
1623 
1624  UT_ASSERT_P(pagenum >= UT_PageNum(0));
1625  UT_ASSERT_P(pagenum < numPages(pages->capacity()));
1626 
1627  if (!SYSisSame<DATA_T,void>())
1628  {
1629  if (TSIZE >= 1)
1630  pages->tryCompressPage(pagenum);
1631  else
1632  pages->tryCompressPage(pagenum, myImpl.getTupleSize());
1633  }
1634  else
1635  {
1636  // Don't want to have to re-check that the table is hardened,
1637  // so call hardenAllPagesUnknownType on hard.
1638  hard.tryCompressPageUnknownType(pagenum);
1639  }
1640  }
1641 
1642  /// Returns a pointer to the data of a page.
1643  /// WARNING: DO NOT call this if DATA_T is void! The pointer returned
1644  /// can depend on the type.
1645  /// This will return nullptr if the page is constant compressed to zero.
1646  /// If it's non-nullptr and isPageConstant(pagenum) returns true, there
1647  /// will only be one element (tuple) at that address.
1648  const NotVoidType *getPageData(UT_PageNum pagenum) const
1649  {
1650  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1651  UT_ASSERT_P(myImpl.getPages() != nullptr);
1652  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1653  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1654  if (!page->isConstant())
1655  return page->getFirstPtr();
1656  if (TSIZE >= 1)
1657  return getConstantPtr(page);
1658  return getConstantPtr(page, 0, getTupleSize());
1659  }
1660 
1661  /// Returns a pointer to the data of a page.
1662  /// WARNING: DO NOT call this if DATA_T is void! The pointer returned
1663  /// can depend on the type.
1664  /// NOTE: DO NOT MODIFY THE DATA if there's any chance the page or the table
1665  /// might be shared. Check isTableHard() and isPageHard(pagenum).
1666  /// This will return nullptr if the page is constant compressed to zero.
1667  /// If it's non-nullptr and isPageConstant(pagenum) returns true, there
1668  /// will only be one element (tuple) at that address.
1670  {
1671  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1672  UT_ASSERT_P(myImpl.getPages() != nullptr);
1673  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1674  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1675  if (!page->isConstant())
1676  return page->getFirstPtr();
1677  if (TSIZE >= 1)
1678  return getConstantPtr(page);
1679  return getConstantPtr(page, 0, getTupleSize());
1680  }
1681 
1682  /// NOTE: Don't use this unless you know what you're doing!
1683  /// This is for the case where a non-POD type that requires destruction
1684  /// is masquerading as a POD type in here, e.g. GA_OffsetList as
1685  /// UT_FixedVector<int64,2>. When destroying a table, this first needs
1686  /// to be called. If it returns true, the table was shared,
1687  /// so it was just decRef'd, meaning that the data don't need to be
1688  /// destructed. If it returns false, the table was not shared,
1689  /// so the data need to be destructed before deleting the table.
1691  {
1692  if (TABLEHARDENED)
1693  return false;
1694 
1695  PageTable *table = myImpl.getPages();
1696  if (!table)
1697  return false;
1698 
1699  bool decremented = table->decRefIffShared();
1700  if (decremented)
1701  myImpl.getPages() = nullptr;
1702  return decremented;
1703  }
1704 
1705  /// NOTE: Don't use this unless you know what you're doing!
1706  /// This is for the case where a non-POD type that requires destruction
1707  /// is masquerading as a POD type in here, e.g. GA_OffsetList as
1708  /// UT_FixedVector<int64,2>. When destroying a page, this first needs
1709  /// to be called. If it returns true, the page was shared,
1710  /// so it was just decRef'd, meaning that the data don't need to be
1711  /// destructed. If it returns false, the page was not shared,
1712  /// so the data need to be destructed before deleting the page.
1714  {
1715  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
1716  UT_ASSERT_P(TSIZE >= 1);
1717  UT_ASSERT_P(myImpl.getPages() != nullptr);
1718  UT_ASSERT_P((exint)pagenum >= 0 && IDX_T((exint)pagenum << thePageBits) < size());
1719 
1720  if (PAGESHARDENED)
1721  return false;
1722 
1723  hardenTable();
1724 
1725  PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1726  if (page->isConstant())
1727  {
1728  if (sizeof(NotVoidType)*TSIZE < sizeof(PageTableEntry))
1729  return false;
1730  }
1731  bool decremented = page->decRefIffShared();
1732  if (decremented)
1733  page->initZero();
1734  return decremented;
1735  }
1736 
1737  /// Only call this if DATA_T is void and you are absolutely, 100% CERTAIN
1738  /// that the storage type is int16, int32, or int64.
1739  /// NOTE: Really! Be careful!
1741  int64 getGuaranteedInt(IDX_T i) const
1742  {
1743  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1744  UT_ASSERT_P(TSIZE == 1);
1746 
1747  UT_PageNum pagenum = pageNum(i);
1748  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1749 
1750  // If we know that the pages are hardened, we can omit
1751  // the constant page case at compile time.
1752  bool isconst = !PAGESHARDENED && page->isConstant();
1753 
1754  const UT_Storage storage = getStorage();
1755  if (!isconst)
1756  {
1757  UT_PageOff pageoff = pageOff(i);
1758  const void *first = page->getFirstPtrVoid();
1759  switch (storage)
1760  {
1761  default:
1762  UT_ASSERT_MSG_P(0, "getGuaranteedInt() only supports int16, int32, and int64.");
1764  case UT_Storage::INT16:
1765  return (int64)((int16*)first)[pageoff];
1766  case UT_Storage::INT32:
1767  return (int64)((int32*)first)[pageoff];
1768  case UT_Storage::INT64:
1769  return ((int64*)first)[pageoff];
1770  }
1771  }
1772 
1773  // Special case for zero.
1774  if (!page->isConstantZero())
1775  return int64(0);
1776 
1777  // Special case for if the constant value is too big to fit in a PageTableEntry.
1778  // At the moment, PageTableEntry is 16 bytes on 64-bit platforms,
1779  // and 8 bytes on 32-bit platforms, so this case will not occur.
1780  if ((sizeof(PageTableEntry) < 9 && storage == UT_Storage::INT64) ||
1781  (sizeof(PageTableEntry) < 5 && storage == UT_Storage::INT32))
1782  {
1783  const void *v = page->getMaskedPtrVoid();
1784  if (sizeof(PageTableEntry) < 5 && storage == UT_Storage::INT32)
1785  {
1786  // This will not be run on 64-bit platforms
1787  return (int64)*(const int32 *)v;
1788  }
1789  return *(const int64 *)v;
1790  }
1791  else
1792  {
1793  // Any types that are strictly smaller than a pointer will be stored inline.
1794  // This is checked at compile time, so there'll be no op at runtime.
1795  if (sizeof(PageTableEntry) >= 9 && storage == UT_Storage::INT64)
1796  {
1797  return *page->template castType<int64>()->getInlinePtr(1);
1798  }
1799  if (sizeof(PageTableEntry) >= 5 && storage == UT_Storage::INT32)
1800  {
1801  return (int64)*page->template castType<int32>()->getInlinePtr(1);
1802  }
1803  return (int64)*page->template castType<int16>()->getInlinePtr(1);
1804  }
1805  }
1806 
1807  /// Returns true iff there is at least one NaN value in the given range.
1808  /// Include UT_PageArrayImpl.h to call this.
1809  bool hasNanInRange(IDX_T start, IDX_T end) const;
1810 
1811 protected:
1812  template<typename DEST_DATA_T=DATA_T>
1813  DEST_DATA_T getUnknownType(IDX_T i,exint component=0) const
1814  {
1815  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1816  UT_ASSERT_P(TSIZE != 0);
1817 
1818  // Make sure common work is extracted out where possible,
1819  // to avoid duplication.
1820 
1821  UT_PageNum pagenum = pageNum(i);
1822  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
1823 
1824  // If we know that the pages are hardened, we can omit
1825  // the constant page case at compile time.
1826  bool isconst = !PAGESHARDENED && page->isConstant();
1827 
1828  const UT_Storage storage = getStorage();
1829  if (!isconst)
1830  {
1831  UT_PageOff pageoff = pageOff(i);
1832  const void *first = page->getFirstPtrVoid();
1833  exint i;
1834  if (TSIZE == 1)
1835  i = (exint)pageoff;
1836  else if (TSIZE > 1)
1837  i = TSIZE*pageoff + component;
1838  else
1839  i = myImpl.getTupleSize()*pageoff + component;
1840  switch (storage)
1841  {
1842  case UT_Storage::INVALID:
1843  UT_ASSERT_MSG_P(0, "Can't read a non-numeric type with a cast to a different type.");
1844  return DEST_DATA_T();
1845  case UT_Storage::INT8:
1846  return (DEST_DATA_T)((int8*)first)[i];
1847  case UT_Storage::UINT8:
1848  return (DEST_DATA_T)((uint8*)first)[i];
1849  case UT_Storage::INT16:
1850  return (DEST_DATA_T)((int16*)first)[i];
1851  case UT_Storage::INT32:
1852  return (DEST_DATA_T)((int32*)first)[i];
1853  case UT_Storage::INT64:
1854  return (DEST_DATA_T)((int64*)first)[i];
1855  case UT_Storage::REAL16:
1856  return (DEST_DATA_T)((fpreal16*)first)[i];
1857  case UT_Storage::REAL32:
1858  return (DEST_DATA_T)((fpreal32*)first)[i];
1859  case UT_Storage::REAL64:
1860  return (DEST_DATA_T)((fpreal64*)first)[i];
1861  }
1862  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1863  return DEST_DATA_T();
1864  }
1865 
1866  switch (storage)
1867  {
1868  case UT_Storage::INVALID:
1869  UT_ASSERT_MSG_P(0, "Can't read a non-numeric type with a cast to a different type.");
1870  return DEST_DATA_T();
1871  case UT_Storage::INT8:
1872  return (DEST_DATA_T)castType<int8>().getConstant(page->template castType<int8>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1873  case UT_Storage::UINT8:
1874  return (DEST_DATA_T)castType<uint8>().getConstant(page->template castType<uint8>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1875  case UT_Storage::INT16:
1876  return (DEST_DATA_T)castType<int16>().getConstant(page->template castType<int16>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1877  case UT_Storage::INT32:
1878  return (DEST_DATA_T)castType<int32>().getConstant(page->template castType<int32>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1879  case UT_Storage::INT64:
1880  return (DEST_DATA_T)castType<int64>().getConstant(page->template castType<int64>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1881  case UT_Storage::REAL16:
1882  return (DEST_DATA_T)castType<fpreal16>().getConstant(page->template castType<fpreal16>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1883  case UT_Storage::REAL32:
1884  return (DEST_DATA_T)castType<fpreal32>().getConstant(page->template castType<fpreal32>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1885  case UT_Storage::REAL64:
1886  return (DEST_DATA_T)castType<fpreal64>().getConstant(page->template castType<fpreal64>(), component, (TSIZE >= 1 ? TSIZE : myImpl.getTupleSize()));
1887  }
1888  return DEST_DATA_T();
1889  }
1890 
1891  template<int OP,typename SRC_DATA_T>
1892  void opUnknownType(IDX_T i,exint component,SRC_DATA_T v)
1893  {
1894  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1895  const UT_Storage storage = getStorage();
1896  switch (storage)
1897  {
1898  case UT_Storage::INVALID:
1899  UT_ASSERT_MSG_P(0, "Can't write a non-numeric type with opUnknownType.");
1900  return;
1901  case UT_Storage::INT8:
1902  castType<int8 >().template op<OP>(i,component, UTconvertStorage<int8>(v)); return;
1903  case UT_Storage::UINT8:
1904  castType<uint8 >().template op<OP>(i,component, UTconvertStorage<uint8>(v)); return;
1905  case UT_Storage::INT16:
1906  castType<int16 >().template op<OP>(i,component, UTconvertStorage<int16>(v)); return;
1907  case UT_Storage::INT32:
1908  castType<int32 >().template op<OP>(i,component, UTconvertStorage<int32>(v)); return;
1909  case UT_Storage::INT64:
1910  castType<int64 >().template op<OP>(i,component, UTconvertStorage<int64>(v)); return;
1911  case UT_Storage::REAL16:
1912  castType<fpreal16>().template op<OP>(i,component, (fpreal16)v); return;
1913  case UT_Storage::REAL32:
1914  castType<fpreal32>().template op<OP>(i,component, (fpreal32)v); return;
1915  case UT_Storage::REAL64:
1916  castType<fpreal64>().template op<OP>(i,component, (fpreal64)v); return;
1917  }
1918  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1919  }
1920  template<int OP,typename SRC_DATA_T,exint SRC_TSIZE>
1922  {
1923  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1924  const UT_Storage storage = getStorage();
1925  switch (storage)
1926  {
1927  case UT_Storage::INVALID:
1928  UT_ASSERT_MSG_P(0, "Can't write a non-numeric type with opVectorUnknownType.");
1929  return;
1930  case UT_Storage::INT8:
1931  castType<int8 >().template opVector<OP>(i,v); return;
1932  case UT_Storage::UINT8:
1933  castType<uint8 >().template opVector<OP>(i,v); return;
1934  case UT_Storage::INT16:
1935  castType<int16 >().template opVector<OP>(i,v); return;
1936  case UT_Storage::INT32:
1937  castType<int32 >().template opVector<OP>(i,v); return;
1938  case UT_Storage::INT64:
1939  castType<int64 >().template opVector<OP>(i,v); return;
1940  case UT_Storage::REAL16:
1941  castType<fpreal16>().template opVector<OP>(i,v); return;
1942  case UT_Storage::REAL32:
1943  castType<fpreal32>().template opVector<OP>(i,v); return;
1944  case UT_Storage::REAL64:
1945  castType<fpreal64>().template opVector<OP>(i,v); return;
1946  }
1947  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1948  }
1949  template<typename DEST_DATA_T,exint DEST_TSIZE>
1951  {
1952  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
1953 
1954  const UT_Storage storage = getStorage();
1955 
1956  // First, check for match on destination type
1958  return castType<DEST_DATA_T>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1959 
1960  // Then, try second-guess type (e.g. reading double as float or vice versa)
1962  return castType<typename UT_StorageNum<DEST_DATA_T>::SecondGuess>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1963 
1964  switch (storage)
1965  {
1966  case UT_Storage::INVALID:
1967  UT_ASSERT_MSG_P(0, "Can't read a non-numeric type with getVectorUnknownType.");
1968  break;
1969  case UT_Storage::INT8:
1970  return castType<int8 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1971  case UT_Storage::UINT8:
1972  return castType<uint8 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1973  case UT_Storage::INT16:
1974  return castType<int16 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1975  case UT_Storage::INT32:
1976  return castType<int32 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1977  case UT_Storage::INT64:
1978  return castType<int64 >().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1979  case UT_Storage::REAL16:
1980  return castType<fpreal16>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1981  case UT_Storage::REAL32:
1982  return castType<fpreal32>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1983  case UT_Storage::REAL64:
1984  return castType<fpreal64>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
1985  }
1986  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
1987  return UT_FixedVector<DEST_DATA_T,DEST_TSIZE>(DEST_DATA_T(0));
1988  }
1989  template<typename DEST_DATA_T,exint DEST_TSIZE>
1991  {
1992  UT_ASSERT_P(TSIZE==-1);
1993  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()));
1994 
1995  const exint tuplesize = getTupleSize();
1996 
1997  // First, check for match on destination type
1998  if (tuplesize == DEST_TSIZE)
1999  return castTupleSize<DEST_TSIZE>().template getVector<DEST_DATA_T,DEST_TSIZE>(i);
2000 
2001  // Then try small sizes
2002  if (tuplesize == 1)
2003  return castTupleSize<1>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
2004  if (tuplesize == 2)
2005  return castTupleSize<2>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
2006  if (tuplesize == 3)
2007  return castTupleSize<3>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
2008  if (tuplesize == 4)
2009  return castTupleSize<4>().template getVectorMismatchedSize<DEST_DATA_T,DEST_TSIZE>(i);
2010 
2011  // Mismatched large, unknown tuplesize (uncommon case)
2012  UT_FixedVector<DEST_DATA_T,DEST_TSIZE> v(DEST_DATA_T(0));
2013  const exint minsize = SYSmin(tuplesize,DEST_TSIZE);
2014  for (exint component = 0; component < minsize; ++component)
2015  v[component] = get<DEST_DATA_T>(i, component);
2016 
2017  return v;
2018  }
2019  template<typename DEST_DATA_T,exint DEST_TSIZE>
2021  {
2022  UT_ASSERT_P(TSIZE!=-1);
2023  UT_ASSERT_P(TSIZE!=DEST_TSIZE);
2024  UT_ASSERT_P((SYSisSame<DEST_DATA_T,DATA_T>()));
2025 
2026  // Types and tuple sizes are known, but don't match
2027  UT_PageNum pagenum = pageNum(i);
2028  const PageTableEntry *page = myImpl.getPages()->getPPage(pagenum);
2029 
2030  // If we know that the pages are hardened, we can omit
2031  // the constant page case at compile time.
2032  bool isconst = !PAGESHARDENED && page->isConstant();
2033 
2034  const NotVoidType *srctuple;
2035  if (!isconst)
2036  {
2037  UT_PageOff pageoff = pageOff(i);
2038  srctuple = (const NotVoidType *)(((const UT_FixedVector<NotVoidType,theSafeTupleSize>*)page->getFirstPtr()) + pageoff);
2039  }
2040  else
2041  {
2042  srctuple = getConstantPtr(page);
2043  if (!srctuple)
2044  return UT_FixedVector<DEST_DATA_T,DEST_TSIZE>(DEST_DATA_T(0));
2045  }
2046 
2047  if (DEST_TSIZE < TSIZE)
2048  return *(const UT_FixedVector<DEST_DATA_T,DEST_TSIZE>*)srctuple;
2049 
2051  for (exint component = 0; component < TSIZE; ++component)
2052  dsttuple[component] = srctuple[component];
2053  for (exint component = TSIZE; component < DEST_TSIZE; ++component)
2054  dsttuple[component] = DEST_DATA_T(0);
2055  return dsttuple;
2056  }
2058  {
2059  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2060  const UT_Storage storage = getStorage();
2061  switch (storage)
2062  {
2063  case UT_Storage::INVALID:
2064  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2065  return;
2066  case UT_Storage::INT8:
2067  castType<int8>().template hardenAllPages<false>(start, end); return;
2068  case UT_Storage::UINT8:
2069  castType<uint8>().template hardenAllPages<false>(start, end); return;
2070  case UT_Storage::INT16:
2071  castType<int16>().template hardenAllPages<false>(start, end); return;
2072  case UT_Storage::INT32:
2073  castType<int32>().template hardenAllPages<false>(start, end); return;
2074  case UT_Storage::INT64:
2075  castType<int64>().template hardenAllPages<false>(start, end); return;
2076  case UT_Storage::REAL16:
2077  castType<fpreal16>().template hardenAllPages<false>(start, end); return;
2078  case UT_Storage::REAL32:
2079  castType<fpreal32>().template hardenAllPages<false>(start, end); return;
2080  case UT_Storage::REAL64:
2081  castType<fpreal64>().template hardenAllPages<false>(start, end); return;
2082  }
2083  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2084  }
2085 
2087  {
2088  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2089  const UT_Storage storage = getStorage();
2090  switch (storage)
2091  {
2092  case UT_Storage::INVALID:
2093  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2094  return;
2095  case UT_Storage::INT8:
2096  castType<int8>().template tryCompressAllPages<false>(start, end); return;
2097  case UT_Storage::UINT8:
2098  castType<uint8>().template tryCompressAllPages<false>(start, end); return;
2099  case UT_Storage::INT16:
2100  castType<int16>().template tryCompressAllPages<false>(start, end); return;
2101  case UT_Storage::INT32:
2102  castType<int32>().template tryCompressAllPages<false>(start, end); return;
2103  case UT_Storage::INT64:
2104  castType<int64>().template tryCompressAllPages<false>(start, end); return;
2105  case UT_Storage::REAL16:
2106  castType<fpreal16>().template tryCompressAllPages<false>(start, end); return;
2107  case UT_Storage::REAL32:
2108  castType<fpreal32>().template tryCompressAllPages<false>(start, end); return;
2109  case UT_Storage::REAL64:
2110  castType<fpreal64>().template tryCompressAllPages<false>(start, end); return;
2111  }
2112  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2113  }
2114 
2116  {
2117  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2118  const UT_Storage storage = getStorage();
2119  switch (storage)
2120  {
2121  case UT_Storage::INVALID:
2122  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2123  return;
2124  case UT_Storage::INT8:
2125  castType<int8>().tryCompressPage(pagenum); return;
2126  case UT_Storage::UINT8:
2127  castType<uint8>().tryCompressPage(pagenum); return;
2128  case UT_Storage::INT16:
2129  castType<int16>().tryCompressPage(pagenum); return;
2130  case UT_Storage::INT32:
2131  castType<int32>().tryCompressPage(pagenum); return;
2132  case UT_Storage::INT64:
2133  castType<int64>().tryCompressPage(pagenum); return;
2134  case UT_Storage::REAL16:
2135  castType<fpreal16>().tryCompressPage(pagenum); return;
2136  case UT_Storage::REAL32:
2137  castType<fpreal32>().tryCompressPage(pagenum); return;
2138  case UT_Storage::REAL64:
2139  castType<fpreal64>().tryCompressPage(pagenum); return;
2140  }
2141  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2142  }
2143 
2144  template<typename SRC_DATA_T>
2146  {
2147  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2148  const UT_Storage storage = getStorage();
2149  switch (storage)
2150  {
2151  case UT_Storage::INVALID:
2152  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2153  return;
2154  case UT_Storage::INT8:
2155  castType<int8>().setPageConstant(pagenum, val); return;
2156  case UT_Storage::UINT8:
2157  castType<uint8>().setPageConstant(pagenum, val); return;
2158  case UT_Storage::INT16:
2159  castType<int16>().setPageConstant(pagenum, val); return;
2160  case UT_Storage::INT32:
2161  castType<int32>().setPageConstant(pagenum, val); return;
2162  case UT_Storage::INT64:
2163  castType<int64>().setPageConstant(pagenum, val); return;
2164  case UT_Storage::REAL16:
2165  castType<fpreal16>().setPageConstant(pagenum, val); return;
2166  case UT_Storage::REAL32:
2167  castType<fpreal32>().setPageConstant(pagenum, val); return;
2168  case UT_Storage::REAL64:
2169  castType<fpreal64>().setPageConstant(pagenum, val); return;
2170  }
2171  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2172  }
2173 
2174  template<typename SRC_DATA_T>
2175  void setPageConstantUnknownType(UT_PageNum pagenum, const SRC_DATA_T *values)
2176  {
2177  UT_ASSERT_P((SYSisSame<DATA_T,void>()));
2178  const UT_Storage storage = getStorage();
2179  switch (storage)
2180  {
2181  case UT_Storage::INVALID:
2182  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
2183  return;
2184  case UT_Storage::INT8:
2185  castType<int8>().setPageConstant(pagenum, values); return;
2186  case UT_Storage::UINT8:
2187  castType<uint8>().setPageConstant(pagenum, values); return;
2188  case UT_Storage::INT16:
2189  castType<int16>().setPageConstant(pagenum, values); return;
2190  case UT_Storage::INT32:
2191  castType<int32>().setPageConstant(pagenum, values); return;
2192  case UT_Storage::INT64:
2193  castType<int64>().setPageConstant(pagenum, values); return;
2194  case UT_Storage::REAL16:
2195  castType<fpreal16>().setPageConstant(pagenum, values); return;
2196  case UT_Storage::REAL32:
2197  castType<fpreal32>().setPageConstant(pagenum, values); return;
2198  case UT_Storage::REAL64:
2199  castType<fpreal64>().setPageConstant(pagenum, values); return;
2200  }
2201  UT_ASSERT_MSG_P(0, "Unhandled UT_Storage enum value!");
2202  }
2203 
2204  template<typename SRC_DATA_T>
2205  void setPageConstantMismatchedType(UT_PageNum pagenum, const SRC_DATA_T *values)
2206  {
2207  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2208  UT_ASSERT_P(!(SYSisSame<DATA_T,SRC_DATA_T>()));
2209 
2210  // This could be done directly, instead of converting
2211  // to a UT_SmallArray<DATA_T>, but this approach is simpler for now.
2212 
2213  UT_SmallArray<NotVoidType, ((TSIZE >= 1) ? TSIZE : 16)*sizeof(NotVoidType)> dest_values;
2214  const exint tuplesize = myImpl.getTupleSize();
2215  dest_values.setSize(tuplesize);
2216 
2217  for (exint i = 0; i < tuplesize; ++i)
2218  dest_values(i) = UTconvertStorage<NotVoidType>(values[i]);
2219 
2220  setPageConstant(pagenum, dest_values.array());
2221  }
2222 
2224  {
2225  public:
2228  static const uintptr_t theConstantPageBit = 1;
2229 
2230  /// This is always valid to call.
2232  {
2233  return (uintptr_t(myData) & theConstantPageBit) != 0;
2234  }
2235  /// This is only valid to call if the type doesn't fit inline.
2237  {
2238  return (uintptr_t(myData) == theConstantPageBit);
2239  }
2240  /// This is always valid to call, and will return true
2241  /// iff this is either inline and all zero or
2242  /// using the constant zero optimization.
2244  {
2245  return (uintptr_t(myData) == theConstantPageBit)
2246  && (myPRefCount == nullptr);
2247  }
2248  /// This is only valid to call when DATA_T is non-void.
2249  SYS_FORCE_INLINE static bool typeFitsInline(exint tuplesize = TSIZE)
2250  {
2252  if (sizeof(PageTableEntry) <= sizeof(NotVoidType))
2253  return false;
2254  return (sizeof(NotVoidType)*((TSIZE >= 0) ? TSIZE : tuplesize) < sizeof(PageTableEntry));
2255  }
2256 
2257  /// This is always valid to call, but be aware that the parameter is the
2258  /// number of *bytes* in a tuple, not the number of components in a
2259  /// tuple.
2260  SYS_FORCE_INLINE bool isRefd(exint tuplebytes) const
2261  {
2262  if (!isConstant())
2263  return true;
2264 
2265  // This will be checked at compile time, when possible.
2266  if (sizeof(PageTableEntry) > ((TSIZE >= 0 && !SYSisSame<DATA_T,void>()) ? TSIZE*sizeof(NotVoidType) : tuplebytes))
2267  return false;
2268 
2269  return !isConstantZero();
2270  }
2271 
2272  /// NOTE: This should only be called when there's an actual allocated page,
2273  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2275  {
2276  UT_ASSERT_P(myPRefCount->relaxedLoad() > 0);
2277  return myPRefCount->relaxedLoad() != 1;
2278  }
2279 
2280  /// NOTE: This should only be called when there's an actual allocated page to decRef,
2281  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2283  {
2284  myPRefCount->add(1);
2285  }
2286 
2287  /// NOTE: This should only be called when there's an actual allocated page to decRef,
2288  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2290  {
2291  int64 new_count = myPRefCount->add(-1);
2292  UT_ASSERT_P(new_count >= 0);
2293  if (new_count == 0)
2294  {
2295  free((NotVoidType*)(uintptr_t(myData) & ~theConstantPageBit));
2296  free(myPRefCount);
2297 #if 0
2298  printf("Free page %d pages total\n", int(thePageTablePageCount.add(-1)));
2299  fflush(stdout);
2300 #endif
2301  }
2302  }
2303 
2304  /// NOTE: This should only be called when there's an actual allocated page,
2305  /// i.e. !isConstant() || (!typeFitsInline(tuplesize) && !isConstantZero()).
2307  {
2308  UT_ASSERT_P(myPRefCount->relaxedLoad() > 0);
2309  return myPRefCount->relaxedLoad();
2310  }
2311 
2312  SYS_FORCE_INLINE void alloc(UT_PageOff nelements, exint tuplesize = TSIZE)
2313  {
2315  myData = (NotVoidType *)malloc(nelements*((TSIZE >= 0) ? TSIZE : tuplesize)*sizeof(NotVoidType));
2316  myPRefCount = (SYS_AtomicCounter *)malloc(sizeof(SYS_AtomicCounter));
2317  myPRefCount->relaxedStore(1);
2318 #if 0
2319  printf("Alloc page %d pages total\n", int(thePageTablePageCount.add(1)));
2320  fflush(stdout);
2321 #endif
2322  }
2323  void realloc(UT_PageOff sizetocopy, UT_PageOff newpagecapacity, exint tuplesize = TSIZE)
2324  {
2326  UT_ASSERT_P(sizetocopy >= 0 && sizetocopy <= newpagecapacity);
2327  UT_ASSERT_P(!isConstant());
2328 
2329  UT_ASSERT_P(tuplesize >= 0);
2330  // This is just to try to force the compiler to constant-value-optimize.
2331  if (TSIZE >= 0)
2332  tuplesize = TSIZE;
2333 
2334  UT_ASSERT_P(myPRefCount->relaxedLoad() > 0);
2335  if (myPRefCount->relaxedLoad() > 1)
2336  {
2337  // If shared, need to make a new page.
2338  PageTableEntry newpage;
2339  newpage.alloc(newpagecapacity, tuplesize);
2340  NotVoidType *__restrict newdata = newpage.myData;
2341  const NotVoidType *__restrict olddata = myData;
2342  // Yes, this loop is an inline memcpy, because memcpy won't inline,
2343  // but the compiler will optimize it if it's like this.
2344  sizetocopy *= tuplesize;
2345  while (sizetocopy)
2346  {
2347  *newdata = *olddata;
2348  ++newdata;
2349  ++olddata;
2350  --sizetocopy;
2351  }
2352  decRef();
2353  myData = newpage.myData;
2354  myPRefCount = newpage.myPRefCount;
2355  }
2356  else
2357  {
2358  // If not shared, we can potentially reuse the same memory.
2359  myData = (NotVoidType*)::realloc(myData, newpagecapacity*(tuplesize*sizeof(NotVoidType)));
2360  // NOTE: Reference count doesn't need to change, since it's 1.
2361  }
2362  }
2363 
2364  /// Atomically decrement the reference count iff the page is shared.
2365  /// See the description of decRefIffSharedPage() above for
2366  /// why this function exists.
2368  bool decRefIffShared(exint tuplesize = TSIZE)
2369  {
2370  if (isConstant())
2371  {
2372  UT_ASSERT_P(tuplesize >= 0);
2373  if (typeFitsInline(tuplesize) || isConstantZero())
2374  return false;
2375  }
2376  return decCounterIffAbove1(*myPRefCount);
2377  }
2378 
2379  /// Returns the data pointer, if not a constant page
2382  {
2383  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtr() is for non-constant pages");
2385 
2386  return myData;
2387  }
2388 
2389  /// Returns the data pointer, if not a constant page
2391  const NotVoidType *getFirstPtr() const
2392  {
2393  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtr() is for non-constant pages");
2395 
2396  return myData;
2397  }
2398 
2399  /// Returns the data pointer, if not a constant page
2402  {
2403  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtrVoid() is for non-constant pages");
2404 
2405  return myData;
2406  }
2407 
2408  /// Returns the data pointer, if not a constant page
2410  const void *getFirstPtrVoid() const
2411  {
2412  UT_ASSERT_MSG_P(!isConstant(), "getFirstPtrVoid() is for non-constant pages");
2413 
2414  return myData;
2415  }
2416 
2417  /// Returns the data pointer, or whatever's in its place
2418  /// with bit 0 set to 1 if it's something constant.
2419  /// If it's constant, it might not be a flagged pointer.
2422  {
2423  return myData;
2424  }
2425 
2426  /// Returns the data pointer, or whatever's in its place
2427  /// with bit 0 set to 1 if it's something constant.
2428  /// If it's constant, it might not be a flagged pointer.
2430  const void *getFirstPtrVoidUnsafe() const
2431  {
2432  return myData;
2433  }
2434 
2435  /// Returns the data pointer, if an inline constant page
2438  {
2439  UT_ASSERT_MSG_P(isConstant() && typeFitsInline(tuplesize), "getInlinePtr() is for inline constant pages");
2441  return (NotVoidType *)(((int8 *)(this+1)) - (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize)));
2442  }
2443 
2444  /// Returns the data pointer, if an inline constant page
2446  const NotVoidType *getInlinePtr(exint tuplesize) const
2447  {
2448  UT_ASSERT_MSG_P(isConstant() && typeFitsInline(tuplesize), "getInlinePtr() is for inline constant pages");
2450  return (const NotVoidType *)(((const int8 *)(this+1)) - (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize)));
2451  }
2452 
2453  /// Returns the data pointer, if a non-inline constant page.
2454  /// Returns nullptr for a zero constant page.
2457  {
2458  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtr() is for constant pages");
2460 
2461  return (NotVoidType *)(uintptr_t(myData) & ~theConstantPageBit);
2462  }
2463 
2464  /// Returns the data pointer, if a non-inline constant page.
2465  /// Returns nullptr for a zero constant page.
2467  const NotVoidType *getMaskedPtr() const
2468  {
2469  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtr() is for constant pages");
2471 
2472  return (const NotVoidType *)(uintptr_t(myData) & ~theConstantPageBit);
2473  }
2474 
2475  /// Returns the data pointer, if a non-inline constant page.
2476  /// Returns nullptr for a zero constant page.
2479  {
2480  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtrVoid() is for constant pages");
2481 
2482  return (void *)(uintptr_t(myData) & ~theConstantPageBit);
2483  }
2484 
2485  /// Returns the data pointer, if a non-inline constant page.
2486  /// Returns nullptr for a zero constant page.
2488  const void *getMaskedPtrVoid() const
2489  {
2490  UT_ASSERT_MSG_P(isConstant(), "getMaskedPtrVoid() is for constant pages");
2491 
2492  return (const void *)(uintptr_t(myData) & ~theConstantPageBit);
2493  }
2494 
2495  /// Initializes a page to constant zero, assuming that it hasn't been
2496  /// initialized yet.
2498  void initZero()
2499  {
2500  myData = (NotVoidType*)theConstantPageBit;
2501  myPRefCount = nullptr;
2502  }
2503 
2506  {
2507  myData = (NotVoidType *)(uintptr_t(myData) | theConstantPageBit);
2508  }
2509 
2510  void tryCompressPage(UT_PageOff pagesize, exint tuplesize = TSIZE)
2511  {
2512  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2513  UT_ASSERT_P(!isConstant());
2514 
2515  tuplesize = (TSIZE >= 0 ? TSIZE : tuplesize);
2516 
2517  const NotVoidType *const first = myData;
2518  const NotVoidType *const end = first + (TSIZE >= 0 ? TSIZE : tuplesize)*exint(pagesize);
2519  for (const NotVoidType *other = first+tuplesize; other < end; other += tuplesize)
2520  {
2521  for (exint component = 0; component < (TSIZE >= 0 ? TSIZE : tuplesize); ++component)
2522  {
2523  if (other[component] != first[component])
2524  return;
2525  }
2526  }
2527  // We have a constant page! Compress it!
2528 
2529  PageTableEntry newpage;
2530  newpage.initZero();
2531  const exint bytesize = sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize);
2532  // Value stored inline if it's strictly smaller than a pointer
2533  // The first check can be done at compile time, so it's only logically redundant,
2534  // not functionally redundant.
2535  if (sizeof(PageTableEntry) > sizeof(NotVoidType) && sizeof(PageTableEntry) > bytesize)
2536  {
2537  memcpy(((int8*)&newpage) + (sizeof(PageTableEntry) - bytesize),
2538  first, bytesize);
2539  }
2540  else
2541  {
2542  if (!isZero(first, (TSIZE >= 0 ? TSIZE : tuplesize)))
2543  {
2544  newpage.alloc(UT_PageOff(1), tuplesize);
2545  NotVoidType *newfirst = newpage.getFirstPtr();
2546  memcpy(newfirst, first, bytesize);
2547  newpage.setConstantBit();
2548  }
2549  }
2550  decRef();
2551  *this = newpage;
2552  }
2553 
2554  /// NOTE: This always excludes sizeof(*this), and is only valid for
2555  /// non-small pages, (okay if constant).
2557  {
2558  if (!isConstant())
2559  return sizeof(SYS_AtomicCounter) + tuplebytes*thePageSize;
2560  if (tuplebytes < sizeof(PageTableEntry) || isConstantZero())
2561  return 0;
2562  return sizeof(SYS_AtomicCounter) + tuplebytes;
2563  }
2564 
2565  template<typename DEST_DATA_T>
2568  castType() const
2569  {
2572  }
2573  template<typename DEST_DATA_T>
2577  {
2580  }
2581 
2582  /// NOTE: This just does a shallow comparison.
2584  bool operator==(const ThisType &that) const
2585  {
2586  return (myData == that.myData) && (myPRefCount == that.myPRefCount);
2587  }
2588  /// NOTE: This just does a shallow comparison.
2590  bool operator!=(const ThisType &that) const
2591  {
2592  return (myData != that.myData) || (myPRefCount != that.myPRefCount);
2593  }
2594 
2595  private:
2596  NotVoidType *myData;
2597  SYS_AtomicCounter *myPRefCount;
2598  };
2599 
2602  {
2603  exint cur_count = counter.relaxedLoad();
2604 
2605  // It should be rare for this to ever loop, because looping
2606  // means that something else referring to the same page has
2607  // done an incRef, decRef, or decRefIffShared on this same page,
2608  // between the relaxedLoad and the compare_swap calls, or on
2609  // subsequent loops, between the compare_swap calls.
2610  while (true)
2611  {
2612  if (cur_count == 1)
2613  return false; // Not shared, so don't decrement
2614  exint new_count = cur_count-1;
2615  exint prev_count = counter.compare_swap(cur_count, new_count);
2616  if (prev_count == cur_count)
2617  return true; // Successfully decremented
2618  cur_count = prev_count;
2619  }
2620  // Execution never gets here, because the only escapes from
2621  // the loops return.
2622  }
2623 
2624  SYS_FORCE_INLINE static NotVoidType getConstant(const PageTableEntry *const page,exint component,exint tuplesize = TSIZE)
2625  {
2626  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2627  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2628  UT_ASSERT_P(component >= 0 && component < tuplesize);
2629  UT_ASSERT_P(page->isConstant());
2630 
2631  // Any types whose tuples are strictly smaller than a pointer will be stored inline.
2632  // If a single one is at least as large as a pointer, it's
2633  // rejected at compile time, so there'll be no op at runtime in that case.
2634  if (PageTableEntry::typeFitsInline(tuplesize))
2635  {
2636  return page->getInlinePtr(tuplesize)[component];
2637  }
2638 
2639  const NotVoidType *masked = page->getMaskedPtr();
2640  if (!masked)
2641  {
2642  NotVoidType zero;
2643  memset(&zero, 0, sizeof(NotVoidType));
2644  return zero;
2645  }
2646 
2647  // Other types have a 1-element page.
2648  return masked[component];
2649  }
2650  /// NOTE: It may seem like overkill to take a PageTableEntry *, but the
2651  /// returned pointer may be *inside* the pointer that the caller owns,
2652  /// so we need the caller's pointer, not a copy. getConstant doesn't
2653  /// have this issue, since it can use the local pointer and read the
2654  /// value.
2655  SYS_FORCE_INLINE static const NotVoidType *getConstantPtr(const PageTableEntry *page,exint component = 0,exint tuplesize = TSIZE)
2656  {
2657  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2658  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2659  UT_ASSERT_P(component >= 0 && component < tuplesize);
2660  UT_ASSERT_P(page->isConstant());
2661 
2662  // Any types whose tuples are strictly smaller than PageTableEntry will be stored inline.
2663  if (PageTableEntry::typeFitsInline(tuplesize))
2664  {
2665  return page->getInlinePtr(tuplesize) + component;
2666  }
2667 
2668  const NotVoidType *masked = page->getMaskedPtr();
2669  if (!masked)
2670  return nullptr;
2671 
2672  // Other types have a 1-element page.
2673  return masked + component;
2674  }
2675  /// NOTE: It may seem like overkill to take a PageTableEntry *, but the
2676  /// returned pointer may be *inside* the pointer that the caller owns,
2677  /// so we need the caller's pointer, not a copy. getConstant doesn't
2678  /// have this issue, since it can use the local pointer and read the
2679  /// value.
2680  SYS_FORCE_INLINE static NotVoidType *getConstantPtr(PageTableEntry *page,exint component = 0,exint tuplesize = TSIZE)
2681  {
2682  return SYSconst_cast(getConstantPtr((const PageTableEntry *)page, component, tuplesize));
2683  }
2685  {
2686  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2687  UT_ASSERT_P(TSIZE >= 1);
2688  UT_ASSERT_P(!PAGESHARDENED);
2689 
2690  if (PAGESHARDENED)
2691  return;
2692 
2694 
2695  // Unref existing non-constant page
2696  bool wasconst = page->isConstant();
2697  if (!wasconst)
2698  page->decRef();
2699 
2700  // Easy for inline case, checked at compile-time.
2701  if (sizeof(PageTableEntry) > sizeof(Tuple))
2702  {
2703  page->initZero();
2704  NotVoidType *tuple = page->getInlinePtr(theSafeTupleSize);
2705  *(Tuple *)tuple = val;
2706  return;
2707  }
2708 
2709  bool iszero = val.isZero();
2710 
2711  // In other cases, we may or may not have to unref constant page
2712  if (wasconst)
2713  {
2714  NotVoidType *masked = page->getMaskedPtr();
2715  if (!masked)
2716  {
2717  // Fairly common case: already zero, making zero.
2718  if (iszero)
2719  return;
2720  }
2721  else if (iszero || (page->isShared() && (*(const Tuple*)masked != val)))
2722  {
2723  // No longer need this old constant page
2724  page->decRef();
2725  }
2726  else if (!page->isShared())
2727  {
2728  // Reuse the unshared constant page
2729  *(Tuple*)masked = val;
2730  return;
2731  }
2732  else
2733  {
2734  // Already equal; nothing to do
2735  UT_ASSERT_P(*(const Tuple*)masked == val);
2736  return;
2737  }
2738  }
2739 
2740  if (iszero)
2741  {
2742  page->initZero();
2743  return;
2744  }
2745 
2746  // Need to allocate new constant page
2747  PageTableEntry newpage;
2748  newpage.alloc(UT_PageOff(1), theSafeTupleSize);
2749  NotVoidType *tuple = newpage.getFirstPtr();
2750  (*(Tuple*)tuple) = val;
2751  newpage.setConstantBit();
2752  (*page) = newpage;
2753  }
2754 
2755  static void makeConstant(PageTableEntry *page, const NotVoidType &val, exint tuplesize = TSIZE)
2756  {
2757  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2758  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2759  UT_ASSERT_P(!PAGESHARDENED);
2760  UT_ASSERT_P(tuplesize >= 1);
2761 
2762  if (PAGESHARDENED)
2763  return;
2764 
2765  // Unref existing non-constant page
2766  bool wasconst = page->isConstant();
2767  if (!wasconst)
2768  page->decRef();
2769 
2770  // Easy for inline case, partly checked at compile-time.
2771  if (PageTableEntry::typeFitsInline(tuplesize))
2772  {
2773  page->initZero();
2774  NotVoidType *tuple = page->getInlinePtr(tuplesize);
2775  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2776  tuple[i] = val;
2777  return;
2778  }
2779 
2780  bool iszero = isZero(val);
2781 
2782  // In other cases, we may or may not have to unref constant page
2783  if (wasconst)
2784  {
2785  NotVoidType *masked = page->getMaskedPtr();
2786  if (!masked)
2787  {
2788  // Fairly common case: already zero, making zero.
2789  if (iszero)
2790  return;
2791  }
2792  else if (iszero)
2793  {
2794  // No longer need this old constant page
2795  page->decRef();
2796  }
2797  else if (page->isShared())
2798  {
2799  bool equal = true;
2800  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2801  {
2802  if (masked[i] != val)
2803  {
2804  equal = false;
2805  break;
2806  }
2807  }
2808 
2809  if (equal)
2810  {
2811  // Already equal; nothing to do
2812  return;
2813  }
2814 
2815  // No longer need this old constant page
2816  page->decRef();
2817  }
2818  else
2819  {
2820  // Reuse the unshared constant page
2821  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2822  masked[i] = val;
2823  return;
2824  }
2825  }
2826 
2827  if (iszero)
2828  {
2829  page->initZero();
2830  return;
2831  }
2832 
2833  // Need to allocate new constant page
2834  PageTableEntry newpage;
2835  newpage.alloc(UT_PageOff(1), tuplesize);
2836  NotVoidType *tuple = newpage.getFirstPtr();
2837  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2838  tuple[i] = val;
2839  newpage.setConstantBit();
2840  (*page) = newpage;
2841  }
2842 
2843  static void makeConstant(PageTableEntry *page, const NotVoidType *values, exint tuplesize)
2844  {
2845  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2846  UT_ASSERT_P(TSIZE == -1 || TSIZE == tuplesize);
2847  UT_ASSERT_P(!PAGESHARDENED);
2848  UT_ASSERT_P(tuplesize >= 1);
2849 
2850  if (PAGESHARDENED)
2851  return;
2852 
2853  // Unref existing non-constant page
2854  bool wasconst = page->isConstant();
2855  if (!wasconst)
2856  page->decRef();
2857 
2858  // Easy for inline case, partly checked at compile-time.
2859  if (PageTableEntry::typeFitsInline(tuplesize))
2860  {
2861  page->initZero();
2862  NotVoidType *tuple = page->getInlinePtr(tuplesize);
2863  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2864  tuple[i] = values[i];
2865  return;
2866  }
2867 
2868  bool iszero = isZero(values, (TSIZE >= 0 ? TSIZE : tuplesize));
2869 
2870  // In other cases, we may or may not have to unref constant page
2871  if (wasconst)
2872  {
2873  NotVoidType *masked = page->getMaskedPtr();
2874  if (!masked)
2875  {
2876  // Fairly common case: already zero, making zero.
2877  if (iszero)
2878  return;
2879  }
2880  else if (iszero)
2881  {
2882  // No longer need this old constant page
2883  page->decRef();
2884  }
2885  else if (page->isShared())
2886  {
2887  bool equal = true;
2888  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2889  {
2890  if (masked[i] != values[i])
2891  {
2892  equal = false;
2893  break;
2894  }
2895  }
2896 
2897  if (equal)
2898  {
2899  // Already equal; nothing to do
2900  return;
2901  }
2902 
2903  // No longer need this old constant page
2904  page->decRef();
2905  }
2906  else
2907  {
2908  // Reuse the unshared constant page
2909  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2910  masked[i] = values[i];
2911  return;
2912  }
2913  }
2914 
2915  if (iszero)
2916  {
2917  page->initZero();
2918  return;
2919  }
2920 
2921  // Need to allocate new constant page
2922  PageTableEntry newpage;
2923  newpage.alloc(UT_PageOff(1), tuplesize);
2924  NotVoidType *tuple = newpage.getFirstPtr();
2925  for (exint i = 0; i < (TSIZE >= 0 ? TSIZE : tuplesize); ++i)
2926  tuple[i] = values[i];
2927  newpage.setConstantBit();
2928  (*page) = newpage;
2929  }
2930 
2931  /// This replaces dest with src. It references if possible and keeps
2932  /// dest constant if easy. If desttuplesize > srctuplesize, it only
2933  /// replaces the first srctuplesize components.
2934  template<typename SrcType>
2935  static void replacePage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destpagesize, UT_PageOff destpagecapacity);
2936 
2937  /// This replaces part of dest with part of src. It keeps
2938  /// dest constant if easy. If desttuplesize > srctuplesize, it only
2939  /// replaces the first srctuplesize components.
2940  /// This will *not* check to see if the whole page is being replaced,
2941  /// so if you want the src page to be referenced, or overwrite a
2942  /// constant value, when writing the whole page, use replacePage.
2943  template<typename SrcType>
2944  static void copyPartialPage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destoff, UT_PageOff srcoff, UT_PageOff ntuples, UT_PageOff destpagecapacity);
2945 
2946  /// This version of makeConstant copies a const page onto this page,
2947  /// converting type if necessary, and supporting a different tuple size,
2948  /// so long as the destination size either isn't larger or is constant.
2949  /// NOTE: This would be templated on the actual templates of UT_PageArray,
2950  /// but somehow, Visual C++ can't deduce the type, so the template type
2951  /// has to be specified by the caller, and one argument is much easier
2952  /// to specify on every call than 4.
2953  template<typename SrcType>
2954  static void makeConstantFrom(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize);
2955 
2956  /// This is just a helper function for writing a single source tuple (or nullptr) to a block of tuple data, with conversion.
2957  /// NOTE: destpagedata must point to an *UNSHARED* page's data.
2958  template<typename SrcNotVoidType>
2959  static void fillNonConstWithConst(NotVoidType *destpagedata, NotVoidType *destpageend, const SrcNotVoidType *stuple, exint mintuplesize, exint desttupleextra);
2960 
2961  /// This is just a helper function for copying a non-constant source page to a non-constant destination page, with conversion.
2962  /// NOTE: destpagedata must point to an *UNSHARED* page's data.
2963  /// NOTE: The two data ranges must be *NON-OVERLAPPING*.
2964  template<typename SrcNotVoidType>
2965  static void copyNonConst(NotVoidType *destpagedata, const SrcNotVoidType *srcpagedata, exint desttuplesize, exint srctuplesize, UT_PageOff ntuples);
2966 
2967  /// This is just a helper function for checking whether two constant pages are equal, up to
2968  /// the smaller of the two, to see if writing is necessary.
2969  template<typename SrcNotVoidType>
2970  static bool isEqualConst(const NotVoidType *tuple, const SrcNotVoidType *stuple, exint mintuplesize);
2971 
2972  static void hardenConstantPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
2973  {
2974  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
2975  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
2976  UT_ASSERT_P(page->isConstant());
2977 
2978  PageTableEntry newpage;
2979  newpage.alloc(pagecapacity, tuplesize);
2980  NotVoidType *dest = newpage.getFirstPtr();
2981  bool isinline = PageTableEntry::typeFitsInline(tuplesize);
2982  if (isinline ? page->isConstantAndZeroSafe() : page->isConstantZero())
2983  {
2984  memset(dest, 0, (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize))*pagecapacity);
2985  }
2986  else
2987  {
2988  const NotVoidType *src;
2989  if (isinline)
2990  src = page->getInlinePtr(tuplesize);
2991  else
2992  src = page->getMaskedPtr();
2993 
2994  for (UT_PageOff i = 0; i < pagecapacity; ++i)
2995  {
2996  for (exint component = 0; component < tuplesize; ++component, ++dest)
2997  *dest = src[component];
2998  }
2999 
3000  if (!isinline)
3001  page->decRef();
3002  }
3003  (*page) = newpage;
3004  }
3005 
3006  static void hardenConstantPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
3007  {
3008  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
3009  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3010  UT_ASSERT_P(page->isConstant());
3011 
3012  PageTableEntry newpage;
3013  newpage.alloc(pagecapacity, tuplesize);
3014  // Compile-time check, to speed up case where even one element doesn't fit inline.
3015  bool isinline = PageTableEntry::typeFitsInline(tuplesize);
3016  if (!isinline && !page->isConstantZero())
3017  page->decRef();
3018  (*page) = newpage;
3019  }
3020 
3021  static void hardenSharedPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
3022  {
3023  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
3024  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3025  UT_ASSERT_P(!page->isConstant());
3026  // This may technically no longer be shared between checking and calling
3027  // due to other threads decRef'ing the page. It's okay to harden anyway.
3028  //UT_ASSERT_P(page->isShared());
3029 
3030  PageTableEntry newpage;
3031  newpage.alloc(pagecapacity, tuplesize);
3032  NotVoidType *dest = newpage.getFirstPtr();
3033  const NotVoidType *src = page->getFirstPtr();
3034  memcpy(dest, src, (sizeof(NotVoidType)*(TSIZE >= 0 ? TSIZE : tuplesize))*pagecapacity);
3035  page->decRef();
3036  (*page) = newpage;
3037  }
3038 
3039  static void hardenSharedPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize = TSIZE)
3040  {
3041  UT_ASSERT_P(TSIZE == -1 || tuplesize == TSIZE);
3042  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3043  UT_ASSERT_P(!page->isConstant());
3044  // This may technically no longer be shared between checking and calling
3045  // due to other threads decRef'ing the page. It's okay to harden anyway.
3046  //UT_ASSERT_P(page->isShared());
3047 
3048  PageTableEntry newpage;
3049  newpage.alloc(pagecapacity, tuplesize);
3050  page->decRef();
3051  (*page) = newpage;
3052  }
3053 
3055  {
3056  public:
3059 
3061  {
3062  UT_ASSERT_P(myCapacity > IDX_T(0));
3063  return reinterpret_cast<PageTableEntry*>(this+1);
3064  }
3066  {
3067  UT_ASSERT_P(myCapacity > IDX_T(0));
3068  return reinterpret_cast<const PageTableEntry *>(this+1);
3069  }
3070  // NOTE: Clang sometimes decides to ignore const references and pass by
3071  // value, so we have to return by pointer, in case the page
3072  // is constant and inlined, so that the address of the constant
3073  // value is accessible through this function.
3074  // Yes, many cases don't rely on this, but to reduce the risk of
3075  // bugs, the only way to access page pointers in a const PageTable
3076  // is to call this to get the page pointer address and explicitly
3077  // dereference, if valid in the given circumstance.
3079  {
3080  UT_ASSERT_P(myCapacity > IDX_T(i << THEPAGEBITS));
3081  return reinterpret_cast<const PageTableEntry *>(this+1) + i;
3082  }
3084  {
3085  UT_ASSERT_P(myCapacity > IDX_T(i << THEPAGEBITS));
3086  return reinterpret_cast<PageTableEntry*>(this+1) + i;
3087  }
3089  {
3090  UT_ASSERT_P(myRefCount.relaxedLoad() > 0);
3091  return myRefCount.relaxedLoad() != 1;
3092  }
3094  {
3095  return myRefCount.relaxedLoad();
3096  }
3097  /// NOTE: This is the size of the full array, not the number of pages.
3098  SYS_FORCE_INLINE IDX_T size() const
3099  {
3100  return mySize;
3101  }
3102  /// NOTE: This is the capacity of the full array, not the capacity of pages.
3104  {
3105  return myCapacity;
3106  }
3107  /// NOTE: This will *assume* the capacity is large enough! The caller *must* check!
3108  /// NOTE: This is the size of the full array, not the number of pages.
3109  SYS_FORCE_INLINE void setSize(IDX_T newsize)
3110  {
3111  UT_ASSERT_P(newsize <= capacity());
3112  UT_ASSERT_MSG_P(myRefCount.relaxedLoad() == 1, "The table must already be hardened before we modify it!");
3113  if (PAGESHARDENED && newsize > mySize)
3114  {
3115  UT_ASSERT_P(TSIZE >= 0);
3116  // Need to ensure pages between mySize and newsize are hardened.
3117  // Pages containing offsets less than or equal to mySize are
3118  // already guaranteed to be hardened.
3119  UT_PageNum startpage = numPages(mySize);
3120  UT_PageNum endpage = numPages(newsize);
3121  UT_PageOff pagecapacity(thePageSize);
3122  if (endpage == UT_PageNum(1) && exint(capacity()) < thePageSize)
3123  pagecapacity = capacity();
3124  PageTableEntry *page = getPPage(startpage);
3125  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3126  {
3127  if (page->isConstant())
3128  hardenConstantPage(page, pagecapacity);
3129  else if (page->isShared())
3130  hardenSharedPage(page, pagecapacity);
3131  }
3132  }
3133  mySize = newsize;
3134  }
3135  /// NOTE: This will *assume* the capacity is large enough! The caller *must* check!
3136  /// NOTE: This is the size of the full array, not the number of pages.
3137  SYS_FORCE_INLINE void setSize(IDX_T newsize, exint tuplesize)
3138  {
3139  UT_ASSERT_P(newsize <= capacity());
3140  UT_ASSERT_MSG_P(myRefCount.relaxedLoad() == 1, "The table must already be hardened before we modify it!");
3141  if (PAGESHARDENED && newsize > mySize)
3142  {
3143  // Need to ensure pages between mySize and newsize are hardened.
3144  // Pages containing offsets less than or equal to mySize are
3145  // already guaranteed to be hardened.
3146  UT_PageNum startpage = numPages(mySize);
3147  UT_PageNum endpage = numPages(newsize);
3148  UT_PageOff pagecapacity(thePageSize);
3149  if (endpage == UT_PageNum(1) && exint(capacity()) < thePageSize)
3150  pagecapacity = capacity();
3151  PageTableEntry *page = getPPage(startpage);
3152  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3153  {
3154  if (page->isConstant())
3155  hardenConstantPage(page, pagecapacity, tuplesize);
3156  else if (page->isShared())
3157  hardenSharedPage(page, pagecapacity, tuplesize);
3158  }
3159  }
3160  mySize = newsize;
3161  }
3162  void fill(IDX_T start, IDX_T end, const NotVoidType &val);
3163  void fill(IDX_T start, IDX_T end, NotVoidType val, exint tuplesize);
3164  void fill(IDX_T start, IDX_T end, const NotVoidType *values, exint tuplesize);
3165  void fill(IDX_T start, IDX_T end, const UT_FixedVector<NotVoidType,theSafeTupleSize> &val);
3166 
3167  void incRef()
3168  {
3169  myRefCount.add(1);
3170  }
3171  void decRef(exint tuplesize)
3172  {
3173  const int64 new_count = myRefCount.add(-1);
3174  if (new_count != 0)
3175  {
3176  UT_ASSERT_P(new_count > 0);
3177  return;
3178  }
3179 
3180  UT_ASSERT_P(!(SYSisSame<DATA_T, void>()));
3181 
3182  const size_t bytesize = sizeof(NotVoidType)*((TSIZE >= 0) ? TSIZE : tuplesize);
3183 
3184  // If we're deleting the table, first, decRef all pages.
3185  UT_PageNum npages = numPages(myCapacity);
3186  PageTableEntry *page = getFirstPage();
3187  for (UT_PageNum p = 0; p < npages; ++p, ++page)
3188  {
3189  if (page->isRefd(bytesize))
3190  page->decRef();
3191  }
3192  free(this);
3193 #if 0
3194  printf("Free table %d pages (case 0); %d tables total\n",int(npages), int(thePageTableCount.add(-1)));
3195  fflush(stdout);
3196 #endif
3197  }
3198 
3199  void hardenAllPages(IDX_T start, IDX_T end, exint tuplesize = TSIZE)
3200  {
3201  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3202  UT_ASSERT_P(tuplesize >= 0);
3203  UT_PageNum startpage = pageNum(start);
3204  UT_PageNum endpage = numPages(end);
3205  UT_PageOff pagecapacity(thePageSize);
3206  if (endpage == UT_PageNum(1) && exint(capacity()) < thePageSize)
3207  pagecapacity = capacity();
3208  PageTableEntry *page = getPPage(startpage);
3209  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3210  {
3211  if (page->isConstant())
3212  hardenConstantPage(page, pagecapacity, tuplesize);
3213  else if (page->isShared())
3214  hardenSharedPage(page, pagecapacity, tuplesize);
3215  }
3216  }
3217 
3218  void tryCompressAllPages(IDX_T start, IDX_T end, exint tuplesize = TSIZE)
3219  {
3220  UT_ASSERT_P(tuplesize >= 0);
3221  UT_PageNum startpage = pageNum(start);
3222  UT_PageNum endpage = numPages(end);
3223  UT_PageNum npages = numPages(mySize);
3224  PageTableEntry *page = getPPage(startpage);
3225  for (UT_PageNum p = startpage; p < endpage; ++p, ++page)
3226  {
3227  if (page->isConstant())
3228  continue;
3229 
3230  UT_PageOff pagesize = thePageSize;
3231  if (p+1 >= npages)
3232  {
3233  if (p+1 == npages)
3234  {
3235  UT_PageOff pagesizetemp = pageOff(mySize);
3236  if (pagesizetemp != UT_PageOff(0))
3237  pagesize = pagesizetemp;
3238  }
3239  else
3240  pagesize = UT_PageOff(0);
3241  }
3242  page->tryCompressPage(pagesize, tuplesize);
3243  }
3244  }
3245 
3246  void tryCompressPage(UT_PageNum pagenum, exint tuplesize = TSIZE)
3247  {
3248  UT_ASSERT_P(tuplesize >= 0);
3249  PageTableEntry *page = getPPage(pagenum);
3250  if (page->isConstant())
3251  return;
3252 
3253  UT_PageOff pagesize = thePageSize;
3254  UT_PageNum npages = numPages(mySize);
3255  if (pagenum+1 >= npages)
3256  {
3257  if (pagenum+1 == npages)
3258  {
3259  UT_PageOff pagesizetemp = pageOff(mySize);
3260  if (pagesizetemp != UT_PageOff(0))
3261  pagesize = pagesizetemp;
3262  }
3263  else
3264  pagesize = UT_PageOff(0);
3265  }
3266  page->tryCompressPage(pagesize, tuplesize);
3267  }
3268 
3269  static ThisType *alloc(UT_PageNum npages)
3270  {
3271  UT_ASSERT_MSG_P(npages > UT_PageNum(0), "Shouldn't have a zero-page page table; nullptr should be used instead.");
3272  ThisType *p = (ThisType *)malloc(sizeof(ThisType) + npages*sizeof(PageTableEntry));
3273  //UT_ASSERT_P(_CrtIsValidHeapPointer(p));
3274  p->myRefCount.relaxedStore(1);
3275 #if 0
3276  printf("Alloc table %d pages; %d tables total\n",int(npages), int(thePageTableCount.add(1)));
3277  fflush(stdout);
3278 #endif
3279  return p;
3280  }
3281  static ThisType *harden(ThisType *pages, IDX_T newcapacity, exint tuplesize=TSIZE)
3282  {
3283  UT_ASSERT_P(!(SYSisSame<DATA_T,void>()));
3284 
3285  // If we're setting capacity to zero, free the table.
3286  if (newcapacity == IDX_T(0))
3287  {
3288  if (pages)
3289  pages->decRef(tuplesize);
3290  return nullptr;
3291  }
3292 
3293  UT_ASSERT_P(tuplesize >= 0);
3294  UT_ASSERT_MSG_P(newcapacity == roundUpCapacity(newcapacity), "Caller should already have adjusted capacity");
3295 
3296  bool wasshared = pages && pages->isShared();
3297  UT_PageNum npages = numPages(newcapacity);
3298  IDX_T oldcapacity = pages ? pages->capacity() : IDX_T(0);
3299  UT_PageNum oldnpages = numPages(oldcapacity);
3300 
3301  // Caller must force capacity to be on a page boundary if more than one page, else power of 2.
3302  UT_ASSERT_P((newcapacity < IDX_T(thePageSize) && SYSisPow2(exint(newcapacity))) || newcapacity == IDX_T(exint(npages)<<thePageBits));
3303 
3304  const bool nonconstfirst = pages && !pages->getFirstPage()->isConstant();
3305 
3306  if (npages == UT_PageNum(1) && oldnpages == UT_PageNum(1) && !wasshared)
3307  {
3308  // Can reuse existing page table if we're staying at one page, but changing its capacity.
3309 
3310  // NOTE: The "newcapacity != oldcapacity" will almost always be true here, but
3311  // we need to make absolutely sure that we don't realloc a shared page when just
3312  // calling hardenTable()! If we didn't have this check, it could happen, because
3313  // between the reference count check in hardenTable() could return 2, and then
3314  // before the pages->isShared() call above, the reference count drops to 1,
3315  // so we have an already hardened table, but we can identify that we don't need
3316  // to do anything because the capacity won't be changing.
3317  // The reason we can't have hardenTable() realloc a shared page is that in
3318  // GA_PrimitiveList, it needs to incDataRef() on all GA_OffsetLists in a shared
3319  // page before ever hardening one.
3320  if (nonconstfirst && newcapacity != oldcapacity)
3321  {
3322  // Just need to realloc the page.
3323  UT_PageOff newpagecapacity = UT_PageOff(exint(newcapacity));
3324  UT_PageOff sizetocopy = SYSmin(UT_PageOff(exint(pages->mySize)), newpagecapacity);
3325  PageTableEntry *page = pages->getFirstPage();
3326  page->realloc(sizetocopy, newpagecapacity, tuplesize);
3327  }
3328  // Don't need to change anything but capacity (and possibly size)
3329  // if constant page, even if it's an allocated constant page.
3330 
3331  pages->myCapacity = newcapacity;
3332  if (pages->mySize > newcapacity)
3333  pages->mySize = newcapacity;
3334  return pages;
3335  }
3336 
3337  PageTable *table = PageTable::alloc(npages);
3338  table->myCapacity = newcapacity;
3339  if (pages)
3340  {
3341  UT_ASSERT_P(pages->getRefCount() > 0 && table->getRefCount() > 0);
3342  UT_ASSERT_P(pages->capacity() > IDX_T(0));
3343  UT_ASSERT_P(oldnpages > UT_PageNum(0));
3344  const bool wassmallfirstpage = pages->capacity() < IDX_T(thePageSize);
3345  const bool willbesmallfirstpage = newcapacity < IDX_T(thePageSize);
3346 
3347  UT_PageNum minnpages = SYSmin(npages, oldnpages);
3348  table->mySize = pages->mySize;
3349  memcpy(table->getFirstPage(), pages->getFirstPage(), (sizeof(PageTableEntry))*minnpages);
3350 
3351  UT_ASSERT_P(!(SYSisSame<DATA_T, void>()));
3352 
3353  const size_t bytesize = sizeof(NotVoidType)*((TSIZE >= 0) ? TSIZE : tuplesize);
3354 
3355  if (wasshared)
3356  {
3357  // Increment all of the reference counts of reused pages,
3358  // if we're unsharing a table.
3359  PageTableEntry *page = table->getFirstPage();
3360  for (UT_PageNum p = 0; p < minnpages; ++p, ++page)
3361  {
3362  if (page->isRefd(bytesize))
3363  page->incRef();
3364  }
3365  pages->decRef(tuplesize);
3366  }
3367  else
3368  {
3369  // pages isn't shared, so we can just decRef() any pages no
3370  // longer referenced and free pages. This avoids calling
3371  // incRef on all kept pages and decRef on all pages like the
3372  // above would do if pages had its reference count go to
3373  // zero.
3374  PageTableEntry *page = pages->getFirstPage() + npages;
3375  for (UT_PageNum p = npages; p < oldnpages; ++p, ++page)
3376  {
3377  if (page->isRefd(bytesize))
3378  page->decRef();
3379  }
3380  UT_ASSERT_MSG_P(!pages->isShared(), "This should still be unshared.");
3381  free(pages);
3382 #if 0
3383  printf("Free table %d pages (case 1); %d tables total\n",int(oldnpages), int(thePageTableCount.add(-1)));
3384  fflush(stdout);
3385 #endif
3386  }
3387  // Can't refer to pages beyond this point!
3388  UT_IF_ASSERT_P(pages = nullptr;)
3389 
3390  if (table->mySize > newcapacity)
3391  table->mySize = newcapacity;
3392 
3393  // NOTE: The "newcapacity != oldcapacity" is to make absolutely
3394  // sure that we don't realloc a shared page when just
3395  // calling hardenTable()! This is because in
3396  // GA_PrimitiveList, it needs to incDataRef() on all
3397  // GA_OffsetLists in a shared page before ever hardening one.
3398  if (nonconstfirst && (wassmallfirstpage || willbesmallfirstpage) && newcapacity != oldcapacity)
3399  {
3400  UT_PageOff newpagecapacity = SYSmin(UT_PageOff(exint(newcapacity)), UT_PageOff(thePageSize));
3401  UT_PageOff sizetocopy = SYSmin(UT_PageOff(exint(table->mySize)), newpagecapacity);
3402  PageTableEntry *page = table->getFirstPage();
3403  page->realloc(sizetocopy, newpagecapacity, tuplesize);
3404  }
3405  }
3406  else
3407  {
3408  oldnpages = 0;
3409  table->mySize = IDX_T(0);
3410  }
3411 
3412  // Set any new pages to be constant zero.
3413  PageTableEntry *page = table->getFirstPage() + oldnpages;
3414  for (UT_PageNum p = oldnpages; p < npages; ++p, ++page)
3415  {
3416  page->initZero();
3417  }
3418 
3419  if (PAGESHARDENED && wasshared)
3420  {
3421  if (TSIZE >= 1)
3422  table->hardenAllPages(IDX_T(0), table->mySize);
3423  else
3424  table->hardenAllPages(IDX_T(0), table->mySize, tuplesize);
3425  }
3426 
3427  return table;
3428  }
3429 
3430  /// Atomically decrement the reference count iff the page is shared.
3431  /// See the description of decRefIffSharedPage() above for
3432  /// why this function exists.
3434  {
3435  return decCounterIffAbove1(myRefCount);
3436  }
3437 
3438  private:
3439  // The constructor/destructor should never be called, since the class
3440  // can only be allocated with malloc.
3441  PageTable() { UT_ASSERT_P(0); }
3442  PageTable(const PageTable &) { UT_ASSERT_P(0); }
3443  ~PageTable() { UT_ASSERT_P(0); }
3444 
3445  SYS_AtomicCounter myRefCount;
3446  IDX_T mySize;
3447  IDX_T myCapacity;
3448  };
3449 
3450  /// NOTE: These have to be below the PageTable class definition, because
3451  /// for no apparent reason, even though you can have a regular local
3452  /// variable of type PageTable before, you can't have a function
3453  /// parameter of type PageTable before. *sigh*
3454  /// @{
3455  static void hardenConstantPage(PageTable *pages, PageTableEntry *page)
3456  {
3457  const exint arraycapacity(exint(pages->capacity()));
3458  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3459  hardenConstantPage(page, pagecapacity);
3460  }
3461  static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page)
3462  {
3463  const exint arraycapacity(exint(pages->capacity()));
3464  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3465  hardenConstantPageNoInit(page, pagecapacity);
3466  }
3467  static void hardenConstantPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
3468  {
3469  const exint arraycapacity(exint(pages->capacity()));
3470  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3471  hardenConstantPage(page, pagecapacity, tuplesize);
3472  }
3473  static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
3474  {
3475  const exint arraycapacity(exint(pages->capacity()));
3476  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3477  hardenConstantPageNoInit(page, pagecapacity, tuplesize);
3478  }
3479  static void hardenSharedPage(PageTable *pages, PageTableEntry *page)
3480  {
3481  const exint arraycapacity(exint(pages->capacity()));
3482  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3483  hardenSharedPage(page, pagecapacity);
3484  }
3485  static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page)
3486  {
3487  const exint arraycapacity(exint(pages->capacity()));
3488  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3489  hardenSharedPageNoInit(page, pagecapacity);
3490  }
3491  static void hardenSharedPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
3492  {
3493  const exint arraycapacity(exint(pages->capacity()));
3494  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3495  hardenSharedPage(page, pagecapacity, tuplesize);
3496  }
3497  static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
3498  {
3499  const exint arraycapacity(exint(pages->capacity()));
3500  UT_PageOff pagecapacity((arraycapacity < thePageSize) ? arraycapacity : thePageSize);
3501  hardenSharedPageNoInit(page, pagecapacity, tuplesize);
3502  }
3503  /// @}
3504 
3505  void forceHardenTable(IDX_T newcapacity)
3506  {
3507  if (SYSisSame<DATA_T,void>())
3508  {
3509  const UT_Storage storage = myImpl.getStorage();
3510  switch (storage)
3511  {
3512  case UT_Storage::INVALID:
3513  UT_ASSERT_MSG_P(0, "Can't have a non-numeric type with a void DATA_T.");
3514  return;
3515  case UT_Storage::INT8:
3516  castType<int8>().forceHardenTable(newcapacity); return;
3517  case UT_Storage::UINT8:
3518  castType<uint8>().forceHardenTable(newcapacity); return;
3519  case UT_Storage::INT16:
3520  castType<int16>().forceHardenTable(newcapacity); return;
3521  case UT_Storage::INT32:
3522  castType<int32>().forceHardenTable(newcapacity); return;
3523  case UT_Storage::INT64:
3524  castType<int64>().forceHardenTable(newcapacity); return;
3525  case UT_Storage::REAL16:
3526  castType<fpreal16>().forceHardenTable(newcapacity); return;
3527  case UT_Storage::REAL32:
3528  castType<fpreal32>().forceHardenTable(newcapacity); return;
3529  case UT_Storage::REAL64:
3530  castType<fpreal64>().forceHardenTable(newcapacity); return;
3531  }
3532  return;
3533  }
3534 
3535  PageTable *&pages = myImpl.getPages();
3536  pages = PageTable::harden(pages, newcapacity, myImpl.getTupleSize());
3537  }
3538 
3539 protected:
3540  template<typename T>
3542  static bool isZero(const T &val)
3543  {
3544  // TODO: Find a way to check for zero that will still compile, but
3545  // that avoids the mess of dealing with memset for POD types.
3546  T zero;
3547  memset(&zero, 0, sizeof(T));
3548  return (val == zero);
3549  }
3550 
3551  template<typename T>
3553  static bool isZero(const T *val,exint tuplesize)
3554  {
3555  UT_ASSERT_P(tuplesize >= 1);
3556  if (tuplesize == 0)
3557  return true;
3558 
3559  // TODO: Find a way to check for zero that will still compile, but
3560  // that avoids the mess of dealing with memset for POD types.
3561  T zero;
3562  memset(&zero, 0, sizeof(T));
3563 
3564  bool iszero = (val[0] == zero);
3565  for (exint i = 1; i < tuplesize; ++i)
3566  iszero &= (val[i] == zero);
3567  return iszero;
3568  }
3569 
3570  template<typename T0,typename T1>
3572  static bool isEqual(const T0 *a,const T1 *b,exint tuplesize)
3573  {
3574  UT_ASSERT_P(tuplesize >= 1);
3575  UT_ASSERT_P(a != nullptr && b != nullptr);
3576 
3577  for (exint i = 0; i < tuplesize; ++i)
3578  {
3579  if (a[i] != b[i])
3580  return false;
3581  }
3582  return true;
3583  }
3584 
3585  template<typename DEST_TYPE,typename SRC_TYPE,exint SRC_TSIZE,bool SRC_INSTANTIATED>
3588  {
3589  if (SYSisSame<DEST_TYPE,SRC_TYPE>())
3590  return src;
3591 
3593  for (exint i = 0; i < SRC_TSIZE; ++i)
3594  dest[i] = UTconvertStorage<DEST_TYPE>(src[i]);
3595  return dest;
3596  }
3597 
3598 private:
3599  /// This class wraps the data of UT_PageArray, just so that if
3600  /// the tuple size and data type are known at compile-time, it doesn't
3601  /// need to store that information as an int and a UT_Storage.
3602  /// myPages is the same in all of them, but can't be extracted out,
3603  /// because the C++ standard, unfortunately, forbids 0-byte classes.
3604  /// @{
3605  struct ImplPageData
3606  {
3607  PageTable *myPages;
3608  };
3609 
3610  template<exint ITSIZE, typename IDATA_T>
3611  class Impl :
3612  private ImplPageData,
3613  private UT_TupleType<ITSIZE, IDATA_T>
3614  {
3615  public:
3616  SYS_FORCE_INLINE const PageTable *getPages() const { return ImplPageData::myPages; }
3617  SYS_FORCE_INLINE PageTable *&getPages() { return ImplPageData::myPages; }
3622  };
3623 
3624  Impl<TSIZE,DATA_T> myImpl;
3625  /// @}
3626 };
3627 
3628 #endif
void hardenAllPages(IDX_T start, IDX_T end, exint tuplesize=TSIZE)
SYS_FORCE_INLINE NotVoidType operator()(IDX_T i, exint component=0) const
Read element i's specified component (default component 0).
Definition: UT_PageArray.h:450
SYS_FORCE_INLINE constexpr const T * data() const noexcept
static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page)
static void copyNonConst(NotVoidType *destpagedata, const SrcNotVoidType *srcpagedata, exint desttuplesize, exint srctuplesize, UT_PageOff ntuples)
SYS_FORCE_INLINE bool isPageConstant(UT_PageNum pagenum) const
Returns true iff the specified page is constant-compressed.
static void hardenConstantPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
SYS_FORCE_INLINE void opVector(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE > &v)
Definition: UT_PageArray.h:735
void printf(internal::basic_buffer< Char > &buf, basic_string_view< Char > format, basic_format_args< Context > args)
Definition: printf.h:570
static SYS_FORCE_INLINE const NotVoidType * getConstantPtr(const PageTableEntry *page, exint component=0, exint tuplesize=TSIZE)
void SecondGuess
Definition: UT_Storage.h:90
SYS_FORCE_INLINE void setSize(IDX_T newsize)
static SYS_FORCE_INLINE bool decCounterIffAbove1(SYS_AtomicCounter &counter)
GLsizeiptr size
Definition: glew.h:1681
GLenum src
Definition: glew.h:2410
int int32
Definition: SYS_Types.h:39
SYS_FORCE_INLINE const NotVoidType * getInlinePtr(exint tuplesize) const
Returns the data pointer, if an inline constant page.
SYS_FORCE_INLINE void * getFirstPtrVoid()
Returns the data pointer, if not a constant page.
UT_Storage
Definition: UT_Storage.h:28
static void hardenSharedPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
void hardenAllPagesUnknownType(IDX_T start, IDX_T end)
void tryCompressPage(UT_PageOff pagesize, exint tuplesize=TSIZE)
static ThisType * harden(ThisType *pages, IDX_T newcapacity, exint tuplesize=TSIZE)
static void hardenSharedPage(PageTable *pages, PageTableEntry *page, exint tuplesize)
SYS_FORCE_INLINE bool isArrayZero() const
Returns true iff the entire array is constant & zero.
static const UT_Storage theStorage
Definition: UT_PageArray.h:252
static SYS_FORCE_INLINE NotVoidType getConstant(const PageTableEntry *const page, exint component, exint tuplesize=TSIZE)
NotVoid< DATA_T >::type NotVoidType
Definition: UT_PageArray.h:244
UT_PageArray(const ThisType &that)
Definition: UT_PageArray.h:310
void setTupleSize(exint newtuplesize, const UT_Defaults &v)
static ThisType * alloc(UT_PageNum npages)
void tryCompressPage(UT_PageNum pagenum, exint tuplesize=TSIZE)
SYS_FORCE_INLINE const NotVoidType * getMaskedPtr() const
void copy(IDX_T dest, IDX_T src)
Copies a single element from src to dest within this.
void tryCompressAllPagesUnknownType(IDX_T start, IDX_T end)
SYS_FORCE_INLINE ~UT_PageArray()
Definition: UT_PageArray.h:324
SYS_FORCE_INLINE bool operator!=(const ThisType &that) const
NOTE: This just does a shallow comparison.
SYS_FORCE_INLINE bool isZero() const noexcept
void tryCompressAllPages(IDX_T start, IDX_T end, exint tuplesize=TSIZE)
GLuint const GLfloat * val
Definition: glew.h:2794
SYS_FORCE_INLINE void clear()
Definition: UT_PageArray.h:330
SYS_FORCE_INLINE ThisType & operator=(const UT_PageArray< SRC_DATA_T, SRC_TSIZE, SRC_TABLEHARDENED, SRC_PAGESHARDENED, THEPAGEBITS, IDX_T > &that)
Definition: UT_PageArray.h:388
static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
GLboolean GLboolean GLboolean GLboolean a
Definition: glew.h:9477
void opVectorUnknownType(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE > &v)
GLenum GLsizei GLenum GLenum const void * table
Definition: glew.h:4940
SYS_FORCE_INLINE IDX_T size() const
Definition: UT_PageArray.h:868
SYS_FORCE_INLINE T * SYSconst_cast(const T *foo)
Definition: SYS_Types.h:136
GLint GLsizei const GLuint64 * values
Definition: glew.h:3612
SYS_FORCE_INLINE void add(IDX_T i, SRC_DATA_T v)
component == 0 in this version
Definition: UT_PageArray.h:620
UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVectorUnknownSize(IDX_T i) const
SYS_FORCE_INLINE bool isShared() const
void countMemory(UT_MemoryCounter &counter, bool inclusive) const
int64 exint
Definition: SYS_Types.h:125
bool iszero(T a, T t)
Definition: ImathFun.h:135
SYS_FORCE_INLINE PageTableEntry * getFirstPage()
const GLint * first
Definition: glew.h:1528
static void hardenSharedPageNoInit(PageTable *pages, PageTableEntry *page)
void setConstant(IDX_T start, IDX_T end, NotVoidType v)
void opUnknownType(IDX_T i, exint component, SRC_DATA_T v)
exint UT_PageNum
Definition: UT_PageArray.h:40
SYS_FORCE_INLINE const PageTableEntry * getPPage(UT_PageNum i) const
static void hardenConstantPage(PageTable *pages, PageTableEntry *page)
SYS_FORCE_INLINE UT_PageArray< DATA_T, TSIZE, true, true, THEPAGEBITS, IDX_T > & hardenAllPages(IDX_T start=IDX_T(0), IDX_T end=IDX_T(-1))
static const exint theTupleSize
Definition: UT_PageArray.h:253
SYS_FORCE_INLINE UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T >::PageTableEntry * castType()
SYS_FORCE_INLINE exint getTupleSize() const
DEST_DATA_T getUnknownType(IDX_T i, exint component=0) const
const GLdouble * v
Definition: glew.h:1391
SYS_SelectType< T, float, SYS_IsSame< T, void >::value >::type type
Definition: UT_PageArray.h:242
#define UT_ASSERT_MSG_P(ZZ,...)
Definition: UT_Assert.h:137
SYS_FORCE_INLINE IDX_T size() const
NOTE: This is the size of the full array, not the number of pages.
SYS_FORCE_INLINE UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVector(IDX_T i) const
Definition: UT_PageArray.h:503
#define UT_IF_ASSERT_P(ZZ)
Definition: UT_Assert.h:161
SYS_FORCE_INLINE void set(IDX_T i, SRC_DATA_T v)
component == 0 in this version
Definition: UT_PageArray.h:614
SYS_FORCE_INLINE UT_Storage getStorage() const
static SYS_FORCE_INLINE IDX_T roundUpCapacity(IDX_T capacity)
Definition: UT_PageArray.h:889
SYS_FORCE_INLINE bool isTableHardened() const
Returns true iff the table isn't shared with any other UT_PageArray's.
float fpreal32
Definition: SYS_Types.h:200
SYS_FORCE_INLINE exint getRefCount() const
SYS_FORCE_INLINE UT_PageArray< DATA_T, DEST_TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castTupleSize()
bool hasNanInRange(IDX_T start, IDX_T end) const
static SYS_FORCE_INLINE UT_FixedVector< DEST_TYPE, SRC_TSIZE > convertVectorStorage(const UT_FixedVector< SRC_TYPE, SRC_TSIZE, SRC_INSTANTIATED > &src)
UT_PageArray(const UT_PageArray< SRC_DATA_T, SRC_TSIZE, SRC_TABLEHARDENED, SRC_PAGESHARDENED, THEPAGEBITS, IDX_T > &that)
Definition: UT_PageArray.h:317
UT_PageArray(exint tuplesize, UT_Storage storage)
Definition: UT_PageArray.h:294
SYS_FORCE_INLINE void addVector(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE, SRC_INSTANTIATED > &v)
Definition: UT_PageArray.h:726
void tryCompressPageUnknownType(UT_PageNum pagenum)
void decRef(exint tuplesize)
SYS_FORCE_INLINE bool isShared() const
void fill(IDX_T start, IDX_T end, const NotVoidType &val)
static bool isEqualConst(const NotVoidType *tuple, const SrcNotVoidType *stuple, exint mintuplesize)
SYS_FORCE_INLINE void setConstantBit()
double fpreal64
Definition: SYS_Types.h:201
#define UT_ASSERT_MSG(ZZ,...)
Definition: UT_Assert.h:138
unsigned char uint8
Definition: SYS_Types.h:36
UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVectorMismatchedSize(IDX_T i) const
UT_FixedVector< DEST_DATA_T, DEST_TSIZE > getVectorUnknownType(IDX_T i) const
SYS_FORCE_INLINE exint getRefCount() const
SYS_FORCE_INLINE bool isConstantZero() const
This is only valid to call if the type doesn't fit inline.
SYS_FORCE_INLINE void setSize(IDX_T newsize, exint tuplesize)
static SYS_FORCE_INLINE UT_PageOff pageOff(IDX_T i)
UT_PageArray(UT_Storage storage)
Definition: UT_PageArray.h:283
static SYS_FORCE_INLINE void call(FUNCTOR functor, ARGS &&...args)
Definition: SYS_CallIf.h:39
SYS_FORCE_INLINE void tryCompressPage(UT_PageNum pagenum)
SYS_FORCE_INLINE ThisType & operator=(const ThisType &that)
Definition: UT_PageArray.h:380
static SYS_FORCE_INLINE bool typeFitsInline(exint tuplesize=TSIZE)
This is only valid to call when DATA_T is non-void.
SYS_FORCE_INLINE const PageTableEntry * getFirstPage() const
SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const UT_FixedVector< SRC_DATA_T, theSafeTupleSize > &val)
static SYS_FORCE_INLINE UT_PageNum numPages(IDX_T nelements)
static SYS_FORCE_INLINE bool isEqual(const T0 *a, const T1 *b, exint tuplesize)
SYS_FORCE_INLINE void getRange(IDX_T srcstart, IDX_T nelements, T *dest) const
GLint GLenum GLsizei GLint GLsizei const void * data
Definition: glew.h:1379
SYS_FORCE_INLINE bool isRefd(exint tuplebytes) const
#define UT_ASSERT_P(ZZ)
Definition: UT_Assert.h:134
SYS_FORCE_INLINE bool operator==(const ThisType &that) const
NOTE: This just does a shallow comparison.
SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const NotVoidType &val)
SYS_FORCE_INLINE IDX_T capacity() const
NOTE: This is the capacity of the full array, not the capacity of pages.
static SYS_FORCE_INLINE bool isZero(const T &val)
SYS_FORCE_INLINE IDX_T capacity() const
Definition: UT_PageArray.h:876
#define SYS_FALLTHROUGH
Definition: SYS_Compiler.h:68
#define SYS_FORCE_INLINE
Definition: SYS_Inline.h:45
GLuint GLuint end
Definition: glew.h:1253
T compare_swap(T expected, T desired)
SYS_FORCE_INLINE NotVoidType * hardenPage(UT_PageNum pagenum)
UT_FixedVector< NotVoidType, theSafeTupleSize > Tuple
DATA_T DataType
Definition: UT_PageArray.h:251
SYS_FORCE_INLINE void op(IDX_T i, exint component, SRC_DATA_T v)
Definition: UT_PageArray.h:640
SYS_FORCE_INLINE NotVoidType * getInlinePtr(exint tuplesize)
Returns the data pointer, if an inline constant page.
void setCapacity(IDX_T newcapacity)
Definition: UT_PageArray.h:904
long long int64
Definition: SYS_Types.h:116
void clearAndDestroy()
Definition: UT_PageArray.h:336
SYS_FORCE_INLINE void setVector(IDX_T i, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE, SRC_INSTANTIATED > &v)
Definition: UT_PageArray.h:721
SYS_FORCE_INLINE void * getMaskedPtrVoid()
static SYS_FORCE_INLINE UT_PageOff pageStart(IDX_T page)
UT_PageArray(exint tuplesize)
Definition: UT_PageArray.h:271
SYS_FORCE_INLINE const void * getFirstPtrVoidUnsafe() const
static void copyPartialPage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destoff, UT_PageOff srcoff, UT_PageOff ntuples, UT_PageOff destpagecapacity)
SYS_FORCE_INLINE bool isConstantAndZeroSafe() const
signed char int8
Definition: SYS_Types.h:35
void setPageConstantUnknownType(UT_PageNum pagenum, const UT_FixedVector< SRC_DATA_T, theSafeTupleSize > &val)
SYS_FORCE_INLINE bool decRefIffShared(exint tuplesize=TSIZE)
static void makeConstant(PageTableEntry *page, const NotVoidType *values, exint tuplesize)
SYS_FORCE_INLINE NotVoidType * hardenPageNoInit(UT_PageNum pagenum)
SYS_FORCE_INLINE void tryCompressAllPages(IDX_T start=IDX_T(0), IDX_T end=IDX_T(-1))
static const exint thePageBits
Definition: UT_PageArray.h:255
SYS_FORCE_INLINE void add(IDX_T i, exint component, SRC_DATA_T v)
Definition: UT_PageArray.h:635
const NotVoidType * getPageData(UT_PageNum pagenum) const
exint UT_PageOff
Definition: UT_PageArray.h:41
static const exint theSafeTupleSize
Definition: UT_PageArray.h:254
SYS_FORCE_INLINE void incRef()
void realloc(UT_PageOff sizetocopy, UT_PageOff newpagecapacity, exint tuplesize=TSIZE)
static void makeConstantFrom(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize)
GLuint start
Definition: glew.h:1253
PageTableEntry ThisType
SYS_FORCE_INLINE void setPageConstant(UT_PageNum pagenum, const SRC_DATA_T *values)
void moveRange(IDX_T srcstart, IDX_T deststart, IDX_T nelements)
static const exint thePageMask
Definition: UT_PageArray.h:257
void getVectorRange(IDX_T srcstart, IDX_T nelements, UT_FixedVector< DEST_DATA_T, DEST_TSIZE, DEST_INSTANTIATED > *dest) const
static void hardenConstantPageNoInit(PageTable *pages, PageTableEntry *page, exint tuplesize)
GLdouble GLdouble GLdouble b
Definition: glew.h:9122
GLfloat GLfloat p
Definition: glew.h:16321
static const uintptr_t theConstantPageBit
SYS_FORCE_INLINE int64 getMemoryUsage(exint tuplebytes) const
SYS_AtomicInt< int32 > SYS_AtomicCounter
Definition: SYS_AtomicInt.h:88
SYS_FORCE_INLINE const UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castType() const
void setSize(IDX_T newsize)
Definition: UT_PageArray.h:928
SYS_FORCE_INLINE const void * getMaskedPtrVoid() const
SYS_FORCE_INLINE const UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T >::PageTableEntry * castType() const
GLuint counter
Definition: glew.h:2740
void swapRange(IDX_T astart, IDX_T bstart, IDX_T nelements)
SYS_FORCE_INLINE void * getFirstPtrVoidUnsafe()
short int16
Definition: SYS_Types.h:37
bool decRefIffSharedPage(UT_PageNum pagenum)
void setPageConstantUnknownType(UT_PageNum pagenum, const SRC_DATA_T *values)
SYS_FORCE_INLINE PageTableEntry * getPPage(UT_PageNum i)
void setStorage(const UT_Storage newstorage)
SYS_FORCE_INLINE void set(IDX_T i, exint component, SRC_DATA_T v)
Definition: UT_PageArray.h:630
static void hardenConstantPageNoInit(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
static void fillNonConstWithConst(NotVoidType *destpagedata, NotVoidType *destpageend, const SrcNotVoidType *stuple, exint mintuplesize, exint desttupleextra)
SYS_FORCE_INLINE NotVoidType * getMaskedPtr()
bool decRefIffSharedTable()
SYS_FORCE_INLINE const UT_PageArray< DATA_T, DEST_TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castTupleSize() const
SYS_FORCE_INLINE bool isConstant() const
This is always valid to call.
SYS_FORCE_INLINE void setRange(IDX_T deststart, IDX_T nelements, const T *src)
static void hardenSharedPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
SYS_FORCE_INLINE const void * getFirstPtrVoid() const
Returns the data pointer, if not a constant page.
static SYS_FORCE_INLINE NotVoidType * getConstantPtr(PageTableEntry *page, exint component=0, exint tuplesize=TSIZE)
static SYS_FORCE_INLINE void makeConstant(PageTableEntry *page, const UT_FixedVector< NotVoidType, theSafeTupleSize > &val)
bool equal(T1 a, T2 b, T3 t)
Definition: ImathFun.h:143
static void makeConstant(PageTableEntry *page, const NotVoidType &val, exint tuplesize=TSIZE)
SYS_FORCE_INLINE UT_PageArray< DATA_T, TSIZE, true, PAGESHARDENED, THEPAGEBITS, IDX_T > & hardenTable()
static const exint thePageSize
Definition: UT_PageArray.h:256
static SYS_FORCE_INLINE bool isZero(const T *val, exint tuplesize)
SYS_FORCE_INLINE void alloc(UT_PageOff nelements, exint tuplesize=TSIZE)
T add(T val)
Atomically adds val to myValue, returning the new value of myValue.
static void hardenConstantPage(PageTableEntry *page, UT_PageOff pagecapacity, exint tuplesize=TSIZE)
getOption("OpenEXR.storage") storage
Definition: HDK_Image.dox:276
Container class for all geometry.
Definition: GA_Detail.h:95
int64 getMemoryUsage(bool inclusive) const
static SYS_FORCE_INLINE UT_PageNum pageNum(IDX_T i)
UT_PageArray< DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > ThisType
Definition: UT_PageArray.h:250
static void hardenSharedPage(PageTable *pages, PageTableEntry *page)
void setCapacityIfNeeded(IDX_T newcapacity)
Definition: UT_PageArray.h:917
void forceHardenTable(IDX_T newcapacity)
SYS_FORCE_INLINE int64 getGuaranteedInt(IDX_T i) const
SYS_FORCE_INLINE void relaxedStore(T val)
Definition: SYS_AtomicInt.h:66
SYS_FORCE_INLINE NotVoidType * unshareConstantPage(UT_PageNum pagenum)
ImageBuf OIIO_API zero(ROI roi, int nthreads=0)
#define SYSmin(a, b)
Definition: SYS_Math.h:1527
SYS_FORCE_INLINE bool isPageHard(UT_PageNum pagenum) const
UT_PageArray< DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > ParentType
SYS_FORCE_INLINE constexpr int UTstorageSize(UT_Storage storage)
Returns the number of bytes in the given storage type.
Definition: UT_Storage.h:57
SYS_FORCE_INLINE void initZero()
SYS_FORCE_INLINE const NotVoidType * getFirstPtr() const
Returns the data pointer, if not a constant page.
SYS_FORCE_INLINE NotVoidType * getFirstPtr()
Returns the data pointer, if not a constant page.
void setPageConstantMismatchedType(UT_PageNum pagenum, const SRC_DATA_T *values)
void replace(const UT_PageArray< SRC_DATA_T, SRC_TSIZE, SRC_TABLEHARDENED, SRC_PAGESHARDENED, THEPAGEBITS, IDX_T > &that)
Definition: UT_PageArray.h:397
#define SYS_CALLIF_AUTO
Definition: SYS_CallIf.h:24
SYS_FORCE_INLINE NotVoidType & operator()(IDX_T i, exint component=0)
Definition: UT_PageArray.h:571
SYS_FORCE_INLINE T relaxedLoad() const
Definition: SYS_AtomicInt.h:59
NotVoidType * getPageData(UT_PageNum pagenum)
void setVectorRange(IDX_T deststart, IDX_T nelements, const UT_FixedVector< SRC_DATA_T, SRC_TSIZE, SRC_INSTANTIATED > *src)
type
Definition: core.h:528
SYS_FORCE_INLINE void decRef()
static void replacePage(PageTableEntry *dest, const typename SrcType::PageTableEntry *src, exint desttuplesize, exint srctuplesize, UT_PageOff destpagesize, UT_PageOff destpagecapacity)
SYS_FORCE_INLINE UT_PageArray< DEST_DATA_T, TSIZE, TABLEHARDENED, PAGESHARDENED, THEPAGEBITS, IDX_T > & castType()