HDK
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
PDG_WorkItem.h
Go to the documentation of this file.
1 /*
2  * PROPRIETARY INFORMATION. This software is proprietary to
3  * Side Effects Software Inc., and is not to be reproduced,
4  * transmitted, or disclosed in any way without written permission.
5  *
6  * COMMENTS:
7  */
8 
9 #ifndef __PDG_WORKITEM_H__
10 #define __PDG_WORKITEM_H__
11 
12 #include "PDG_API.h"
13 
14 #include "PDG_AttributeOwner.h"
15 #include "PDG_AttributeTypes.h"
16 #include "PDG_EventEmitter.h"
17 #include "PDG_EventTypes.h"
18 #include "PDG_File.h"
19 #include "PDG_LoopInfo.h"
20 #include "PDG_RegisteredType.h"
21 #include "PDG_SchedulerTypes.h"
22 #include "PDG_WorkItemDirty.h"
23 #include "PDG_WorkItemStats.h"
24 #include "PDG_WorkItemTypes.h"
25 
26 #include <PDGE/PDGE_Dependency.h>
28 #include <PDGE/PDGE_Resolutions.h>
29 #include <PDGT/PDGT_Value.h>
30 
31 #include <UT/UT_ArrayStringSet.h>
32 #include <UT/UT_Options.h>
33 #include <UT/UT_StringHolder.h>
34 #include <UT/UT_TBBSpinLock.h>
35 #include <UT/UT_WorkBuffer.h>
36 
38 class PDG_BatchWorkItem;
39 class PDG_Graph;
40 class PDG_GraphContext;
41 struct PDG_MemoryInfo;
42 class PDG_Node;
43 class PDG_Scheduler;
44 class PDG_WorkItemData;
45 
47  public PDG_AttributeOwner,
49 {
50 public:
51  /// Default, invalid work item ID value.
52  static constexpr PDG_WorkItemID theInvalidId = -1;
53 
54  /// Enumeration of pending dirty states
56  {
57  /// The item does not require any sort of dirtying
59 
60  /// The item needs to be dirtied, but not deleted
62 
63  /// the item needs to be deleted
64  ePendingDelete
65  };
66 
67 public:
68  /// Constructs and initialize a new work item. This method will get a
69  /// unique id for the item from the PDG_Graph, unless a valid id is
70  /// passed in
72  PDG_GraphContext* context,
73  bool is_static = true,
75  PDG_WorkItemID id = theInvalidId);
76 
77  /// Deletes this work item. Does not delete any dependencies or remove
78  /// references in other items or node. The appropriate dirty method must
79  /// be called first.
80  ~PDG_WorkItem() override;
81 
82  /// Returns the list of supported event types
83  const PDG_EventFilter& supportedEventTypes() const override;
84 
85  /// Returns memory usage for this work item
86  int64 getMemoryUsage(bool inclusive) const override;
87 
88  /// Returns a break down of the memory used by the work item
89  virtual void memoryInfo(PDG_MemoryInfo& memory_info,
90  bool inclusive) const;
91 
92  /// Returns the owning node's attribute lock
93  UT_RWLock* attribGlobalLock() const override;
94 
95  /// Returns true if the work item's attributes are writeable
96  bool attribCanWrite(
97  const PDG_EvaluationContext* ctx) const override;
98 
99  /// Called when the attributes have been unlocked
100  void attribUnlock(bool modified) const override;
101 
102  /// Called when an attributed owned by the work item is modified
103  void attribModify(
104  PDG_AttributeType attrib_type,
105  const UT_StringHolder& attrib_name) const override;
106 
107  /// Called when an attribute operation produces a non-fatal warning
108  void attribWarn(
109  const UT_StringHolder& message) const override;
110 
111  /// Returns a descriptive name of the attribute owner
112  UT_StringHolder attribOwnerName() const override;
113 
114  /// Resets the work item's dependency objects
115  void resetOwner() override;
116 
117  /// Returns the debug name for this work item, used by PDGE for
118  /// logging and debuggin features
119  UT_StringHolder debugName() const override
120  { return name(); }
121 
122  /// Returns the debug group for this work item, used by PDGE for
123  /// outputting a DOT graph.
124  UT_StringHolder debugGroup() const override;
125 
126 
127  /// Returns the can cook dependency object for this work item
129  { return &myCanCookDep; }
130 
131  /// Returns the is cooked dependency object for this work item
133  { return &myIsCookedDep; }
134 
135  /// Returns the work item type
137  { return myType; }
138 
139  /// Returns the work item execution type
141  { return myExecutionType; }
142 
143  /// Returns the work item cook type
145  { return myCookType; }
146 
147  /// Returns the id of the work item, guaranteed to be unique
149  { return myId; }
150 
151 
152  /// Returns the name of the work item, guaranteed to be unique
153  UT_StringHolder name() const;
154 
155  /// Returns true if the work item has a command line string set, for
156  /// any platform
157  bool hasCommand() const;
158 
159  /// Returns true if the work item has platform-specific command line
160  /// strings
161  bool hasPlatformCommand() const;
162 
163  /// Returns the generic command associated with the work item
164  const UT_StringHolder& command() const;
165 
166  /// Returns the platform specific command associated with the work item,
167  /// or the command for the current platform if no platform is specified
168  const UT_StringHolder& platformCommand(
169  PDG_Platform platform=
171 
172  /// Returns true if the command should run in the shell
173  bool shouldRunInShell() const;
174 
175  /// Returns the label assocaited with the work item
176  UT_StringHolder label() const;
177 
178  /// Returns true if the work item has a custom label
179  bool hasLabel() const;
180 
181  /// Returns the custom state string associated with the work item, if
182  /// one has be set.
183  const UT_StringHolder& customState() const;
184 
185  /// Returns true if the work item has a custom state string
186  bool hasCustomState() const;
187 
188 
189  /// Returns the node that owns the work item
190  PDG_Node* node() const
191  { return myNode; }
192 
193  /// Returns the context that owns the work item
195  { return myContext; }
196 
197  /// Returns the graph that the work item belongs to
198  PDG_Graph* graph() const;
199 
200 
201  /// Returns the batch item that owns the work item, or nullptr if
202  /// the item is not in a batch
204  { return myBatchParent; }
205 
206  /// Returns the clone target for the work item, or nullptr if a specific
207  /// target was not set
208  const PDG_WorkItem* cloneTarget() const
209  { return myCloneTarget; }
210 
211  /// Const acessor to the work item's parent
212  const PDG_WorkItem* parent() const;
213 
214  /// Returns the work item that this work item copied its attributes
215  /// from. If the work item is a partition, this method returns nullptr.
216  const PDG_WorkItem* attributeSource() const;
217 
218 
219  /// Returns the work item index
220  int index() const
221  { return myIndex; }
222 
223  /// Returns the work item's internal index
224  int internalIndex() const
225  { return myInternalIndex; }
226 
227  /// Returns the work items index within its batch, or -1 if the work item
228  /// is not in a batch
229  int batchIndex() const
230  { return myBatchIndex; }
231 
232  /// Returns the work item priority
233  int priority() const
234  { return myPriority; }
235 
236 
237  /// Returns the cache id of the work item
239  { return myCacheId; }
240 
241  /// Returns the work item frame
242  fpreal frame() const
243  { return myFrame; }
244 
245  /// Returns true if the frame has been set
246  bool hasFrame() const
247  { return myHasFrame; }
248 
249  /// Returns the work item step size, primarily used with batch items
251  { return myFrameStep; }
252 
253  /// Returns true if the work item has any warning log messages
254  bool hasWarnings() const
255  { return myHasWarnings; }
256 
257  /// Returns true if the item is a static work item
258  bool isStatic() const
259  { return myIsStatic; }
260 
261  /// Returns true if the work item is a regular item or batch item
262  bool isRegular() const;
263 
264  /// Returns true if the work item is a partition type
265  bool isPartition() const;
266 
267  /// Returns true if the work item is a dynamic, regular work item
268  bool isDynamic() const;
269 
270  /// Returns true if the work tiem is a batch
271  bool isBatch() const;
272 
273  /// Returns true if the work item is frozen
274  bool isFrozen() const
275  { return myIsFrozen; }
276 
277  /// Returns true if the work item is in process
278  bool isInProcess() const
279  { return (myCookType ==
281 
282  /// Returns true if the work item is marked as a service task
283  bool isServiceMode() const
284  { return (myCookType ==
286 
287  /// Returns true if the work item is scheduled out of process
288  bool isOutOfProcess() const
289  { return (myCookType ==
291 
292  /// Returns true if the work item is scheduled (either service, in, or out
293  /// of process).
294  bool isScheduled() const
295  {
296  if (myCookType ==
298  {
299  return false;
300  }
301 
302  if (myCookType ==
304  {
305  return false;
306  }
307 
308  return true;
309  }
310 
311  /// Returns true if the work item has post-cook logic
312  bool isPostCook() const
313  { return myIsPostCook; }
314 
315  /// Returns true if the work item is flagged as not being allowed to
316  /// generate child items.
317  bool isNoGenerate() const
318  { return myIsNoGenerate; }
319 
320  /// Returns true if the work item is marked as needing to clone output
321  /// files from its parent
322  bool isCloneOutputFiles() const
323  { return myIsCloneOutputFiles; }
324 
325  /// Returns true if the work item has to cook on the main thread, when
326  /// it cooks in process. This is determined by the script info from
327  /// the owning node
328  bool isMainThread() const;
329 
330  /// Returns the work item cook state
332  { return myState; }
333 
334  /// Returns the worst dependency state, e.g. if any of the deps have failed
335  /// or canceled.
337  { return myDepState; }
338 
339  /// Returns true if the work item is in a cooked state
340  bool isCooked() const;
341 
342  /// Returns true if the work item is in a successful cooked state
343  bool isSuccessful() const;
344 
345  /// Returns true if the work item is in a unsucessful cooked state
346  bool isUnsuccessful() const;
347 
348  /// Returns true if the work item is uncooked
349  bool isUncooked() const;
350 
351  /// Returns true if the work item has an unsucessful dependency
352  bool isUnsuccessfulDep() const;
353 
354  /// Returns true if the work item is an cooking/scheduled state
355  bool isProcessing() const;
356 
357  /// Returns true if the work item is long running
358  bool isLongRunning() const;
359 
360  /// Returns an enum indicating if work item is ready to cook, blocked, or
361  /// has failed dependencies
362  PDG_WorkItemEvalState evaluationState() const;
363 
364 
365  /// Returns the edge lock for the work item
367  { return myEdgeLock; }
368 
369  /// Returns the dependencies, e.g. work items that this item depends on. Not
370  /// thread safe.
371  const PDG_WorkItemIDMap& dependenciesUnsafe() const;
372 
373  /// Returns the dependents, e.g. work items that depend on this item. Not
374  /// thread safe.
375  const PDG_WorkItemIDMap& dependentsUnsafe() const;
376 
377  /// Returns a copy of the dependencies in an array
378  void dependencies(
379  PDG_WorkItemConstArray& dependency_array) const;
380 
381  /// Returns the dependency ids in a set
382  void dependencyIds(
383  PDG_WorkItemIDSet& dependency_ids) const;
384 
385  /// Returns the number of dependencies
386  int dependencySize() const;
387 
388  /// Returns a copy of the dependents in an array
389  void dependents(
390  PDG_WorkItemConstArray& dependent_array) const;
391 
392  /// Returns the dependent ids in a set
393  void dependentIds(
394  PDG_WorkItemIDSet& dependent_ids) const;
395 
396  /// Returns the root failed dependencies for this work item. This doesn't
397  /// include intermediate tasks in the dep chain that also failed because of
398  /// the same upstream failure
399  void failedDependencies(
400  PDG_WorkItemConstArray& dependecies) const;
401 
402  /// Returns a copy of the "visual" dependencies, e.g. the ones the user
403  /// would see if they used TOPs or wanted to print out the DAG itself. This
404  /// strips out any intenal items/dependencies used by TOPs
405  void visualDependencies(
406  PDG_WorkItemSet& deps,
407  bool expand,
408  PDG_Scheduler* filter) const;
409 
410  /// Inverse of the above
411  void visualDependents(
412  PDG_WorkItemSet& deps,
413  bool expand,
414  PDG_Scheduler* filter) const;
415 
416  /// Returns the require dependencies, e.g. deps that own this item. This is
417  /// used with partitioning to force the partition to by delete when one of
418  /// it's required dependencies is deleted
419  const PDG_WorkItemIDBoolMap&
420  requiredDependencies() const;
421 
422  /// Returns true if the item is a partition, and stores the partition
423  /// components into the array passed in as a parameter
424  bool partitionItems(
425  PDG_WorkItemConstArray& components) const;
426 
427  /// Flattens dependencies by query all dependencies of the work item,
428  /// and if the the dependency is a wrapper expands it into real items that
429  /// the wrapper depends on.
430  void sortedDependencies(
431  PDG_WorkItemConstArray& sorted_depencies,
432  bool only_upstream=false) const;
433 
434  /// Adds a dependency to the work item, and a corresponding dependent
435  /// back pointer to the incoming item.
436  ///
437  /// Returns false if the dependency already exists, and returns true if the
438  /// dependency does not
439  bool addDependency(
440  PDG_WorkItem* work_item,
441  bool required = false,
442  bool user_defined = false);
443 
444  /// Marks an existing dependency as a required dep. The work item passed in
445  /// *must* already be a dependency
446  void addRequiredDependency(
447  const PDG_WorkItem* work_item,
448  bool user_defined);
449 
450  /// Checks if the work item has a given item as a dependency
451  bool hasDependency(
452  const PDG_WorkItem* work_item) const;
453 
454  /// Removes a dependency and the corresponding back pointer from the target
455  /// work item.
456  bool removeDependency(PDG_WorkItem* work_item);
457 
458  /// Updates the loop stack, with the specified parent
459  void pushLoopInfoStack(
460  const PDG_WorkItem* parent_item);
461 
462  /// Sets the loop stack contents
463  void setLoopInfoStack(
464  const PDG_LoopInfo::Stack& loop_stack);
465 
466  /// Returns the loop stack entry at the specified index;
467  PDG_WorkItem* loopInfoStackEntry(int index) const;
468 
469  /// Returns the loop iteration of the work item
470  int loopInfoIteration() const;
471 
472  /// Returns the loop number of the work item
473  int loopInfoNumber() const;
474 
475  /// Returns the size of the loop that contains the work item
476  int loopInfoSize() const;
477 
478  /// Returns the loop depth of the work item
479  int loopInfoDepth() const;
480 
481  /// Returns the loop service lock ID of the work item
482  PDG_WorkItemID loopInfoLock() const;
483 
484  /// Returns the loop stack for this work item
486  { return myLoopInfoStack; }
487 
488  /// Compares the loop info stack of another work item with this one,
489  /// returning true if they put in the same partition or false if they're
490  /// from independent loops.
491  bool compareLoopInfoStack(
492  const PDG_WorkItem* other) const;
493 
494  /// Sets the is post cook flag
495  void setIsPostCook(bool postcook);
496 
497  /// Sets the work item no generate flag
498  void setIsNoGenerate(bool no_gen);
499 
500  /// Sets the work item's clone output files flag
501  void setIsCloneOutputFiles(bool clone);
502 
503  /// Sets the work item state
504  bool setState(
505  PDG_WorkItemState state,
506  bool emit=true);
507 
508  /// Sets the frozen flag on the work item
509  void setIsFrozen(bool is_frozen);
510 
511  /// Sets the execution type of the work item
512  void setExecutionType(
513  PDG_WorkItemExecutionType execution_type);
514 
515  /// Sets the cook type for the work item
516  void setCookType(PDG_WorkItemCookType cook_type);
517 
518  /// Returns the work item's modified flag
519  bool wasModified() const
520  { return myWasModified; }
521 
522  /// Sets the work item index
523  void setIndex(int index);
524 
525  /// Sets the work items internal index, used for dirtying
527  { myInternalIndex = index; }
528 
529  /// Sets the work item priority
530  void setPriority(int priority, bool emit=true);
531 
532  /// Boosts the priority of the work item and its dependencies, if they
533  /// have not yet started to cook
534  void boostPriority();
535 
536  /// Sets the work item frame. This methods takes a step size since it gets
537  /// overridden by batch work items to initialize the frame of each sub item
538  /// as frame + index*step_size;
539  bool clearFrame();
540  bool setFrame(fpreal frame);
541  bool setFrame(fpreal frame, fpreal frame_step);
542 
543  /// Sets the work item node
544  void setNode(PDG_Node* node, bool emit);
545 
546  /// Sets the pending dirty flag on the work item, used for regeneration
547  virtual void setPendingDirty(PendingDirty pending);
548 
549  /// Called when the work item should begin regenerating
550  void startRegenerating();
551 
552  /// Called when the work item is finished regenerating, in order to
553  /// determine if it needs to be deleted or dirtied
554  PendingDirty doneRegenerating();
555 
556  /// Sets the clone target, e.g. the item from which this item will
557  /// clone attributes
558  void setCloneTarget(
559  const PDG_WorkItem* clone_target);
560 
561  /// Sets the batch item info, e.g. the batch item pointer and the index
562  /// within the batch.
563  void setBatchInfo(PDG_BatchWorkItem* item,
564  int index,
565  int batch_index,
566  int priority);
567 
568  /// Updates the loop iteration and number, at the head of the current
569  /// loop stack
570  void setLoopInfo(
571  int iteration,
572  int number,
573  int size,
574  PDG_WorkItemID service_lock=theInvalidId);
575 
576  /// Sets the work item command directly
577  void setCommand(
578  const UT_StringHolder& command,
579  bool shell=false);
580 
581  /// Sets a per-platform work item command
582  void setPlatformCommands(
583  const UT_StringHolder& linux,
584  const UT_StringHolder& mac,
585  const UT_StringHolder& windows,
586  bool shell=false);
587 
588  /// Sets the work item label
589  void setLabel(const UT_StringHolder& label);
590 
591  /// Clears the work item label
592  void clearLabel();
593 
594  /// Sets the custom state string. Note that custom states are cleared
595  /// any time the actual work item state changes.
596  void setCustomState(const UT_StringHolder& state);
597 
598  /// Clears the custom state string
599  void clearCustomState();
600 
601  bool hasCookPercent() const;
602  fpreal cookPercent() const;
603  void setCookPercent(const fpreal &cook_percent);
604 
605  /// Appends a formatted error message to the work item's in process
606  /// log buffer
607  template <typename... Args>
609  const char* fmt,
610  Args&&... args) const
611  {
612  appendLogFmt(
613  true,
614  0,
615  "ERROR",
616  fmt,
617  std::forward<Args>(args)...);
618  }
619 
620  /// Appends a plain error message to the work item's in process
621  /// log buffer
622  void addError(
623  const UT_StringHolder& log,
624  bool timestamp=true) const
625  {
626  appendLogPrefix(
627  timestamp,
628  0,
629  "ERROR",
630  log);
631  }
632 
633  /// Appends a formatted warning message to the work item's in process
634  /// log buffer
635  template <typename... Args>
637  const char* fmt,
638  Args&&... args) const
639  {
640  appendLogFmt(
641  true,
642  0,
643  "WARNING",
644  fmt,
645  std::forward<Args>(args)...);
646  myHasWarnings = true;
647  }
648 
649  /// Appends a plain warning message to the work item's in process
650  /// log buffer
652  const UT_StringHolder& log,
653  bool timestamp=true,
654  int verbosity=0) const
655  {
656  appendLogPrefix(
657  timestamp,
658  verbosity,
659  "WARNING",
660  log);
661  myHasWarnings = true;
662  }
663 
664  /// Appends a formatted message to the work item's in process
665  /// log buffer
666  template <typename... Args>
668  const char* fmt,
669  Args&&... args) const
670  {
671  appendLogFmt(
672  true,
673  0,
675  fmt,
676  std::forward<Args>(args)...);
677  }
678 
679  /// Appends a plain message to the work item's in process log buffer
681  const UT_StringHolder& log,
682  bool timestamp=true,
683  int verbosity=0) const
684  {
685  appendLogPrefix(
686  timestamp,
687  verbosity,
689  log);
690  }
691 
692  /// Appends text data to the log with the given log type
693  void addLog(
695  const UT_StringHolder& log,
696  bool timestamp=true,
697  int verbosity=0) const;
698 
699  /// Returns the current in process log buffer
700  const UT_WorkBuffer& logMessages() const
701  { return myLogMessages; }
702 
703  /// Returns the application path to the work item, e.g. /node/workitem
704  void appPath(UT_WorkBuffer& buffer) const;
705 
706  /// Dirties the work item by initiating a dirty operation from the graph.
707  /// Probably the method you want to call to dirty the item if you aren't
708  /// working in $PDG directly
709  void dirty(bool should_delete, bool remove_outputs);
710 
711  /// Prepares the work item for dirtying, but does not actually modify the
712  /// item or its dependencies in any way.
713  PDG_WorkItemDirty prepareDirty(bool should_delete);
714 
715  /// Cancels the work item's execution, if it's running with a scheduler
716  void cancel();
717 
718  /// Freezes the attributes of the work item and clears out all references
719  /// to upstream work items and data.
720  void freeze();
721 
722  /// Invalidates the file cache of this work item
723  void invalidateCache();
724 
725  /// Syncs work item data with its parent
726  void syncData(bool force_recursive);
727 
728  /// Getter/setter for the backing work item data object, which might be a
729  /// user defined class
730  PDG_WorkItemData* data() const;
731  void setData(PDG_WorkItemDataPtr data);
732  bool compareDataType(
733  const PDG_BaseType* other_type,
734  bool deep=true) const;
735  SYS_HashType hash() const;
736 
737  /// Does a numeric data lookup, for use with the @ operator
738  PDG_AttributeCast numericData(
739  fpreal& param,
740  const PDG_AttributeEvaluator& evaluator,
741  fpreal frame,
742  int thread) const;
743 
744  /// Does a string data lookup, for use with the @ operator
745  PDG_AttributeCast stringData(
747  const PDG_AttributeEvaluator& evaluator,
748  fpreal frame,
749  int thread) const;
750 
751  /// Serialization of underlying PDG_WorkItemData object
752  UT_StringHolder serializeData() const;
753  bool serializeDataToFile(
754  const UT_StringHolder& file) const;
755 
756  /// Compares the work item with another item
757  bool compare(PDG_WorkItem* item) const;
758 
759 
760  /// Runs selection logic from selecting a work item in TOPs
761  bool uiSelect(bool select);
762 
763 
764  /// Returns true if the work item has at least one output file
765  bool hasOutputFiles() const;
766 
767  /// Returns the output file cache state of this work item
768  virtual PDG_WorkItemCacheState
769  outputCacheState() const;
770 
771  /// Returns the raw output file array for the work item. The paths in this
772  /// array are unlocalized, i.e. they may contain __PDG_DIR__
773  bool outputFiles(
774  PDG_File::Array& files) const;
775 
776  /// Localizes output files and returns it back in the supplied array.
777  /// Returns false if the work item has no node or the node has no
778  /// scheduler.
779  bool localizedOutputFiles(
780  PDG_File::Array& files) const;
781 
782  /// Returns the local temporary file directory from the sechduler
783  /// associated with this work item.
784  UT_StringHolder tempDir() const;
785 
786  /// Returns all output files that are a substring-match for the given tag
787  bool outputFilesForTag(
788  PDG_File::Array& files,
789  const UT_StringHolder& tag,
790  bool include_expected = false) const;
791 
792  /// Localizes a file path with the scheduler associated with the work
793  /// item
794  UT_StringHolder localizePath(
795  const UT_StringHolder& path) const;
796 
797  /// Returns the first output file that is a substring-match for the given
798  /// tag
799  UT_StringHolder firstOutputFileForTag(
800  const UT_StringHolder& tag) const;
801 
802  /// Returns the list of expected output files
803  bool expectedOutputFiles(
804  PDG_File::Array& files) const;
805 
806  /// Adds an expected output file to the work item
807  void addExpectedOutputFile(
808  const UT_StringHolder& path,
809  const UT_StringHolder& tag,
810  bool own);
811 
812  /// Adds an array of expected output file paths to the work item
813  void addExpectedOutputFiles(
814  const UT_StringArray& paths,
815  const UT_StringHolder& tag,
816  bool own);
817 
818  /// Adds an expected output file to the work item
819  void addExpectedOutputFile(const PDG_File& file);
820 
821  /// Appends the given output file to the work item
822  void addOutputFile(
823  const UT_StringHolder& path,
824  const UT_StringHolder& tag,
825  PDG_File::Hash hash_code,
826  bool own);
827 
828  /// Appends the given array of output files to the work item. If the
829  /// length of the hashes array is less than the paths, a default value
830  /// of 0 will be used for paths without a corresponding hash code. I.e
831  /// an empty hash array can be passed in to use 0 for each file. Assumes
832  /// that the same tag is used for all work items
833  void addOutputFiles(
834  const UT_StringArray& paths,
835  const UT_StringHolder& tag,
836  const PDG_File::HashArray& hashes,
837  bool own);
838 
839  /// Appends the given array of output files to the work item. If the length
840  /// of the hashes or tags array is less than the paths, a default value of
841  /// 0 and "" are used respectively.
842  void addOutputFiles(
843  const UT_StringArray& paths,
844  const UT_StringArray& tags,
845  const PDG_File::HashArray& hashes,
846  bool own);
847 
848  /// Replaces the output file at the given index, if that index is valid
849  void updateOutputFile(
850  int index,
851  const UT_StringHolder& path,
852  const UT_StringHolder& tag,
853  PDG_File::Hash hash_code,
854  bool owner);
855 
856  /// Returns the input files for immediate dependencies
857  void inputFiles(
858  PDG_File::Array& files,
859  bool include_expected) const;
860 
861  /// Returns all input files that are a substring-match for the given tag
862  bool inputFilesForTag(
863  PDG_File::Array& files,
864  const UT_StringHolder& tag,
865  bool include_expected = false) const;
866 
867  /// Checks if the work items output files are valid, e.g. they exist
868  bool validateOutputFiles();
869 
870  /// Transfers all dependent files to the remote directory, using the
871  /// supplied scheduler
872  bool transferFiles(bool update_cache);
873 
874  /// Clears all output files
875  void clearOutputFiles();
876 
877  /// Clears expected outputs
878  void clearExpectedOutputFiles();
879 
880  /// Clears output files added during the cook. Note that this method is
881  /// only safe to call if the item has not yet been marked as cooked.
882  void clearRuntimeOutputFiles();
883 
884  /// Sets the clear environment flagging, effectively stopping any upstream
885  /// env vars from being considered beyond this work item
886  void clearEnvironment();
887 
888  /// Adds an environment variable and value to this work items env var map
889  bool addEnvironmentVar(
890  const UT_StringHolder& var_name,
891  const PDGT_Value& value,
892  UT_WorkBuffer& errors);
893 
894  /// Returns true if the work item has the specified env var
895  bool hasEnvironmentVar(
896  const UT_StringHolder& var_name) const;
897 
898  /// Globs the full set of environment variables for this work item by
899  /// searching through it's parents.
900  void globEnvironment(
901  PDGT_Value::Map& environment) const;
902 
903  /// Looks up a key in the environment, returns true if the key exists
904  bool envLookup(const char* key,
905  PDGT_Value& value) const;
906 
907  /// Adds a warning to the owning node
908  void addNodeWarning(
909  const UT_StringHolder& message) const;
910 
911  /// Starts cooking a sub item in a batch, and optionally waits for it to
912  /// be available. Returns false if the item cannot start due to a failure.
913  bool startSubItem(bool wait, UT_WorkBuffer& errors);
914 
915  /// Returns true if the sub item is ready
916  bool checkSubItem(UT_WorkBuffer& errors);
917 
918  /// Marks a work item is cooking, updates it cook start time, and prints
919  /// log messages.
920  bool startWorkItem();
921 
922  /// Marks the work item as cooked, if it is an in-process batch item
923  bool cookSubItem(
924  PDG_WorkItemState state,
925  fpreal duration);
926 
927  /// Returns the scheduler associated with the work item, looked up from the
928  /// work item's node.
929  PDG_Scheduler* scheduler() const;
930 
931  /// Returns true if the work item is schduled by the specified schduler
932  bool isScheduledBy(PDG_Scheduler* scheduler) const;
933 
934  /// Reports a performance monitor event for this work item
935  void performanceEvent() const;
936 
937  /// Sets a stat to the current time
938  virtual fpreal setTimeStat(PDG_WorkItemStats::StatType stat,
939  bool batched);
940 
941  /// Returns the perf stats object for the work item
942  const PDG_WorkItemStats& stats() const
943  { return myStats; }
945  { return myStats; }
946 
947  /// Replaces env vars in the target string with respect to the environment
948  /// and properties of this work item. If py_replace is true, replaces the
949  /// env vars with a valid python token instead of
950  void varReplace(const char* input,
951  UT_WorkBuffer& output) const;
952 
953  /// Saves the work item to a .json file or string
954  bool saveJSONFile(
955  const UT_StringHolder& path,
956  bool pretty_print,
957  bool global_attributes,
958  bool skip_defaults) const;
959  UT_StringHolder saveJSONString(
960  bool pretty_print,
961  bool global_attributes,
962  bool skip_defaults) const;
963 
964  /// Saves the work item to a UT_OptionsHolder
965  UT_OptionsHolder saveDict(
966  bool global_attributes,
967  bool skip_defaults) const;
968 
969  /// Saves the array of work items to a .json file or string
970  static bool saveArrayJSONFile(
971  const UT_StringHolder& path,
972  const PDG_WorkItemArray& items,
973  bool pretty_print,
974  bool global_attributes,
975  bool skip_defaults);
976  static UT_StringHolder saveArrayJSONString(
977  const PDG_WorkItemArray& items,
978  bool pretty_print,
979  bool global_attributes,
980  bool skip_defaults);
981 
982  /// Saves the list of work items to a UT_OptionsHolder
983  static UT_OptionsHolder saveArrayDict(
984  const PDG_WorkItemArray& items,
985  bool global_attributes,
986  bool skip_defaults);
987 
988  /// Loads a work item from a .json file or string
989  static PDG_WorkItem* loadJSONFile(
990  const UT_StringHolder& path,
991  bool make_active,
992  UT_WorkBuffer& errors);
993  static PDG_WorkItem* loadJSONString(
994  const UT_StringHolder& data,
995  bool make_active,
996  UT_WorkBuffer& errors);
997 
998  /// Creates a JSON patch from the work item as a string
999  UT_StringHolder createJSONPatch(bool pretty_print) const;
1000 
1001 protected:
1002  friend class PDG_BatchWorkItem;
1003  friend class PDG_Scheduler;
1004  friend class PDG_WorkItemHolder;
1005  friend class PDG_WorkItemDirty;
1006 
1007  /// Dirties or deletes the work item and propogates the necessary changes
1008  /// to downstream items and dependents.
1009  void prepareDirty(PDG_WorkItemDirty& dirty_state,
1010  bool is_delete,
1011  bool is_dirty_data) const;
1012 
1013  /// Applies a dirty operation that was stored to the dirty state
1014  void applyDirty(
1015  const PDG_WorkItemDirty& dirty_state,
1016  bool remove_outputs);
1017 
1018  /// Runs custom dirty handlers and deletes output files
1019  void handleDirty(
1020  const UT_ArrayStringSet& keep_set,
1021  bool should_delete,
1022  bool remove_outputs);
1023 
1024  /// Returns the list of files owned by this work item
1025  void ownedFiles(
1026  PDG_File::Set& files,
1027  bool include_expected) const;
1028 
1029  /// Updates the file cache of this work item so the ID is at least as
1030  /// large as the input ID
1031  void updateCacheId(PDG_CacheID cache_id);
1032 
1033 
1034  /// Called when one of the dependencies owned by this work item
1035  /// is resolved.
1037  PDGE_Resolutions& resolutions,
1038  const PDGE_Evaluator& evaluator,
1039  PDGE_Dependency* dependency) override;
1040 
1041  /// Called when one of the dependencies owned by this work item
1042  /// is unresolved;
1043  int evalUnresolve(
1044  PDGE_Dependency::Array& extras,
1045  PDGE_Dependency* dependency) override;
1046 
1047  /// Called when an item is about to be scheduled, to check if it has
1048  /// cached outputs on disk. Returns Uncooked if the item is not cached
1049  /// and should cook, Failure if the read files mode fails, or CookedCache
1050  /// if the item is cooked from cache.
1051  virtual PDG_WorkItemState checkCached(bool add_outputs);
1052 
1053  /// Upgrades expected outputs to runtime outputs for this work item
1054  bool upgradeExpectedOutputFiles(
1055  bool check_disk,
1056  bool clear);
1057 
1058  /// Called by the scheduler to mark out of process items as cooked
1059  void postCook(PDGE_Resolutions& resolutions,
1061  int batch_index,
1062  fpreal duration,
1063  bool notify_all,
1064  bool static_cook,
1065  bool unresolve);
1066 
1067  /// Notifies this work item that is has cooked.
1068  virtual void notifyCook(PDGE_Resolutions& resolutions,
1070  int batch_index,
1071  fpreal duration,
1072  bool notify_all,
1073  bool static_cook,
1074  bool unresolve);
1075 
1076  /// Syncs a work item's cache with its dependents
1077  virtual void syncCacheId();
1078 
1079  /// Does an inprocess cook of the work item, returning true on success
1080  bool cook();
1081 
1082  /// Internal method for setting the frame
1083  virtual bool setFrame(fpreal frame,
1084  fpreal frame_step,
1085  bool check_dirty);
1086 
1087  /// Adds a parent and copies attributes by inheriting them from the parent
1088  /// of another work item. This is used for generating dynamic sub items
1089  /// in a batch, and copying the parent information from the last batch
1090  /// item onto the new item
1091  void inheritParent(const PDG_WorkItem* work_item);
1092 
1093 private:
1094  /// Removes all dependencies from the work item, and removes them work item
1095  /// from the dependents list of all dependencies
1096  void removeAllDependencies();
1097 
1098  /// Updates the dependency state, given the incoming state
1099  void updateDependencyState(
1100  PDG_WorkItemState dep_state);
1101 
1102  /// Returns a copy of the dependencies in an array, and filters out
1103  /// work items from the target node
1104  void upstreamDependencies(
1105  PDG_WorkItemConstArray& dependency_array) const;
1106 
1107  /// Helper method to query the list of failed upstream dependencies
1108  void failedDependencies(
1109  PDG_WorkItemConstArray& dependencies,
1110  PDG_WorkItemIDSet& visited) const;
1111 
1112  /// Called before the work item is scheduled
1113  bool prepareTask();
1114 
1115  /// Called when a work item should be scheduled for execution
1116  PDG_WorkItemState schedule();
1117 
1118  /// Appends a log message with a time stamp and a new line
1119  void appendLogPrefix(
1120  bool timestamp,
1121  int verbosity,
1122  const UT_StringHolder& prefix,
1123  const UT_StringHolder& msg) const;
1124 
1125  /// Appends a format string to the work item's log
1126  template <typename... Args>
1127  void appendLogFmt(
1128  bool timestamp,
1129  int verbosity,
1130  const UT_StringHolder& prefix,
1131  const char* fmt,
1132  Args&&... args) const
1133  {
1135  message.format(
1136  fmt, std::forward<Args>(args)...);
1137  appendLogPrefix(
1138  timestamp,
1139  verbosity,
1140  prefix,
1141  message.buffer());
1142  }
1143 
1144  /// Processes this work item as a priority boost dep
1145  void processBoostDependency(PDG_WorkItemSet& deps);
1146 
1147  /// Gathers the input dependencies from priority boosting
1148  void gatherBoostDependencies(PDG_WorkItemSet& deps);
1149 
1150 protected:
1153 
1155 
1159 
1161 
1163 
1166 
1169 
1171 
1174 
1176  int myIndex;
1181 
1188 
1190 
1191  bool myHasFrame:1;
1192  mutable bool myHasWarnings:1;
1193 
1194  bool myAttributesDirty:1;
1195  bool myWasModified:1;
1196  bool myDidCook:1;
1197 
1198  bool myIsPostCook:1;
1199  bool myIsStatic:1;
1200  bool myIsNoGenerate:1;
1201  bool myIsCloneOutputFiles:1;
1202  bool myIsFrozen:1;
1203  bool myIsBatchReady:1;
1204 };
1205 
1206 #endif /* __PDG_WORKITEM_H__ */
exint PDG_WorkItemID
Type defs for unique work item IDs.
bool isNoGenerate() const
Definition: PDG_WorkItem.h:317
const PDG_LoopInfo::Stack & loopInfoStack() const
Returns the loop stack for this work item.
Definition: PDG_WorkItem.h:485
bool isScheduled() const
Definition: PDG_WorkItem.h:294
GLuint GLsizei const GLchar * message
Definition: glcorearb.h:2543
bool isPostCook() const
Returns true if the work item has post-cook logic.
Definition: PDG_WorkItem.h:312
PDG_WorkItemIDBoolMap myRequiredDependencies
bool isServiceMode() const
Returns true if the work item is marked as a service task.
Definition: PDG_WorkItem.h:283
GLuint GLsizei const GLchar * label
Definition: glcorearb.h:2545
void addMessageFmt(const char *fmt, Args &&...args) const
Definition: PDG_WorkItem.h:667
PDG_Node * node() const
Returns the node that owns the work item.
Definition: PDG_WorkItem.h:190
bool hasFrame() const
Returns true if the frame has been set.
Definition: PDG_WorkItem.h:246
void addMessage(const UT_StringHolder &log, bool timestamp=true, int verbosity=0) const
Appends a plain message to the work item's in process log buffer.
Definition: PDG_WorkItem.h:680
GLboolean * data
Definition: glcorearb.h:131
void addErrorFmt(const char *fmt, Args &&...args) const
Definition: PDG_WorkItem.h:608
int index() const
Returns the work item index.
Definition: PDG_WorkItem.h:220
virtual UT_RWLock * attribGlobalLock() const
UT_StringHolder debugName() const override
Definition: PDG_WorkItem.h:119
#define PDG_API
Definition: PDG_API.h:23
GLsizei const GLchar *const * path
Definition: glcorearb.h:3341
virtual void attribUnlock(bool modified) const
virtual UT_StringHolder attribOwnerName() const
SYS_FORCE_INLINE const char * buffer() const
std::size_t SYS_HashType
Define the type for hash values.
Definition: SYS_Hash.h:19
int batchIndex() const
Definition: PDG_WorkItem.h:229
**But if you need a result
Definition: thread.h:613
bool isOutOfProcess() const
Returns true if the work item is scheduled out of process.
Definition: PDG_WorkItem.h:288
PDG_CacheID myCacheId
PDG_WorkItemType myType
PDG_WorkItemEvalState
bool isCloneOutputFiles() const
Definition: PDG_WorkItem.h:322
PDG_WorkItemStats myStats
void addWarning(const UT_StringHolder &log, bool timestamp=true, int verbosity=0) const
Definition: PDG_WorkItem.h:651
bool wasModified() const
Returns the work item's modified flag.
Definition: PDG_WorkItem.h:519
unsigned char uint8
Definition: SYS_Types.h:36
Unspecified/automatic cook type.
CompareResults OIIO_API compare(const ImageBuf &A, const ImageBuf &B, float failthresh, float warnthresh, ROI roi={}, int nthreads=0)
PDG_WorkItemType
Enum of work item types.
PDG_Node * myNode
virtual bool attribCanWrite(const PDG_EvaluationContext *context) const
int64 Hash
The file hash/modtime type.
Definition: PDG_File.h:39
Definition: core.h:760
PDGE_Dependency myIsCookedDep
PDG_WorkItemState myState
PDGE_Dependency * isCookedDep()
Returns the is cooked dependency object for this work item.
Definition: PDG_WorkItem.h:132
vint4 select(const vbool4 &mask, const vint4 &a, const vint4 &b)
Definition: simd.h:4816
PDG_GraphContext * context() const
Returns the context that owns the work item.
Definition: PDG_WorkItem.h:194
PDG_WorkItemState
Enum of possible work item states.
fpreal frame() const
Returns the work item frame.
Definition: PDG_WorkItem.h:242
static const UT_StringHolder theEmptyString
virtual void attribModify(PDG_AttributeType attrib_type, const UT_StringHolder &attrib_name) const
PDG_AttributeCast
Enumeration of attribute cast results.
The work item cooks as a service.
The work item cooks out of process.
const UT_WorkBuffer & logMessages() const
Returns the current in process log buffer.
Definition: PDG_WorkItem.h:700
PDG_BatchWorkItem * batchParent() const
Definition: PDG_WorkItem.h:203
long long int64
Definition: SYS_Types.h:116
PDG_AttributeType
Enumeration of possible attribute types.
void applyDirty(bool remove_outputs)
Applies the stored dirty operation.
bool isFrozen() const
Returns true if the work item is frozen.
Definition: PDG_WorkItem.h:274
virtual int64 getMemoryUsage(bool inclusive) const
Returns the memory usage of this owner instance.
PDG_WorkItemCookType
Enumeration of work item cook types.
The work item cooks in process.
fpreal frameStep() const
Returns the work item step size, primarily used with batch items.
Definition: PDG_WorkItem.h:250
bool isInProcess() const
Returns true if the work item is in process.
Definition: PDG_WorkItem.h:278
PDG_WorkItemType type() const
Returns the work item type.
Definition: PDG_WorkItem.h:136
PDG_Platform
Enumeration of supported platforms, for work item command line array.
PDG_WorkItemID myId
PDG_WorkItemIDMap myDependencies
void addError(const UT_StringHolder &log, bool timestamp=true) const
Definition: PDG_WorkItem.h:622
PDG_WorkItemLogType
Enumeration of work item log message types.
PDG_WorkItemCacheState
fpreal myFrameStep
StatType
Enumeration of stats stored in this object.
*tasks wait()
PendingDirty
Enumeration of pending dirty states.
Definition: PDG_WorkItem.h:55
UT_TBBSpinLock myEdgeLock
No specified platform.
GLsizeiptr size
Definition: glcorearb.h:664
**Note that the tasks the is the thread number *for the or if it s being executed by a non pool thread(this *can happen in cases where the whole pool is occupied and the calling *thread contributes to running the work load).**Thread pool.Have fun
PDG_BatchWorkItem * myBatchParent
UT_SharedPtr< PDG_WorkItemData > PDG_WorkItemDataPtr
Type def for registered type objects.
GLenum GLfloat param
Definition: glcorearb.h:104
size_t format(const char *fmt, const Args &...args)
PDG_CacheID cacheId() const
Returns the cache id of the work item.
Definition: PDG_WorkItem.h:238
PDG_WorkItemExecutionType myExecutionType
The item needs to be dirtied, but not deleted.
Definition: PDG_WorkItem.h:61
fpreal64 fpreal
Definition: SYS_Types.h:277
UT_TBBSpinLock & edgeLock() const
Returns the edge lock for the work item.
Definition: PDG_WorkItem.h:366
PDG_WorkItemDataPtr myData
PDG_LoopInfo::Stack myLoopInfoStack
PDG_WorkItemID id() const
Returns the id of the work item, guaranteed to be unique.
Definition: PDG_WorkItem.h:148
GLuint index
Definition: glcorearb.h:786
PDG_WorkItemIDMap myDependents
PDGE_Dependency * canCookDep()
Returns the can cook dependency object for this work item.
Definition: PDG_WorkItem.h:128
PDG_WorkItemState dependencyState() const
Definition: PDG_WorkItem.h:336
PDGE_Dependency myCanCookDep
void addWarningFmt(const char *fmt, Args &&...args) const
Definition: PDG_WorkItem.h:636
virtual PDGE_Dependency::State evalResolve(PDGE_Resolutions &, const PDGE_Evaluator &, PDGE_Dependency *)
Called when a dependency owned by this object is resolved.
void setInternalIndex(int index)
Sets the work items internal index, used for dirtying.
Definition: PDG_WorkItem.h:526
PDG_WorkItemState myDepState
virtual UT_StringHolder debugGroup() const
bool hasWarnings() const
Returns true if the work item has any warning log messages.
Definition: PDG_WorkItem.h:254
**If you just want to fire and args
Definition: thread.h:609
PDG_WorkItemStats & stats()
Definition: PDG_WorkItem.h:944
PDG_WorkItemExecutionType
Enum of work item runtime types.
void updateCacheId(const PDG_WorkItem *dependent, const PDG_WorkItem *dependency)
Updates the cache ID of the work item for a dependency.
PDG_WorkItemCookType myCookType
The item does not require any sort of dirtying.
Definition: PDG_WorkItem.h:58
Definition: core.h:1131
virtual void attribWarn(const UT_StringHolder &msg) const
UT_WorkBuffer myLogMessages
const PDG_WorkItemStats & stats() const
Returns the perf stats object for the work item.
Definition: PDG_WorkItem.h:942
PDG_WorkItemState state() const
Returns the work item cook state.
Definition: PDG_WorkItem.h:331
PDG_GraphContext * myContext
int priority() const
Returns the work item priority.
Definition: PDG_WorkItem.h:233
PDG_WorkItemCookType cookType() const
Returns the work item cook type.
Definition: PDG_WorkItem.h:144
virtual int evalUnresolve(PDGE_Dependency::Array &, PDGE_Dependency *)
OIIO_FORCEINLINE T log(const T &v)
Definition: simd.h:7688
bool isStatic() const
Returns true if the item is a static work item.
Definition: PDG_WorkItem.h:258
Reader/Writer mutex class.
Definition: UT_RWLock.h:48
type
Definition: core.h:1059
virtual void resetOwner()
Resets the owner.
PDG_WorkItemExecutionType executionType() const
Returns the work item execution type.
Definition: PDG_WorkItem.h:140
const PDG_WorkItem * myCloneTarget
void prepareDirty(PDG_WorkItem *work_item)
Adds a work item and its dependencies to the dirty sets.
Definition: format.h:895
PendingDirty myPendingDirty
const PDG_WorkItem * cloneTarget() const
Definition: PDG_WorkItem.h:208
virtual const PDG_EventFilter & supportedEventTypes() const =0
Returns the list of supported event types for this emitter.
int internalIndex() const
Returns the work item's internal index.
Definition: PDG_WorkItem.h:224
int PDG_CacheID
Definition: PDG_NodeTypes.h:30
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
Definition: glcorearb.h:1297