HDK
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
PDG_Scheduler Class Referenceabstract

#include <PDG_Scheduler.h>

+ Inheritance diagram for PDG_Scheduler:

Public Types

enum  ScheduleResult {
  eCookCanceled, eCookFailed, eCookSucceeded, eFailed,
  eSucceeded, eDeferred, eFullDeferred
}
 
enum  TickResult { eSchedulerReady, eSchedulerBusy, eSchedulerCancelCook }
 
enum  AcceptResult { eSchedulerAccept, eSchedulerReject, eSchedulerDefault }
 
enum  PathMappingMode { eDefaultPathMapping, eNoPathMapping }
 
enum  WorkItemDataSource { eSourceJSONFile, eSourceRPCMessage }
 
enum  TempDirCleanup { eCleanupNone, eCleanupShutdown, eCleanupCook }
 
- Public Types inherited from PDG_NodeInterface
using ParameterGroupArray = UT_Array< ParameterGroup >
 
using DependencyArray = UT_Array< AutoFileDependency >
 
- Public Types inherited from PDGT_TypeInstance< EnumType >
using BaseType = PDGT_BaseType< EnumType >
 
- Public Types inherited from PDGE_DependencyOwner
using Array = UT_Array< PDGE_DependencyOwner * >
 
using Set = UT_ArraySet< PDGE_DependencyOwner * >
 

Public Member Functions

 PDG_Scheduler (const PDG_BaseType *type, const PDGT_ValueArgs &extra_args, const UT_StringHolder &name)
 
 ~PDG_Scheduler () override
 
const PDG_EventFiltersupportedEventTypes () const override
 Returns the list of supported event types. More...
 
int64 getMemoryUsage (bool inclusive) const override
 Returns the memory usage of this owner instance. More...
 
UT_StringHolder debugName () const override
 Debug name method, required by PDGE_DependencyOwner. More...
 
void resetOwner () override
 Resets the dependency owner. More...
 
PDGE_DependencyisCookedDep ()
 Returns the is cooked dep for this scheduler. More...
 
PDGE_DependencyisSetupDep ()
 Returns the is cooked dep for this scheduler. More...
 
const UT_StringHoldername () const
 Returns the name of the shceudler. More...
 
bool reloadInstance (UT_WorkBuffer &errors) override
 
const PDG_NodeInterfacetemplateInterface () const override
 
bool canSchedule (const PDG_WorkItem *work_item)
 
virtual void onScheduleStatic (const PDG_WorkItemMap &dependencies, const PDG_WorkItemMap &dependents, const PDG_WorkItemArray &ready_items)=0
 Called when the scheduler should process a static dependency graph. More...
 
virtual ScheduleResult onSchedule (PDG_WorkItem *work_item)=0
 
virtual void onCancelWorkItems (const PDG_WorkItemArray &work_item, PDG_Node *node)=0
 Called when a specific work item should be canceled. More...
 
virtual bool onStart ()=0
 
virtual bool onStop ()=0
 
virtual bool onStartCook (bool static_cook, const PDG_NodeSet &)=0
 Called when cooking begins. More...
 
virtual bool onStopCook (bool cancel)=0
 Called when cooking completes or is canceled. More...
 
virtual void onConfigureCook (PDG_CookOptions *cook_ptions)=0
 
virtual bool onSetupCook ()=0
 
virtual TickResult onTick ()=0
 Called periodically during cooking to update state. More...
 
virtual AcceptResult onAcceptWorkItem (const PDG_WorkItem *work_item)=0
 
virtual bool onTransferFile (const UT_StringHolder &file_path)
 Custom file transfer logic, implemented per-scheduler. More...
 
bool transferFile (const UT_StringHolder &file_path, PDG_FileUtils::TransferType default_type, bool check_type, UT_WorkBuffer &errors) const
 
bool transferFile (const UT_StringHolder &root_path, const UT_StringHolder &file_path, const UT_StringHolder &sub_path, PDG_FileUtils::TransferType default_type, bool check_type, UT_WorkBuffer &errors) const
 
UT_StringHolder formatTransferPath (const UT_StringHolder &local_path, PDG_FileUtils::TransferType default_type, bool local, bool check_type) const
 
UT_StringHolder formatTransferPath (const UT_StringHolder &root_path, const UT_StringHolder &local_path, const UT_StringHolder &sub_path, PDG_FileUtils::TransferType default_type, bool local, bool check_type) const
 
UT_StringHolder localizePath (const UT_StringHolder &deloc_path) const
 Returns the local version of the given path. More...
 
UT_StringHolder delocalizePath (const UT_StringHolder &local_path) const
 Returns the delocalized (remote) version of the given local path. More...
 
UT_StringHolder jobName (const PDG_WorkItem *work_item) const
 Returns the job name for the specified work item. More...
 
virtual UT_StringHolder expandCommandTokens (const UT_StringHolder &command, const PDG_WorkItem *work_item)=0
 Expands special tokens in the command string. More...
 
virtual UT_StringHolder submitAsJob (const UT_StringHolder &graph_file, const UT_StringHolder &node_name)=0
 cook the output node in the graph context of the given file as a single job More...
 
void setWorkingDir (const UT_StringHolder &local_path, const UT_StringHolder &remote_path)
 
void setTempDir (const UT_StringHolder &local_path, const UT_StringHolder &remote_path)
 
void setScriptDir (const UT_StringHolder &local_path, const UT_StringHolder &remote_path)
 
void setAcceptInProcess (bool in_process)
 sets whether or not the scheduler accepts in-process work items More...
 
UT_StringHolder workingDir (bool local) const
 
UT_StringHolder tempDir (bool local) const
 
UT_StringHolder scriptDir (bool local) const
 
UT_StringHolder logDir (bool local) const
 
UT_StringHolder dataDir (bool local) const
 
virtual UT_StringHolder applicationBin (const UT_StringHolder &name, const PDG_WorkItem *work_item)
 
virtual UT_StringHolder workItemResultServerAddr ()=0
 
virtual UT_StringHolder getLogURI (const PDG_WorkItem *work_item)=0
 returns a URI for the workitem cook log (text file) More...
 
virtual UT_StringHolder getStatusURI (const PDG_WorkItem *work_item)=0
 returns a URI for the workitem status page if relevant More...
 
virtual bool endSharedServer (const UT_StringHolder &sharedserver_name)=0
 Terminate the give shared server. More...
 
virtual void clearSharedServerInfo (const UT_StringHolder &sharedserver_name)
 Clears metadata associated with the given shared server name. More...
 
virtual UT_StringHolder setSharedServerInfo (const UT_StringHolder &sharedserver_name, const PDGT_ValueArgs &info)
 Sets metadata associated with the given shared server name, returns full name. More...
 
PDGT_ValueArgs getSharedServerInfo (const UT_StringHolder &sharedserver_name)
 Gets metadata associated with the given shared server name. More...
 
UT_StringArray getSharedServers ()
 Gets the list of shared servers. More...
 
virtual void cleanTempDirectory ()
 Delete the temporary directory and all its contents. More...
 
void stop (bool exiting)
 Called to stop the scheduler. More...
 
bool startCook (PDG_CookState &cook_state)
 Called when a cook begins with the cook state data for that cook. More...
 
void stopCook ()
 Called when a cook stops. More...
 
void cancelCook (bool pause)
 Cancels an active cook. More...
 
bool tick (PDG_Scheduler::TickResult &tick_result)
 Ticks the scheduler. More...
 
bool cookWorkItem (PDG_WorkItem *work_item)
 Cooks a specific work item using it's internal cook method. More...
 
void submitStatic ()
 Submits a static cook with this scheduler. More...
 
virtual PDG_WorkItemState queueWorkItem (PDG_WorkItem *work_item)
 
void removeWorkItem (const PDG_WorkItem *work_item)
 Removes a work item from any internal tracking lists. More...
 
void cancelWorkItem (PDG_WorkItem *work_item)
 Cancels a cook for a specific work tiem. More...
 
void cancelNode (PDG_Node *node)
 Cancels a cook for a specific node. More...
 
void onWorkItemPriorityChanged (PDG_WorkItem *work_item)
 
void process (bool skip_ready_items)
 
bool isRunning (int count=0)
 
bool isRunning (const PDG_WorkItem *work_item)
 
bool isDefault ()
 
bool isWaitForFailures () const
 
bool isValidateOutputFiles () const
 
bool isCheckExpectedOutputFiles () const
 
bool isCompressWorkItemData () const
 
virtual
PDGN::PDGN_PollingClientNNG * 
getPollingClient ()
 
WorkItemDataSource workItemDataSource () const
 
int numRunningItems () const
 
int numFailedItems () const
 
int numQueuedItems () const
 
void setContext (PDG_GraphContext *context)
 
PDG_GraphContextcontext () const
 
void dependencyGraph (PDG_WorkItemMap &dependencies, PDG_WorkItemMap &dependents, PDG_WorkItemArray &ready, bool expand)
 
virtual bool startService (UT_WorkBuffer &errors, PDG_Service *service)
 Starts a service using this scheduler. More...
 
virtual bool stopService (UT_WorkBuffer &errors, PDG_Service *service)
 Stops a service that was started with this scheduler. More...
 
template<typename T , typename D = T>
bool evaluateOverride (T &result, PDG_NodeInterface *node, const UT_StringHolder &prefix, const UT_StringHolder &parm, PDG_WorkItem *work_item, const D &default_value, UT_WorkBuffer &errors) const
 
void onWorkItemSetStringArray (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &attribute_name, const PDG_AttributeString::Array &value)
 
void onWorkItemSetFloatArray (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &attribute_name, const PDG_AttributeFloat::Array &value)
 
void onWorkItemSetIntArray (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &attribute_name, const PDG_AttributeInteger::Array &value)
 
void onWorkItemSetFileArray (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &attribute_name, const PDG_AttributeFile::Array &value)
 
void onWorkItemSetDictArray (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &attribute_name, const UT_Array< UT_OptionsHolder > &value)
 
void onWorkItemSetPyObject (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &attribute_name, const UT_StringHolder &pyobject_repr)
 
void onWorkItemSetString (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &attribute_name, const UT_StringHolder &value, int attrib_index)
 
void onWorkItemSetFloat (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &attribute_name, fpreal value, int attrib_index)
 
void onWorkItemSetInt (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &attribute_name, exint value, int attrib_index)
 
void onWorkItemSetFile (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &attribute_name, const PDG_AttributeFile::Data &value, int attrib_index)
 
void onWorkItemSetDict (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &attribute_name, const UT_OptionsHolder &dict_repr, int attrib_index)
 
void onWorkItemAddOutput (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &path, const UT_StringHolder &tag, PDG_File::Hash hash_code, bool active_only)
 
void onWorkItemAddOutputs (PDG_WorkItemID work_item_id, int index, const UT_StringArray &paths, const UT_StringHolder &tags, const PDG_File::HashArray &hashes, bool active_only)
 
void onWorkItemAddOutputs (PDG_WorkItemID work_item_id, int index, const UT_StringArray &paths, const UT_StringArray &tags, const PDG_File::HashArray &hashes, bool active_only)
 
void onWorkItemInvalidateCache (PDG_WorkItemID work_item_id, int index)
 
void onWorkItemSucceeded (PDG_WorkItemID work_item_id, int index, fpreal cook_duration)
 
void onWorkItemFailed (PDG_WorkItemID work_item_id, int index)
 
void onWorkItemCanceled (PDG_WorkItemID work_item_id, int index)
 
void onWorkItemStartCook (PDG_WorkItemID work_item_id, int index, bool clear_outputs)
 
void onWorkItemSetCustomState (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &custom_state)
 
void onWorkItemSetCookPercent (PDG_WorkItemID work_item_id, int index, fpreal &cook_percent)
 
void onWorkItemAppendLog (PDG_WorkItemID work_item_id, int index, const UT_StringHolder &log_data, PDG_WorkItemLogType log_type)
 
PDG_WorkItemEvalState isWorkItemReady (PDG_WorkItemID work_item_id, int index)
 
void addError (const UT_StringHolder &message) const override
 Emits errors or warnings from the scheduler as PDG_Events. More...
 
void addWarning (const UT_StringHolder &message) const override
 Adds a warning to the node interface – implemented in subclasses. More...
 
bool asJSON (UT_JSONWriter &writer, bool skip_defaults) const
 
virtual fpreal tickPeriod () const
 
virtual int maxItemsPerTick () const
 
virtual int maxConcurrentTasks () const
 
PathMappingMode mapMode () const
 
- Public Member Functions inherited from PDG_NodeInterface
 PDG_NodeInterface (UT_StringHolder name="", UT_StringHolder data_type="")
 
virtual ~PDG_NodeInterface ()
 
int64 getMemoryUsage (bool inclusive) const
 
void cloneFrom (const PDG_NodeInterface &interface)
 
void addBuiltinPorts (PDG_NodeType type, PDG_NodeSubtype sub_type)
 Adds builtin ports for the specified node type and subtype. More...
 
PDG_WorkItemDataTypeworkItemDataType ()
 Queries the work item data type associated with this node. More...
 
const PDG_WorkItemDataTypeworkItemDataType () const
 
bool resolveWorkItemDataType (UT_WorkBuffer &errors)
 Resolves and caches the work item data type. More...
 
const PDG_PortArrayports (PDG_PortType type) const
 Returns the list of all ports of the specified type. More...
 
const PDG_PortArrayinputPorts () const
 
const PDG_PortArrayoutputPorts () const
 
const PDG_PortArrayparameters () const
 
const ParameterGroupArrayparameterGroups () const
 Returns the list of parameter groups for the node. More...
 
bool parametersForTag (const UT_StringHolder &tag, PDG_PortArray &parameters) const
 Returns the list of parameters with the specified tag. More...
 
const PDG_PortArraycustomParameters () const
 Returns the list of custom parameter ports. More...
 
PDG_Portport (PDG_PortType type, int index) const
 
PDG_Portport (PDG_PortType type, const UT_StringHolder &name) const
 
PDG_Portport (const UT_StringHolder &name) const
 
PDG_PortinputPort (int index) const
 Specialized functions for finding input ports by name or index. More...
 
PDG_PortinputPort (const UT_StringHolder &name) const
 
bool inputPortsForNode (PDG_Node *node, PDG_PortArray &ports) const
 
bool inputPortsForWorkItem (const PDG_WorkItem *work_item, PDG_PortArray &ports) const
 
bool isWorkItemFromInputPort (const PDG_WorkItem *work_item, const UT_StringHolder &name) const
 
PDG_PortoutputPort (int index) const
 Sepcialized functions for finding output ports by name or index. More...
 
PDG_PortoutputPort (const UT_StringHolder &name) const
 
PDG_Portparameter (const UT_StringHolder &name, int multi=-1) const
 
PDGT_ValueparamValue (const UT_StringHolder &name, int index=0) const
 
PDGT_ValuedefaultParamValue (const UT_StringHolder &name, int index=0) const
 
int portCount (PDG_PortType type) const
 
int inputCount () const
 
int outputCount () const
 
int parameterCount () const
 
bool hasUnorderedPorts () const
 
int numOrderedPorts () const
 
PDG_PortaddPort (PDG_PortType type, const PDG_WorkItemDataType *data_type, const UT_StringHolder &name, const UT_StringHolder &label, const UT_StringArray &tags=UT_StringArray(), int size=0, bool ordered=true, bool custom=false)
 
PDG_PortaddParameter (PDGT_Value::DataType data_type, const UT_StringHolder &name, const UT_StringHolder &label, const UT_StringArray &tags, int size=1, bool custom=false)
 
PDG_PortaddParameter (PDGT_Value::DataType data_type, const UT_StringHolder &name, const UT_StringHolder &label, int size=1, bool custom=false)
 Adds a parameter with a data type, name, label and optional size. More...
 
bool removeParameter (const UT_StringHolder &name)
 
void addFileDependency (const UT_StringHolder &file_path)
 Adds a new file dependency to the node interface. More...
 
void addParameterGroup (const UT_StringHolder &name, bool root=true, bool multiparm=false)
 Adds a new parameter group to the node interface. More...
 
void endParameterGroup ()
 Ends the current parameter group. More...
 
const UT_StringHoldertemplateName () const
 
bool isValid () const
 Returns true if the node interface is valid. More...
 
const DependencyArrayautoDependencies () const
 Returns the list of external files that this node depends on. More...
 
const PDG_Port::ExpressionVarscommonVars () const
 
PDG_ApplicationShim::NodeappNode () const
 
template<typename T , typename... Args>
void setAppNode (Args &&...args)
 Sets the underlying app node. More...
 
const PDG_AttributeInfoattributeInfo () const
 
void setAttributeInfo (const PDG_AttributeInfo &info)
 Clears and sets the attribute info for this interface. More...
 
const PDG_JobScriptInfoscriptInfo () const
 Returns the script info for this node interface. More...
 
void setScriptInfo (const PDG_JobScriptInfo &info)
 Sets the script info for the interface. More...
 
void setServiceCompatibility (const UT_StringHolder &service_type_name, bool compatible)
 
template<typename... Args>
void addErrorFmt (const char *fmt, Args &&...args) const
 Adds an error using a format string, instead of plain string. More...
 
template<typename... Args>
void addWarningFmt (const char *fmt, Args &&...args) const
 Adds a warning using a format string, instead of plain string. More...
 
template<typename T >
bool evaluate (T &result, const UT_StringHolder &name, const PDG_WorkItem *work_item=nullptr, int index=0, int multi=-1) const
 Evaluates a parameter and reports errors if desired. More...
 
bool evaluateRaw (UT_StringHolder &result, const UT_StringHolder &name, const PDG_WorkItem *work_item=nullptr, int index=0, int multi=-1) const
 
template<typename T >
bool evaluateArray (T &result, const UT_StringHolder &name, const PDG_WorkItem *work_item=nullptr, int multi=-1) const
 Evaluates a parameter as an array and reports errors if desired. More...
 
- Public Member Functions inherited from PDG_EventEmitter
 PDG_EventEmitter (PDG_EventQueue *event_queue, bool block_destruction)
 
virtual ~PDG_EventEmitter ()
 
int64 getMemoryUsage (bool inclusive) const
 Returns memory usage for this event emitter instance. More...
 
bool addEventHandler (PDG_EventHandler *handler)
 Adds an event handlers that listens for all types of events. More...
 
bool addEventHandler (PDG_EventHandler *handler, PDG_EventType event)
 Adds an event handlers that listens for a specific event. More...
 
bool addEventHandler (PDG_EventHandler *handler, const PDG_EventFilter &filter)
 
void removeEventHandler (PDG_EventHandler *handler)
 Removes an event handler from this emitter. More...
 
void removeAllEventHandlers (bool user_handler_only=false)
 
void emitEvent (const PDG_Event &event, bool immediate=false) const
 Emit an event to all handlers. More...
 
void eventHandlers (PDG_EventHandlerArray &handlers) const
 
void setEventQueue (PDG_EventQueue *event_queue)
 
void clearEventQueue ()
 
bool shouldEmit (const PDG_Event &event) const
 
- Public Member Functions inherited from PDGT_TypeInstance< EnumType >
 PDGT_TypeInstance ()
 
 PDGT_TypeInstance (const PDGT_BaseType< EnumType > *base_type, const PDGT_ValueArgs &extra_args)
 
virtual ~PDGT_TypeInstance ()
 
const BaseTypetype () const
 
const UT_StringHoldertypeName () const
 
const PDGT_ValueArgsextraArgs () const
 
bool compareType (const BaseType *other_type, bool deep=true) const
 
void setReloading (bool is_reloading)
 
bool isReloading () const
 
- Public Member Functions inherited from PDGE_DependencyOwner
 PDGE_DependencyOwner ()
 Constructs a new dependency owner, which must have a string name. More...
 
virtual ~PDGE_DependencyOwner ()
 
virtual UT_StringHolder debugGroup () const
 

Static Public Member Functions

static PDG_SchedulerfromJSON (const UT_JSONValue *value, PDG_GraphContext *context, UT_WorkBuffer &errors)
 Reads and constructs a scheduler from JSON and returns the instance. More...
 
- Static Public Member Functions inherited from PDGE_DependencyOwner
template<typename T >
static void cast (UT_Array< T > &out, const Array &in)
 Converts an owner array to an array of the specified type. More...
 
template<typename T >
static void cast (Array &out, const UT_Array< T > &in)
 Converts an array of the specified type to an owner array. More...
 

Static Public Attributes

static const UT_StringHolder theNameKey
 The key for the name of the scheduler when writing it to JSON. More...
 
static const UT_StringHolder theTypeKey
 The key for the type of the scheduler when writing it to JSON. More...
 
static const UT_StringHolder theParametersKey
 The key for the parameters of the scheduler when writing them to JSON. More...
 
- Static Public Attributes inherited from PDG_NodeInterface
static const UT_StringHolder theProcessorCommandParm
 
static const UT_StringHolder theProcessorWorkItemGenerationParm
 
static const UT_StringHolder theProcessorCacheModeParm
 
static const UT_StringHolder theProcessorCookTypeParm
 
static const UT_StringHolder theProcessorServiceNameParm
 
static const UT_StringHolder theProcessorServiceResetWhenParm
 
static const UT_StringHolder theProcessorServiceResetTypeParm
 
static const UT_StringHolder theProcessorUseServiceBlockParm
 
static const UT_StringHolder theProcessorPriorityTypeParm
 
static const UT_StringHolder theProcessorPriorityExpressionParm
 
static const UT_StringHolder theProcessorLabelTypeParm
 
static const UT_StringHolder theProcessorLabelExpressionParm
 
static const UT_StringHolder theProcessorUseScheduleWhenParm
 
static const UT_StringHolder theProcessorScheduleWhenParm
 
static const UT_StringHolder theFeedbackBeginParm
 
static const UT_StringHolder theFeedbackFilesParm
 
static const UT_StringHolder theFeedbackAttribsParm
 
static const UT_StringHolder theFeedbackAttribPatternParm
 
static const UT_StringHolder thePartitionerSortParm
 
static const UT_StringHolder thePartitionerSortAttribParm
 
static const UT_StringHolder thePartitionerSortDirectionParm
 
static const UT_StringHolder thePartitionerSortRequiredParm
 
static const UT_StringHolder thePartitionerWhenParm
 
static const UT_StringHolder thePartitionerIgnoreFailures
 
static const UT_StringHolder thePartitionerSplitEnabledParm
 
static const UT_StringHolder thePartitionerSplitAttributeParm
 
static const UT_StringHolder thePartitionerSplitMissingParm
 
static const UT_StringHolder thePartitionerSplitDefaultParm
 
static const UT_StringHolder thePartitionerSplitPartialParm
 
static const UT_StringHolder thePartitionerTargetTypeParm
 
static const UT_StringHolder thePartitionerTargetParm
 
static const UT_StringHolder thePartitionerMergeParm
 
static const UT_StringHolder thePartitionerMergeOutputsParm
 
static const UT_StringHolder thePartitionerMergeAttribsParm
 
static const UT_StringHolder thePartitionerMergeOpParm
 
static const UT_StringHolder thePartitionerMergePatternParm
 
static const UT_StringHolder thePartitionerMergePreserveParm
 
static const UT_StringHolder thePartitionerStoreIDsParm
 
static const UT_StringHolder thePartitionerIDAttribParm
 
static const UT_StringHolder thePartitionerSetFrameParm
 
static const UT_StringHolder theSchedulerWorkingDirParm
 
static const UT_StringHolder theSchedulerDataSourceParm
 
static const UT_StringHolder theSchedulerValidateOutputsParm
 
static const UT_StringHolder theSchedulerCheckExpectedOutputsParm
 
static const UT_StringHolder theSchedulerCompressWorkItemDataParm
 
static const UT_StringHolder theSchedulerTickPeriodParm
 
static const UT_StringHolder theSchedulerMaxItemsParm
 
static const UT_StringHolder theSchedulerUseMaxTasksParm
 
static const UT_StringHolder theSchedulerMaxTasksParm
 
static const UT_StringHolder theSchedulerMapModeParm
 
static const UT_StringHolder theSchedulerMapZoneParm
 
static const UT_StringHolder theSchedulerUseMapZoneParm
 
static const UT_StringHolder theSchedulerWaitForFailures
 
static const UT_StringHolder theSchedulerDeleteTempDir
 
static const UT_StringHolder theSchedulerRpcTimeout
 
static const UT_StringHolder theSchedulerRpcRetries
 
static const UT_StringHolder theSchedulerRpcBackoff
 
static const UT_StringHolder theSchedulerRpcBatch
 
static const UT_StringHolder theSchedulerRpcRelease
 
static const UT_StringHolder theSchedulerRpcIgnoreErrors
 
static const UT_StringHolder theSchedulerRpcMaxErrors
 
static const UT_StringHolder theTagExpandName
 
static const UT_StringHolder theTagOpDepDirty
 
static const UT_StringHolder theTagOpDepRegen
 
static const UT_StringHolder theTagExtraInput
 
static const UT_StringHolder theSeparatorLabel
 
static const UT_StringHolder theGenerateMenuScript
 
static const UT_StringHolder theCacheModeMenuScript
 

Protected Member Functions

PDG_WorkItemworkItemChecked (PDG_WorkItemID work_item_id, int index)
 Find the workitem and report error if not found. More...
 
PDGE_Dependency::State evalResolve (PDGE_Resolutions &resolutions, const PDGE_Evaluator &evaluator, PDGE_Dependency *dependency) override
 Called when a dependency owned by this object is resolved. More...
 
- Protected Member Functions inherited from PDGE_DependencyOwner
virtual int evalUnresolve (PDGE_Dependency::Array &, PDGE_Dependency *)
 
virtual PDGE_Dependency::State evalPartial (PDGE_Resolutions &, const PDGE_Evaluator &, PDGE_Dependency *, const Array &)
 
const PDGE_Dependency::SetpropagateSet ()
 Returns the propagate set for this owner. More...
 
void addPropagate (PDGE_Dependency *dependency)
 Adds an entry to the propagate set. More...
 

Additional Inherited Members

- Protected Attributes inherited from PDG_NodeInterface
PDG_AttributeInfo myAttributeInfo
 
- Protected Attributes inherited from PDGT_TypeInstance< EnumType >
const BaseTypemyBaseType
 
PDGT_ValueArgs myExtraArgs
 
bool myReloading
 

Detailed Description

Definition at line 60 of file PDG_Scheduler.h.

Member Enumeration Documentation

Enumerator
eSchedulerAccept 

The scheduler is able to handle the specified work item.

eSchedulerReject 

The scheduler is not able to handle the specified work item.

eSchedulerDefault 

The scheduler cannot tell if can/can't schedule the specified work item, and the default logic for checking should be used instead

Definition at line 114 of file PDG_Scheduler.h.

Enumerator
eDefaultPathMapping 

Use the default, global path mapping.

eNoPathMapping 

Disable path mapping.

Definition at line 131 of file PDG_Scheduler.h.

Enumerator
eCookCanceled 

The work item was cooked, and should be marked as canceled.

eCookFailed 

The work item was cooked, and should be marked as failed.

eCookSucceeded 

The work item was cooked, and should be marked as succeeded.

eFailed 

The call to schedule the work item failed.

eSucceeded 

The call to schedule the work item succeeded.

eDeferred 

The scheduler cannot handle the work item at this time.

eFullDeferred 

The scheduler cannot handle any work items at this time, and all further scheduling requests will be deferred

Definition at line 69 of file PDG_Scheduler.h.

Enumerator
eCleanupNone 

The temp file dir is not cleaned up automatically.

eCleanupShutdown 

The temp file dir is cleaned up when the scheduler is removed or the session shuts down

eCleanupCook 

The temp file dir is cleaned up after the scheduler finishes cooking

Definition at line 156 of file PDG_Scheduler.h.

Enumerator
eSchedulerReady 

The scheduler is ready to accept work items.

eSchedulerBusy 

The scheduler is busy, and cannot process additional work items.

eSchedulerCancelCook 

The scheduler has encountered some sort of fatal error or request to cancel, and will not longer accept work items for the remainder of the cook.

Definition at line 97 of file PDG_Scheduler.h.

Enumerator
eSourceJSONFile 

Work item data is transfered to the farm using a shared drive, by writing it to .json file on disk.

eSourceRPCMessage 

Work item data is transfered via RPC.

Definition at line 143 of file PDG_Scheduler.h.

Constructor & Destructor Documentation

PDG_Scheduler::PDG_Scheduler ( const PDG_BaseType type,
const PDGT_ValueArgs extra_args,
const UT_StringHolder name 
)
PDG_Scheduler::~PDG_Scheduler ( )
override

Member Function Documentation

void PDG_Scheduler::addError ( const UT_StringHolder message) const
overridevirtual

Emits errors or warnings from the scheduler as PDG_Events.

Reimplemented from PDG_NodeInterface.

void PDG_Scheduler::addWarning ( const UT_StringHolder message) const
overridevirtual

Adds a warning to the node interface – implemented in subclasses.

Reimplemented from PDG_NodeInterface.

virtual UT_StringHolder PDG_Scheduler::applicationBin ( const UT_StringHolder name,
const PDG_WorkItem work_item 
)
virtual

Returns the path to an application for the given name, for example "python", "hython"

bool PDG_Scheduler::asJSON ( UT_JSONWriter writer,
bool  skip_defaults 
) const

Writes the current parameter value configuration of this scheduler to JSON

void PDG_Scheduler::cancelCook ( bool  pause)

Cancels an active cook.

void PDG_Scheduler::cancelNode ( PDG_Node node)

Cancels a cook for a specific node.

void PDG_Scheduler::cancelWorkItem ( PDG_WorkItem work_item)

Cancels a cook for a specific work tiem.

bool PDG_Scheduler::canSchedule ( const PDG_WorkItem work_item)

Returns true if the scheduler is able to queue the specified work item, else false.

virtual void PDG_Scheduler::cleanTempDirectory ( )
virtual

Delete the temporary directory and all its contents.

virtual void PDG_Scheduler::clearSharedServerInfo ( const UT_StringHolder sharedserver_name)
virtual

Clears metadata associated with the given shared server name.

PDG_GraphContext* PDG_Scheduler::context ( ) const
bool PDG_Scheduler::cookWorkItem ( PDG_WorkItem work_item)

Cooks a specific work item using it's internal cook method.

UT_StringHolder PDG_Scheduler::dataDir ( bool  local) const

Returns the path to the directory where work items are serialized before their commands are executed. When local is true, returns the aboslute path on the local file system.

UT_StringHolder PDG_Scheduler::debugName ( ) const
inlineoverridevirtual

Debug name method, required by PDGE_DependencyOwner.

Implements PDGE_DependencyOwner.

Definition at line 192 of file PDG_Scheduler.h.

UT_StringHolder PDG_Scheduler::delocalizePath ( const UT_StringHolder local_path) const

Returns the delocalized (remote) version of the given local path.

void PDG_Scheduler::dependencyGraph ( PDG_WorkItemMap dependencies,
PDG_WorkItemMap dependents,
PDG_WorkItemArray ready,
bool  expand 
)
virtual bool PDG_Scheduler::endSharedServer ( const UT_StringHolder sharedserver_name)
pure virtual

Terminate the give shared server.

PDGE_Dependency::State PDG_Scheduler::evalResolve ( PDGE_Resolutions ,
const PDGE_Evaluator ,
PDGE_Dependency  
)
overrideprotectedvirtual

Called when a dependency owned by this object is resolved.

Reimplemented from PDGE_DependencyOwner.

template<typename T , typename D = T>
bool PDG_Scheduler::evaluateOverride ( T result,
PDG_NodeInterface node,
const UT_StringHolder prefix,
const UT_StringHolder parm,
PDG_WorkItem work_item,
const D &  default_value,
UT_WorkBuffer errors 
) const
inline

Definition at line 496 of file PDG_Scheduler.h.

virtual UT_StringHolder PDG_Scheduler::expandCommandTokens ( const UT_StringHolder command,
const PDG_WorkItem work_item 
)
pure virtual

Expands special tokens in the command string.

UT_StringHolder PDG_Scheduler::formatTransferPath ( const UT_StringHolder local_path,
PDG_FileUtils::TransferType  default_type,
bool  local,
bool  check_type 
) const

Returns the path on the scheduler that a local file will be copied to, using the standard file transfer mechanism

UT_StringHolder PDG_Scheduler::formatTransferPath ( const UT_StringHolder root_path,
const UT_StringHolder local_path,
const UT_StringHolder sub_path,
PDG_FileUtils::TransferType  default_type,
bool  local,
bool  check_type 
) const

Returns the path on the scheduler that a local file will be copied to. Unlike the above method, this method preserves the directory structure relative to the specified root directory

static PDG_Scheduler* PDG_Scheduler::fromJSON ( const UT_JSONValue value,
PDG_GraphContext context,
UT_WorkBuffer errors 
)
static

Reads and constructs a scheduler from JSON and returns the instance.

virtual UT_StringHolder PDG_Scheduler::getLogURI ( const PDG_WorkItem work_item)
pure virtual

returns a URI for the workitem cook log (text file)

int64 PDG_Scheduler::getMemoryUsage ( bool  inclusive) const
overridevirtual

Returns the memory usage of this owner instance.

Reimplemented from PDGE_DependencyOwner.

virtual PDGN::PDGN_PollingClientNNG* PDG_Scheduler::getPollingClient ( )
inlinevirtual

Definition at line 466 of file PDG_Scheduler.h.

PDGT_ValueArgs PDG_Scheduler::getSharedServerInfo ( const UT_StringHolder sharedserver_name)

Gets metadata associated with the given shared server name.

UT_StringArray PDG_Scheduler::getSharedServers ( )

Gets the list of shared servers.

virtual UT_StringHolder PDG_Scheduler::getStatusURI ( const PDG_WorkItem work_item)
pure virtual

returns a URI for the workitem status page if relevant

bool PDG_Scheduler::isCheckExpectedOutputFiles ( ) const
bool PDG_Scheduler::isCompressWorkItemData ( ) const
PDGE_Dependency* PDG_Scheduler::isCookedDep ( )
inline

Returns the is cooked dep for this scheduler.

Definition at line 199 of file PDG_Scheduler.h.

bool PDG_Scheduler::isDefault ( )
bool PDG_Scheduler::isRunning ( int  count = 0)
bool PDG_Scheduler::isRunning ( const PDG_WorkItem work_item)
PDGE_Dependency* PDG_Scheduler::isSetupDep ( )
inline

Returns the is cooked dep for this scheduler.

Definition at line 203 of file PDG_Scheduler.h.

bool PDG_Scheduler::isValidateOutputFiles ( ) const
bool PDG_Scheduler::isWaitForFailures ( ) const
PDG_WorkItemEvalState PDG_Scheduler::isWorkItemReady ( PDG_WorkItemID  work_item_id,
int  index 
)
UT_StringHolder PDG_Scheduler::jobName ( const PDG_WorkItem work_item) const

Returns the job name for the specified work item.

UT_StringHolder PDG_Scheduler::localizePath ( const UT_StringHolder deloc_path) const

Returns the local version of the given path.

UT_StringHolder PDG_Scheduler::logDir ( bool  local) const

Returns the path to the directory where log files generated during the cook should be written. When local is true, returns the aboslute path on the local file system.

PathMappingMode PDG_Scheduler::mapMode ( ) const
virtual int PDG_Scheduler::maxConcurrentTasks ( ) const
virtual
virtual int PDG_Scheduler::maxItemsPerTick ( ) const
virtual
const UT_StringHolder& PDG_Scheduler::name ( void  ) const
inline

Returns the name of the shceudler.

Definition at line 207 of file PDG_Scheduler.h.

int PDG_Scheduler::numFailedItems ( ) const
int PDG_Scheduler::numQueuedItems ( ) const
int PDG_Scheduler::numRunningItems ( ) const
virtual AcceptResult PDG_Scheduler::onAcceptWorkItem ( const PDG_WorkItem work_item)
pure virtual

Returns an enum indicating whether or not the scheduler accepts the specified work item

virtual void PDG_Scheduler::onCancelWorkItems ( const PDG_WorkItemArray work_item,
PDG_Node node 
)
pure virtual

Called when a specific work item should be canceled.

virtual void PDG_Scheduler::onConfigureCook ( PDG_CookOptions cook_ptions)
pure virtual

Called once at the start of the cook, to allow the scheduler to set any custom cook options

virtual ScheduleResult PDG_Scheduler::onSchedule ( PDG_WorkItem work_item)
pure virtual

Called whenever the scheduler should schedule a work item. This is a required callback.

virtual void PDG_Scheduler::onScheduleStatic ( const PDG_WorkItemMap dependencies,
const PDG_WorkItemMap dependents,
const PDG_WorkItemArray ready_items 
)
pure virtual

Called when the scheduler should process a static dependency graph.

virtual bool PDG_Scheduler::onSetupCook ( )
pure virtual

Called at the before cooking on the background thread to setup anything needed for a cook.

virtual bool PDG_Scheduler::onStart ( )
pure virtual

Called when the scheduler is first started, should be used to initialize scheduler state, threads, etc. This is an optional callback.

virtual bool PDG_Scheduler::onStartCook ( bool  static_cook,
const PDG_NodeSet  
)
pure virtual

Called when cooking begins.

virtual bool PDG_Scheduler::onStop ( )
pure virtual

Called when the scheduler is stopped, to clean up any resources held by the scheduler (sockets, threads, etc). Optional callback.

virtual bool PDG_Scheduler::onStopCook ( bool  cancel)
pure virtual

Called when cooking completes or is canceled.

virtual TickResult PDG_Scheduler::onTick ( )
pure virtual

Called periodically during cooking to update state.

virtual bool PDG_Scheduler::onTransferFile ( const UT_StringHolder file_path)
virtual

Custom file transfer logic, implemented per-scheduler.

void PDG_Scheduler::onWorkItemAddOutput ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder path,
const UT_StringHolder tag,
PDG_File::Hash  hash_code,
bool  active_only 
)
void PDG_Scheduler::onWorkItemAddOutputs ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringArray paths,
const UT_StringHolder tags,
const PDG_File::HashArray hashes,
bool  active_only 
)
void PDG_Scheduler::onWorkItemAddOutputs ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringArray paths,
const UT_StringArray tags,
const PDG_File::HashArray hashes,
bool  active_only 
)
void PDG_Scheduler::onWorkItemAppendLog ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder log_data,
PDG_WorkItemLogType  log_type 
)
void PDG_Scheduler::onWorkItemCanceled ( PDG_WorkItemID  work_item_id,
int  index 
)
void PDG_Scheduler::onWorkItemFailed ( PDG_WorkItemID  work_item_id,
int  index 
)
void PDG_Scheduler::onWorkItemInvalidateCache ( PDG_WorkItemID  work_item_id,
int  index 
)
void PDG_Scheduler::onWorkItemPriorityChanged ( PDG_WorkItem work_item)
void PDG_Scheduler::onWorkItemSetCookPercent ( PDG_WorkItemID  work_item_id,
int  index,
fpreal cook_percent 
)
void PDG_Scheduler::onWorkItemSetCustomState ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder custom_state 
)
void PDG_Scheduler::onWorkItemSetDict ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder attribute_name,
const UT_OptionsHolder dict_repr,
int  attrib_index 
)
void PDG_Scheduler::onWorkItemSetDictArray ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder attribute_name,
const UT_Array< UT_OptionsHolder > &  value 
)
void PDG_Scheduler::onWorkItemSetFile ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder attribute_name,
const PDG_AttributeFile::Data value,
int  attrib_index 
)
void PDG_Scheduler::onWorkItemSetFileArray ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder attribute_name,
const PDG_AttributeFile::Array value 
)
void PDG_Scheduler::onWorkItemSetFloat ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder attribute_name,
fpreal  value,
int  attrib_index 
)
void PDG_Scheduler::onWorkItemSetFloatArray ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder attribute_name,
const PDG_AttributeFloat::Array value 
)
void PDG_Scheduler::onWorkItemSetInt ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder attribute_name,
exint  value,
int  attrib_index 
)
void PDG_Scheduler::onWorkItemSetIntArray ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder attribute_name,
const PDG_AttributeInteger::Array value 
)
void PDG_Scheduler::onWorkItemSetPyObject ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder attribute_name,
const UT_StringHolder pyobject_repr 
)
void PDG_Scheduler::onWorkItemSetString ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder attribute_name,
const UT_StringHolder value,
int  attrib_index 
)
void PDG_Scheduler::onWorkItemSetStringArray ( PDG_WorkItemID  work_item_id,
int  index,
const UT_StringHolder attribute_name,
const PDG_AttributeString::Array value 
)
void PDG_Scheduler::onWorkItemStartCook ( PDG_WorkItemID  work_item_id,
int  index,
bool  clear_outputs 
)
void PDG_Scheduler::onWorkItemSucceeded ( PDG_WorkItemID  work_item_id,
int  index,
fpreal  cook_duration 
)
void PDG_Scheduler::process ( bool  skip_ready_items)
virtual PDG_WorkItemState PDG_Scheduler::queueWorkItem ( PDG_WorkItem work_item)
virtual

Queues a work item and returns a state that represents the result of the operation

bool PDG_Scheduler::reloadInstance ( UT_WorkBuffer errors)
overridevirtual

Called when the type object should reload itself because of a change to the underlying Python module

Reimplemented from PDGT_TypeInstance< EnumType >.

void PDG_Scheduler::removeWorkItem ( const PDG_WorkItem work_item)

Removes a work item from any internal tracking lists.

void PDG_Scheduler::resetOwner ( )
overridevirtual

Resets the dependency owner.

Reimplemented from PDGE_DependencyOwner.

UT_StringHolder PDG_Scheduler::scriptDir ( bool  local) const

returns the temp script directory (network) path, equivalent to the PDG_SCRIPTDIR command token when local is false. When local is true, returns the absolute path to the temp script on the local file system

void PDG_Scheduler::setAcceptInProcess ( bool  in_process)

sets whether or not the scheduler accepts in-process work items

void PDG_Scheduler::setContext ( PDG_GraphContext context)
void PDG_Scheduler::setScriptDir ( const UT_StringHolder local_path,
const UT_StringHolder remote_path 
)

set the script directory local and network path, equivalent to the PDG_SCRIPTDIR

virtual UT_StringHolder PDG_Scheduler::setSharedServerInfo ( const UT_StringHolder sharedserver_name,
const PDGT_ValueArgs info 
)
virtual

Sets metadata associated with the given shared server name, returns full name.

void PDG_Scheduler::setTempDir ( const UT_StringHolder local_path,
const UT_StringHolder remote_path 
)

set the temp directory local and network path, equivalent to the PDG_TEMP

void PDG_Scheduler::setWorkingDir ( const UT_StringHolder local_path,
const UT_StringHolder remote_path 
)

set the working directory local and network path, equivalent to the PDG_DIR

bool PDG_Scheduler::startCook ( PDG_CookState cook_state)

Called when a cook begins with the cook state data for that cook.

virtual bool PDG_Scheduler::startService ( UT_WorkBuffer errors,
PDG_Service service 
)
inlinevirtual

Starts a service using this scheduler.

Definition at line 484 of file PDG_Scheduler.h.

void PDG_Scheduler::stop ( bool  exiting)

Called to stop the scheduler.

void PDG_Scheduler::stopCook ( )

Called when a cook stops.

virtual bool PDG_Scheduler::stopService ( UT_WorkBuffer errors,
PDG_Service service 
)
inlinevirtual

Stops a service that was started with this scheduler.

Definition at line 490 of file PDG_Scheduler.h.

virtual UT_StringHolder PDG_Scheduler::submitAsJob ( const UT_StringHolder graph_file,
const UT_StringHolder node_name 
)
pure virtual

cook the output node in the graph context of the given file as a single job

void PDG_Scheduler::submitStatic ( )

Submits a static cook with this scheduler.

const PDG_EventFilter& PDG_Scheduler::supportedEventTypes ( ) const
overridevirtual

Returns the list of supported event types.

Implements PDG_EventEmitter.

UT_StringHolder PDG_Scheduler::tempDir ( bool  local) const

returns the temp directory (network) path, equivalent to the PDG_TEMP command token when local is false. When local is true, returns the absolute path to the shared temp on the local file system

const PDG_NodeInterface* PDG_Scheduler::templateInterface ( ) const
overridevirtual

Returns the node interface/template that describes the default parameters and connections for the scheduler

Reimplemented from PDG_NodeInterface.

bool PDG_Scheduler::tick ( PDG_Scheduler::TickResult tick_result)

Ticks the scheduler.

virtual fpreal PDG_Scheduler::tickPeriod ( ) const
virtual
bool PDG_Scheduler::transferFile ( const UT_StringHolder file_path,
PDG_FileUtils::TransferType  default_type,
bool  check_type,
UT_WorkBuffer errors 
) const

Transfers a file from the local machine to the machine that will be executing processor tasks, e.g. a remote machine on the farm.

bool PDG_Scheduler::transferFile ( const UT_StringHolder root_path,
const UT_StringHolder file_path,
const UT_StringHolder sub_path,
PDG_FileUtils::TransferType  default_type,
bool  check_type,
UT_WorkBuffer errors 
) const

Transfers a file from the local machine to the machine that will be executing processor tasks, e.g. a remote machine on the farm. This variant of the method preserves the directory structure relative to the specified root directory

UT_StringHolder PDG_Scheduler::workingDir ( bool  local) const

returns the working directory (network) path, equivalent to the PDG_DIR command token when local is false. When local is true, returns the absolute path to the shared root on the local file system

PDG_WorkItem* PDG_Scheduler::workItemChecked ( PDG_WorkItemID  work_item_id,
int  index 
)
protected

Find the workitem and report error if not found.

WorkItemDataSource PDG_Scheduler::workItemDataSource ( ) const
virtual UT_StringHolder PDG_Scheduler::workItemResultServerAddr ( )
pure virtual

returns the server endpoint for work item results, in format <HOST>:<PORT>, equivalent to the PDG_RESULT_SERVER command token

Member Data Documentation

const UT_StringHolder PDG_Scheduler::theNameKey
static

The key for the name of the scheduler when writing it to JSON.

Definition at line 171 of file PDG_Scheduler.h.

const UT_StringHolder PDG_Scheduler::theParametersKey
static

The key for the parameters of the scheduler when writing them to JSON.

Definition at line 177 of file PDG_Scheduler.h.

const UT_StringHolder PDG_Scheduler::theTypeKey
static

The key for the type of the scheduler when writing it to JSON.

Definition at line 174 of file PDG_Scheduler.h.


The documentation for this class was generated from the following file: