PDG_WorkItemSerialization.h

Go to the documentation of this file.

 /*
 * PROPRIETARY INFORMATION.  This software is proprietary to
 * Side Effects Software Inc., and is not to be reproduced,
 * transmitted, or disclosed in any way without written permission.
 *
 * COMMENTS:
 */

#ifndef __PDG_WORKITEM_SERIALIZATION_H__
#define __PDG_WORKITEM_SERIALIZATION_H__

#include "PDG_API.h"

#include "PDG_AttributeMap.h"
#include "PDG_File.h"
#include "PDG_WorkItemTypes.h"

#include <UT/UT_Array.h>
#include <UT/UT_ArrayStringMap.h>
#include <UT/UT_StringArray.h>

#include <SYS/SYS_TypeDecorate.h>

#include <iosfwd>

class PDG_GraphContext;
class PDG_Graph;
struct PDG_SerializationOptions;
class PDG_WorkItem;

class UT_AutoJSONParser;
class UT_IStream;
class UT_JSONValueMap;
class UT_JSONValue;
class UT_JSONWriter;
class UT_Options;
class UT_OptionsHolder;
class UT_WorkBuffer;

/**
 * Helper class for loading and storing work items to serialization
 * formats
 */
class PDG_API PDG_WorkItemSerialization
{
public:
    using Array = UT_Array<PDG_WorkItemSerialization>;
    using Map   = UT_ArrayStringMap<PDG_WorkItem*>;

    /// Returned by the statJSON methods, to provide information about
    /// the type and number of work items in a .json file or string
    struct Stat
    {
        int     myVersion;
        int     myNumWorkItems;

        bool    myIsArray;
        bool    myHasGraphAttributes;
    };

    /// Enumeration of checkpoint save types, which are used to determine
    /// which set of fields to write to the JSON object for a serialized
    /// work item
    enum CheckpointType
    {
        /// Serialize all fields -- no checkpointing is in progress
        eCheckpointNone,

        /// Serialize the initial baseline fields for the work item
        eCheckpointInitial,

        /// Serialize only fields that can be updated dynamically, when the
        /// work item cooks
        eCheckpointUpdate
    };

public:
    /// Constructs an empty, default-initialized serialization object
                            PDG_WorkItemSerialization(int version=0);

    /// Loads work item attributes from a JSON value
    bool                    loadAttributes(
                                const UT_JSONValue& value,
                                PDG_AttributeSaveType save_type,
                                bool reset);

    /// Loads work item attributres from a buffer
    bool                    loadAttributes(
                                const UT_WorkBuffer& buffer,
                                PDG_AttributeSaveType save_type,
                                bool reset);

    /// Converts the serialization object to a PDG_WorkItem, in the given
    /// graph context
    PDG_WorkItem*           toWorkItem(
                                PDG_GraphContext* context,
                                PDG_WorkItemID base_id,
                                bool strict,
                                Map& compat_map,
                                UT_WorkBuffer& errors) const;

    /// Initializes this instance from JSON string data
    bool                    initializeString(
                                const UT_StringHolder& data,
                                UT_WorkBuffer& errors);

    /// Initializes this instance from a stream of JSON data
    bool                    initializeStream(
                                UT_IStream& istream,
                                UT_WorkBuffer& errors);

    /// Initializes this instance from a JSON file
    bool                    initializeFile(
                                const UT_StringHolder& file_path,
                                UT_WorkBuffer& errors);

    /// Initializes this instance from a dictionary
    bool                    initializeDict(
                                const UT_OptionsHolder& dict,
                                UT_WorkBuffer& errors);

    /// Returns the work item name. If the serialization object was created
    /// from the old, name-based format this returns the work item's name
    /// as listed in the json/python data. Otherwise, this method computes
    /// the name from the work item's node name and unique ID.
    UT_StringHolder         workItemName() const;

    
    /// Saves the specified work item to the buffer, as JSON
    static bool             saveJSON(
                                UT_WorkBuffer& buffer,
                                const PDG_WorkItem* work_item,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);

    /// Saves the specified work item to the stream, as JSON
    static bool             saveJSON(
                                std::ostream& ostream,
                                const PDG_WorkItem* work_item,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);

    /// Saves the specified work item to the file, as JSON
    static bool             saveJSON(
                                const UT_StringHolder& file_path,
                                const PDG_WorkItem* work_item,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);

    /// Saves the specified work item to the UT_Options as a dictionary
    static bool             saveDict(
                                UT_OptionsHolder& dict,
                                const PDG_WorkItem* work_item,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);

    /// Saves the specified work item array to the buffer, as JSON
    static bool             saveArrayJSON(
                                UT_WorkBuffer& buffer,
                                const PDG_WorkItemArray& work_items,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);
    static bool             saveArrayJSON(
                                UT_WorkBuffer& buffer,
                                const PDG_WorkItemPtrArray& work_items,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);

    /// Saves the specified work item array to the stream, as JSON
    static bool             saveArrayJSON(
                                std::ostream& ostream,
                                const PDG_WorkItemArray& work_items,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);
    static bool             saveArrayJSON(
                                std::ostream& ostream,
                                const PDG_WorkItemPtrArray& work_items,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);

    /// Saves the specified work item array to the file, as JSON
    static bool             saveArrayJSON(
                                const UT_StringHolder& file_path,
                                const PDG_WorkItemArray& work_items,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);
    static bool             saveArrayJSON(
                                const UT_StringHolder& file_path,
                                const PDG_WorkItemPtrArray& work_items,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);

    /// Saves the specified work item array to UT_Options
    static bool             saveArrayDict(
                                UT_OptionsHolder& dict,
                                const PDG_WorkItemArray& work_items,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);


    /// Saves the specified work item dependency pair to the buffer, as JSON
    static bool             saveDependencyJSON(
                                UT_WorkBuffer& buffer,
                                const PDG_WorkItem* from,
                                const PDG_WorkItem* to,
                                bool required,
                                bool user_defined,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);

    /// Saves the specified work item dependency pair to the stream, as JSON
    static bool             saveDependencyJSON(
                                std::ostream& ostream,
                                const PDG_WorkItem* from,
                                const PDG_WorkItem* to,
                                bool required,
                                bool user_defined,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);

    /// Saves the specified work item dependency pair to the file, as JSON
    static bool             saveDependencyJSON(
                                const UT_StringHolder& file_path,
                                const PDG_WorkItem* from,
                                const PDG_WorkItem* to,
                                bool required,
                                bool user_defined,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint=eCheckpointNone);

    
    /// Saves a map with the data version to the buffer, as JSON
    static bool             saveVersion(
                                UT_WorkBuffer& buffer,
                                const PDG_SerializationOptions& options);

    /// Saves a map with the data version to the stream, as JSON
    static bool             saveVersion(
                                std::ostream& ostream,
                                const PDG_SerializationOptions& options);

    /// Saves a map with the data version to the file, as JSON
    static bool             saveVersion(
                                const UT_StringHolder& file_path,
                                const PDG_SerializationOptions& options);

    /// Saves the specified work item to the buffer, as a Python script
    static bool             savePython(
                                UT_WorkBuffer& buffer,
                                const PDG_WorkItem* work_item,
                                const PDG_SerializationOptions& options);


    /// Loads a work item from a buffer containing JSON string data
    static PDG_WorkItem*    fromJSON(
                                const UT_StringHolder& buffer,
                                PDG_GraphContext* context,
                                Map& compat_map,
                                bool make_active,
                                UT_WorkBuffer& errors);

    /// Loads a work item from a stream containing JSON data
    static PDG_WorkItem*    fromJSON(
                                UT_IStream& istream,
                                bool make_active,
                                UT_WorkBuffer& errors);

    /// Loads a work item from a file path
    static PDG_WorkItem*    fromJSON(
                                const UT_StringHolder& file_path,
                                bool make_active,
                                UT_WorkBuffer& errors);

    /// Loads a work item from a dict
    static PDG_WorkItem*    fromDict(
                                const UT_OptionsHolder& dict,
                                bool make_active,
                                UT_WorkBuffer& errors);


    /// Loads a work item array from a stream containing JSON data
    static bool             fromArrayJSON(
                                PDG_WorkItemArray& work_items,
                                UT_IStream& istream,
                                PDG_GraphContext* context,
                                bool is_compressed,
                                UT_WorkBuffer& errors);


    /// Loads a serialization object array from a buffer containing JSON data
    static bool             fromArrayJSON(
                                Array& serialization_array,
                                const UT_StringHolder& buffer,
                                UT_WorkBuffer& errors);

    /// Loads a serialization object array from a file containing JSON data
    static bool             fromArrayJSON(
                                Array& serialization_array,
                                const UT_StringHolder& file_path,
                                bool is_compressed,
                                UT_WorkBuffer& errors);

    /// Loads a serialization object array from a dictionary
    static bool             fromArrayDict(
                                Array& serialization_array,
                                const UT_OptionsHolder& dict,
                                UT_WorkBuffer& errors);


    /// Stats the JSON data in the specified string
    static bool             statJSON(
                                Stat& json_stat,
                                const UT_StringHolder& buffer,
                                UT_WorkBuffer& errors);

    /// Stats the JSON data in the specified file
    static bool             statJSON(
                                Stat& json_stat,
                                const UT_StringHolder& file_path,
                                bool is_compressed,
                                UT_WorkBuffer& errors);

    /// Stats the JSON data in the stream
    static bool             statJSON(
                                Stat& json_stat,
                                UT_IStream& istream,
                                bool is_compressed,
                                UT_WorkBuffer& errors);

    /// Stats the dictionary data
    static bool             statDict(
                                Stat& dict_stat,
                                const UT_OptionsHolder& dict,
                                UT_WorkBuffer& errors);

    /// Creates a JSON patch from changed attributes and outputs on the work
    /// item
    static UT_StringHolder  createJSONPatch(
                                const PDG_WorkItem* work_item,
                                bool pretty_print);

    /// Updates work item outputs and attributes from the patch stored in
    /// the json map
    static bool             applyJSONPatch(
                                PDG_WorkItem* work_item,
                                const UT_JSONValueMap& patch_map,
                                UT_WorkBuffer& errors);

    /// Loads a checkpoint .json file
    static bool             loadCheckpoint(
                                PDG_GraphContext* context,
                                const UT_StringHolder& file_path,
                                UT_WorkBuffer& errors);

private:
    /// Fields for a dependency pair that was serialized to JSON
    struct SerializedDependency
    {
        PDG_WorkItemID  myTo;
        PDG_WorkItemID  myFrom;
        bool            myRequired;
        bool            myUserDefined;
    };

    /// Populates this object from the json parser object
    bool                    initializeJSONParser(
                                UT_AutoJSONParser& parser,
                                UT_WorkBuffer& errors);

    /// Populates this object from the json map, using the V1 format
    bool                    initializeJSONV1(
                                const UT_JSONValueMap& item_map,
                                UT_WorkBuffer& errors);

    /// Populates this object from the json map, using the V2 format
    bool                    initializeJSONV2(
                                const UT_JSONValueMap& item_map,
                                UT_WorkBuffer& errors);

    /// Sets the contents of the serialization object by loading the fields
    /// from JSON
    bool                    initializeJSONValue(
                                const UT_JSONValue& value,
                                bool validate,
                                UT_WorkBuffer& errors);

    /// Sets the contents of the serialization object by loading the fields
    /// from a dict
    bool                    initializeDict(
                                const UT_OptionsHolder& dict,
                                bool validate,
                                UT_WorkBuffer& errors);

    /// Initializes the attribute table in this serialization objeect based
    /// on the attributes found in the JSON graph section
    bool                    loadGraphJSONValue(
                                const UT_JSONValue& value,
                                UT_WorkBuffer& errors);

    /// Initializes the attribute table in this serialization object based
    /// on the attributes found in the dict graph section
    bool                    loadGraphDict(
                                const UT_OptionsHolder& value,
                                UT_WorkBuffer& errors);

    /// Internal wrapper that configures the JSON writer based on options,
    /// and saves the specified work item
    static bool             saveJSON(
                                UT_JSONWriter& writer,
                                const PDG_WorkItem* work_item,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint);

    /// Internal wrapper that configures the JSON writer based on options,
    /// save saves the entire list of work items to an array
    static bool             saveArrayJSON(
                                UT_JSONWriter& writer,
                                const PDG_WorkItemArray& work_items,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint);
    static bool             saveArrayJSON(
                                UT_JSONWriter& writer,
                                const PDG_WorkItemPtrArray& work_items,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint);

    /// Internal wrapper that configures the JSON writer based on options,
    /// and saves a dependency pair
    static bool             saveDependencyJSON(
                                UT_JSONWriter& writer,
                                const PDG_WorkItem* from,
                                const PDG_WorkItem* to,
                                bool required,
                                bool user_defined,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint);

    /// Internal wrapper that configures the JSON writer based on options,
    /// and saves the data version
    static bool             saveVersion(
                                UT_JSONWriter& writer,
                                const PDG_SerializationOptions& options);

    /// Internal implementation of JSON saving. Writes a work item to the 
    /// specified writer, and writes batch sub items if the recurse flag is
    /// set to true.
    static bool             saveJSONWorkItem(
                                UT_JSONWriter& writer,
                                const PDG_WorkItem* work_item,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint);

    /// Internal implementation of UT_Options saving. Writes a work item to the 
    /// specified options, and writes batch sub items if the recurse flag is
    /// set to true.
    static bool             saveDictWorkItem(
                                UT_OptionsHolder& dict,
                                const PDG_WorkItem* work_item,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint);

    /// Saves attributes from a PDG_Graph to the specified writer
    static bool             saveJSONGraph(
                                UT_JSONWriter& writer,
                                const PDG_Graph* graph,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint);

    /// Saves attributes from a PDG_Graph to the specified UT_Options
    static bool             saveDictGraph(
                                UT_OptionsHolder& dict,
                                const PDG_Graph* graph,
                                const PDG_SerializationOptions& options,
                                CheckpointType checkpoint);

    /// Internal implementation of the fromJSON and fromDict public interface
    static PDG_WorkItem*    fromSerialization(
                                const PDG_WorkItemSerialization& serialization,
                                PDG_GraphContext* context,
                                Map& compat_map,
                                bool make_active,
                                UT_WorkBuffer& errors);

    /// Internal implementation of the array-based fromJSON method
    static bool             fromArrayJSONParser(
                                Array& serialization_array,
                                UT_AutoJSONParser& parser,
                                UT_WorkBuffer& errors);

    /// Internal implementation that interprets a JSONValue as an array
    /// of work items
    static bool             fromArrayJSONValue(
                                Array& serialization_array,
                                const UT_JSONValue& value,
                                int version_num,
                                UT_WorkBuffer& errors);

    /// Internal implementation of the statJSON method that uses a JSON
    /// parser to check for version, graph and item array fields.
    static bool             statJSONParser(
                                Stat& json_stat,
                                UT_AutoJSONParser& parser,
                                UT_WorkBuffer& errors);

    /// Loads a serialized dependency pair from a JSON value
    static bool             fromJSONDependency(
                                SerializedDependency& dependency,
                                const UT_JSONValue& value,
                                UT_WorkBuffer& errors);

    /// Loads checkpoint data from a stream
    static bool             loadCheckpoint(
                                PDG_GraphContext* context,
                                UT_IStream& istream,
                                UT_WorkBuffer& errors);

    /// Internal logic for creating a work item from this serialization
    /// object
    PDG_WorkItem*           toWorkItem(
                                PDG_GraphContext* context,
                                PDG_WorkItemID base_id,
                                bool strict,
                                Map& compat_map,
                                bool make_active,
                                PDG_BatchWorkItem* batch_parent,
                                UT_WorkBuffer& errors) const;

public:
    // Full serialization keys
    static const UT_StringHolder        theNodeKey;
    static const UT_StringHolder        theSchedulerKey;
    static const UT_StringHolder        theIdKey;
    static const UT_StringHolder        theBatchParentIdKey;
    static const UT_StringHolder        theCloneTargetIdKey;
    static const UT_StringHolder        theLoopStackIdsKey;
    static const UT_StringHolder        theLoopStackNumbersKey;
    static const UT_StringHolder        theLoopStackItersKey;
    static const UT_StringHolder        theLoopStackSizesKey;
    static const UT_StringHolder        theLoopStackLockIdsKey;
    static const UT_StringHolder        theCustomDataTypeKey;
    static const UT_StringHolder        theCustomDataKey;
    static const UT_StringHolder        theAttributesKey;
    static const UT_StringHolder        theIndexKey;
    static const UT_StringHolder        theBatchIndexKey;
    static const UT_StringHolder        thePriorityKey;
    static const UT_StringHolder        theFrameKey;
    static const UT_StringHolder        theFrameStepKey;
    static const UT_StringHolder        theHasFrameKey;
    static const UT_StringHolder        theIsPostCookKey;
    static const UT_StringHolder        theIsStaticKey;
    static const UT_StringHolder        theIsNoGenerateKey;
    static const UT_StringHolder        theIsCloneOutputFilesKey;
    static const UT_StringHolder        theIsFrozenKey;
    static const UT_StringHolder        theStateKey;
    static const UT_StringHolder        theTypeKey;
    static const UT_StringHolder        theExecutionTypeKey;
    static const UT_StringHolder        theCookTypeKey;
    static const UT_StringHolder        theBatchNameKey;
    static const UT_StringHolder        theBatchCountKey;
    static const UT_StringHolder        theBatchOffsetKey;
    static const UT_StringHolder        theBatchStartKey;
    static const UT_StringHolder        theIsPartialCookKey;
    static const UT_StringHolder        theIsDynamicBatchKey;
    static const UT_StringHolder        theActivationModeKey;
    static const UT_StringHolder        theActivationCountKey;
    static const UT_StringHolder        theSubItemsKey;

    // Attribute patch serialization
    static const UT_StringHolder        thePatchOutputsKey;
    static const UT_StringHolder        thePatchAttributesKey;
    static const UT_StringHolder        thePatchGraphKey;

    // Dependency serialization
    static const UT_StringHolder        theDependencyFromKey;
    static const UT_StringHolder        theDependencyToKey;
    static const UT_StringHolder        theDependencyRequiredKey;

    // Section keys
    static const UT_StringHolder        theVersionSectionKey;
    static const UT_StringHolder        theWorkItemSectionKey;
    static const UT_StringHolder        theWorkItemsSectionKey;
    static const UT_StringHolder        theGraphSectionKey;
    static const UT_StringHolder        theDependencySectionKey;

    // Deprecated json keys, no longer emitted during serialization
    // but still used when loading olds format files.
    static const UT_StringHolder        theNameKey;
    static const UT_StringHolder        theBaseNameKey;
    static const UT_StringHolder        theParentKey;
    static const UT_StringHolder        theParentIdKey;
    static const UT_StringHolder        theBatchParentKey;
    static const UT_StringHolder        theCloneTargetKey;
    static const UT_StringHolder        theAncestorsKey;
    static const UT_StringHolder        theLoopStackKey;
    static const UT_StringHolder        theIsClearEnvKey;
    static const UT_StringHolder        theEnvironmentKey;
    static const UT_StringHolder        theIsControlKey;
    static const UT_StringHolder        theExpansionKey;
    static const UT_StringHolder        theCommandKey;
    static const UT_StringHolder        theIsInProcessKey;

    // Constants for detecting serialization format verison

    // The current verison
    static const int                    theAttribVersionCurrent;
    
    // 1 -- The initial version of the PDG serialization format, added in H17.5
    static const int                    theAttribV1;

    // 2 -- The new PDG attribute storage format, added in H18.0.
    static const int                    theAttribV2;

    // 3 -- V2 attribute format, with IDs instead of string names. Compatible
    // with V2.
    static const int                    theAttribV2Ids;

    // 4 -- V2 attribute format, with work item command stored as an attribute
    // instead of a separate field. Compatible with V2.
    static const int                    theAttribV2Command;

    // 5 -- V2 attribute format, with the new loop storage format. Compatible
    // with earlier V2 versions, but loop items will need to recook due to
    // internal format changes
    static const int                    theAttribV2LoopInfo;

    // 6 -- V2 attribute format, with the new PDG_WorkItemCookType to track
    // how the work item should execute. Compatible from older version, but
    // cook type has to be derived from a number of existing fields.
    static const int                    theAttribV2CookType;

    // 7 -- V2 attribute format, with service block loops that can be run using
    // services. This introduces a new field that locks a loop block to a
    // particular service
    static const int                    theAttribV2LoopLock;

public:
    UT_StringArray                      myLoopBeginStackNames;
    UT_StringArray                      myStaticAncestorNames;

    UT_Array<int64>                     myLoopBeginStackIds;
    UT_Array<int64>                     myLoopBeginStackNumbers;
    UT_Array<int64>                     myLoopBeginStackIters;
    UT_Array<int64>                     myLoopBeginStackSizes;
    UT_Array<int64>                     myLoopBeginStackLockIds;

    Array                               mySubItems;

    PDG_File::Array                     myInputs;

    UT_StringHolder                     myNodeName;
    UT_StringHolder                     mySchedulerName;
    UT_StringHolder                     myName;
    UT_StringHolder                     myBaseName;
    UT_StringHolder                     myBatchName;
    UT_StringHolder                     myCommand;
    UT_StringHolder                     myParentName;
    UT_StringHolder                     myBatchParentName;
    UT_StringHolder                     myCloneTargetName;

    UT_StringHolder                     myCustomDataType;
    UT_StringHolder                     myCustomData;

    mutable PDG_AttributeMap            myAttributes;
    PDGT_Value::Map                     myEnvironment;

    PDG_WorkItemID                      myParentId;
    PDG_WorkItemID                      myBatchParentId;
    PDG_WorkItemID                      myCloneTargetId;
    PDG_WorkItemID                      myId;

    int                                 myVersion;
    int                                 myIndex;
    int                                 myBatchIndex;
    int                                 myPriority;
    int                                 myInputExpansion;
    int                                 myBatchCount;
    int                                 myBatchOffset;
    int                                 myBatchStart;
    int                                 myBatchActivationCount;

    fpreal                              myFrame;
    fpreal                              myFrameStep;
    bool                                myHasFrame;
    
    bool                                myIsPostCook;
    bool                                myIsStatic;
    bool                                myIsControl;
    bool                                myIsNoGenerate;
    bool                                myIsClearEnvironment;
    bool                                myIsCloneOutputFiles;
    bool                                myIsFrozen;
    bool                                myIsPartialCook;
    bool                                myIsDynamicBatch;

    PDG_WorkItemState                   myState;
    PDG_WorkItemType                    myType;
    PDG_WorkItemExecutionType           myExecutionType;
    PDG_WorkItemCookType                myCookType;
    PDG_BatchActivation                 myActivationMode;
};

SYS_DECLARE_LEGACY_TR(PDG_WorkItemSerialization)

#endif /* __PDG_WORKITEM_SERIALIZATION_H__ */