PDG_FileUtils.h

Go to the documentation of this file.

/*
 * PROPRIETARY INFORMATION.  This software is proprietary to
 * Side Effects Software Inc., and is not to be reproduced,
 * transmitted, or disclosed in any way without written permission.
 *
 * COMMENTS:
 */

#ifndef PDG_FILE_UTILS_H
#define PDG_FILE_UTILS_H

#include "PDG_API.h"

#include "PDG_File.h"

#include <UT/UT_Array.h>
#include <UT/UT_ArrayStringMap.h>
#include <UT/UT_NonCopyable.h>
#include <UT/UT_StringHolder.h>
#include <UT/UT_StringMap.h>
#include <UT/UT_StringSet.h>
#include <UT/UT_TBBSpinLock.h>

class PDG_CustomHandlers;
class PDG_Scheduler;
class PDG_WorkItem;

class UT_FileStat;

/*
 * Struct for defining a file type attribute in PDG, and utility functions for
 * doing extension -> tag lookup and saving file data.
 */
class PDG_API PDG_FileUtils : public UT_NonCopyable
{
public:
    /// Pair of viewer application, and whether the application supports
    /// open multiple files
    using ViewerPair = std::pair<UT_StringHolder, bool>;

    /// Default tag for any file
    static const UT_StringHolder theFileTag;

    /// Default tag for an archive
    static const UT_StringHolder theArchiveTag;

    /// Default tag for an archive
    static const UT_StringHolder theCsvTag;

    /// Default tag for geometry
    static const UT_StringHolder theGeoTag;

    /// Default tag for USD
    static const UT_StringHolder theUSDTag;

    /// Default tag for imgages
    static const UT_StringHolder theImageTag;

    /// Default tag for text data
    static const UT_StringHolder theTextTag;

    /// Default tag for videos
    static const UT_StringHolder theVideoTag;

    /// Default tag for hip files
    static const UT_StringHolder theHipTag;

    /// Default tag for hda files
    static const UT_StringHolder theHdaTag;

    /// Default tag for Python script files
    static const UT_StringHolder thePythonScriptTag;

    /// Default tag for directories
    static const UT_StringHolder theDirectoryTag;

    /// Default tag for sym links
    static const UT_StringHolder theSymLinkTag;

    /// Default viewer application
    static const ViewerPair theDefaultViewer;

    /// Enumeration of file transfer destination paths
    enum TransferType
    {
        /// The default temp file directory. Most files go here, unless
        /// they're one of the special cases below
        eTransferTempDir,

        /// The file goes into the scheduler's workingDir instead of the
        /// temporary file directory
        eTransferWorkingDir,

        /// The file is a python script and goes to the scheduler's
        /// scriptDir, which normally a special path on the temp dir that
        /// gets added to the Python path
        eTransferScript,

        /// The file is a digital asset and goes to the tempDir/otls
        /// directory, which is added to the OTL search path.
        eTransferAsset,
    };

public:
    /// Returns the set of available tags
    const UT_StringSet&             tags() const;

    /// Returns the sorted list of tags for the given prefix
    void                            tags(const UT_StringHolder& prefix,
                                         UT_StringArray& sorted_tags) const;

    /// Returns true if the tag is viewable
    bool                            isViewable(const UT_StringHolder& tag) const;

    /// Looks up the viewer application for a tag
    const ViewerPair&               tagToViewer(const UT_StringHolder& tag) const;

    /// Loops up the viewer application for a path
    const ViewerPair&               pathToViewer(const UT_StringHolder& path) const;


    /// Looks up the file tag for a full file path
    const UT_StringHolder&          pathToTag(const UT_StringHolder& path) const;

    /// Looks up the file tag for an extension
    const UT_StringHolder&          extensionToTag(const char* ext,
                                                   const UT_StringHolder& def) const;

    /// Return the extension of the given path which may include an archive dot suffix
    /// like .bgeo.sc
    UT_StringHolder                 pathToExtension(const UT_StringHolder& path) const;

    /// Adds a tag to the system, without establishing an extension - >tag
    /// mapping. For non-file based tags
    void                            addTag(const UT_StringHolder& tag);

    /// Adds an extension -> tag mapping 
    void                            addExtensionTag(const UT_StringHolder& ext,
                                                    const UT_StringHolder& tag);

    /// Adds a tag -> viewer application mapping
    void                            addTagViewer(const UT_StringHolder& tag,
                                                 const UT_StringHolder& viewer,
                                                 bool is_multiviewer=false);

    /// Checks if a given file path is cached
    bool                            checkCached(
                                        const UT_StringHolder& local_path,
                                        const PDG_File& file,
                                        const PDG_WorkItem* work_item) const;

    /// Returns the hash/mod time of the file
    bool                            statFile(
                                        const UT_StringHolder& local_path,
                                        const PDG_File& file,
                                        const PDG_WorkItem* work_item,
                                        PDG_File::Hash& file_hash,
                                        exint& file_size) const;

    /// Called when a work item should delete files, and runs any custom
    /// handlers that match the item
    void                            dirtyFiles(
                                        const PDG_WorkItem* work_item,
                                        const PDG_File::Set& files,
                                        bool delete_items,
                                        bool remove_outputs) const;

    /// Transfers a file from a local path to a remote path accessible by the
    /// specified scheduler. The work item indicates what depends on the file,
    /// and may be null if the file transfer was initiated for a global file
    /// dependency.
    bool                            transferFile(
                                        const PDG_Scheduler* scheduler,
                                        const PDG_WorkItem* work_item,
                                        const PDG_File& file,
                                        TransferType default_type,
                                        bool check_type,
                                        UT_WorkBuffer& errors) const;

    /// Transfers a file from a local path to a remote path accessible by the
    /// specified scheduler, and preserves the directory structure relative to
    /// the specified root path. The work item indicates what depends on the
    /// file, and may be null if the file transfer was initiated for a global
    /// file dependency.
    bool                            transferFile(
                                        const PDG_Scheduler* scheduler,
                                        const PDG_WorkItem* work_item,
                                        const PDG_File& file,
                                        const UT_StringHolder& root_path,
                                        const UT_StringHolder& sub_path,
                                        TransferType default_type,
                                        bool check_type,
                                        UT_WorkBuffer& errors) const;

    /// Returns the path that a local file will be transfered to, based on
    /// the transfer type and the file extension. Always looks up the extension
    /// in the transfer type mapping, and only uses the default transfer type
    /// as a fallback.
    UT_StringHolder                 formatTransferPath(
                                        const PDG_Scheduler* scheduler,
                                        const UT_StringHolder& local_path,
                                        TransferType default_type,
                                        bool local,
                                        bool check_type) const;

    /// Returns the path that a local file will be transfer to, based on
    /// the transfer type, extension, and the relative path of the local file
    /// to the root file path. For example, if the root path is $HIP and the
    /// local_path is $HIP/sub_dir/file.bgeo.sc, the file will be copied tp
    /// {dest}/{sub_path}/sub_dir/file.bgeo.sc where "dest" is determined by
    /// the specified scheduler and file type.
    UT_StringHolder                 formatTransferPath(
                                        const PDG_Scheduler* scheduler,
                                        const UT_StringHolder& root_path,
                                        const UT_StringHolder& local_path,
                                        const UT_StringHolder& sub_path,
                                        TransferType default_type,
                                        bool local,
                                        bool check_type) const;

    /// Compares the file paths and returns 0 if they're equal, -1 if the left
    /// path is ordered first, and 1 if the right path is ordered first. The
    /// paths are first compared directly as strings, and if they're not equal
    /// the paths are normalized to remove extra slashes or .. tokens, then
    /// compared again.
    static int                      compareFilePaths(
                                        const UT_StringHolder& left,
                                        const UT_StringHolder& right);

    /// Makes intermediate directories
    static bool                     makeIntermediateDirs(const char* full_path);

    /// Formats a URL given a scheme and url body
    static UT_StringHolder          makeURL(const UT_StringHolder& scheme,
                                            const UT_StringHolder& path);

    /// Returns true if the specified list of files has any duplicates, and
    /// returns the set of duplicates.
    static bool                     hasDuplicateFiles(
                                        const PDG_File::Array& files,
                                        PDG_File::Set& duplicates);

protected:
    friend class PDG_TypeRegistry;

    /// Initializes the file mapping object and optionally loads in the
    /// default, built-in tags/extension mappings.
                                    PDG_FileUtils(
                                        const PDG_CustomHandlers* handlers,
                                        bool init_defaults);

private:
    /// Caches an id for completed transfers, to potentially skip future
    /// transfers of the same file path. By default, the mod time of the
    /// file is used as the cache value.
    struct TransferCacheEntry
    {
        UT_ArrayStringMap<exint>    myCache;
        UT_TBBSpinLock              myLock;
    };

    using TransferCache = UT_StringMap<TransferCacheEntry>;

private:
    void                            init();


    bool                            transferFile(
                                        TransferCacheEntry* transfer_cache,
                                        const PDG_Scheduler* scheduler,
                                        const PDG_WorkItem* work_item,
                                        const UT_FileStat& local_stat,
                                        const PDG_File& file,
                                        const UT_StringHolder& local_path,
                                        const UT_StringHolder& dest_path,
                                        UT_WorkBuffer& errors) const;
    exint                           transferFileDefault(
                                        const UT_FileStat& local_stat,
                                        const UT_StringHolder& local_path,
                                        const UT_StringHolder& dest_path,
                                        exint last_update,
                                        UT_WorkBuffer& errors) const;
    UT_StringHolder                 formatTransferPath(
                                        const PDG_Scheduler* scheduler,
                                        const UT_StringHolder& local_name,
                                        const UT_StringHolder& sub_path,
                                        TransferType transfer_type,
                                        bool local) const;

private:
    // Custom handler table, for methods that may need to call into user-
    // defined handlers before running the standard logic.
    const PDG_CustomHandlers*       myHandlers;

    // Mapping of file extension to file tag, e.g. .bgeo -> file/geo or
    // .png -> file/image
    UT_StringMap<UT_StringHolder>   myExtMap;

    // Mapping of tags to result viewers, e.g. file/image -> mplay
    UT_StringMap<ViewerPair>        myViewerMap;

    // Scheduler name -> file transfer cache
    mutable TransferCache           myTransferCache;

    // Map of file extension -> transfer path
    UT_StringMap<TransferType>      myTransferTypes;

    // Set of all available tags
    UT_StringSet                    myTags;

    // Lock for guarding concurrent access to myTransferCache
    mutable UT_TBBSpinLock          myTransferCacheLock;
};

#endif