HDK
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
PDG_FileUtils.h
Go to the documentation of this file.
1 /*
2  * PROPRIETARY INFORMATION. This software is proprietary to
3  * Side Effects Software Inc., and is not to be reproduced,
4  * transmitted, or disclosed in any way without written permission.
5  *
6  * COMMENTS:
7  */
8 
9 #ifndef PDG_FILE_UTILS_H
10 #define PDG_FILE_UTILS_H
11 
12 #include "PDG_API.h"
13 
14 #include "PDG_File.h"
15 
16 #include <UT/UT_Array.h>
17 #include <UT/UT_NonCopyable.h>
18 #include <UT/UT_StringHolder.h>
19 #include <UT/UT_StringMap.h>
20 #include <UT/UT_StringSet.h>
21 #include <UT/UT_TBBSpinLock.h>
22 
23 class PDG_CustomHandlers;
24 class PDG_Scheduler;
25 class PDG_WorkItem;
26 
27 class UT_FileStat;
28 
29 /*
30  * Struct for defining a file type attribute in PDG, and utility functions for
31  * doing extension -> tag lookup and saving file data.
32  */
34 {
35 public:
36  /// Pair of viewer application, and whether the application supports
37  /// open multiple files
38  using ViewerPair = std::pair<UT_StringHolder, bool>;
39 
40  /// Default tag for any file
42 
43  /// Default tag for an archive
45 
46  /// Default tag for an archive
47  static const UT_StringHolder theCsvTag;
48 
49  /// Default tag for geometry
50  static const UT_StringHolder theGeoTag;
51 
52  /// Default tag for USD
53  static const UT_StringHolder theUSDTag;
54 
55  /// Default tag for imgages
57 
58  /// Default tag for text data
60 
61  /// Default tag for videos
63 
64  /// Default tag for hip files
65  static const UT_StringHolder theHipTag;
66 
67  /// Default tag for hda files
68  static const UT_StringHolder theHdaTag;
69 
70  /// Default tag for Python script files
72 
73  /// Default tag for directories
75 
76  /// Default tag for sym links
78 
79  /// Default viewer application
81 
82  /// Enumeration of file transfer destination paths
84  {
85  /// The default temp file directory. Most files go here, unless
86  /// they're one of the special cases below
88 
89  /// The file goes into the scheduler's workingDir instead of the
90  /// temporary file directory
92 
93  /// The file is a python script and goes to the scheduler's
94  /// scriptDir, which normally a special path on the temp dir that
95  /// gets added to the Python path
97 
98  /// The file is a digital asset and goes to the tempDir/otls
99  /// directory, which is added to the OTL search path.
101  };
102 
103 public:
104  /// Returns the set of available tags
105  const UT_StringSet& tags() const;
106 
107  /// Returns the sorted list of tags for the given prefix
108  void tags(const UT_StringHolder& prefix,
109  UT_StringArray& sorted_tags) const;
110 
111  /// Returns true if the tag is viewable
112  bool isViewable(const UT_StringHolder& tag) const;
113 
114  /// Looks up the viewer application for a tag
115  const ViewerPair& tagToViewer(const UT_StringHolder& tag) const;
116 
117  /// Loops up the viewer application for a path
118  const ViewerPair& pathToViewer(const UT_StringHolder& path) const;
119 
120 
121  /// Looks up the file tag for a full file path
122  const UT_StringHolder& pathToTag(const UT_StringHolder& path) const;
123 
124  /// Looks up the file tag for an extension
125  const UT_StringHolder& extensionToTag(const char* ext,
126  const UT_StringHolder& def) const;
127 
128  /// Return the extension of the given path which may include an archive dot suffix
129  /// like .bgeo.sc
130  UT_StringHolder pathToExtension(const UT_StringHolder& path) const;
131 
132  /// Adds a tag to the system, without establishing an extension - >tag
133  /// mapping. For non-file based tags
134  void addTag(const UT_StringHolder& tag);
135 
136  /// Adds an extension -> tag mapping
137  void addExtensionTag(const UT_StringHolder& ext,
138  const UT_StringHolder& tag);
139 
140  /// Adds a tag -> viewer application mapping
141  void addTagViewer(const UT_StringHolder& tag,
142  const UT_StringHolder& viewer,
143  bool is_multiviewer=false);
144 
145  /// Checks if a given file path is cached
146  bool checkCached(
147  const UT_StringHolder& local_path,
148  const PDG_File& file,
149  const PDG_WorkItem* work_item) const;
150 
151  /// Returns the hash/mod time of the file
152  bool statFile(
153  const UT_StringHolder& local_path,
154  const PDG_File& file,
155  const PDG_WorkItem* work_item,
156  PDG_File::Hash& file_hash,
157  exint& file_size) const;
158 
159  /// Called when a work item should delete files, and runs any custom
160  /// handlers that match the item
161  void dirtyFiles(
162  const PDG_WorkItem* work_item,
163  const PDG_File::Set& files,
164  bool delete_items,
165  bool remove_outputs) const;
166 
167  /// Transfers a file from a local path to a remote path accessible by the
168  /// specified scheduler. The work item indicates what depends on the file,
169  /// and may be null if the file transfer was initiated for a global file
170  /// dependency.
171  bool transferFile(
172  const PDG_Scheduler* scheduler,
173  const PDG_WorkItem* work_item,
174  const PDG_File& file,
175  TransferType default_type,
176  UT_WorkBuffer& errors) const;
177 
178  /// Returns the path that a local file will be transfered to, based on
179  /// the transfer type and the file extension.
180  UT_StringHolder formatTransferPath(
181  const PDG_Scheduler* scheduler,
182  const UT_StringHolder& local_path,
183  TransferType default_type,
184  bool local) const;
185 
186  /// Compares the file paths and returns 0 if they're equal, -1 if the left
187  /// path is ordered first, and 1 if the right path is ordered first. The
188  /// paths are first compared directly as strings, and if they're not equal
189  /// the paths are normalized to remove extra slashes or .. tokens, then
190  /// compared again.
191  static int compareFilePaths(
192  const UT_StringHolder& left,
193  const UT_StringHolder& right);
194 
195  /// Makes intermediate directories
196  static bool makeIntermediateDirs(const char* full_path);
197 
198  /// Formats a URL given a scheme and url body
199  static UT_StringHolder makeURL(const UT_StringHolder& scheme,
200  const UT_StringHolder& path);
201 
202  /// Returns true if the specified list of files has any duplicates, and
203  /// returns the set of duplicates.
204  static bool hasDuplicateFiles(
205  const PDG_File::Array& files,
206  PDG_File::Set& duplicates);
207 
208 protected:
209  friend class PDG_TypeRegistry;
210 
211  /// Initializes the file mapping object and optionally loads in the
212  /// default, built-in tags/extension mappings.
214  const PDG_CustomHandlers* handlers,
215  bool init_defaults);
216 
217 private:
218  /// Caches an id for completed transfers, to potentially skip future
219  /// transfers of the same file path. By default, the mod time of the
220  /// file is used as the cache value.
221  struct TransferCacheEntry
222  {
223  UT_ArrayStringMap<exint> myCache;
224  UT_TBBSpinLock myLock;
225  };
226 
227  using TransferCache = UT_StringMap<TransferCacheEntry>;
228 
229 private:
230  void init();
231 
232  exint transferFileDefault(
233  const UT_FileStat& local_stat,
234  const UT_StringHolder& local_path,
235  const UT_StringHolder& dest_path,
236  exint last_update,
237  UT_WorkBuffer& errors) const;
238 
239 private:
240  // Custom handler table, for methods that may need to call into user-
241  // defined handlers before running the standard logic.
242  const PDG_CustomHandlers* myHandlers;
243 
244  // Mapping of file extension to file tag, e.g. .bgeo -> file/geo or
245  // .png -> file/image
247 
248  // Mapping of tags to result viewers, e.g. file/image -> mplay
249  UT_StringMap<ViewerPair> myViewerMap;
250 
251  // Scheduler name -> file transfer cache
252  mutable TransferCache myTransferCache;
253 
254  // Map of file extension -> transfer path
255  UT_StringMap<TransferType> myTransferTypes;
256 
257  // Set of all available tags
258  UT_StringSet myTags;
259 
260  // Lock for guarding concurrent access to myTransferCache
261  mutable UT_TBBSpinLock myTransferCacheLock;
262 };
263 
264 #endif
static const UT_StringHolder theTextTag
Default tag for text data.
Definition: PDG_FileUtils.h:59
static const UT_StringHolder theSymLinkTag
Default tag for sym links.
Definition: PDG_FileUtils.h:77
static const UT_StringHolder theUSDTag
Default tag for USD.
Definition: PDG_FileUtils.h:53
static const UT_StringHolder theHdaTag
Default tag for hda files.
Definition: PDG_FileUtils.h:68
static const UT_StringHolder thePythonScriptTag
Default tag for Python script files.
Definition: PDG_FileUtils.h:71
GLint left
Definition: glcorearb.h:2005
static const UT_StringHolder theImageTag
Default tag for imgages.
Definition: PDG_FileUtils.h:56
#define PDG_API
Definition: PDG_API.h:23
GLsizei const GLchar *const * path
Definition: glcorearb.h:3341
GLdouble right
Definition: glad.h:2817
int64 exint
Definition: SYS_Types.h:125
static const ViewerPair theDefaultViewer
Default viewer application.
Definition: PDG_FileUtils.h:80
static const UT_StringHolder theDirectoryTag
Default tag for directories.
Definition: PDG_FileUtils.h:74
int64 Hash
The file hash/modtime type.
Definition: PDG_File.h:39
OIIO_UTIL_API uint64_t file_size(string_view path) noexcept
std::pair< UT_StringHolder, bool > ViewerPair
Definition: PDG_FileUtils.h:38
static const UT_StringHolder theArchiveTag
Default tag for an archive.
Definition: PDG_FileUtils.h:44
static const UT_StringHolder theFileTag
Default tag for any file.
Definition: PDG_FileUtils.h:41
static const UT_StringHolder theCsvTag
Default tag for an archive.
Definition: PDG_FileUtils.h:47
TransferType
Enumeration of file transfer destination paths.
Definition: PDG_FileUtils.h:83
static const UT_StringHolder theGeoTag
Default tag for geometry.
Definition: PDG_FileUtils.h:50
static const UT_StringHolder theHipTag
Default tag for hip files.
Definition: PDG_FileUtils.h:65
static const UT_StringHolder theVideoTag
Default tag for videos.
Definition: PDG_FileUtils.h:62