HDK
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
fileSystem.h
Go to the documentation of this file.
1 //
2 // Copyright 2016 Pixar
3 //
4 // Licensed under the terms set forth in the LICENSE.txt file available at
5 // https://openusd.org/license.
6 //
7 #ifndef PXR_BASE_ARCH_FILE_SYSTEM_H
8 #define PXR_BASE_ARCH_FILE_SYSTEM_H
9 
10 /// \file arch/fileSystem.h
11 /// \ingroup group_arch_SystemFunctions
12 /// Architecture dependent file system access
13 
14 #include "pxr/pxr.h"
15 #include "pxr/base/arch/api.h"
16 #include "pxr/base/arch/defines.h"
17 #include "pxr/base/arch/inttypes.h"
18 #include <memory>
19 #include <cstdio>
20 #include <string>
21 #include <set>
22 
23 #include <fcntl.h>
24 #include <sys/types.h>
25 #include <sys/stat.h>
26 
27 #if defined(ARCH_OS_LINUX)
28 #include <unistd.h>
29 #include <sys/statfs.h>
30 #include <glob.h>
31 #elif defined(ARCH_OS_DARWIN)
32 #include <unistd.h>
33 #include <sys/mount.h>
34 #include <glob.h>
35 #elif defined(ARCH_OS_WINDOWS)
36 #include <io.h>
37 #include <windows.h>
38 #include <stringapiset.h>
39 #endif
40 
42 
43 /// \addtogroup group_arch_SystemFunctions
44 ///@{
45 #if !defined(ARCH_OS_WINDOWS)
46  #ifdef _POSIX_VERSION
47  #include <limits.h> /* for PATH_MAX */
48  #else
49  #include <sys/param.h> /* for MAXPATHLEN */
50  #endif
51 #else
52  // XXX -- Should probably have ARCH_ macro for this.
53  #define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
54 
55  // See https://msdn.microsoft.com/en-us/library/1w06ktdy.aspx
56  // XXX -- Should probably have Arch enum for these.
57  #define F_OK 0 // Test for existence.
58  #define X_OK 1 // Test for execute permission.
59  #define W_OK 2 // Test for write permission.
60  #define R_OK 4 // Test for read permission.
61 #endif
62 
63 #if defined(ARCH_OS_WINDOWS)
64  #define ARCH_GLOB_NOCHECK 1
65  #define ARCH_GLOB_MARK 2
66  #define ARCH_GLOB_NOSORT 4
67 #else
68  #define ARCH_GLOB_NOCHECK GLOB_NOCHECK
69  #define ARCH_GLOB_MARK GLOB_MARK
70  #define ARCH_GLOB_NOSORT GLOB_NOSORT
71 #endif
72 #define ARCH_GLOB_DEFAULT (ARCH_GLOB_NOCHECK | ARCH_GLOB_MARK)
73 
74 #ifndef ARCH_PATH_MAX
75  #ifdef PATH_MAX
76  #define ARCH_PATH_MAX PATH_MAX
77  #else
78  #ifdef MAXPATHLEN
79  #define ARCH_PATH_MAX MAXPATHLEN
80  #else
81  #ifdef _MAX_PATH
82  #define ARCH_PATH_MAX _MAX_PATH
83  #else
84  #define ARCH_PATH_MAX 1024
85  #endif
86  #endif
87  #endif
88 #endif
89 
90 #if defined(ARCH_OS_WINDOWS)
91  #define ARCH_PATH_SEP "\\"
92  #define ARCH_PATH_LIST_SEP ";"
93  #define ARCH_REL_PATH_IDENT ".\\"
94 #else
95  #define ARCH_PATH_SEP "/"
96  #define ARCH_PATH_LIST_SEP ":"
97  #define ARCH_REL_PATH_IDENT "./"
98 #endif
99 
100 #if defined(ARCH_OS_WINDOWS)
101 typedef struct __stat64 ArchStatType;
102 #else
103 typedef struct stat ArchStatType;
104 #endif
105 
106 /// \file fileSystem.h
107 /// Architecture dependent file system access
108 /// \ingroup group_arch_SystemFunctions
109 ///
110 
111 /// Opens a file.
112 ///
113 /// Opens the file that is specified by filename.
114 /// Returning true if the file was opened successfully; false otherwise.
115 ///
116 ARCH_API FILE*
117 ArchOpenFile(char const* fileName, char const* mode);
118 
119 #if defined(ARCH_OS_WINDOWS)
120 # define ArchChmod(path, mode) _chmod(path, mode)
121 #else
122 # define ArchChmod(path, mode) chmod(path, mode)
123 #endif
124 
125 #if defined(ARCH_OS_WINDOWS)
126 # define ArchCloseFile(fd) _close(fd)
127 #else
128 # define ArchCloseFile(fd) close(fd)
129 #endif
130 
131 #if defined(ARCH_OS_WINDOWS)
132 # define ArchUnlinkFile(path) _unlink(path)
133 #else
134 # define ArchUnlinkFile(path) unlink(path)
135 #endif
136 
137 #if defined(ARCH_OS_WINDOWS)
138  ARCH_API int ArchFileAccess(const char* path, int mode);
139 #else
140 # define ArchFileAccess(path, mode) access(path, mode)
141 #endif
142 
143 #if defined(ARCH_OS_WINDOWS)
144 # define ArchFdOpen(fd, mode) _fdopen(fd, mode)
145 #else
146 # define ArchFdOpen(fd, mode) fdopen(fd, mode)
147 #endif
148 
149 #if defined(ARCH_OS_WINDOWS)
150 # define ArchFileNo(stream) _fileno(stream)
151 #else
152 # define ArchFileNo(stream) fileno(stream)
153 #endif
154 
155 #if defined(ARCH_OS_WINDOWS)
156 # define ArchFileIsaTTY(stream) _isatty(stream)
157 #else
158 # define ArchFileIsaTTY(stream) isatty(stream)
159 #endif
160 
161 #if defined(ARCH_OS_WINDOWS)
162  ARCH_API int ArchRmDir(const char* path);
163 #else
164 # define ArchRmDir(path) rmdir(path)
165 #endif
166 
167 /// Return the length of a file in bytes.
168 ///
169 /// Returns -1 if the file cannot be opened/read.
170 ARCH_API int64_t ArchGetFileLength(const char* fileName);
171 ARCH_API int64_t ArchGetFileLength(FILE *file);
172 
173 /// Return a filename for this file, if one can be obtained. Note that there
174 /// are many reasons why it may be impossible to obtain a filename, even for an
175 /// opened FILE *. Whenever possible avoid using this function and instead
176 /// store the filename for future use.
177 ARCH_API std::string ArchGetFileName(FILE *file);
178 
179 /// Returns true if the data in \c stat struct \p st indicates that the target
180 /// file or directory is writable.
181 ///
182 /// This returns true if the struct pointer is valid, and the stat indicates
183 /// the target is writable by the effective user, effective group, or all
184 /// users.
186 
187 /// Returns the modification time (mtime) in seconds for a file.
188 ///
189 /// This function stores the modification time with as much precision as is
190 /// available in the stat structure for the current platform in \p time and
191 /// returns \c true on success, otherwise just returns \c false.
192 ARCH_API bool ArchGetModificationTime(const char* pathname, double* time);
193 
194 /// Returns the modification time (mtime) in seconds from the stat struct.
195 ///
196 /// This function returns the modification time with as much precision as is
197 /// available in the stat structure for the current platform.
199 
200 /// Normalizes the specified path, eliminating double slashes, etc.
201 ///
202 /// This canonicalizes paths, removing any double slashes, and eliminiating
203 /// '.', and '..' components of the path. This emulates the behavior of
204 /// os.path.normpath in Python.
205 ///
206 /// On Windows, all backslashes are converted to forward slashes and drive
207 /// specifiers (e.g., "C:") are lower-cased. If \p stripDriveSpecifier
208 /// is \c true, these drive specifiers are removed from the path.
209 ARCH_API std::string ArchNormPath(const std::string& path,
210  bool stripDriveSpecifier = false);
211 
212 /// Returns the canonical absolute path of the specified filename.
213 ///
214 /// This makes the specified path absolute, by prepending the current working
215 /// directory. If the path is already absolute, it is returned unmodified.
216 ARCH_API std::string ArchAbsPath(const std::string& path);
217 
218 /// Returns the permissions mode (mode_t) for the given pathname.
219 ///
220 /// This function stats the given pathname and returns the permissions flags
221 /// for it and returns true. If the stat fails, returns false.
222 ///
223 ARCH_API bool ArchGetStatMode(const char *pathname, int *mode);
224 
225 /// Return the path to a temporary directory for this platform.
226 ///
227 /// The returned temporary directory will be a location that will normally
228 /// be cleaned out on a reboot. This is /var/tmp on Linux machines (for
229 /// legacy reasons), but /tmp on Darwin machines (/var/tmp on Darwin is
230 /// specified as a location where files are kept between system reboots -
231 /// see "man hier"). The returned string will not have a trailing slash.
232 ///
233 /// This routine is threadsafe and will not perform any memory allocations.
234 ARCH_API const char *ArchGetTmpDir();
235 
236 /// Make a temporary file name, in a system-determined temporary directory.
237 ///
238 /// The result returned has the form TMPDIR/prefix.pid[.n]suffix where TMPDIR
239 /// is a system-determined temporary directory (typically /tmp or /usr/tmp),
240 /// pid is the process id of the process, and the optional .n records the
241 /// number of times this function has been called by a process (and is ommited
242 /// the first time this function is called).
243 ///
244 /// The call is threadsafe.
245 ///
246 /// \warning This call opens a security hole because of the race between
247 /// choosing the name and opening the file. This call should be avoided in
248 /// favor of \c ArchMakeTmpFile().
249 ARCH_API
250 std::string ArchMakeTmpFileName(const std::string& prefix,
251  const std::string& suffix = std::string());
252 
253 /// Create a temporary file, in a system-determined temporary directory.
254 ///
255 /// The result returned has the form TMPDIR/prefix.XXXXXX where TMPDIR is a
256 /// system-determined temporary directory (typically /tmp or /usr/tmp) and
257 /// XXXXXX is a unique suffix. Returns the file descriptor of the new file
258 /// and, if pathname isn't NULL, returns the full path to the file in
259 /// pathname. Returns -1 on failure and errno is set.
260 ///
261 /// The call is threadsafe.
262 ARCH_API
263 int ArchMakeTmpFile(const std::string& prefix, std::string* pathname = 0);
264 
265 /// Create a temporary file, in a given temporary directory.
266 ///
267 /// The result returned has the form TMPDIR/prefix.XXXXXX where TMPDIR is the
268 /// given temporary directory and XXXXXX is a unique suffix. Returns the file
269 /// descriptor of the new file and, if pathname isn't NULL, returns the full
270 /// path to the file in pathname. Returns -1 on failure and errno is set.
271 ///
272 /// The call is threadsafe.
273 ARCH_API
274 int ArchMakeTmpFile(const std::string& tmpdir,
275  const std::string& prefix, std::string* pathname = 0);
276 
277 /// Create a temporary sub-direcrory, in a given temporary directory.
278 ///
279 /// The result returned has the form TMPDIR/prefix.XXXXXX/ where TMPDIR is the
280 /// given temporary directory and XXXXXX is a unique suffix. Returns the the
281 /// full path to the subdir in pathname. Returns empty string on failure and
282 /// errno is set.
283 ///
284 /// The call is threadsafe.
285 ARCH_API
286 std::string ArchMakeTmpSubdir(const std::string& tmpdir,
287  const std::string& prefix);
288 
289 // Helper 'deleter' for use with std::unique_ptr for file mappings.
291  Arch_Unmapper() : _length(~0) {}
292  explicit Arch_Unmapper(size_t length) : _length(length) {}
293  ARCH_API void operator()(char *mapStart) const;
294  ARCH_API void operator()(char const *mapStart) const;
295  size_t GetLength() const { return _length; }
296 private:
297  size_t _length;
298 };
299 
300 /// ArchConstFileMapping and ArchMutableFileMapping are std::unique_ptr<char
301 /// const *, ...> and std::unique_ptr<char *, ...> respectively. The functions
302 /// ArchMapFileReadOnly() and ArchMapFileReadWrite() return them and provide
303 /// access to memory-mapped file contents.
304 using ArchConstFileMapping = std::unique_ptr<char const, Arch_Unmapper>;
305 using ArchMutableFileMapping = std::unique_ptr<char, Arch_Unmapper>;
306 
307 /// Return the length of an ArchConstFileMapping.
308 inline size_t
310  return m.get_deleter().GetLength();
311 }
312 
313 /// Return the length of an ArchMutableFileMapping.
314 inline size_t
316  return m.get_deleter().GetLength();
317 }
318 
319 /// Privately map the passed \p file into memory and return a unique_ptr to the
320 /// read-only mapped contents. The contents may not be modified. If mapping
321 /// fails, return a null unique_ptr and if errMsg is not null fill it with
322 /// information about the failure.
323 ARCH_API
325 ArchMapFileReadOnly(FILE *file, std::string *errMsg=nullptr);
326 
327 /// \overload
328 ARCH_API
330 ArchMapFileReadOnly(std::string const& path, std::string *errMsg=nullptr);
331 
332 /// Privately map the passed \p file into memory and return a unique_ptr to the
333 /// copy-on-write mapped contents. If modified, the affected pages are
334 /// dissociated from the underlying file and become backed by the system's swap
335 /// or page-file storage. Edits are not carried through to the underlying file.
336 /// If mapping fails, return a null unique_ptr and if errMsg is not null fill it
337 /// with information about the failure.
338 ARCH_API
340 ArchMapFileReadWrite(FILE *file, std::string *errMsg=nullptr);
341 
342 /// \overload
343 ARCH_API
345 ArchMapFileReadWrite(std::string const& path, std::string *errMsg=nullptr);
346 
348  ArchMemAdviceNormal, // Treat range with default behavior.
349  ArchMemAdviceWillNeed, // OS may prefetch this range.
350  ArchMemAdviceDontNeed, // OS may free resources related to this range.
351  ArchMemAdviceRandomAccess, // Prefetching may not be beneficial.
352 };
353 
354 /// Advise the OS regarding how the application intends to access a range of
355 /// memory. See ArchMemAdvice. This is primarily useful for mapped file
356 /// regions. This call does not change program semantics. It is only an
357 /// optimization hint to the OS, and may be a no-op on some systems.
358 ARCH_API
359 void ArchMemAdvise(void const *addr, size_t len, ArchMemAdvice adv);
360 
361 /// Report whether or not the mapped virtual memory pages starting at \p addr
362 /// for \p len bytes are resident in RAM. Pages that are resident will not,
363 /// when accessed, cause a page fault while those that are not will. Return
364 /// true on success and false in case of an error. The \p addr argument must be
365 /// a multiple of ArchGetPageSize(). The \p len argument need not be a multiple
366 /// of the page size; it will be rounded up to the next page boundary. Fill
367 /// \p pageMap with 0s for pages not resident in memory and 1s for pages that
368 /// are. The \p pageMap argument must therefore point to at least (\p len +
369 /// ArchGetPageSize()-1)/ArchGetPageSize() bytes.
370 ///
371 /// Note that currently this function is only implemented on Linux and Darwin.
372 /// On Windows it currently always returns false.
373 ARCH_API
374 bool
376  void const *addr, size_t len, unsigned char *pageMap);
377 
378 /// Read up to \p count bytes from \p offset in \p file into \p buffer. The
379 /// file position indicator for \p file is not changed. Return the number of
380 /// bytes read, or zero if at end of file. Return -1 in case of an error, with
381 /// errno set appropriately.
382 ARCH_API
383 int64_t ArchPRead(FILE *file, void *buffer, size_t count, int64_t offset);
384 
385 /// Write up to \p count bytes from \p buffer to \p file at \p offset. The file
386 /// position indicator for \p file is not changed. Return the number of bytes
387 /// written, possibly zero if none written. Return -1 in case of an error, with
388 /// errno set appropriately.
389 ARCH_API
390 int64_t ArchPWrite(FILE *file, void const *bytes, size_t count, int64_t offset);
391 
392 /// Returns the value of the symbolic link at \p path. Returns the empty
393 /// string on error or if \p path does not refer to a symbolic link.
394 ARCH_API
395 std::string ArchReadLink(const char* path);
396 
398  ArchFileAdviceNormal, // Treat range with default behavior.
399  ArchFileAdviceWillNeed, // OS may prefetch this range.
400  ArchFileAdviceDontNeed, // OS may free resources related to this range.
401  ArchFileAdviceRandomAccess, // Prefetching may not be beneficial.
402 };
403 
404 /// Advise the OS regarding how the application intends to access a range of
405 /// bytes in a file. See ArchFileAdvice. This call does not change program
406 /// semantics. It is only an optimization hint to the OS, and may be a no-op on
407 /// some systems.
408 ARCH_API
409 void ArchFileAdvise(FILE *file, int64_t offset, size_t count,
410  ArchFileAdvice adv);
411 
412 #if defined(ARCH_OS_WINDOWS)
413 
414 /// Converts UTF-16 windows string to regular std::string - Windows-only
415 inline std::string ArchWindowsUtf16ToUtf8(const std::wstring &wstr)
416 {
417  if (wstr.empty()) return std::string();
418  // first call is only to get required size for string
419  int size = WideCharToMultiByte(
420  CP_UTF8, 0, wstr.data(), (int)wstr.size(), NULL, 0, NULL, NULL);
421  if (size == 0) return std::string();
422  std::string str(size, 0);
423  if (WideCharToMultiByte(CP_UTF8, 0, wstr.data(), (int)wstr.size(),
424  &str[0], size, NULL, NULL) == 0) {
425  return std::string();
426  }
427  return str;
428 }
429 
430 /// Converts regular std::string to UTF-16 windows string - Windows-only
431 inline std::wstring ArchWindowsUtf8ToUtf16(const std::string &str)
432 {
433  if (str.empty()) return std::wstring();
434  // first call is only to get required size for wstring
435  int size = MultiByteToWideChar(
436  CP_UTF8, 0, str.data(), (int)str.size(), NULL, 0);
437  if (size == 0) return std::wstring();
438  std::wstring wstr(size, 0);
439  if(MultiByteToWideChar(
440  CP_UTF8, 0, str.data(), (int)str.size(), &wstr[0], size) == 0) {
441  return std::wstring();
442  }
443  return wstr;
444 }
445 
446 #endif
447 
448 ///@}
449 
451 
452 #endif // PXR_BASE_ARCH_FILE_SYSTEM_H
ArchFileAdvice
Definition: fileSystem.h:397
typedef int(APIENTRYP RE_PFNGLXSWAPINTERVALSGIPROC)(int)
ARCH_API FILE * ArchOpenFile(char const *fileName, char const *mode)
ARCH_API void ArchMemAdvise(void const *addr, size_t len, ArchMemAdvice adv)
GT_API const UT_StringHolder time
GLsizei const GLchar *const * path
Definition: glcorearb.h:3341
ARCH_API std::string ArchAbsPath(const std::string &path)
ARCH_API void ArchFileAdvise(FILE *file, int64_t offset, size_t count, ArchFileAdvice adv)
ARCH_API std::string ArchNormPath(const std::string &path, bool stripDriveSpecifier=false)
GLuint GLsizei GLsizei * length
Definition: glcorearb.h:795
ARCH_API ArchConstFileMapping ArchMapFileReadOnly(FILE *file, std::string *errMsg=nullptr)
GLuint buffer
Definition: glcorearb.h:660
std::unique_ptr< char const, Arch_Unmapper > ArchConstFileMapping
Definition: fileSystem.h:304
Arch_Unmapper(size_t length)
Definition: fileSystem.h:292
ARCH_API ArchMutableFileMapping ArchMapFileReadWrite(FILE *file, std::string *errMsg=nullptr)
ARCH_API const char * ArchGetTmpDir()
ARCH_API int ArchMakeTmpFile(const std::string &prefix, std::string *pathname=0)
GLintptr offset
Definition: glcorearb.h:665
std::unique_ptr< char, Arch_Unmapper > ArchMutableFileMapping
Definition: fileSystem.h:305
ARCH_API std::string ArchReadLink(const char *path)
ARCH_API int64_t ArchGetFileLength(const char *fileName)
size_t ArchGetFileMappingLength(ArchConstFileMapping const &m)
Return the length of an ArchConstFileMapping.
Definition: fileSystem.h:309
ARCH_API bool ArchGetStatMode(const char *pathname, int *mode)
GLenum mode
Definition: glcorearb.h:99
ARCH_API bool ArchStatIsWritable(const ArchStatType *st)
GLsizeiptr size
Definition: glcorearb.h:664
struct stat ArchStatType
Definition: fileSystem.h:103
ARCH_API bool ArchQueryMappedMemoryResidency(void const *addr, size_t len, unsigned char *pageMap)
GT_API const UT_StringHolder st
PXR_NAMESPACE_CLOSE_SCOPE PXR_NAMESPACE_OPEN_SCOPE
Definition: path.h:1425
#define ArchRmDir(path)
Definition: fileSystem.h:164
#define PXR_NAMESPACE_CLOSE_SCOPE
Definition: pxr.h:74
ARCH_API std::string ArchGetFileName(FILE *file)
#define ARCH_API
Definition: api.h:23
#define ArchFileAccess(path, mode)
Definition: fileSystem.h:140
ARCH_API int64_t ArchPRead(FILE *file, void *buffer, size_t count, int64_t offset)
ARCH_API void operator()(char *mapStart) const
size_t GetLength() const
Definition: fileSystem.h:295
ArchMemAdvice
Definition: fileSystem.h:347
ARCH_API std::string ArchMakeTmpSubdir(const std::string &tmpdir, const std::string &prefix)
ARCH_API int64_t ArchPWrite(FILE *file, void const *bytes, size_t count, int64_t offset)
GLint GLsizei count
Definition: glcorearb.h:405
Definition: format.h:4365
ARCH_API bool ArchGetModificationTime(const char *pathname, double *time)
ARCH_API std::string ArchMakeTmpFileName(const std::string &prefix, const std::string &suffix=std::string())