CE_Context.h

Go to the documentation of this file.

/*
 * PROPRIETARY INFORMATION.  This software is proprietary to
 * Side Effects Software Inc., and is not to be reproduced,
 * transmitted, or disclosed in any way without written permission.
 *
 * NAME: CE_Context.h ( CE Library, C++)
 *
 * COMMENTS: Compute Engine Contexts.
 */

#ifndef __CE_Context__
#define __CE_Context__

#include "CE_API.h"
#include "CE_Tracing.h"
#include "CE_Error.h"

#include <UT/UT_Array.h>
#include <UT/UT_Error.h>
#include <UT/UT_Lock.h>
#include <UT/UT_Map.h>
#include <UT/UT_NonCopyable.h>
#include <UT/UT_StringMap.h>
#include <UT/UT_UniquePtr.h>
#include <SYS/SYS_Types.h>
#include <SYS/SYS_Handle.h>
#include <iosfwd>

class CE_MemoryPool;
class UT_MemoryResource;

typedef void (*CE_ErrorCB)(const char *errmsg, UT_ErrorSeverity severity,
                           void *data);

class CE_API CE_DelayedOGLBindBuffer
{
public:
    CE_DelayedOGLBindBuffer(){}
    virtual ~CE_DelayedOGLBindBuffer(){}

    UT_NON_COPYABLE(CE_DelayedOGLBindBuffer)

    virtual void rebindOGLBuffer( uint buf_obj ) = 0;
    virtual void unbindOGLBuffer() = 0;
    virtual bool isBinded() = 0;
};

/// An OpenCL buffer that is backed by a external memory. External memory is accessed
/// through a platform specific memory handle. 
/// Requires the OpenCL driver implements the following extensions:
/// cl_khr_external_memory
/// cl_khr_external_memory_opaque_fd (only on Linux and possibly Mac)
/// cl_khr_external_memory_win32 (only on Windows)
class CE_API CE_ExternalBuffer : public cl::Buffer
{
public:
    CE_ExternalBuffer(
            const cl::Context &context,
            cl_mem_flags flags,
            ::size_t size,
            SYS_Handle handle,
            cl_int *err = nullptr);

    CE_ExternalBuffer() : cl::Buffer() {}
};
/// An OpenCL image that is backed by external memory, similar to
/// CE_ExternalBuffer.
class CE_API CE_ExternalImage : public cl::Image
{
public:
    CE_ExternalImage(
            const cl::Context &context,
            cl_mem_flags flags,
            const cl_image_format *format,
            const cl_image_desc *desc,
            SYS_Handle handle,
            cl_int *err = nullptr);

    CE_ExternalImage() : cl::Image() {}
};

#ifndef CL_UUID_SIZE_KHR
#define CL_UUID_SIZE_KHR 16
#endif

/// CE_Context encapsulates the OpenCL context and provides various convenience
/// functions for loading kernel programs and allocating GPU memory.
class CE_API CE_Context
{
public:
    CE_Context();
    virtual ~CE_Context();

    UT_NON_COPYABLE(CE_Context)

    /// Returns a pointer to the singleton CE_Context object.  This function
    /// attempts to initialize OpenCL if it has not yet been.
    /// gl_shared should be true if the context will be expected to interoperate
    /// with the OpenGL context. If both gl_shared and shared_fallback are true,
    /// then the function will try to make an unshared context in case the
    /// shared context fails to create.
    static CE_Context *getContext(bool gl_shared = true,
                                  bool shared_fallback = true);
    /// Returns true if interoperability between CL and GL is possible.
    static bool        isGLSharingPossible();

    /// Returns true if interoperability between CL and VK is possible.
    bool               isSameDevice(const uint8 (&uuid)[16]);

    /// Returns the underlying cl::Context object.
    cl::Context getCLContext() const {return myContext;}

    /// Returns the cl::Queue object that is used to enqueue OpenCL kernels
    /// and memory transfers.
    cl::CommandQueue getQueue() const {return myQueue;}

    /// Returns the OpenCL Device object.
    cl::Device getDevice() const {return myDevice;}

    ceTraceCtx getTraceContext() const {return myTraceCtx;}

    // Write OpenCL Device info to the supplied buffer.
    static void getInfo(const cl::Device &device, UT_WorkBuffer &buffer );
    static void getExtendedInfo(const cl::Device &device, UT_WorkBuffer &buffer );

    // Write info for all available OpenCL platforms to the supplied buffer.
    static void getAllPlatformsInfo(UT_WorkBuffer &buffer);

    /// Get the suggested global and local ranges for the given 1-D kernel over
    /// the specified number of items.
    void get1DRanges(const cl::Kernel &k, size_t items,
                     cl::NDRange &g, cl::NDRange &l);

    /// Get the suggested global and local ranges for a reduction for the
    /// given 1-D kernel over the specified number of items.
    void getReductionRanges(const cl::Kernel &k, size_t items, size_t accumsize,
                            cl::NDRange &global_range, cl::NDRange &local_range,
                            uint &groupsize, uint &ngroups) const;

    /// Get the maximum workgroup size for the given kernel.
    size_t getMaxWorkgroupSize(const cl::Kernel &k);
    /// Get the array of maximum work items along each dimension supported by the
    /// compute device.
    std::vector<size_t> getMaxWorkItemSizes();

    /// Round up a provided group size to a larger clean one as some
    /// driers die with prime-based groups sizes.
    /// Less than 1024 is raised to next power of 2, greater is to a multiple
    /// of 1024.
    static size_t roundUpGroupSize(size_t gsize);

    /// Round up a 2D group size to something that can factor nicely.
    /// Same rule as roundUpGroupSize of ensuring there is a 1024 factor
    /// present if group greater than 1024, else power of 2.
    static void roundUpGroupSize2D(exint &width, exint &height);

    /// Loads the OpenCL program specified by progname.  This functions searches
    /// for the file in the HOUDINI_OCL_PATH environment variable.  Any compile-
    /// time options can be passed in the options parameter.  If the program
    /// load succeeds, the progname will be cached, using the progrname and
    /// options strings together as a hash value lookup.  In this way the same
    /// OpenCL program can be loaded several times with different compile-time
    /// flags.
    cl::Program loadProgram(const char *progname,
                            const char *options = nullptr,
                            bool recompile = false);

    /// Like loadProgram() except it creates a program from code directly.
    /// `progname` should be provided where possible so that errors are easier
    /// to find.
    cl::Program compileProgram(const char *progtext,
                               const char *options = nullptr,
                               bool recompile = false,
                               const char *progname = nullptr);

    /// Create an OpenCL kernel named kernelname from the program  specified by
    /// progname.  For some types of devices these kernels will be cached, as
    /// kernels can be expensive to create.   This is the recommended method
    /// for creating kernels.
    cl::Kernel loadKernel(const cl::Program &prog, const UT_StringRef &kernelname);
    cl::Kernel loadKernel(const char *progname, const UT_StringRef &kernelname,
                          const char *options = nullptr)
    { return loadKernel(loadProgram(progname, options), kernelname); }

    /// Returns whether the CE_Context has been successfully initialized.
    bool isValid() const {return myIsValid;}

    /// Returns whether the singleton CE_Context has been initialized yet.  This
    /// can be used to test whether OpenCL has been initialized without calling
    /// getContext and forcing an attempt at initialization.
    static bool isInitialized(bool gl_shared=false);

    /// Returns true if the OpenCL device is running on the CPU.
    bool isCPU() const;

    /// Returns true if the OpenCL device supports double precision.
    bool hasDoubleSupport() const {return mySupportsDouble;}
    /// Returns true if the OpenCL device supports writing to 3D image objects.
    bool has3DImageWriteSupport() const {return mySupports3DImageWrites;}

    /// Block until any outstanding kernel or memory transfers on the main
    /// CommandQueue have executed.  If sweep_pool is true, the context's
    /// CE_MemoryPool will sweep for any buffers that were in use when their
    /// CE_Grid's went out of scope, but that were still active in kernels.
    void finish(bool sweep_pool=true);

    /// Allocate a buffer of specified size on the CE_Device.
    /// use_pool= true, attempts to use the underlying CE_MemoryPool to possibly return
    /// an already allocated, unused buffer.
    /// read=true, creates a buffer that is readable inside kernels.
    /// write=true, creates a buffer that is writable inside kernels.
    /// ogl_bind, specifies an OGL buffer to bind to.
    cl::Buffer allocBuffer(int64 size, bool use_pool=true, bool read=true, bool write=true, uint32 ogl_bind=SYS_UINT32_MAX);

    /// Release the specified buffer, possibly to the CE_MemoryPool.
    void releaseBuffer(cl::Buffer &&buf, bool use_pool=true);

    /// Read the specified number of bytes from the buffer.
    void readBuffer(const cl::Buffer &buf, size_t size, void *p, bool blocking = true, size_t offset = 0);

    /// Write the specified number of bytes to the buffer.
    void writeBuffer(const cl::Buffer &buf, size_t size, const void *p, bool blocking = true, size_t offset = 0);

    /// Copy the specified amount of data from source buffer to destination
    /// buffer.
    void copyBuffer(const cl::Buffer &src, const cl::Buffer &dst, size_t size,
                    size_t src_offset = 0, size_t dst_offset = 0);

    /// Copy data from the source buffer into the specified region of the
    /// destination image.
    void copyBufferToImage(const cl::Buffer &src, const cl::Image &dst,
                           const cl::size_t<3>& dst_origin,
                           const cl::size_t<3>& dst_region,
                           size_t src_offset = 0);

    /// Copy data from the specified region of the source image to the
    /// destination buffer.
    void copyImageToBuffer(const cl::Image &src, const cl::Buffer &dst,
                           const cl::size_t<3>& src_origin,
                           const cl::size_t<3>& src_region,
                           size_t dst_offset = 0);
     
     /// Enqueue the kernel over the provided ranges.
    void enqueueKernel(const cl::Kernel &kernel, const cl::NDRange &global, const cl::NDRange &local);

    /// Keep a map buffer to bind at render time
    /// The first time a CL::Buffer is created it can be registered to rebing to a OGL vertex buffer at drawing time.
    /// The uint returned by the register call can be attached to a detail attribute and the drawing code can convert
    /// the CL Buffer to a CL BufferGL.
    uint32 registerDelayedOGLBindBuffer(CE_DelayedOGLBindBuffer* buffer);
    void unregisterDelayedOGLBindBuffer(uint32 id);
    CE_DelayedOGLBindBuffer* lookupDelayedOGLBindBuffer( uint id );

    /// Returns true if the context supports querying for device and driver
    /// UUIDs that are unique across APIs (allowing for example, to match Vulkan
    /// device selection). This requires the cl_khr_device_uuid extension.
    bool supportsUUID();
    /// Writes the OpenCL device UUID. Check that supportsUUID() returns true
    /// before trying to query the UUID.
    void getDeviceUUID(cl_uchar (&uuid)[CL_UUID_SIZE_KHR]);
    /// Writes the OpenCL driver UUID. Check that supportsUUID() returns true
    /// before trying to query the UUID.
    void getDriverUUID(cl_uchar (&uuid)[CL_UUID_SIZE_KHR]);

    /// Returns true if the context supports the creation of buffers backed by
    /// external memory. Mainly for use in sharing buffers with Vulkan.
    bool supportsExternalMemory();

    /// Create a buffer backed by external memory. The handle will be the reference
    /// to the actual memory object, and its lifetime is externally managed. The
    /// memory might be owned by another GPU API, such as Vulkan. In those cases,
    /// that handle can be obtained by a call to
    /// vkGetMemoryFdKHR or vkGetMemoryWin32HandleKHR
    /// This requires a few extensions to be available. Check that the context
    /// supports these by calling CE_Context::supportsExternalMemory first.
    CE_ExternalBuffer createExternalMemoryBuffer(SYS_Handle handle, int64_t size, bool read=true, bool write=true);

    CE_ExternalImage createExternalImage(SYS_Handle handle,
                                         const cl_image_format& format,
                                         const cl_image_desc& image_desc,
                                         bool read = true, bool write = true);

    /// Clear the CE_MemoryPool object.
    void clearMemoryPool();

    /// Sweep the memory pool for in use buffers.  We call this on every finish
    /// call as well, but if we have a large sequence of operations that use
    /// temporary buffers without a finish, it's a good idea to call this directly
    /// occasionally.
    void sweepInUseMemory();

    /// Return a pointer to pinned (page-locked) host memory. On some devices
    /// (Nvidia), using this type of memory for the PCI/E host/device transfers
    /// can double the throughput.  Will return NULL if the memory can't be
    /// allocated, or if the device is not a GPU.
    fpreal32 *getPinnedBuffer(int64 size);

    cl::Buffer getXNoiseData();

    /// Standard error reporting for OpenCL exceptions.  They should generally
    /// take the form:
    /// @code
    /// try
    /// {
    ///      OpenCL calls...
    /// }
    /// catch(cl::Error &err)
    /// {
    ///      CE_Context::reportError(err);
    ///      ///cleanup
    /// }
    /// @endcode
    /// This will not capture delayed errors, however.  Instead
    /// you will need to add a callback to intercept them.
    static void reportError(const cl::Error &err);
    static void outputErrorMessage(const char *err_msg);
    static void setErrorCB(CE_ErrorCB callback, void *data);
    static void outputWarningMessage(const char *err_msg);

    static void initMainSharedGLContext( int devicetype, void* context, void* display, const cl_uchar (&uuid)[16]);
    static bool useHalfNormalDelayedBindBuffer();

    /// Marks that an operation has run out of memory, allowing us
    /// to report elsewhere.
    void setOutOfMemoryFailure(bool hasfailed = true) { myOutOfMemoryFailure = true; }
    bool hasOutOfMemoryFailureHappened() const { return myOutOfMemoryFailure; }

    /// Combination of environment variable, but overriden to be false by known buggy drivers
    static bool shouldUseOCLOGLInterop();

    /// This structure holds a device name, vendor, device number, and drive version with respect to
    /// its vendor platform.
    struct DeviceDescriptor
    {
        UT_StringHolder label;
        UT_StringHolder vendor;
        UT_StringHolder platformVendor;
        int             number;
        UT_StringHolder driverVersion;
    };
    /// Get the vector of available devices of the given type.
    static void getDevices(UT_Array<DeviceDescriptor>&, cl_device_type t);

    /// Get an index to the preferred/default device for the specified device
    /// type and the list of available devices.
    static int getDefaultDevice(
        cl_device_type t, const UT_Array<DeviceDescriptor> &devices);

    /// Returns true if environment variables are set that override preferences.
    static bool isEnvironmentOverride();

    // Queries the device by calling clGetDeviceInfo, but returning false and setting
    // result to zero for unknown flags or flags that are disabled with environment
    // variables.
    template <class T>
    static bool getDeviceInfoRestricted(cl_device_id device, cl_uint flag, T &result);

    /// Queries the current device give the specified flag using clGetDeviceInfo,
    /// used by ocldeviceinfo EXPR function. Returns false for unrecognized flag.
    bool getDeviceInfo(const char *flag, fpreal &result);

    /// This function returns the total size of addressable compute memory for
    /// the current device.
    size_t getAddressableMemory() const;

    /// Report memory usage
    void        reportUsage(std::ostream &os) const;

    /// Requests the specified amount of memory to be freed from the OpenCL
    /// device and returns the amount actually freed.
    int64 requestMemoryFromResource(int64 amount);
    /// Returns the memory resource object for the OpenCL device. All objects
    /// that use OpenCL memory should ideally be registered as clients for this
    /// resource. NOTE: this only exists if the context is valid!
    UT_MemoryResource* getMemoryResource() const
    {
        return myMemResource;
    }

private:
    UT_UniquePtr<cl::Program> doCompileProgram(
                                    const char *progname,
                                    const char *progtext,
                                    const char *options,
                                    bool recompile);

    /// Initialize the context for the given device.
    void init(cl::Context &context, cl::Device &device, bool gl_shared);

    /// Releases the pinned, page-locked memory buffer.
    void releasePinnedBuffer();


    cl::Context             myContext;
    cl::CommandQueue        myQueue;
    cl::CommandQueue        myDeviceQueue;
    ceTraceCtx              myTraceCtx;
    cl::Device              myDevice;
    bool                    myIsValid;
    bool                    mySupportsDouble;
    bool                    mySupports3DImageWrites;
    cl::Buffer              myXNoiseData;
    UT_Lock                 myXNoiseLock;

    // Indirection to avoid changing reference counts when caches resize/rehash.
    using ProgramUPtr = UT_UniquePtr<cl::Program>;
    using KernelUPtr = UT_UniquePtr<cl::Kernel>;

    struct KernelInfo
    {
        UT_StringHolder          name;
        UT_Array<KernelUPtr>     kernels;
    };

    UT_StringMap<ProgramUPtr>    myProgramTable;
    UT_Map<const _cl_program *, UT_Array<KernelInfo>> myKernelTable;
    UT_Lock                      myKernelCacheLock;

    UT_UniquePtr<CE_MemoryPool>  myMemPool;
    UT_MemoryResource           *myMemResource;

    // The pinned buffer is unique to the main thread.
    cl::Buffer              myPinnedBuffer;
    fpreal32                *myPinnedData;

    bool                    myOutOfMemoryFailure;

    UT_Map<uint32,CE_DelayedOGLBindBuffer*> myDelayedOGLBindBuffers;

    static void*                        theGLContext;
    static void*                        theGLDisplay;
    static int                          theGLDeviceType;
    static char                         theGLDeviceUUID[16];
};

/// NOTE: this function will retry if it fails on allocation failure, after
/// freeing some memory.
CE_API cl_int
ce_enqueueKernel(const cl::CommandQueue& queue, const cl::Kernel &kernel,
                 const cl::NDRange &offset, const cl::NDRange &global, const cl::NDRange &local,
                 const std::vector<cl::Event>* events,
                 cl::Event* event);

#endif