UT_TestManager.h

Go to the documentation of this file.

/*
 * PROPRIETARY INFORMATION.  This software is proprietary to
 * Side Effects Software Inc., and is not to be reproduced,
 * transmitted, or disclosed in any way without written permission.
 *
 * NAME:        UT_TestManager.h (C++)
 *
 * COMMENTS:    This class provides a test harness framework.
 *
 * USAGE:
 *      1.  Create a main() method for your test application like so:
 *              int main(int argc, char *argv[])
 *              {
 *                  return UT_TestManager::get().run(argc, argv);
 *              }
 *          The return code will be the number of test failures.
 *          To obtain the command line usage, use "testapp -h".
 *
 *      2.  For each test case, use one of the TEST_REGISTER*() macros for
 *          registering a test function. The function should return true if the
 *          test passed.
 */

#ifndef __UT_TESTMANAGER_H__
#define __UT_TESTMANAGER_H__

#include "UT_API.h"
#include "UT_Functor.h"
#include "UT_Function.h"
#include "UT_Main.h"
#include "UT_NonCopyable.h"
#include "UT_StopWatch.h"
#include "UT_WorkArgs.h"
#include "UT_WorkBuffer.h"
#include <SYS/SYS_Types.h>

class UT_API UT_TestManager
{
public:
             UT_TestManager() { }
    virtual ~UT_TestManager() { }

    UT_NON_COPYABLE(UT_TestManager)

    /// Access the global test manager
    static UT_TestManager & get();

    /// Determine if the test was interactively invoked
    virtual bool            isInteractive() const = 0;

    /// Determine if the tests are being run for performance
    virtual bool            isPerformance() const = 0;

    /// Determine the number of threads requested
    virtual int             numThreads() const = 0;

    /// Called by main() to run the tests. Returns the number of test failures.
    virtual int             run(int argc, char *argv[]) = 0;

    /// Called by the TEST_REGISTER*() macros to automatically add a new test
    /// case
    virtual void            addTest(const char *name,
                                    UT_Functor<bool> &callback) = 0;

    /// Allow for logging of intermediate test results.  If logging is turned
    /// on in the test manager, the test name, status and run time will be
    /// saved.
    virtual void        logResult(const char *name, bool success,
                                    fpreal64 run_time) = 0;

    /// Get the arguments passed into this program.
    virtual const UT_WorkArgs& getArgs() const = 0;
};

/// @brief Status maintainer used to run a test.
///
/// This class simplifies the book-keeping when running a test.  The unit test
/// will automatically log information (i.e. performance, memory) if the
/// UT_TestManager is so configured.
///
/// Example 1: @code
/// static bool
/// unitTest(int argument)
/// {
///     UT_TestUnit         status("UnitTest: %d", argument);
/// 
///     if (!doTest(...))
///         return status.fail("Test failed: %s", some_string);
/// 
///     return status.ok();
/// }
/// @endcode
/// 
/// Example 2: @code
/// static bool
/// unitTest(int argument)
/// {
///     UT_TestUnit unit(UT_TestUnit::OK, "UnitTest: %d", argument);
/// 
///     if (!doTest1(...))
///         unit.fail("Test 1 failed: %s", some_string);
///     if (!doTest2(...))
///         unit.fail("Test 2 failed: %s", some_string);
/// 
///     return unit.status();
/// }
/// @endcode
class UT_API UT_TestUnit
{
public:
    enum InitEnum { OK, FAIL };

     UT_TestUnit(const char *format, ...) SYS_PRINTF_CHECK_ATTRIBUTE(2, 3);
     UT_TestUnit(InitEnum init_enum, const char *format, ...)
         SYS_PRINTF_CHECK_ATTRIBUTE(3, 4);
    ~UT_TestUnit();

    /// Print out the message given by a format.  This includes the test name,
    /// and other information.  No newline should be included in the format.
    ///
    /// The function @b always returns @b false.
    bool        fail(const char *format=NULL, ...)
                    SYS_PRINTF_CHECK_ATTRIBUTE(2, 3);

    /// Print out the message given by a format.  This includes the test name,
    /// and other information.  No newline should be included in the format.
    ///
    /// The function @b always returns @b true.
    bool        ok(const char *format=NULL, ...)
                    SYS_PRINTF_CHECK_ATTRIBUTE(2, 3);

    /// Restart the timer. If format is non-NULL, then it will print the time
    /// as well from when the timer last started. Note that the timer is
    /// automatically started in the constructor.
    void        restartTimer(const char *format=NULL, ...)
                    SYS_PRINTF_CHECK_ATTRIBUTE(2, 3);

    /// Enable printing of success/failure when this unit test is destroyed.
    void        setAlwaysPrint(bool f)  { myAlwaysPrint = f; }

    /// Query the status of the test
    bool        status() const          { return myStatus; }
    bool        getStatus() const       { return myStatus; }
    bool        setStatus(bool b)       { myStatus = b; return myStatus; }
private:
    UT_WorkBuffer       myName;         // Test-name
    UT_Timer            myTimer;
    bool                myStatus;
    bool                myPrint;
    bool                myAlwaysPrint;
};


// Forward-declaration of nanobench classes.
namespace ankerl::nanobench
{
    class Bench;
}

/// @brief Wrapper around the nanobench object for unit performance tests.
///
/// This class wraps the nanobench::Bench object for performance testing,
/// and streamlines the workflows for deploying it in unit tests.
///
/// Example: @code
/// static void
/// performanceTest(UT_TestUnit &test_unit)
/// {
///     int result = expensiveComputation();
///     if (result != 42) 
///         test_unit.fail("Wrong computation: got %d, expected 42", result);
/// }
///
/// static bool
/// test()
/// {
///     UT_TestBench    bench;
///     bool            ok = true;
///
///     // Time the call to the function.
///     ok &= bench.run("Forty two", performanceTest));
///
///     // Time the call to the lambda.
///     ok &= bench.run("Loop", [](UT_testUnit &test_unit)
///             auto sum = 0;
///             for(auto i=0; i<1000; i++)
///                 sum += i;
///         });
/// 
///     // Prints the nanobench timing report, saves it to file for the future,
///     // tries to read a corresponding report from the ./correct subdir,
///     // and returns true if the performance has not deteriorated
///     // (thus passing the performance measurement test).
///     ok &= bench.saveAndCompareResults("MyPerformanceTest");
///
///     return ok;
/// }
/// @endcode
///
class UT_API UT_TestBench
{
public:
     UT_TestBench(const UT_StringRef &title);
    ~UT_TestBench();

    UT_NON_COPYABLE(UT_TestBench)


    /// Runs and times the given function @p f.
    /// Prints the unit test wall clock time to the output as a summary.
    ///
    /// Stores the nanobench timing in the internal structures for later 
    /// comparisons and detection of the performance regressions.
    ///
    /// It passes a UT_TestUnit to that function, for setting status.
    /// I.e., bad results may be fast, but they should not be considered
    /// for parformance timing, and hence status should be set to error.
    /// Returns true if that status is a success or false otherwise.
    /// 
    /// @param test_name  The name of the timing test to appear in the reports.
    /// @param fn         The test function to time.
    bool    run( const UT_StringRef &test_name,
                 UT_Function<void(UT_TestUnit &)> fn );

    
    /// Saves the nanobench timings from the previously performed runs
    /// to a file with the given name in the current working directory. 
    /// Compares the timings with a similarly named file in a ./correct subdir,
    /// if it exists. If it does not exists, it creates one as a base line
    /// for the future runs.
    /// comparing to the baseline times, and false otherise.
    /// Prints the report of the current run times comparison to the
    /// times recorded as a baseline in the ./correct file.
    /// Returns true if the current timings are within an acceptable delta
    ///
    /// @param file_name_root The base name to use for constructing the
    ///         timing result file name.
    /// @return True if the current timings are within an acceptable delta;
    ///         False otherwise.
    bool    saveAndCompareResults(const UT_StringRef &file_name_root) const;

private:
    // Wrapping Bench using pimpl idiom. 
    using Bench = ankerl::nanobench::Bench;
    UT_UniquePtr<Bench>  myBench;
};


/// Use this define to log information into the result table
#define TEST_LOGRESULT(NAME, SUCCESS, TIME)     \
            UT_TestManager::get().logResult(NAME, SUCCESS, TIME)

/// Use this macro after your test function is declared. It will register
/// your method with the test manager. Your method should return true if the
/// test succeeded, false otherwise. The name is the label that will be used
/// for invoking this specific test.
#define TEST_REGISTER_FN(name, method)                                  \
            class method##Registrar                                     \
            {                                                           \
                public:                                                 \
                    method##Registrar()                                 \
                    {                                                   \
                        UT_Functor<bool> callback(&method);             \
                        UT_TestManager::get().addTest(name, callback);  \
                    }                                                   \
            };                                                          \
            static method##Registrar    the##method##Registrar;

/// Use this macro after your test class is declared. It will create a static
/// instance of your class that then registers your 'bool classname::method()'
/// with the test manager. Your method should return true if the test
/// succeeded, false otherwise. The name is the label that will be used for
/// invoking this specific test.
#define TEST_REGISTER(name, classname, method)                          \
            class classname##method##Registrar                          \
            {                                                           \
                public:                                                 \
                    classname##method##Registrar()                      \
                    {                                                   \
                        UT_Functor<bool> callback(&myTest,              \
                                                  &classname::method);  \
                        UT_TestManager::get().addTest(name, callback);  \
                    }                                                   \
                private:                                                \
                    classname   myTest;                                 \
            };                                                          \
            static classname##method##Registrar                         \
                                    the##classname##method##Registrar;

#define TEST_UNIT_MAIN()                                                       \
    int theMain(int argc, char *argv[])                                        \
    {                                                                          \
        /*Ensure we reset the task scheduler to avoid hangning on exit in      \
         * Windows.*/                                                          \
        UT_Thread::resetNumProcessors();                                       \
        return UT_TestManager::get().run(argc, argv);                          \
    }                                                                          \
    UT_MAIN(theMain);

#endif // __UT_TESTMANAGER_H__