/**************************************************************************** ** ** Copyright (C) 2016 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the QtTest module of the Qt Toolkit. ** ** $QT_BEGIN_LICENSE:LGPL$ ** Commercial License Usage ** Licensees holding valid commercial Qt licenses may use this file in ** accordance with the commercial license agreement provided with the ** Software or, alternatively, in accordance with the terms contained in ** a written agreement between you and The Qt Company. For licensing terms ** and conditions see https://www.qt.io/terms-conditions. For further ** information use the contact form at https://www.qt.io/contact-us. ** ** GNU Lesser General Public License Usage ** Alternatively, this file may be used under the terms of the GNU Lesser ** General Public License version 3 as published by the Free Software ** Foundation and appearing in the file LICENSE.LGPL3 included in the ** packaging of this file. Please review the following information to ** ensure the GNU Lesser General Public License version 3 requirements ** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. ** ** GNU General Public License Usage ** Alternatively, this file may be used under the terms of the GNU ** General Public License version 2.0 or (at your option) the GNU General ** Public license version 3 or any later version approved by the KDE Free ** Qt Foundation. The licenses are as published by the Free Software ** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 ** included in the packaging of this file. Please review the following ** information to ensure the GNU General Public License requirements will ** be met: https://www.gnu.org/licenses/gpl-2.0.html and ** https://www.gnu.org/licenses/gpl-3.0.html. ** ** $QT_END_LICENSE$ ** ****************************************************************************/ #include "qxctestlogger_p.h" #include #include #include #import // --------------------------------------------------------- @interface XCTestProbe (Private) + (BOOL)isTesting; + (void)runTests:(id)unusedArgument; + (NSString*)testScope; + (BOOL)isInverseTestScope; @end @interface XCTestDriver : NSObject + (XCTestDriver*)sharedTestDriver; @property (readonly, assign) NSObject *IDEConnection; @end @interface XCTest (Private) - (NSString *)nameForLegacyLogging; @end QT_WARNING_PUSH // Ignore XCTestProbe deprecation QT_WARNING_DISABLE_DEPRECATED // --------------------------------------------------------- @interface QtTestLibWrapper : XCTestCase @end @interface QtTestLibTests : XCTestSuite + (XCTestSuiteRun*)testRun; @end @interface QtTestLibTest : XCTestCase @property (nonatomic, retain) NSString* testObjectName; @property (nonatomic, retain) NSString* testFunctionName; @end // --------------------------------------------------------- class ThreadBarriers { public: enum Barrier { XCTestCanStartTesting, XCTestHaveStarted, QtTestsCanStartTesting, QtTestsHaveCompleted, XCTestsHaveCompleted, BarrierCount }; static ThreadBarriers *get() { static ThreadBarriers instance; return &instance; } static void initialize() { get(); } void wait(Barrier barrier) { dispatch_semaphore_wait(barriers[barrier], DISPATCH_TIME_FOREVER); } void signal(Barrier barrier) { dispatch_semaphore_signal(barriers[barrier]); } private: #define FOREACH_BARRIER(cmd) for (int i = 0; i < BarrierCount; ++i) { cmd } ThreadBarriers() { FOREACH_BARRIER(barriers[i] = dispatch_semaphore_create(0);) } ~ThreadBarriers() { FOREACH_BARRIER(dispatch_release(barriers[i]);) } dispatch_semaphore_t barriers[BarrierCount]; }; #define WAIT_FOR_BARRIER(b) ThreadBarriers::get()->wait(ThreadBarriers::b); #define SIGNAL_BARRIER(b) ThreadBarriers::get()->signal(ThreadBarriers::b); // --------------------------------------------------------- @implementation QtTestLibWrapper + (void)load { NSAutoreleasePool *autoreleasepool = [[NSAutoreleasePool alloc] init]; if (![XCTestProbe isTesting]) return; if (Q_UNLIKELY(!([NSDate timeIntervalSinceReferenceDate] > 0))) qFatal("error: Device date '%s' is bad, likely set to update automatically. Please correct.", [[NSDate date] description].UTF8String); XCTestDriver *testDriver = nil; if ([QtTestLibWrapper usingTestManager]) testDriver = [XCTestDriver sharedTestDriver]; // Spawn off task to run test infrastructure on separate thread so that we can // let main() execute like normal on the main thread. The queue will never be // destroyed, so there's no point in trying to keep a proper retain count. dispatch_async(dispatch_queue_create("io.qt.QTestLib.xctest-wrapper", DISPATCH_QUEUE_SERIAL), ^{ Q_ASSERT(![NSThread isMainThread]); [XCTestProbe runTests:nil]; Q_UNREACHABLE(); }); // Initialize barriers before registering exit handler so that the // semaphores stay alive until after the exit handler completes. ThreadBarriers::initialize(); // We register an exit handler so that we can intercept when main() completes // and let the XCTest thread finish up. For main() functions that never started // testing using QtTestLib we also need to signal that xcTestsCanStart. atexit_b(^{ Q_ASSERT([NSThread isMainThread]); // In case not started by startLogging SIGNAL_BARRIER(XCTestCanStartTesting); // [XCTestProbe runTests:] ends up calling [XCTestProbe exitTests:] after // all test suites have completed, which calls exit(). We use that to signal // to the main thread that it's free to continue its exit handler. atexit_b(^{ Q_ASSERT(![NSThread isMainThread]); SIGNAL_BARRIER(XCTestsHaveCompleted); // Block forever so that the main thread does all the cleanup dispatch_semaphore_wait(dispatch_semaphore_create(0), DISPATCH_TIME_FOREVER); }); SIGNAL_BARRIER(QtTestsHaveCompleted); // Ensure XCTest complets the top level tests suite WAIT_FOR_BARRIER(XCTestsHaveCompleted); }); // Let test driver (Xcode) connection setup complete before continuing if ([[[NSProcessInfo processInfo] arguments] containsObject:@"--use-testmanagerd"]) { while (!testDriver.IDEConnection) [[NSRunLoop mainRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate distantFuture]]; } // Wait for our QtTestLib test suite to run before running main WAIT_FOR_BARRIER(QtTestsCanStartTesting); // Prevent XCTestProbe from re-launching runTests on application startup [[NSNotificationCenter defaultCenter] removeObserver:[XCTestProbe class] name:[NSString stringWithFormat:@"%@DidFinishLaunchingNotification", #if defined(Q_OS_OSX) @"NSApplication" #else @"UIApplication" #endif ] object:nil]; [autoreleasepool release]; } + (id)defaultTestSuite { return [[QtTestLibTests alloc] initWithName:@"QtTestLib"]; } + (BOOL)usingTestManager { return [[[NSProcessInfo processInfo] arguments] containsObject:@"--use-testmanagerd"]; } @end // --------------------------------------------------------- static XCTestSuiteRun *s_qtTestSuiteRun = 0; @implementation QtTestLibTests - (void)performTest:(XCTestSuiteRun *)testSuiteRun { Q_ASSERT(![NSThread isMainThread]); Q_ASSERT(!s_qtTestSuiteRun); s_qtTestSuiteRun = testSuiteRun; SIGNAL_BARRIER(QtTestsCanStartTesting); // Wait for main() to complete, or a QtTestLib test to start, so we // know if we should start the QtTestLib test suite. WAIT_FOR_BARRIER(XCTestCanStartTesting); if (QXcodeTestLogger::isActive()) [testSuiteRun start]; SIGNAL_BARRIER(XCTestHaveStarted); // All test reporting happens on main thread from now on. Wait until // main() completes before allowing the XCTest thread to continue. WAIT_FOR_BARRIER(QtTestsHaveCompleted); if ([testSuiteRun startDate]) [testSuiteRun stop]; } + (XCTestSuiteRun*)testRun { return s_qtTestSuiteRun; } @end // --------------------------------------------------------- @implementation QtTestLibTest - (id)initWithInvocation:(NSInvocation *)invocation { if (self = [super initWithInvocation:invocation]) { // The test object name and function name are used by XCTest after QtTestLib has // reset them, so we need to store them up front for each XCTestCase. self.testObjectName = [NSString stringWithUTF8String:QTestResult::currentTestObjectName()]; self.testFunctionName = [NSString stringWithUTF8String:QTestResult::currentTestFunction()]; } return self; } - (NSString *)testClassName { return self.testObjectName; } - (NSString *)testMethodName { return self.testFunctionName; } - (NSString *)nameForLegacyLogging { NSString *name = [NSString stringWithFormat:@"%@::%@", [self testClassName], [self testMethodName]]; if (QTestResult::currentDataTag() || QTestResult::currentGlobalDataTag()) { const char *currentDataTag = QTestResult::currentDataTag() ? QTestResult::currentDataTag() : ""; const char *globalDataTag = QTestResult::currentGlobalDataTag() ? QTestResult::currentGlobalDataTag() : ""; const char *filler = (currentDataTag[0] && globalDataTag[0]) ? ":" : ""; name = [name stringByAppendingString:[NSString stringWithFormat:@"(%s%s%s)", globalDataTag, filler, currentDataTag]]; } return name; } @end // --------------------------------------------------------- bool QXcodeTestLogger::canLogTestProgress() { return [XCTestProbe isTesting]; // FIXME: Exclude xctool } int QXcodeTestLogger::parseCommandLineArgument(const char *argument) { if (strncmp(argument, "-NS", 3) == 0 || strncmp(argument, "-Apple", 6) == 0) return 2; // -NSTreatUnknownArgumentsAsOpen, -ApplePersistenceIgnoreState, etc, skip argument else if (strcmp(argument, "--use-testmanagerd") == 0) return 2; // Skip UID argument else if (strncmp(argument, "-XCTest", 7) == 0) return 2; // -XCTestInvertScope, -XCTest scope, etc, skip argument else if (strcmp(argument + (strlen(argument) - 7), ".xctest") == 0) return 1; // Skip test bundle else return 0; } // --------------------------------------------------------- QXcodeTestLogger *QXcodeTestLogger::s_currentTestLogger = 0; // --------------------------------------------------------- QXcodeTestLogger::QXcodeTestLogger() : QAbstractTestLogger(0) , m_testRuns([[NSMutableArray arrayWithCapacity:2] retain]) { Q_ASSERT(!s_currentTestLogger); s_currentTestLogger = this; } QXcodeTestLogger::~QXcodeTestLogger() { s_currentTestLogger = 0; [m_testRuns release]; } void QXcodeTestLogger::startLogging() { SIGNAL_BARRIER(XCTestCanStartTesting); static dispatch_once_t onceToken; dispatch_once (&onceToken, ^{ WAIT_FOR_BARRIER(XCTestHaveStarted); }); // Scope test object suite under top level QtTestLib test run [m_testRuns addObject:[QtTestLibTests testRun]]; NSString *suiteName = [NSString stringWithUTF8String:QTestResult::currentTestObjectName()]; pushTestRunForTest([XCTestSuite testSuiteWithName:suiteName], true); } void QXcodeTestLogger::stopLogging() { popTestRun(); } static bool isTestFunctionInActiveScope(const char *function) { static NSString *testScope = [XCTestProbe testScope]; enum TestScope { Unknown, All, None, Self, Selected }; static TestScope activeScope = Unknown; if (activeScope == Unknown) { if ([testScope isEqualToString:@"All"]) activeScope = All; else if ([testScope isEqualToString:@"None"]) activeScope = None; else if ([testScope isEqualToString:@"Self"]) activeScope = Self; else activeScope = Selected; } if (activeScope == All) return true; else if (activeScope == None) return false; else if (activeScope == Self) return true; // Investigate Q_ASSERT(activeScope == Selected); static NSArray *forcedTests = [@[ @"initTestCase", @"initTestCase_data", @"cleanupTestCase" ] retain]; if ([forcedTests containsObject:[NSString stringWithUTF8String:function]]) return true; static NSArray *testsInScope = [[testScope componentsSeparatedByString:@","] retain]; bool inScope = [testsInScope containsObject:[NSString stringWithFormat:@"%s/%s", QTestResult::currentTestObjectName(), function]]; if ([XCTestProbe isInverseTestScope]) inScope = !inScope; return inScope; } void QXcodeTestLogger::enterTestFunction(const char *function) { if (!isTestFunctionInActiveScope(function)) QTestResult::setSkipCurrentTest(true); XCTest *test = [QtTestLibTest testCaseWithInvocation:nil]; pushTestRunForTest(test, !QTestResult::skipCurrentTest()); } void QXcodeTestLogger::leaveTestFunction() { popTestRun(); } void QXcodeTestLogger::addIncident(IncidentTypes type, const char *description, const char *file, int line) { XCTestRun *testRun = [m_testRuns lastObject]; // The 'expected' argument to recordFailureWithDescription refers to whether // the failure was a regular failed assertion, or an unexpected exception, // so in our case it's always 'YES', and we need to explicitly ignore XFail. if (type == QAbstractTestLogger::XFail) { QTestCharBuffer buf; NSString *testCaseName = [[testRun test] nameForLegacyLogging]; QTest::qt_asprintf(&buf, "Test Case '%s' failed expectedly (%s).\n", [testCaseName UTF8String], description); outputString(buf.constData()); return; } if (type == QAbstractTestLogger::Pass) { // We ignore non-data passes, as we're already reporting that as part of the // normal test case start/stop cycle. if (!(QTestResult::currentDataTag() || QTestResult::currentGlobalDataTag())) return; QTestCharBuffer buf; NSString *testCaseName = [[testRun test] nameForLegacyLogging]; QTest::qt_asprintf(&buf, "Test Case '%s' passed.\n", [testCaseName UTF8String]); outputString(buf.constData()); return; } // FIXME: Handle blacklisted tests if (!file || !description) return; // Or report? [testRun recordFailureWithDescription:[NSString stringWithUTF8String:description] inFile:[NSString stringWithUTF8String:file] atLine:line expected:YES]; } void QXcodeTestLogger::addMessage(MessageTypes type, const QString &message, const char *file, int line) { QTestCharBuffer buf; if (QTestLog::verboseLevel() > 0 && (file && line)) { QTest::qt_asprintf(&buf, "%s:%d: ", file, line); outputString(buf.constData()); } if (type == QAbstractTestLogger::Skip) { XCTestRun *testRun = [m_testRuns lastObject]; NSString *testCaseName = [[testRun test] nameForLegacyLogging]; QTest::qt_asprintf(&buf, "Test Case '%s' skipped (%s).\n", [testCaseName UTF8String], message.toUtf8().constData()); } else { QTest::qt_asprintf(&buf, "%s\n", message.toUtf8().constData()); } outputString(buf.constData()); } void QXcodeTestLogger::addBenchmarkResult(const QBenchmarkResult &result) { Q_UNUSED(result); } void QXcodeTestLogger::pushTestRunForTest(XCTest *test, bool start) { XCTestRun *testRun = [[test testRunClass] testRunWithTest:test]; [m_testRuns addObject:testRun]; if (start) [testRun start]; } XCTestRun *QXcodeTestLogger::popTestRun() { XCTestRun *testRun = [[m_testRuns lastObject] retain]; [m_testRuns removeLastObject]; if ([testRun startDate]) [testRun stop]; [[m_testRuns lastObject] addTestRun:testRun]; [testRun release]; return testRun; } bool QXcodeTestLogger::isActive() { return s_currentTestLogger; } QT_WARNING_POP