1
/****************************************************************************
2
**
3
** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
4
** All rights reserved.
5
** Contact: Nokia Corporation (qt-info@nokia.com)
6
**
7
** This file is part of the test suite of the Qt Toolkit.
8
**
9
** $QT_BEGIN_LICENSE:LGPL$
10
** GNU Lesser General Public License Usage
11
** This file may be used under the terms of the GNU Lesser General Public
12
** License version 2.1 as published by the Free Software Foundation and
13
** appearing in the file LICENSE.LGPL included in the packaging of this
14
** file. Please review the following information to ensure the GNU Lesser
15
** General Public License version 2.1 requirements will be met:
16
** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
17
**
18
** In addition, as a special exception, Nokia gives you certain additional
19
** rights. These rights are described in the Nokia Qt LGPL Exception
20
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
21
**
22
** GNU General Public License Usage
23
** Alternatively, this file may be used under the terms of the GNU General
24
** Public License version 3.0 as published by the Free Software Foundation
25
** and appearing in the file LICENSE.GPL included in the packaging of this
26
** file. Please review the following information to ensure the GNU General
27
** Public License version 3.0 requirements will be met:
28
** http://www.gnu.org/copyleft/gpl.html.
29
**
30
** Other Usage
31
** Alternatively, this file may be used in accordance with the terms and
32
** conditions contained in a signed written agreement between you and Nokia.
33
**
34
**
35
**
36
**
37
**
38
** $QT_END_LICENSE$
39
**
40
****************************************************************************/
41
42
43
#include <atWrapper.h>
44
#include <datagenerator/datagenerator.h>
45
46
#include <QString>
47
#include <QHash>
48
#include <QFile>
49
#include <QFtp>
50
#include <QObject>
51
#include <QHostInfo>
52
#include <QWidget>
53
#include <QImage>
54
#include <QtTest/QSignalSpy>
55
#include <QLibraryInfo>
56
57
static const char *ArthurDir = "../../arthur";
58
59
#include <string.h>
60
61
atWrapper::atWrapper()
62
{
63
64
 //   initTests();
65
66
}
67
68
bool atWrapper::initTests(bool *haveBaseline)
69
{
70
    qDebug() << "Running test on buildkey:" << QLibraryInfo::buildKey() << "  qt version:" << qVersion();
71
72
    qDebug( "Initializing tests..." );
73
74
    if (!loadConfig( QHostInfo::localHostName().split( "." ).first() + ".ini" ))
75
        return false;
76
77
    //Reset the FTP environment where the results are stored
78
    *haveBaseline = setupFTP();
79
80
    // Retrieve the latest test result baseline from the FTP server.
81
    downloadBaseline();
82
    return true;
83
}
84
85
void atWrapper::downloadBaseline()
86
{
87
88
    qDebug() << "Now downloading baseline...";
89
90
    QFtp ftp;
91
92
    QObject::connect( &ftp, SIGNAL( listInfo( const QUrlInfo & ) ), this, SLOT( ftpMgetAddToList(const QUrlInfo & ) ) );
93
94
    //Making sure that the needed local directories exist.
95
96
    QHashIterator<QString, QString> j(enginesToTest);
97
98
    while ( j.hasNext() )
99
    {
100
        j.next();
101
102
        QDir dir( output );
103
104
        if ( !dir.cd( j.key() + ".baseline" ) )
105
            dir.mkdir( j.key() + ".baseline" );
106
107
    }
108
109
    //FTP to the host specified in the config file, and retrieve the test result baseline.
110
    ftp.connectToHost( ftpHost );
111
    ftp.login( ftpUser, ftpPass );
112
113
    ftp.cd( ftpBaseDir );
114
115
    QHashIterator<QString, QString> i(enginesToTest);
116
    while ( i.hasNext() )
117
    {
118
        i.next();
119
        mgetDirList.clear();
120
        mgetDirList << i.key() + ".baseline";
121
        ftp.cd( i.key() + ".baseline" );
122
        ftp.list();
123
        ftp.cd( ".." );
124
125
        while ( ftp.hasPendingCommands() )
126
            QCoreApplication::instance()->processEvents();
127
128
        ftpMgetDone( true );
129
    }
130
131
    ftp.close();
132
    ftp.close();
133
134
    while ( ftp.hasPendingCommands() )
135
        QCoreApplication::instance()->processEvents();
136
137
}
138
139
void atWrapper::ftpMgetAddToList( const QUrlInfo &urlInfo )
140
{
141
    //Simply adding to the list of files to download.
142
    mgetDirList << urlInfo.name();
143
144
}
145
146
void atWrapper::ftpMgetDone( bool error)
147
{
148
    Q_UNUSED( error );
149
150
    //Downloading the files listed in mgetDirList...
151
    QFtp ftp;
152
    ftp.connectToHost( ftpHost );
153
    ftp.login( ftpUser, ftpPass );
154
155
    QFile* file;
156
157
    if ( mgetDirList.size() > 1 )
158
        for ( int i = 1; i < mgetDirList.size(); ++i )
159
        {
160
            file = new QFile( QString( output ) + "/" + mgetDirList.at( 0 ) + "/" + mgetDirList.at( i ) );
161
            if (file->open(QIODevice::WriteOnly)) {
162
                ftp.get( ftpBaseDir + "/" + mgetDirList.at( 0 ) + "/" + mgetDirList.at( i ), file );
163
                ftp.list(); //Only there to fill up a slot in the pendingCommands queue.
164
                while ( ftp.hasPendingCommands() )
165
                    QCoreApplication::instance()->processEvents();
166
                file->close();
167
            } else {
168
                qDebug() << "Couldn't open file for writing: " << file->fileName();
169
            }
170
        }
171
172
173
    while ( ftp.hasPendingCommands() )
174
        QCoreApplication::instance()->processEvents();
175
}
176
177
void atWrapper::uploadFailed( QString dir, QString filename, QByteArray filedata )
178
{
179
    //Upload a failed test case image to the FTP server.
180
    QFtp ftp;
181
    ftp.connectToHost( ftpHost );
182
    ftp.login( ftpUser, ftpPass );
183
184
    ftp.cd( ftpBaseDir );
185
    ftp.cd( dir );
186
187
    ftp.put( filedata, filename, QFtp::Binary );
188
189
    ftp.close();
190
191
    while ( ftp.hasPendingCommands() )
192
        QCoreApplication::instance()->processEvents();
193
}
194
195
// returns false if no baseline exists
196
bool atWrapper::setupFTP()
197
{
198
    qDebug( "Setting up FTP environment" );
199
200
    QString dir = "";
201
    ftpMkDir( ftpBaseDir );
202
203
    ftpBaseDir += "/" + QLibraryInfo::buildKey();
204
205
    ftpMkDir( ftpBaseDir );
206
207
    ftpBaseDir += "/" + QString( qVersion() );
208
209
    ftpMkDir( ftpBaseDir );
210
211
    QHashIterator<QString, QString> i(enginesToTest);
212
    QHashIterator<QString, QString> j(enginesToTest);
213
214
    bool haveBaseline = true;
215
    //Creating the baseline directories for each engine
216
    while ( i.hasNext() )
217
    {
218
        i.next();
219
        //qDebug() << "Creating dir with key:" << i.key();
220
        ftpMkDir( ftpBaseDir + "/" +  QString( i.key() ) + ".failed" );
221
        ftpMkDir( ftpBaseDir + "/" +  QString( i.key() ) + ".diff" );
222
        if (!ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".baseline" ))
223
            haveBaseline = false;
224
    }
225
226
227
    QFtp ftp;
228
    ftp.connectToHost( ftpHost );
229
    ftp.login( ftpUser, ftpPass );
230
231
    ftp.cd( ftpBaseDir );
232
    //Deleting previous failed directory and all the files in it, then recreating it.
233
    while ( j.hasNext() )
234
    {
235
        j.next();
236
        rmDirList.clear();
237
        rmDirList << ftpBaseDir + "/" + j.key() + ".failed" + "/";
238
        ftpRmDir( j.key() + ".failed" );
239
        ftp.rmdir( j.key() + ".failed" );
240
        ftp.mkdir( j.key() + ".failed" );
241
        ftp.list();
242
243
        while ( ftp.hasPendingCommands() )
244
            QCoreApplication::instance()->processEvents();
245
246
        rmDirList.clear();
247
        rmDirList << ftpBaseDir + "/" + j.key() + ".diff" + "/";
248
        ftpRmDir( j.key() + ".diff" );
249
        ftp.rmdir( j.key() + ".diff" );
250
        ftp.mkdir( j.key() + ".diff" );
251
        ftp.list();
252
253
        while ( ftp.hasPendingCommands() )
254
            QCoreApplication::instance()->processEvents();
255
256
    }
257
258
    ftp.close();
259
260
    while ( ftp.hasPendingCommands() )
261
        QCoreApplication::instance()->processEvents();
262
263
    return haveBaseline;
264
}
265
266
void atWrapper::ftpRmDir( QString dir )
267
{
268
    //Hack to remove a populated directory. (caveat: containing only files and empty dirs, not recursive!)
269
    qDebug() << "Now removing directory: " << dir;
270
    QFtp ftp;
271
    QObject::connect( &ftp, SIGNAL( listInfo( const QUrlInfo & ) ), this, SLOT( ftpRmDirAddToList(const QUrlInfo & ) ) );
272
    QObject::connect( &ftp, SIGNAL( done( bool ) ), this, SLOT( ftpRmDirDone( bool ) ) );
273
274
    ftp.connectToHost( ftpHost );
275
    ftp.login( ftpUser, ftpPass );
276
277
    ftp.list( ftpBaseDir + "/" +  dir );
278
    ftp.close();
279
    ftp.close();
280
281
    while ( ftp.hasPendingCommands() )
282
                QCoreApplication::instance()->processEvents();
283
}
284
285
void atWrapper::ftpRmDirDone( bool error )
286
{
287
    //Deleting each file in the directory listning, rmDirList.
288
    Q_UNUSED( error );
289
290
    QFtp ftp;
291
    ftp.connectToHost( ftpHost );
292
    ftp.login( ftpUser, ftpPass );
293
294
    if ( rmDirList.size() > 1 )
295
        for (int i = 1; i < rmDirList.size(); ++i)
296
            ftp.remove( rmDirList.at(0) + rmDirList.at( i ) );
297
298
    ftp.close();
299
300
    while ( ftp.hasPendingCommands() )
301
        QCoreApplication::instance()->processEvents();
302
}
303
304
// returns false if the directory already exists
305
bool atWrapper::ftpMkDir( QString dir )
306
{
307
    //Simply used to avoid QFTP from bailing out and loosing a queue of commands.
308
    // IE: conveniance.
309
    QFtp ftp;
310
311
    QSignalSpy commandSpy(&ftp, SIGNAL(commandFinished(int, bool)));
312
313
    ftp.connectToHost( ftpHost );
314
    ftp.login( ftpUser, ftpPass );
315
    const int command = ftp.mkdir( dir );
316
    ftp.close();
317
318
    while ( ftp.hasPendingCommands() )
319
        QCoreApplication::instance()->processEvents();
320
321
    for (int i = 0; i < commandSpy.count(); ++i)
322
        if (commandSpy.at(i).at(0) == command)
323
            return commandSpy.at(i).at(1).toBool();
324
325
    return false;
326
}
327
328
329
void atWrapper::ftpRmDirAddToList( const QUrlInfo &urlInfo )
330
{
331
    //Just adding the file to the list for deletion
332
    rmDirList << urlInfo.name();
333
}
334
335
336
bool atWrapper::executeTests()
337
{
338
    qDebug("Executing the tests...");
339
340
    QHashIterator<QString, QString> i(enginesToTest);
341
342
    DataGenerator generator;
343
344
    //Running datagenerator against all the frameworks specified in the config file.
345
    while ( i.hasNext() )
346
    {
347
348
        i.next();
349
350
        qDebug( "Now testing: " + i.key().toLatin1() );
351
352
        char* params[13];
353
        //./bin/datagenerator  -framework data/framework.ini  -engine OpenGL -suite 1.1 -output outtest
354
355
356
        QByteArray eng = i.key().toLatin1();
357
        QByteArray fwk = framework.toLatin1();
358
        QByteArray sut = suite.toLatin1();
359
        QByteArray out = output.toLatin1();
360
        QByteArray siz = size.toLatin1();
361
        QByteArray fill = fillColor.toLatin1();
362
363
        params[1] = "-framework";
364
        params[2] = fwk.data();
365
        params[3] = "-engine";
366
        params[4] = eng.data();
367
        params[5] = "-suite";
368
        params[6] = sut.data();
369
        params[7] = "-output";
370
        params[8] = out.data();
371
        params[9] = "-size";
372
        params[10] = siz.data();
373
        params[11] = "-fill";
374
        params[12] = fill.data();
375
376
        generator.run( 13, params );
377
    }
378
379
    return true;
380
}
381
382
void atWrapper::createBaseline()
383
{
384
    qDebug( "Now uploading a baseline of only the latest test values" );
385
386
    QHashIterator<QString, QString> i(enginesToTest);
387
388
    QDir dir( output );
389
    QFtp ftp;
390
    ftp.connectToHost( ftpHost );
391
    ftp.login( ftpUser, ftpPass );
392
    ftp.cd( ftpBaseDir );
393
    //Upload all the latest test results to the FTP server's baseline directory.
394
    while ( i.hasNext() )
395
    {
396
397
        i.next();
398
        dir.cd( i.key() );
399
        ftp.cd( i.key() + ".baseline" );
400
        dir.setFilter(QDir::Files | QDir::Hidden | QDir::NoSymLinks);
401
        dir.setNameFilters( QStringList() << "*.png" );
402
        QFileInfoList list = dir.entryInfoList();
403
        dir.cd( ".." );
404
        for (int n = 0; n < list.size(); n++)
405
        {
406
            QFileInfo fileInfo = list.at( n );
407
            QFile file( QString( output ) + "/" + i.key() + "/" + fileInfo.fileName() );
408
            file.open( QIODevice::ReadOnly );
409
            QByteArray fileData = file.readAll();
410
            //qDebug() << "Sending up:" << fileInfo.fileName() << "with file size" << fileData.size();
411
            file.close();
412
            ftp.put( fileData, fileInfo.fileName(), QFtp::Binary );
413
        }
414
415
        ftp.cd( ".." );
416
    }
417
418
    ftp.close();
419
420
    while ( ftp.hasPendingCommands() )
421
        QCoreApplication::instance()->processEvents();
422
}
423
424
bool atWrapper::compare()
425
{
426
    qDebug( "Now comparing the results to the baseline" );
427
428
    QHashIterator<QString, QString> i(enginesToTest);
429
430
    while ( i.hasNext() )
431
    {
432
        i.next();
433
434
        compareDirs( output , i.key() );
435
436
    }
437
438
    return true;
439
}
440
441
void atWrapper::compareDirs( QString basedir, QString target )
442
{
443
444
    QDir dir( basedir );
445
446
    /* The following should be redundant now.
447
448
    if ( !dir.cd( target + ".failed" ) )
449
        dir.mkdir( target + ".failed" );
450
    else
451
        dir.cdUp();
452
453
    */
454
455
     if ( !dir.cd( target + ".diff" ) )
456
         dir.mkdir( target + ".diff" );
457
     else
458
         dir.cdUp();
459
460
461
462
    //Perform comparisons between the two directories.
463
464
    dir.setFilter(QDir::Files | QDir::Hidden | QDir::NoSymLinks);
465
    dir.setNameFilters( QStringList() << "*.png" );
466
    dir.cd( target + ".baseline" );
467
    QFileInfoList list = dir.entryInfoList();
468
469
    for (int i = 0; i < list.size(); ++i)
470
    {
471
        QFileInfo fileInfo = list.at(i);
472
        diff ( basedir, target, fileInfo.fileName() );
473
    }
474
}
475
476
bool atWrapper::diff( QString basedir, QString dir, QString target )
477
{
478
    //Comparing the two specified files, and then uploading them to
479
    //the ftp server if they differ
480
481
    basedir += "/" + dir;
482
    QString one = basedir + ".baseline/" + target;
483
    QString two = basedir + "/" + target;
484
485
    QFile file( one );
486
487
    file.open( QIODevice::ReadOnly );
488
    QByteArray contentsOfOne = file.readAll();
489
    file.close();
490
491
    file.setFileName( two );
492
493
    file.open( QIODevice::ReadOnly );
494
    QByteArray contentsOfTwo = file.readAll();
495
    file.close();
496
497
    if ( contentsOfTwo.size() == 0 )
498
    {
499
        qDebug() << "No test result found for baseline: " << one;
500
        file.setFileName( one );
501
        file.open( QIODevice::ReadOnly );
502
        file.copy( basedir + ".failed/" + target + "_missing"  );
503
        uploadFailed( dir + ".failed", target + "_missing", contentsOfTwo );
504
        return false;
505
    }
506
507
508
    if ( ( memcmp( contentsOfOne, contentsOfTwo, contentsOfOne.size() ) ) == 0 )
509
        return true;
510
    else
511
    {
512
        qDebug() << "Sorry, the result did not match: " << one;
513
        file.setFileName( two );
514
        file.open( QIODevice::ReadOnly );
515
        file.copy( basedir + ".failed/" + target  );
516
        file.close();
517
        uploadFailed( dir + ".failed", target, contentsOfTwo );
518
        uploadDiff( basedir, dir, target );
519
        return false;
520
    }
521
}
522
523
void atWrapper::uploadDiff( QString basedir, QString dir, QString filename )
524
{
525
526
    qDebug() << basedir;
527
    QImage im1( basedir + ".baseline/" + filename );
528
    QImage im2( basedir + "/" + filename );
529
530
    QImage im3(im1.size(), QImage::Format_ARGB32);
531
532
    im1 = im1.convertToFormat(QImage::Format_ARGB32);
533
    im2 = im2.convertToFormat(QImage::Format_ARGB32);
534
535
    for ( int y=0; y<im1.height(); ++y )
536
    {
537
        uint *s = (uint *) im1.scanLine(y);
538
        uint *d = (uint *) im2.scanLine(y);
539
        uint *w = (uint *) im3.scanLine(y);
540
541
        for ( int x=0; x<im1.width(); ++x )
542
        {
543
            if (*s != *d)
544
                *w = 0xff000000;
545
            else
546
                *w = 0xffffffff;
547
        w++;
548
        s++;
549
        d++;
550
        }
551
    }
552
553
    im3.save( basedir + ".diff/" + filename ,"PNG");
554
555
    QFile file( basedir + ".diff/" + filename );
556
    file.open( QIODevice::ReadOnly );
557
    QByteArray contents = file.readAll();
558
    file.close();
559
560
    uploadFailed( dir + ".diff", filename, contents );
561
562
}
563
564
bool atWrapper::loadConfig( QString path )
565
{
566
    qDebug() << "Loading config file from ... " << path;
567
    configPath = path;
568
    //If there is no config file, dont proceed;
569
    if ( !QFile::exists( path ) )
570
    {
571
        return false;
572
    }
573
574
575
    QSettings settings( path, QSettings::IniFormat, this );
576
577
578
    //FIXME: Switch to QStringList or something, hash is not needed!
579
    int numEngines = settings.beginReadArray("engines");
580
581
    for ( int i = 0; i < numEngines; ++i )
582
    {
583
        settings.setArrayIndex(i);
584
        enginesToTest.insert( settings.value( "engine" ).toString(), "Info here please :p" );
585
    }
586
587
    settings.endArray();
588
589
    framework = QString(ArthurDir) + QDir::separator() + settings.value( "framework" ).toString();
590
    suite = settings.value( "suite" ).toString();
591
    output = settings.value( "output" ).toString();
592
    size = settings.value( "size", "480,360" ).toString();
593
    fillColor = settings.value( "fill", "white" ).toString();
594
    ftpUser = settings.value( "ftpUser" ).toString();
595
    ftpPass = settings.value( "ftpPass" ).toString();
596
    ftpHost = settings.value( "ftpHost" ).toString();
597
    ftpBaseDir = settings.value( "ftpBaseDir" ).toString();
598
599
600
    QDir::current().mkdir( output );
601
602
    output += "/" + QLibraryInfo::buildKey();
603
604
    QDir::current().mkdir( output );
605
606
    output += "/" + QString( qVersion() );
607
608
    QDir::current().mkdir( output );
609
610
611
    ftpBaseDir += "/" + QHostInfo::localHostName().split( "." ).first();
612
613
614
/*
615
    framework = "data/framework.ini";
616
    suite = "1.1";
617
    output = "testresults";
618
    ftpUser = "anonymous";
619
    ftpPass = "anonymouspass";
620
    ftpHost = "kramer.troll.no";
621
    ftpBaseDir = "/arthurtest";
622
*/
623
    return true;
624
}
625
626
bool atWrapper::runAutoTests()
627
{
628
    //SVG needs this widget...
629
    QWidget dummy;
630
631
    bool haveBaseline = false;
632
633
    if (!initTests(&haveBaseline))
634
        return false;
635
    executeTests();
636
637
    if ( !haveBaseline )
638
    {
639
        qDebug( " First run! Creating baseline..." );
640
        createBaseline();
641
    }
642
    else
643
    {
644
        qDebug( " Comparing results..." );
645
        compare();
646
    }
647
    return true;
648
}