diff --git a/src/collection/sqlcollection/ScanManager.cpp b/src/collection/sqlcollection/ScanManager.cpp index b775ba75c2..156a7726b7 100644 --- a/src/collection/sqlcollection/ScanManager.cpp +++ b/src/collection/sqlcollection/ScanManager.cpp @@ -1,679 +1,679 @@ /* * Copyright (c) 2003-2008 Mark Kretschmann * Copyright (c) 2007 Maximilian Kossick * Copyright (c) 2007 Casey Link * Copyright (c) 2008 Leo Franchi * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "ScanManager.h" #include "App.h" #include "Debug.h" #include "MountPointManager.h" #include "ScanResultProcessor.h" #include "SqlCollection.h" #include "SqlCollectionDBusHandler.h" #include "amarokconfig.h" #include "meta/MetaConstants.h" #include "meta/MetaUtility.h" #include "playlistmanager/PlaylistManager.h" #include "statusbar/StatusBar.h" #include #include #include #include #include #include #include #include #include #include static const int MAX_RESTARTS = 80; static const int MAX_FAILURE_PERCENTAGE = 5; static const int WATCH_INTERVAL = 60 * 1000; // = 60 seconds ScanManager::ScanManager( SqlCollection *parent ) : QObject( parent ) , m_collection( parent ) , m_dbusHandler( 0 ) , m_scanner( 0 ) , m_parser( 0 ) , m_restartCount( 0 ) , m_isIncremental( false ) , m_blockScan( false ) { DEBUG_BLOCK // If Amarok is not installed in standard directory const QString binDir = KStandardDirs::findExe( "amarok" ); m_amarokCollectionScanDir = binDir.left( binDir.lastIndexOf( '/' ) + 1 ); QTimer *watchFoldersTimer = new QTimer( this ); connect( watchFoldersTimer, SIGNAL( timeout() ), SLOT( slotWatchFolders() ) ); watchFoldersTimer->start( WATCH_INTERVAL ); } ScanManager::~ScanManager() { DEBUG_BLOCK stopParser(); } void ScanManager::startFullScan() { DEBUG_BLOCK if( m_parser ) { debug() << "scanner already running"; return; } if( m_blockScan ) { debug() << "scanning currently blocked"; return; } cleanTables(); m_scanner = new AmarokProcess( this ); *m_scanner << m_amarokCollectionScanDir + "amarokcollectionscanner" << "--nocrashhandler" << "-p"; if( AmarokConfig::scanRecursively() ) *m_scanner << "-r"; *m_scanner << MountPointManager::instance()->collectionFolders(); m_scanner->setOutputChannelMode( KProcess::OnlyStdoutChannel ); connect( m_scanner, SIGNAL( readyReadStandardOutput() ), this, SLOT( slotReadReady() ) ); connect( m_scanner, SIGNAL( finished( int ) ), SLOT( slotFinished( ) ) ); connect( m_scanner, SIGNAL( error( QProcess::ProcessError ) ), SLOT( slotError( QProcess::ProcessError ) ) ); m_scanner->start(); if( m_parser ) { stopParser(); } m_parser = new XmlParseJob( this, m_collection ); m_parser->setIsIncremental( false ); m_isIncremental = false; connect( m_parser, SIGNAL( done( ThreadWeaver::Job* ) ), SLOT( slotJobDone() ) ); ThreadWeaver::Weaver::instance()->enqueue( m_parser ); } void ScanManager::startIncrementalScan() { DEBUG_BLOCK if( m_parser ) { debug() << "scanner already running"; return; } if( m_blockScan ) { debug() << "scanning currently blocked"; return; } const QStringList dirs = getDirsToScan(); debug() << "GOING TO SCAN:"; foreach( const QString &dir, dirs ) debug() << " " << dir; if( dirs.isEmpty() ) { debug() << "Scanning nothing, return."; return; } if( !m_dbusHandler ) { m_dbusHandler = new SqlCollectionDBusHandler( m_collection ); } m_scanner = new AmarokProcess( this ); *m_scanner << m_amarokCollectionScanDir + "amarokcollectionscanner" << "--nocrashhandler" << "-i" << "--collectionid" << m_collection->collectionId(); if( AmarokConfig::scanRecursively() ) *m_scanner << "-r"; if( pApp->isUniqueInstance() ) *m_scanner << "--pid" << QString::number( QApplication::applicationPid() ); *m_scanner << dirs; m_scanner->setOutputChannelMode( KProcess::OnlyStdoutChannel ); connect( m_scanner, SIGNAL( readyReadStandardOutput() ), this, SLOT( slotReadReady() ) ); connect( m_scanner, SIGNAL( finished( int ) ), SLOT( slotFinished() ) ); connect( m_scanner, SIGNAL( error( QProcess::ProcessError ) ), SLOT( slotError( QProcess::ProcessError ) ) ); m_scanner->start(); if( m_parser ) { stopParser(); } m_parser = new XmlParseJob( this, m_collection ); m_parser->setIsIncremental( true ); m_isIncremental = true; connect( m_parser, SIGNAL( done( ThreadWeaver::Job* ) ), SLOT( slotJobDone() ) ); ThreadWeaver::Weaver::instance()->enqueue( m_parser ); } bool ScanManager::isDirInCollection( QString path ) { // In the database all directories have a trailing slash, so we must add that if ( !path.endsWith( '/' ) ) path += '/'; const int deviceid = MountPointManager::instance()->getIdForUrl( path ); const QString rpath = MountPointManager::instance()->getRelativePath( deviceid, path ); const QStringList values = m_collection->query( QString( "SELECT changedate FROM directories WHERE dir = '%2' AND deviceid = %1;" ) .arg( QString::number( deviceid ), m_collection->escape( rpath ) ) ); //debug() << "dir " << rpath << " is in collection? " << !values.isEmpty(); return !values.isEmpty(); } bool ScanManager::isFileInCollection( const QString &url ) { const int deviceid = MountPointManager::instance()->getIdForUrl( url ); const QString rpath = MountPointManager::instance()->getRelativePath( deviceid, url ); QString sql = QString( "SELECT id FROM urls WHERE rpath = '%2' AND deviceid = %1" ) .arg( QString::number( deviceid ), m_collection->escape( rpath ) ); if ( deviceid == -1 ) { sql += ';'; } else { QString rpath2 = '.' + url; sql += QString( " OR rpath = '%1' AND deviceid = -1;" ) .arg( m_collection->escape( rpath2 ) ); } const QStringList values = m_collection->query( sql ); //debug() << "File " << rpath << " is in collection? " << !values.isEmpty(); return !values.isEmpty(); } void ScanManager::setBlockScan( bool blockScan ) { m_blockScan = blockScan; if( m_parser ) { warning() << "Scanner is running while scan got blocked"; } //TODO what happens if the collection scanner is currently running? } void ScanManager::slotWatchFolders() { if( AmarokConfig::monitorChanges() ) startIncrementalScan(); } void ScanManager::slotReadReady() { QByteArray line; QString newData; line = m_scanner->readLine(); while( !line.isEmpty() ) { // amarokcollectionscanner outputs UTF-8 regardless of local encoding QString data = QTextCodec::codecForName( "UTF-8" )->toUnicode( line ); if( !data.startsWith( "exepath=" ) ) // skip binary location info from scanner newData += data; line = m_scanner->readLine(); } //debug() << "Parsing all the following data:\n" << newData; if( m_parser ) m_parser->addNewXmlData( newData ); } void ScanManager::slotFinished( ) { DEBUG_BLOCK //make sure that we read the complete buffer slotReadReady(); m_scanner->deleteLater(); m_scanner = 0; m_restartCount = 0; if( m_dbusHandler ) { m_dbusHandler->deleteLater(); m_dbusHandler = 0; } } void ScanManager::slotError( QProcess::ProcessError error ) { DEBUG_BLOCK debug() << "Error: " << error; if( error == QProcess::Crashed ) { handleRestart(); } } QStringList ScanManager::getDirsToScan() const { DEBUG_BLOCK const IdList list = MountPointManager::instance()->getMountedDeviceIds(); QString deviceIds; foreach( int id, list ) { if ( !deviceIds.isEmpty() ) deviceIds += ','; deviceIds += QString::number( id ); } const QStringList values = m_collection->query( QString( "SELECT id, deviceid, dir, changedate FROM directories WHERE deviceid IN (%1);" ) .arg( deviceIds ) ); QList changedFolderIds; QStringList result; for( QListIterator iter( values ); iter.hasNext(); ) { int id = iter.next().toInt(); int deviceid = iter.next().toInt(); const QString folder = MountPointManager::instance()->getAbsolutePath( deviceid, iter.next() ); const uint mtime = iter.next().toUInt(); QFileInfo info( folder ); if( info.exists() ) { if( info.lastModified().toTime_t() != mtime ) { result << folder; changedFolderIds << id; } } else { // this folder has been removed result << folder; changedFolderIds << id; } } { QString ids; foreach( int id, changedFolderIds ) { if( !ids.isEmpty() ) ids += ','; ids += QString::number( id ); } if( !ids.isEmpty() ) { QString query = QString( "SELECT id FROM urls WHERE directory IN ( %1 );" ).arg( ids ); QStringList urlIds = m_collection->query( query ); ids.clear(); foreach( const QString &id, urlIds ) { if( !ids.isEmpty() ) ids += ','; ids += id; } if( !ids.isEmpty() ) { QString sql = QString( "DELETE FROM tracks WHERE url IN ( %1 );" ).arg( ids ); m_collection->query( sql ); } } } //debug() << "Scanning the following dirs: " << result; return result; } void ScanManager::slotJobDone() { m_parser->deleteLater(); m_parser = 0; } void ScanManager::handleRestart() { DEBUG_BLOCK m_restartCount++; debug() << "Collection scanner crashed, restart count is " << m_restartCount; slotReadReady(); //make sure that we read the complete buffer disconnect( m_scanner, SIGNAL( readyReadStandardOutput() ), this, SLOT( slotReadReady() ) ); disconnect( m_scanner, SIGNAL( finished( int ) ), this, SLOT( slotFinished( ) ) ); disconnect( m_scanner, SIGNAL( error( QProcess::ProcessError ) ), this, SLOT( slotError( QProcess::ProcessError ) ) ); m_scanner->deleteLater(); m_scanner = 0; if( m_restartCount >= MAX_RESTARTS ) { - KMessageBox::error( 0, i18n( "

Sorry, the collection scan had to be aborted.

Too many errors were encountered during the scan.

" ), + KMessageBox::error( 0, i18n( "

Sorry, the collection scan had to be aborted.

Too many errors were encountered during the scan.

" ), i18n( "Collection Scan Error" ) ); stopParser(); return; } QTimer::singleShot( 0, this, SLOT( restartScanner() ) ); } void ScanManager::restartScanner() { DEBUG_BLOCK m_scanner = new AmarokProcess( this ); *m_scanner << m_amarokCollectionScanDir + "amarokcollectionscanner" << "--nocrashhandler"; if( m_isIncremental ) { *m_scanner << "-i" << "--collectionid" << m_collection->collectionId(); if( pApp->isUniqueInstance() ) *m_scanner << "--pid" << QString::number( QApplication::applicationPid() ); } *m_scanner << "-s"; // "--restart" m_scanner->setOutputChannelMode( KProcess::OnlyStdoutChannel ); connect( m_scanner, SIGNAL( readyReadStandardOutput() ), this, SLOT( slotReadReady() ) ); connect( m_scanner, SIGNAL( finished( int ) ), SLOT( slotFinished( ) ) ); connect( m_scanner, SIGNAL( error( QProcess::ProcessError ) ), SLOT( slotError( QProcess::ProcessError ) ) ); m_scanner->start(); } void ScanManager::cleanTables() { DEBUG_BLOCK m_collection->query( "DELETE FROM tracks;" ); m_collection->query( "DELETE FROM genres;" ); m_collection->query( "DELETE FROM years;" ); m_collection->query( "DELETE FROM composers;" ); m_collection->query( "DELETE FROM albums;" ); m_collection->query( "DELETE FROM artists;" ); } void ScanManager::stopParser() { DEBUG_BLOCK if( m_parser ) { ThreadWeaver::Weaver::instance()->dequeue( m_parser ); m_parser->requestAbort(); while( !m_parser->isFinished() ) usleep( 100000 ); // Sleep 100 msec m_parser->deleteLater(); m_parser = 0; } } /////////////////////////////////////////////////////////////////////////////// // class XmlParseJob /////////////////////////////////////////////////////////////////////////////// XmlParseJob::XmlParseJob( ScanManager *parent, SqlCollection *collection ) : ThreadWeaver::Job( parent ) , m_collection( collection ) , m_abortRequested( false ) , m_isIncremental( false ) { DEBUG_BLOCK The::statusBar()->newProgressOperation( this, i18n( "Scanning music" ) ) ->setAbortSlot( parent, SLOT( deleteLater() ) ); connect( this, SIGNAL( incrementProgress() ), The::statusBar(), SLOT( incrementProgress() ), Qt::QueuedConnection ); } XmlParseJob::~XmlParseJob() { DEBUG_BLOCK The::statusBar()->endProgressOperation( this ); } void XmlParseJob::setIsIncremental( bool incremental ) { m_isIncremental = incremental; } void XmlParseJob::run() { DEBUG_BLOCK QList directoryData; bool firstTrack = true; QString currentDir; ScanResultProcessor processor( m_collection ); if( m_isIncremental ) { processor.setScanType( ScanResultProcessor::IncrementalScan ); } else { processor.setScanType( ScanResultProcessor::FullScan ); } do { m_abortMutex.lock(); const bool abort = m_abortRequested; m_abortMutex.unlock(); if( abort ) break; //debug() << "Get new xml data or wait till new xml data is available"; m_mutex.lock(); if( m_nextData.isEmpty() ) { m_wait.wait( &m_mutex ); } if( m_nextData.isEmpty() ) break; m_reader.addData( m_nextData ); m_nextData.clear(); m_mutex.unlock(); while( !m_reader.atEnd() ) { m_reader.readNext(); if( m_reader.isStartElement() ) { QStringRef localname = m_reader.name(); if( localname == "dud" || localname == "tags" || localname == "playlist" || localname == "image" ) { emit incrementProgress(); } if( localname == "itemcount" ) { The::statusBar()->incrementProgressTotalSteps( this, m_reader.attributes().value( "count" ).toString().toInt() ); } else if( localname == "tags" ) { //debug() << "Parsing FILE:\n"; QXmlStreamAttributes attrs = m_reader.attributes(); QList list = attrs.toList(); //foreach( QXmlStreamAttribute l, list ) // debug() << " TAG: " << l.name().toString() << '\t' << l.value().toString() << '\n'; //debug() << "End FILE"; QVariantMap data; data.insert( Meta::Field::URL, attrs.value( "path" ).toString() ); data.insert( Meta::Field::TITLE, attrs.value( "title" ).toString() ); data.insert( Meta::Field::ARTIST, attrs.value( "artist" ).toString() ); data.insert( Meta::Field::COMPOSER, attrs.value( "composer" ).toString() ); data.insert( Meta::Field::ALBUM, attrs.value( "album" ).toString() ); data.insert( Meta::Field::COMMENT, attrs.value( "comment" ).toString() ); data.insert( Meta::Field::GENRE, attrs.value( "genre" ).toString() ); data.insert( Meta::Field::YEAR, attrs.value( "year" ).toString() ); data.insert( Meta::Field::TRACKNUMBER, attrs.value( "track" ).toString() ); data.insert( Meta::Field::DISCNUMBER, attrs.value( "discnumber" ).toString() ); data.insert( Meta::Field::BPM, attrs.value( "bpm" ).toString() ); //filetype and uniqueid are missing in the fields, compilation is not used here if( attrs.value( "audioproperties" ) == "true" ) { data.insert( Meta::Field::BITRATE, attrs.value( "bitrate" ).toString() ); data.insert( Meta::Field::LENGTH, attrs.value( "length" ).toString() ); data.insert( Meta::Field::SAMPLERATE, attrs.value( "samplerate" ).toString() ); } if( !attrs.value( "filesize" ).isEmpty() ) data.insert( Meta::Field::FILESIZE, attrs.value( "filesize" ).toString() ); data.insert( Meta::Field::UNIQUEID, attrs.value( "uniqueid" ).toString() ); KUrl url( data.value( Meta::Field::URL ).toString() ); if( firstTrack ) { currentDir = url.directory(); firstTrack = false; } if( url.directory() == currentDir ) { directoryData.append( data ); } else { processor.processDirectory( directoryData ); directoryData.clear(); directoryData.append( data ); currentDir = url.directory(); } } else if( localname == "folder" ) { //debug() << "Parsing FOLDER:\n"; QXmlStreamAttributes attrs = m_reader.attributes(); QList list = attrs.toList(); //foreach( QXmlStreamAttribute l, list ) // debug() << " ATTRIBUTE: " << l.name().toString() << '\t' << l.value().toString() << '\n'; //debug() << "End FOLDER"; const QString folder = attrs.value( "path" ).toString(); const QFileInfo info( folder ); processor.addDirectory( folder, info.lastModified().toTime_t() ); } else if( localname == "playlist" ) { //TODO check for duplicates //debug() << "Saving playlist with path: " << m_reader.attributes().value( "path" ).toString(); The::playlistManager()->save( m_reader.attributes().value( "path" ).toString() ); } else if( localname == "image" ) { debug() << "Parsing IMAGE:\n"; QXmlStreamAttributes attrs = m_reader.attributes(); QList thisList = attrs.toList(); //foreach( QXmlStreamAttribute l, thisList ) // debug() << " ATTR: " << l.name().toString() << '\t' << l.value().toString() << '\n'; //debug() << "End IMAGE"; // Deserialize CoverBundle list QStringList list = attrs.value( "list" ).toString().split( "AMAROK_MAGIC" ); QList< QPair > covers; // Don't iterate if the list only has one element if( list.size() > 1 ) { for( int i = 0; i < list.count(); i += 2 ) covers += qMakePair( list[i], list[i + 1] ); processor.addImage( attrs.value( "path" ).toString(), covers ); } } } } if( m_reader.error() != QXmlStreamReader::PrematureEndOfDocumentError ) { debug() << "do-while done with error: " << m_reader.error(); //the error cannot be PrematureEndOfDocumentError, so handle an unrecoverable error here // At this point, most likely the scanner has crashed and is about to get restarted. // Reset the XML-reader and try to get new data: debug() << "Trying to restart the QXmlStreamReader.."; m_reader.clear(); continue; } } while( m_reader.error() == QXmlStreamReader::PrematureEndOfDocumentError ); if( m_abortRequested ) { debug() << "Abort requested."; processor.rollback(); } else { debug() << "Success. Committing result to database."; if( !directoryData.isEmpty() ) processor.processDirectory( directoryData ); processor.commit(); } } void XmlParseJob::addNewXmlData( const QString &data ) { m_mutex.lock(); //append the new xml data because the parser thread //might not have retrieved all xml data yet m_nextData += data; m_wait.wakeOne(); m_mutex.unlock(); } void XmlParseJob::requestAbort() { DEBUG_BLOCK m_abortMutex.lock(); m_abortRequested = true; m_abortMutex.unlock(); m_wait.wakeOne(); } #include "ScanManager.moc"