[KPhotoAlbum] Patch to speed up scanning for new images

[Robert L Krawitz]

Here's my updated patch based on the review comments.  I've added it
to the bug report obsoleting the previous one.

Index: DB/NewImageFinder.cpp
===================================================================
--- DB/NewImageFinder.cpp       (revision 663342)
+++ DB/NewImageFinder.cpp       (working copy)
@@ -20,7 +20,9 @@
 #include <qfileinfo.h>
 #include "Settings/SettingsData.h"
 #include "Browser/BrowserWidget.h"
-#include <qdir.h>
+#include "ImageManager/RawImageDecoder.h"
+#include <sys/types.h>
+#include <dirent.h>
 #include "Utilities/Util.h"
 #include <qprogressdialog.h>
 #include <klocale.h>
@@ -56,6 +58,57 @@
     return (!_pendingLoad.isEmpty()); // returns if new images was found.
 }
 
+// FastDir is used in place of QDir because QDir stat()s every file in
+// the directory, even if we tell it not to restrict anything.  When
+// scanning for new images, we don't want to look at files we already
+// have in our database, and we also don't want to look at files whose
+// names indicate that we don't care about them.  So what we do is
+// simply read the names from the directory and let the higher layers
+// decide what to do with them.
+//
+// On my sample database with ~20,000 images, this improves the time
+// to rescan for images on a cold system from about 100 seconds to
+// about 3 seconds.
+class FastDir
+{
+public:
+    FastDir(const QString &path);
+    QStringList entryList(void) const;
+private:
+    const QString path_;
+};
+
+FastDir::FastDir(const QString &path)
+  : path_(path)
+{
+}
+
+QStringList FastDir::entryList(void) const
+{
+    QStringList answer;
+    DIR *dir;
+    dirent *file;
+    dir = opendir( QFile::encodeName(path_) );
+    if ( !dir )
+       return answer; // cannot read the directory
+
+#if defined(QT_THREAD_SUPPORT) && defined(_POSIX_THREAD_SAFE_FUNCTIONS) && !defined(Q_OS_CYGWIN)
+    union dirent_buf {
+       struct dirent mt_file;
+       char b[sizeof(struct dirent) + MAXNAMLEN + 1];
+    } *u = new union dirent_buf;
+    while ( readdir_r(dir, &(u->mt_file), &file ) == 0 && file )
+#else
+    while ( (file = readdir(dir)) )
+#endif // QT_THREAD_SUPPORT && _POSIX_THREAD_SAFE_FUNCTIONS
+       answer.append(QFile::decodeName(file->d_name));
+#if defined(QT_THREAD_SUPPORT) && defined(_POSIX_THREAD_SAFE_FUNCTIONS) && !defined(Q_OS_CYGWIN)
+    delete u;
+#endif
+    (void) closedir(dir);
+    return answer;
+}
+
 void NewImageFinder::searchForNewFiles( const QDict<void>& loadedFiles, QString directory )
 {
     if ( directory.endsWith( QString::fromLatin1("/") ) )
@@ -65,18 +118,24 @@
     if ( imageDir.endsWith( QString::fromLatin1("/") ) )
         imageDir = imageDir.mid( 0, imageDir.length()-1 );
 
-    QDir dir( directory );
-    QStringList dirList = dir.entryList( QDir::All );
+    FastDir dir( directory );
+    QStringList dirList = dir.entryList( );
+    ImageManager::RAWImageDecoder dec;
     for( QStringList::Iterator it = dirList.begin(); it != dirList.end(); ++it ) {
         QString file = directory + QString::fromLatin1("/") + *it;
-        QFileInfo fi( file );
         if ( (*it) == QString::fromLatin1(".") || (*it) == QString::fromLatin1("..") ||
              (*it) == QString::fromLatin1("ThumbNails") ||
              (*it) == QString::fromLatin1("CategoryImages") ||
-             !fi.isReadable() )
+            loadedFiles.find( file ) ||
+            dec._skipThisFile(loadedFiles, file) )
             continue;
 
-        if ( fi.isFile() && loadedFiles.find( file ) == 0) {
+        QFileInfo fi( file );
+
+       if ( !fi.isReadable() )
+           continue;
+
+        if ( fi.isFile() ) {
             QString baseName = file.mid( imageDir.length()+1 );
             if ( ! DB::ImageDB::instance()->isBlocking( baseName ) ) {
                 if ( Utilities::canReadImage(file) )
Index: ImageManager/RawImageDecoder.cpp
===================================================================
--- ImageManager/RawImageDecoder.cpp    (revision 663342)
+++ ImageManager/RawImageDecoder.cpp    (working copy)
@@ -94,4 +94,52 @@
        return false;
 }
 
+bool RAWImageDecoder::_skipThisFile( const QDict<void>& loadedFiles, const QString& imageFile )
+{
+       /* Known RAW file extensions. TODO: Complete */
+       static const QString extensions[] = { QString::fromLatin1("crw"),
+                                                                                QString::fromLatin1("cr2"),
+                                                                                QString::fromLatin1("nef"),
+                                                                                QString::fromLatin1("bay"),
+                                                                                QString::fromLatin1("mos"),
+                                                                                QString::fromLatin1("mrw"),
+                                                                                QString::fromLatin1("orf"),
+                                                                                QString::fromLatin1("cs1"),
+                                                                                QString::fromLatin1("dc2"),
+                                                                                QString::fromLatin1("kdc"),
+                                                                                QString::fromLatin1("raf"),
+                                                                                QString::fromLatin1("rdc"),
+                                                                                QString::fromLatin1("x3f"),
+                                                                                QString::null };
+       if (imageFile.endsWith(QString::fromLatin1(".thm")) ||
+           imageFile.endsWith(QString::fromLatin1(".THM")))
+         return true;
+       if (! (Settings::SettingsData::instance()->dontReadRawFilesWithOtherMatchingFile()))
+         return false;
+       bool isRaw = false;
+       for( int i = 0; !extensions[i].isNull(); ++i ) {
+         if( imageFile.endsWith( extensions[i], false ) ) {
+           isRaw = true;
+           break;
+         }
+       }
+       if (!isRaw)
+         return false;
+       static const QString standardExtensions[] = {
+         QString::fromLatin1("jpg"),
+         QString::fromLatin1("JPG"),
+         QString::fromLatin1("tif"),
+         QString::fromLatin1("TIF"),
+         QString::fromLatin1("png"),
+         QString::fromLatin1("PNG"),
+         QString::null };
+       QString baseFileName = imageFile;
+       baseFileName.remove(baseFileName.length() - 3, 3);
+       for (int i = 0; !standardExtensions[i].isNull(); ++i) {
+         if (loadedFiles.find(baseFileName + standardExtensions[i]))
+           return true;
+       }
+       return false;
 }
+
+}
Index: ImageManager/RawImageDecoder.h
===================================================================
--- ImageManager/RawImageDecoder.h      (revision 663342)
+++ ImageManager/RawImageDecoder.h      (working copy)
@@ -19,6 +19,7 @@
 #define RAWIMAGEDECODER_H
 
 #include "ImageDecoder.h"
+#include <qdict.h>
 
 namespace ImageManager
 {
@@ -29,6 +30,7 @@
 
        virtual bool _decode(QImage *img, const QString& imageFile, QSize* fullSize, int dim=-1);
        virtual bool _mightDecode( const QString& imageFile );
+       virtual bool _skipThisFile( const QDict<void>& loadedFiles, const QString& imageFile );
 };
 
 }