[KPhotoAlbum] Speeding up directory scanning

[Robert L Krawitz]

As an example of what I mean about speeding up searching for new
images, see the enclosed patch.  On my system (which actually has
about 25000 files, including 21000 active images -- most of the rest
of the files are RAW files with an accompanying JPEG), this patch
speeds up scanning of a freshly mounted filesystem from 95 seconds to
3 seconds.  If the filesystem has already been scanned once (so the
inodes are all in RAM), there's no performance issue either way -- it
takes less than a second either way.  The problem is that there's no
way to know at user level whether the inodes in question are cached or
not.

This may not be the ideal final form for this kind of thing -- it is a
bit ugly, but in my defense it's just a prototype -- but I do think
that after the release this optimization should be considered.  The
performance may well be different on different filesystems; if the
image directory is remote, the difference may well be far greater.

Index: DB/NewImageFinder.cpp
===================================================================
--- DB/NewImageFinder.cpp       (revision 616536)
+++ DB/NewImageFinder.cpp       (working copy)
@@ -20,7 +20,9 @@
 #include <qfileinfo.h>
 #include "Settings/SettingsData.h"
 #include "Browser/BrowserWidget.h"
-#include <qdir.h>
+#include "ImageManager/RawImageDecoder.h"
+#include <sys/types.h>
+#include <dirent.h>
 #include "Utilities/Util.h"
 #include <qprogressdialog.h>
 #include <klocale.h>
@@ -56,6 +58,54 @@
     return (!_pendingLoad.isEmpty()); // returns if new images was found.
 }
 
+class MyDir
+{
+public:
+    MyDir();
+    MyDir(const QString &path);
+    virtual QStringList entryList(void) const;
+    virtual ~MyDir();
+private:
+    QString path_;
+};
+
+MyDir::MyDir()
+  : path_(QString::fromLatin1("."))
+{
+}
+
+MyDir::MyDir(const QString &path)
+  : path_(path)
+{
+}
+
+MyDir::~MyDir()
+{
+}
+
+QStringList MyDir::entryList(void) const
+{
+    QStringList answer;
+    DIR *dir;
+    dirent *file;
+    dir = opendir( QFile::encodeName(path_) );
+    if ( !dir )
+       return answer; // cannot read the directory
+
+#if defined(QT_THREAD_SUPPORT) && defined(_POSIX_THREAD_SAFE_FUNCTIONS) && !defined(Q_OS_CYGWIN)
+    union {
+       struct dirent mt_file;
+       char b[sizeof(struct dirent) + MAXNAMLEN + 1];
+    } u;
+    while ( readdir_r(dir, &u.mt_file, &file ) == 0 && file )
+#else
+    while ( (file = readdir(dir)) )
+#endif // QT_THREAD_SUPPORT && _POSIX_THREAD_SAFE_FUNCTIONS
+       answer.append(QFile::decodeName(file->d_name));
+    (void) closedir(dir);
+    return answer;
+}
+
 void NewImageFinder::searchForNewFiles( const QDict<void>& loadedFiles, QString directory )
 {
     if ( directory.endsWith( QString::fromLatin1("/") ) )
@@ -65,18 +115,26 @@
     if ( imageDir.endsWith( QString::fromLatin1("/") ) )
         imageDir = imageDir.mid( 0, imageDir.length()-1 );
 
-    QDir dir( directory );
-    QStringList dirList = dir.entryList( QDir::All );
+    MyDir dir( directory );
+    QStringList dirList = dir.entryList( );
+    ImageManager::RAWImageDecoder dec;
     for( QStringList::Iterator it = dirList.begin(); it != dirList.end(); ++it ) {
         QString file = directory + QString::fromLatin1("/") + *it;
-        QFileInfo fi( file );
         if ( (*it) == QString::fromLatin1(".") || (*it) == QString::fromLatin1("..") ||
              (*it) == QString::fromLatin1("ThumbNails") ||
              (*it) == QString::fromLatin1("CategoryImages") ||
-             !fi.isReadable() )
+            loadedFiles.find( file ) ||
+            dec._skipThisFile(loadedFiles, file) ||
+            file.endsWith(QString::fromLatin1(".thm")) ||
+            file.endsWith(QString::fromLatin1(".THM")) )
             continue;
 
-        if ( fi.isFile() && loadedFiles.find( file ) == 0) {
+        QFileInfo fi( file );
+
+       if ( !fi.isReadable() )
+           continue;
+
+        if ( fi.isFile() ) {
             QString baseName = file.mid( imageDir.length()+1 );
             if ( ! DB::ImageDB::instance()->isBlocking( baseName ) ) {
                 if ( Utilities::canReadImage(file) )
Index: ImageManager/RawImageDecoder.cpp
===================================================================
--- ImageManager/RawImageDecoder.cpp    (revision 616536)
+++ ImageManager/RawImageDecoder.cpp    (working copy)
@@ -109,4 +109,49 @@
        return false;
 }
 
+bool RAWImageDecoder::_skipThisFile( const QDict<void>& loadedFiles, const QString& imageFile )
+{
+       /* Known RAW file extensions. TODO: Complete */
+       static const QString extensions[] = { QString::fromLatin1("crw"),
+                                                                                QString::fromLatin1("cr2"),
+                                                                                QString::fromLatin1("nef"),
+                                                                                QString::fromLatin1("bay"),
+                                                                                QString::fromLatin1("mos"),
+                                                                                QString::fromLatin1("mrw"),
+                                                                                QString::fromLatin1("orf"),
+                                                                                QString::fromLatin1("cs1"),
+                                                                                QString::fromLatin1("dc2"),
+                                                                                QString::fromLatin1("kdc"),
+                                                                                QString::fromLatin1("raf"),
+                                                                                QString::fromLatin1("rdc"),
+                                                                                QString::fromLatin1("x3f"),
+                                                                                QString::null };
+       if (! (Settings::SettingsData::instance()->dontReadRawFilesWithOtherMatchingFile()))
+         return false;
+       bool isRaw = false;
+       for( int i = 0; !extensions[i].isNull(); ++i ) {
+         if( imageFile.endsWith( extensions[i], false ) ) {
+           isRaw = true;
+           break;
+         }
+       }
+       if (!isRaw)
+         return false;
+       static const QString standardExtensions[] = {
+         QString::fromLatin1("jpg"),
+         QString::fromLatin1("JPG"),
+         QString::fromLatin1("tif"),
+         QString::fromLatin1("TIF"),
+         QString::fromLatin1("png"),
+         QString::fromLatin1("PNG"),
+         QString::null };
+       QString baseFileName = imageFile;
+       baseFileName.remove(baseFileName.length() - 3, 3);
+       for (int i = 0; !standardExtensions[i].isNull(); ++i) {
+         if (loadedFiles.find(baseFileName + standardExtensions[i]))
+           return true;
+       }
+       return false;
 }
+
+}
Index: ImageManager/RawImageDecoder.h
===================================================================
--- ImageManager/RawImageDecoder.h      (revision 616536)
+++ ImageManager/RawImageDecoder.h      (working copy)
@@ -19,6 +19,7 @@
 #define RAWIMAGEDECODER_H
 
 #include "ImageDecoder.h"
+#include <qdict.h>
 
 namespace ImageManager
 {
@@ -29,6 +30,7 @@
 
        virtual bool _decode(QImage *img, const QString& imageFile, QSize* fullSize, int dim=-1);
        virtual bool _mightDecode( const QString& imageFile );
+       virtual bool _skipThisFile( const QDict<void>& loadedFiles, const QString& imageFile );
 };
 
 }