[KPhotoAlbum] Speeding up directory scanning

[Shawn Willden]

I thought for a moment that this patch made starting kphotoalbum *hugely* 
faster on OS X.  It used to be impossiblly slow -- taking over five minutes 
when a similarly-powered Linux box started in 20-30 seconds.  This is with 
just under 12,000 images, on a 1.2GHz G4 iBook.

Wondering if maybe OS X had an insanely slow stat() call, I updated to SVN 
head and then applied the patch.

I was excited to see that startup time dropped to ~30 seconds.  Just to get 
some good numbers, I removed the patch and rebuilt and retested -- still 30 
seconds.  I rebooted to make sure there weren't a bunch of cached inodes.  
Still 30 seconds.

AFAICT, the patch made no difference at all, but something did.  There was an 
OS X kernel update a few days ago, so I'm wondering if that was it.

	Shawn.

On Thursday 28 December 2006 07:28, Robert L Krawitz wrote:
> I think this is cleaner, since .thm files are related to raw images.
>
> Index: DB/NewImageFinder.cpp
> ===================================================================
> --- DB/NewImageFinder.cpp       (revision 617031)
> +++ DB/NewImageFinder.cpp       (working copy)
> @@ -20,7 +20,9 @@
>  #include <qfileinfo.h>
>  #include "Settings/SettingsData.h"
>  #include "Browser/BrowserWidget.h"
> -#include <qdir.h>
> +#include "ImageManager/RawImageDecoder.h"
> +#include <sys/types.h>
> +#include <dirent.h>
>  #include "Utilities/Util.h"
>  #include <qprogressdialog.h>
>  #include <klocale.h>
> @@ -56,6 +58,54 @@
>      return (!_pendingLoad.isEmpty()); // returns if new images was found.
>  }
>
> +class MyDir
> +{
> +public:
> +    MyDir();
> +    MyDir(const QString &path);
> +    virtual QStringList entryList(void) const;
> +    virtual ~MyDir();
> +private:
> +    QString path_;
> +};
> +
> +MyDir::MyDir()
> +  : path_(QString::fromLatin1("."))
> +{
> +}
> +
> +MyDir::MyDir(const QString &path)
> +  : path_(path)
> +{
> +}
> +
> +MyDir::~MyDir()
> +{
> +}
> +
> +QStringList MyDir::entryList(void) const
> +{
> +    QStringList answer;
> +    DIR *dir;
> +    dirent *file;
> +    dir = opendir( QFile::encodeName(path_) );
> +    if ( !dir )
> +       return answer; // cannot read the directory
> +
> +#if defined(QT_THREAD_SUPPORT) && defined(_POSIX_THREAD_SAFE_FUNCTIONS) &&
> !defined(Q_OS_CYGWIN) +    union {
> +       struct dirent mt_file;
> +       char b[sizeof(struct dirent) + MAXNAMLEN + 1];
> +    } u;
> +    while ( readdir_r(dir, &u.mt_file, &file ) == 0 && file )
> +#else
> +    while ( (file = readdir(dir)) )
> +#endif // QT_THREAD_SUPPORT && _POSIX_THREAD_SAFE_FUNCTIONS
> +       answer.append(QFile::decodeName(file->d_name));
> +    (void) closedir(dir);
> +    return answer;
> +}
> +
>  void NewImageFinder::searchForNewFiles( const QDict<void>& loadedFiles,
> QString directory ) {
>      if ( directory.endsWith( QString::fromLatin1("/") ) )
> @@ -65,18 +115,24 @@
>      if ( imageDir.endsWith( QString::fromLatin1("/") ) )
>          imageDir = imageDir.mid( 0, imageDir.length()-1 );
>
> -    QDir dir( directory );
> -    QStringList dirList = dir.entryList( QDir::All );
> +    MyDir dir( directory );
> +    QStringList dirList = dir.entryList( );
> +    ImageManager::RAWImageDecoder dec;
>      for( QStringList::Iterator it = dirList.begin(); it != dirList.end();
> ++it ) { QString file = directory + QString::fromLatin1("/") + *it; -      
>  QFileInfo fi( file );
>          if ( (*it) == QString::fromLatin1(".") || (*it) ==
> QString::fromLatin1("..") || (*it) == QString::fromLatin1("ThumbNails") ||
>               (*it) == QString::fromLatin1("CategoryImages") ||
> -             !fi.isReadable() )
> +            loadedFiles.find( file ) ||
> +            dec._skipThisFile(loadedFiles, file) )
>              continue;
>
> -        if ( fi.isFile() && loadedFiles.find( file ) == 0) {
> +        QFileInfo fi( file );
> +
> +       if ( !fi.isReadable() )
> +           continue;
> +
> +        if ( fi.isFile() ) {
>              QString baseName = file.mid( imageDir.length()+1 );
>              if ( ! DB::ImageDB::instance()->isBlocking( baseName ) ) {
>                  if ( Utilities::canReadImage(file) )
> Index: ImageManager/RawImageDecoder.cpp
> ===================================================================
> --- ImageManager/RawImageDecoder.cpp    (revision 617031)
> +++ ImageManager/RawImageDecoder.cpp    (working copy)
> @@ -109,4 +109,52 @@
>         return false;
>  }
>
> +bool RAWImageDecoder::_skipThisFile( const QDict<void>& loadedFiles, const
> QString& imageFile ) +{
> +       /* Known RAW file extensions. TODO: Complete */
> +       static const QString extensions[] = { QString::fromLatin1("crw"),
> +                                                                          
>      QString::fromLatin1("cr2"), +                                         
>                                       QString::fromLatin1("nef"), +        
>                                                                       
> QString::fromLatin1("bay"), +                                              
>                                  QString::fromLatin1("mos"), +             
>                                                                  
> QString::fromLatin1("mrw"), +                                              
>                                  QString::fromLatin1("orf"), +             
>                                                                  
> QString::fromLatin1("cs1"), +                                              
>                                  QString::fromLatin1("dc2"), +             
>                                                                  
> QString::fromLatin1("kdc"), +                                              
>                                  QString::fromLatin1("raf"), +             
>                                                                  
> QString::fromLatin1("rdc"), +                                              
>                                  QString::fromLatin1("x3f"), +             
>                                                                  
> QString::null }; +       if
> (imageFile.endsWith(QString::fromLatin1(".thm")) ||
> +           imageFile.endsWith(QString::fromLatin1(".THM")))
> +         return true;
> +       if (!
> (Settings::SettingsData::instance()->dontReadRawFilesWithOtherMatchingFile(
>))) +         return false;
> +       bool isRaw = false;
> +       for( int i = 0; !extensions[i].isNull(); ++i ) {
> +         if( imageFile.endsWith( extensions[i], false ) ) {
> +           isRaw = true;
> +           break;
> +         }
> +       }
> +       if (!isRaw)
> +         return false;
> +       static const QString standardExtensions[] = {
> +         QString::fromLatin1("jpg"),
> +         QString::fromLatin1("JPG"),
> +         QString::fromLatin1("tif"),
> +         QString::fromLatin1("TIF"),
> +         QString::fromLatin1("png"),
> +         QString::fromLatin1("PNG"),
> +         QString::null };
> +       QString baseFileName = imageFile;
> +       baseFileName.remove(baseFileName.length() - 3, 3);
> +       for (int i = 0; !standardExtensions[i].isNull(); ++i) {
> +         if (loadedFiles.find(baseFileName + standardExtensions[i]))
> +           return true;
> +       }
> +       return false;
>  }
> +
> +}
> Index: ImageManager/RawImageDecoder.h
> ===================================================================
> --- ImageManager/RawImageDecoder.h      (revision 617031)
> +++ ImageManager/RawImageDecoder.h      (working copy)
> @@ -19,6 +19,7 @@
>  #define RAWIMAGEDECODER_H
>
>  #include "ImageDecoder.h"
> +#include <qdict.h>
>
>  namespace ImageManager
>  {
> @@ -29,6 +30,7 @@
>
>         virtual bool _decode(QImage *img, const QString& imageFile, QSize*
> fullSize, int dim=-1); virtual bool _mightDecode( const QString& imageFile
> );
> +       virtual bool _skipThisFile( const QDict<void>& loadedFiles, const
> QString& imageFile ); };
>
>  }
> _______________________________________________
> KPhotoAlbum mailing list
> KPhotoAlbum at kdab.net
> http://mail.kdab.net/mailman/listinfo/kphotoalbum