[KPhotoAlbum] Speeding up directory scanning
Jesper K. Pedersen
blackie at blackie.dk
Thu Dec 28 05:55:15 GMT 2006
Thanks for the patch. I'll keep it around.
For the release, it is of course completely out of the question (I know you
know), for longer time frame, I'd still like to first push this onto a
thread. The reason is that this code (I assume) makes it harder to port KPA
to windows (a request I get over and over and over and over again, and
something perhaps doable when KPA has been ported to Qt4)
Cheers
Jesper.
On Wednesday 27 December 2006 21:02, Robert L Krawitz wrote:
| As an example of what I mean about speeding up searching for new
| images, see the enclosed patch. On my system (which actually has
| about 25000 files, including 21000 active images -- most of the rest
| of the files are RAW files with an accompanying JPEG), this patch
| speeds up scanning of a freshly mounted filesystem from 95 seconds to
| 3 seconds. If the filesystem has already been scanned once (so the
| inodes are all in RAM), there's no performance issue either way -- it
| takes less than a second either way. The problem is that there's no
| way to know at user level whether the inodes in question are cached or
| not.
|
| This may not be the ideal final form for this kind of thing -- it is a
| bit ugly, but in my defense it's just a prototype -- but I do think
| that after the release this optimization should be considered. The
| performance may well be different on different filesystems; if the
| image directory is remote, the difference may well be far greater.
|
| Index: DB/NewImageFinder.cpp
| ===================================================================
| --- DB/NewImageFinder.cpp (revision 616536)
| +++ DB/NewImageFinder.cpp (working copy)
| @@ -20,7 +20,9 @@
| #include <qfileinfo.h>
| #include "Settings/SettingsData.h"
| #include "Browser/BrowserWidget.h"
| -#include <qdir.h>
| +#include "ImageManager/RawImageDecoder.h"
| +#include <sys/types.h>
| +#include <dirent.h>
| #include "Utilities/Util.h"
| #include <qprogressdialog.h>
| #include <klocale.h>
| @@ -56,6 +58,54 @@
| return (!_pendingLoad.isEmpty()); // returns if new images was found.
| }
|
| +class MyDir
| +{
| +public:
| + MyDir();
| + MyDir(const QString &path);
| + virtual QStringList entryList(void) const;
| + virtual ~MyDir();
| +private:
| + QString path_;
| +};
| +
| +MyDir::MyDir()
| + : path_(QString::fromLatin1("."))
| +{
| +}
| +
| +MyDir::MyDir(const QString &path)
| + : path_(path)
| +{
| +}
| +
| +MyDir::~MyDir()
| +{
| +}
| +
| +QStringList MyDir::entryList(void) const
| +{
| + QStringList answer;
| + DIR *dir;
| + dirent *file;
| + dir = opendir( QFile::encodeName(path_) );
| + if ( !dir )
| + return answer; // cannot read the directory
| +
| +#if defined(QT_THREAD_SUPPORT) && defined(_POSIX_THREAD_SAFE_FUNCTIONS) &&
| !defined(Q_OS_CYGWIN) + union {
| + struct dirent mt_file;
| + char b[sizeof(struct dirent) + MAXNAMLEN + 1];
| + } u;
| + while ( readdir_r(dir, &u.mt_file, &file ) == 0 && file )
| +#else
| + while ( (file = readdir(dir)) )
| +#endif // QT_THREAD_SUPPORT && _POSIX_THREAD_SAFE_FUNCTIONS
| + answer.append(QFile::decodeName(file->d_name));
| + (void) closedir(dir);
| + return answer;
| +}
| +
| void NewImageFinder::searchForNewFiles( const QDict<void>& loadedFiles,
| QString directory ) {
| if ( directory.endsWith( QString::fromLatin1("/") ) )
| @@ -65,18 +115,26 @@
| if ( imageDir.endsWith( QString::fromLatin1("/") ) )
| imageDir = imageDir.mid( 0, imageDir.length()-1 );
|
| - QDir dir( directory );
| - QStringList dirList = dir.entryList( QDir::All );
| + MyDir dir( directory );
| + QStringList dirList = dir.entryList( );
| + ImageManager::RAWImageDecoder dec;
| for( QStringList::Iterator it = dirList.begin(); it != dirList.end();
| ++it ) { QString file = directory + QString::fromLatin1("/") + *it; -
| QFileInfo fi( file );
| if ( (*it) == QString::fromLatin1(".") || (*it) ==
| QString::fromLatin1("..") || (*it) == QString::fromLatin1("ThumbNails") ||
| (*it) == QString::fromLatin1("CategoryImages") ||
| - !fi.isReadable() )
| + loadedFiles.find( file ) ||
| + dec._skipThisFile(loadedFiles, file) ||
| + file.endsWith(QString::fromLatin1(".thm")) ||
| + file.endsWith(QString::fromLatin1(".THM")) )
| continue;
|
| - if ( fi.isFile() && loadedFiles.find( file ) == 0) {
| + QFileInfo fi( file );
| +
| + if ( !fi.isReadable() )
| + continue;
| +
| + if ( fi.isFile() ) {
| QString baseName = file.mid( imageDir.length()+1 );
| if ( ! DB::ImageDB::instance()->isBlocking( baseName ) ) {
| if ( Utilities::canReadImage(file) )
| Index: ImageManager/RawImageDecoder.cpp
| ===================================================================
| --- ImageManager/RawImageDecoder.cpp (revision 616536)
| +++ ImageManager/RawImageDecoder.cpp (working copy)
| @@ -109,4 +109,49 @@
| return false;
| }
|
| +bool RAWImageDecoder::_skipThisFile( const QDict<void>& loadedFiles, const
| QString& imageFile ) +{
| + /* Known RAW file extensions. TODO: Complete */
| + static const QString extensions[] = { QString::fromLatin1("crw"),
| +
| QString::fromLatin1("cr2"), +
| QString::fromLatin1("nef"), +
|
| QString::fromLatin1("bay"), +
| QString::fromLatin1("mos"), +
|
| QString::fromLatin1("mrw"), +
| QString::fromLatin1("orf"), +
|
| QString::fromLatin1("cs1"), +
| QString::fromLatin1("dc2"), +
|
| QString::fromLatin1("kdc"), +
| QString::fromLatin1("raf"), +
|
| QString::fromLatin1("rdc"), +
| QString::fromLatin1("x3f"), +
|
| QString::null }; + if (!
| (Settings::SettingsData::instance()->dontReadRawFilesWithOtherMatchingFile(
|))) + return false;
| + bool isRaw = false;
| + for( int i = 0; !extensions[i].isNull(); ++i ) {
| + if( imageFile.endsWith( extensions[i], false ) ) {
| + isRaw = true;
| + break;
| + }
| + }
| + if (!isRaw)
| + return false;
| + static const QString standardExtensions[] = {
| + QString::fromLatin1("jpg"),
| + QString::fromLatin1("JPG"),
| + QString::fromLatin1("tif"),
| + QString::fromLatin1("TIF"),
| + QString::fromLatin1("png"),
| + QString::fromLatin1("PNG"),
| + QString::null };
| + QString baseFileName = imageFile;
| + baseFileName.remove(baseFileName.length() - 3, 3);
| + for (int i = 0; !standardExtensions[i].isNull(); ++i) {
| + if (loadedFiles.find(baseFileName + standardExtensions[i]))
| + return true;
| + }
| + return false;
| }
| +
| +}
| Index: ImageManager/RawImageDecoder.h
| ===================================================================
| --- ImageManager/RawImageDecoder.h (revision 616536)
| +++ ImageManager/RawImageDecoder.h (working copy)
| @@ -19,6 +19,7 @@
| #define RAWIMAGEDECODER_H
|
| #include "ImageDecoder.h"
| +#include <qdict.h>
|
| namespace ImageManager
| {
| @@ -29,6 +30,7 @@
|
| virtual bool _decode(QImage *img, const QString& imageFile, QSize*
| fullSize, int dim=-1); virtual bool _mightDecode( const QString& imageFile
| );
| + virtual bool _skipThisFile( const QDict<void>& loadedFiles, const
| QString& imageFile ); };
|
| }
| _______________________________________________
| KPhotoAlbum mailing list
| KPhotoAlbum at kdab.net
| http://mail.kdab.net/mailman/listinfo/kphotoalbum
--
Having trouble finding a given image in your collection containing
thousands of images?
http://www.kphotoalbum.org might be the answer.
More information about the Kphotoalbum
mailing list