[Marble-commits] KDE/kdeedu/marble/src/lib
Jens-Michael Hoffmann
jensmh at gmx.de
Thu Oct 1 11:57:36 CEST 2009
SVN commit 1029978 by jmhoffmann:
This patch adds basic infrastructure for host and usage specific download policies.
It does not add features, only adds infrastructure and changes HttpDonwloadManager
to use the new infrastructure.
Motivation:
===========
OpenStreetMap allows for bulk downloads (planned "Download region" feature)
only two concurrent connections. For browsing it is not a problem to have 10+
connections (like web browsers do).
So we need a way to distinguish between bulk downloading and browsing the map
per host/set of hosts.
What this patch contains:
=========================
- basic classes (DownloadQueueSet, DownloadPolicy)
- changes to HttpDownloadManager to use DownloadQueueSet internally
- one default download policy, which allows 20 concurrent connections
- change acceptJob to canAcceptJob and only create HttpJob if the job can be
accepted
What has to be done:
====================
- agree on additional DGML elements/attributes for defining policies
- implement those DGML extensions
- make HttpJobs aware of their usage
...
M +2 -0 CMakeLists.txt
A DownloadPolicy.cpp [License: LGPL (v2.1+)]
A DownloadPolicy.h [License: LGPL (v2.1+)]
A DownloadQueueSet.cpp [License: LGPL (v2.1+)]
A DownloadQueueSet.h [License: LGPL (v2.1+)]
M +79 -178 HttpDownloadManager.cpp
M +17 -28 HttpDownloadManager.h
--- trunk/KDE/kdeedu/marble/src/lib/CMakeLists.txt #1029977:1029978
@@ -76,6 +76,8 @@
QtMarbleConfigDialog.cpp
ClipPainter.cpp
+ DownloadPolicy.cpp
+ DownloadQueueSet.cpp
GeoPainter.cpp
GeoPolygon.cpp
HttpDownloadManager.cpp
--- trunk/KDE/kdeedu/marble/src/lib/HttpDownloadManager.cpp #1029977:1029978
@@ -28,7 +28,6 @@
HttpDownloadManager::HttpDownloadManager( const QUrl& serverUrl,
StoragePolicy *policy )
: m_downloadEnabled( true ), //enabled for now
- m_activatedJobsLimit( 40 ),
m_jobQueueLimit( 1000 ),
m_serverUrl( serverUrl ),
m_storagePolicy( policy ),
@@ -37,28 +36,17 @@
m_requeueTimer = new QTimer( this );
m_requeueTimer->setInterval( requeueTime );
connect( m_requeueTimer, SIGNAL( timeout() ), this, SLOT( requeue() ) );
+
+ // default download policy
+ DownloadPolicy defaultDownloadPolicy;
+ defaultDownloadPolicy.setMaximumConnections( 20 );
+ addDownloadPolicy( defaultDownloadPolicy );
}
HttpDownloadManager::~HttpDownloadManager()
{
m_downloadEnabled = false;
-
- qDeleteAll( m_jobQueue );
- m_jobQueue.clear();
-
- // activated jobs have to be deleted using deleteLater()
- // because they may be connected to signals
- QList<HttpJob*>::const_iterator pos = m_activatedJobList.constBegin();
- QList<HttpJob*>::const_iterator const end = m_activatedJobList.constEnd();
- for (; pos != end; ++pos ) {
- (*pos)->deleteLater();
- }
- m_activatedJobList.clear();
-
- qDeleteAll( m_waitingQueue );
- m_waitingQueue.clear();
-
delete m_storagePolicy;
delete m_networkPlugin;
}
@@ -73,9 +61,9 @@
m_jobQueueLimit = jobQueueLimit;
}
+// FIXME: remove this method
void HttpDownloadManager::setActivatedJobsLimit( int activatedJobsLimit )
{
- m_activatedJobsLimit = activatedJobsLimit;
}
void HttpDownloadManager::setDownloadEnabled( const bool enable )
@@ -83,35 +71,31 @@
m_downloadEnabled = enable;
}
-StoragePolicy* HttpDownloadManager::storagePolicy() const
+void HttpDownloadManager::addDownloadPolicy( const DownloadPolicy& policy )
{
- return m_storagePolicy;
+ DownloadQueueSet * const queueSet = new DownloadQueueSet( policy );
+ connect( queueSet, SIGNAL( jobFinished( QByteArray, QString, QString )),
+ SLOT( finishJob( QByteArray, QString, QString )));
+ connect( queueSet, SIGNAL( jobRetry() ), SLOT( startRetryTimer() ));
+ connect( queueSet, SIGNAL( jobRedirected( QUrl, QString, QString )),
+ SLOT( addJob( QUrl, QString, QString )));
+ // relay jobAdded/jobRemoved signals (interesting for progress bar)
+ connect( queueSet, SIGNAL( jobAdded() ), SIGNAL( jobAdded() ));
+ connect( queueSet, SIGNAL( jobRemoved() ), SIGNAL( jobRemoved() ));
+ m_queueSets.insert( queueSet->downloadPolicy().key(), queueSet );
}
-void HttpDownloadManager::addJob( HttpJob * job )
+StoragePolicy* HttpDownloadManager::storagePolicy() const
{
- if ( acceptJob( job ) ) {
- m_jobQueue.push( job );
- emit jobAdded();
- activateJobs();
- }
- else {
- job->deleteLater();
- }
+ return m_storagePolicy;
}
void HttpDownloadManager::addJob( const QString& relativeUrlString, const QString &id )
{
- if ( !m_downloadEnabled )
- return;
-
QUrl sourceUrl( m_serverUrl );
QString path = sourceUrl.path();
sourceUrl.setPath( path + relativeUrlString );
-
- HttpJob *job = createJob( sourceUrl, relativeUrlString, id );
- if ( job )
- addJob( job );
+ addJob( sourceUrl, relativeUrlString, id );
}
void HttpDownloadManager::addJob( const QUrl& sourceUrl, const QString& destFileName,
@@ -120,153 +104,48 @@
if ( !m_downloadEnabled )
return;
- HttpJob *job = createJob( sourceUrl, destFileName, id );
- if ( job )
- addJob( job );
-}
-
-bool HttpDownloadManager::acceptJob( HttpJob *job )
-{
- // We update the initiatorId as the previous initiator
- // likely doesn't exist anymore
-
- QStack<HttpJob*>::iterator j = m_jobQueue.begin();
- QStack<HttpJob*>::iterator const jEnd = m_jobQueue.end();
- for (; j != jEnd; ++j ) {
- if ( job->destinationFileName() == (*j)->destinationFileName() ) {
- qDebug() << "Download rejected: It's in the queue already.";
- (*j)->setInitiatorId( job->initiatorId() );
- return false;
+ DownloadQueueSet * const queueSet = findQueues( sourceUrl.host() );
+ if ( queueSet->canAcceptJob( sourceUrl, destFileName )) {
+ HttpJob * const job = createJob( sourceUrl, destFileName, id );
+ if ( job ) {
+ queueSet->addJob( job );
}
}
-
- QList<HttpJob*>::iterator i = m_waitingQueue.begin();
- QList<HttpJob*>::iterator iEnd = m_waitingQueue.end();
- for (; i != iEnd; ++i) {
- if ( job->destinationFileName() == (*i)->destinationFileName() ) {
- qDebug() << "Download rejected: Will try to download again in some time.";
- (*i)->setInitiatorId( job->initiatorId() );
- return false;
- }
- }
-
- i = m_activatedJobList.begin();
- iEnd = m_activatedJobList.end();
- for (; i != iEnd; ++i ) {
- if ( job->destinationFileName() == (*i)->destinationFileName() ) {
- qDebug() << "Download rejected: It's being downloaded already:" << job->destinationFileName();
- (*i)->setInitiatorId( job->initiatorId() );
- return false;
- }
- }
-
- QSet<QString>::const_iterator const pos = m_jobBlackList.find( job->sourceUrl().toString() );
- if ( pos != m_jobBlackList.end() ) {
- qDebug() << "Download rejected: Blacklisted.";
- return false;
- }
-
- return true;
}
-void HttpDownloadManager::removeJob( HttpJob* job )
+void HttpDownloadManager::finishJob( const QByteArray& data, const QString& destinationFileName,
+ const QString& id )
{
- int pos = m_activatedJobList.indexOf( job );
-
- if ( pos > 0 ) {
- m_activatedJobList.removeAt( pos );
-// qDebug() << "Removing: " << job->initiatorId();
- job->deleteLater();
+ qDebug() << "emitting downloadComplete( QByteArray, " << id << ")";
+ emit downloadComplete( data, id );
+ if ( m_storagePolicy ) {
+ const bool saved = m_storagePolicy->updateFile( destinationFileName, data );
+ if ( saved ) {
+ qDebug() << "emitting downloadComplete( " << destinationFileName << ", " << id << ")";
+ emit downloadComplete( destinationFileName, id );
+ } else {
+ qWarning() << "Could not save:" << destinationFileName;
+ }
}
- emit jobRemoved();
-
- activateJobs();
}
-void HttpDownloadManager::activateJob( HttpJob * const job )
+void HttpDownloadManager::requeue()
{
- // qDebug() << "On activatedJobList: " << job->sourceUrl().toString()
- // << job->destinationFileName();
- m_activatedJobList.push_back( job );
+ m_requeueTimer->stop();
- // No duplicate connections please
- // FIXME: move calls to disconnect to the right place, here it just
- // unconditionally disconnects even if there was no connection before
- disconnect( job, SIGNAL( jobDone( Marble::HttpJob*, int ) ),
- this, SLOT( reportResult( Marble::HttpJob*, int ) ) );
- disconnect( job, SIGNAL( redirected( HttpJob *, QUrl ) ),
- this, SLOT( jobRedirected( HttpJob *, QUrl ) ) );
- disconnect( job, SIGNAL( dataReceived( HttpJob *, QByteArray ) ),
- this, SLOT( jobDataReceived( HttpJob *, QByteArray ) ) );
- connect( job, SIGNAL( jobDone( Marble::HttpJob*, int ) ),
- this, SLOT( reportResult( Marble::HttpJob*, int ) ) );
- connect( job, SIGNAL( redirected( HttpJob *, QUrl ) ),
- SLOT( jobRedirected( HttpJob *, QUrl ) ) );
- connect( job, SIGNAL( dataReceived( HttpJob *, QByteArray ) ),
- SLOT( jobDataReceived( HttpJob *, QByteArray ) ) );
-
- job->execute();
-}
-
-void HttpDownloadManager::activateJobs()
-{
- while ( m_jobQueue.count() > 0
- && m_activatedJobList.count() < m_activatedJobsLimit )
- {
- HttpJob *job = m_jobQueue.pop();
- activateJob( job );
+ QMap<DownloadPolicyKey, DownloadQueueSet *>::iterator pos = m_queueSets.begin();
+ QMap<DownloadPolicyKey, DownloadQueueSet *>::iterator const end = m_queueSets.end();
+ for (; pos != end; ++pos ) {
+ pos.value()->retryJobs();
}
}
-void HttpDownloadManager::reportResult( HttpJob* job, int err )
+void HttpDownloadManager::startRetryTimer()
{
- if ( err != 0 ) {
- int pos = m_activatedJobList.indexOf( job );
-
- if ( pos >= 0 ) {
- m_activatedJobList.removeAt( pos );
- emit jobRemoved();
-
- // This should always return true
- if( !m_waitingQueue.contains( job ) ) {
- if( job->tryAgain() ) {
- qDebug() << QString( "Download of %1 failed, but trying again soon" )
- .arg( job->destinationFileName() );
- m_waitingQueue.enqueue( job );
- if( !m_requeueTimer->isActive() )
- m_requeueTimer->start();
- }
- else {
- qDebug() << "JOB-address: " << job << "Blacklist-size:" << m_jobBlackList.size() << "err:" << err;
- m_jobBlackList.insert( job->sourceUrl().toString() );
- qDebug() << QString( "Download of %1 Blacklisted. Number of blacklist items: %2" )
- .arg( job->destinationFileName() ).arg( m_jobBlackList.size() );
-
- job->deleteLater();
- }
- }
- }
- }
- else {
-// qDebug() << "HttpDownloadManager: Download Complete:"
-// << job->destinationFileName() << job->initiatorId();
- emit downloadComplete( job->destinationFileName(), job->initiatorId() );
- removeJob( job );
- }
-
- activateJobs();
+ if ( !m_requeueTimer->isActive() )
+ m_requeueTimer->start();
}
-void HttpDownloadManager::requeue()
-{
- m_requeueTimer->stop();
- while( !m_waitingQueue.isEmpty() ) {
- HttpJob* job = m_waitingQueue.dequeue();
- qDebug() << QString( "Requeuing %1." ).arg( job->destinationFileName() );
- addJob( job );
- }
-}
-
HttpJob *HttpDownloadManager::createJob( const QUrl& sourceUrl, const QString& destFileName,
const QString &id )
{
@@ -289,21 +168,43 @@
return m_networkPlugin->createJob( sourceUrl, destFileName, id );
}
-void HttpDownloadManager::jobRedirected( HttpJob *job, QUrl newLocation )
+DownloadQueueSet const * HttpDownloadManager::findQueues( const QString& hostName ) const
{
- qDebug() << "jobRedirected" << job->sourceUrl() << " -> " << newLocation;
- HttpJob * const newJob = m_networkPlugin->createJob( newLocation, job->destinationFileName(),
- job->initiatorId() );
- removeJob( job );
- activateJob( newJob );
+ DownloadQueueSet const * result = 0;
+ QMap<DownloadPolicyKey, DownloadQueueSet*>::const_iterator pos = m_queueSets.constBegin();
+ QMap<DownloadPolicyKey, DownloadQueueSet*>::const_iterator const end = m_queueSets.constEnd();
+ for (; pos != end; ++pos ) {
+ if ( pos.key().matches( hostName )) {
+ result = pos.value();
+ break;
+ }
+ }
+ // FIXME:
+ if ( !result ) {
+ Q_ASSERT( !m_queueSets.isEmpty() );
+ result = m_queueSets.constBegin().value();
+ }
+ return result;
}
-void HttpDownloadManager::jobDataReceived( HttpJob *job, QByteArray data )
+DownloadQueueSet *HttpDownloadManager::findQueues( const QString& hostName )
{
- emit downloadComplete( data, job->initiatorId() );
- const bool error = storagePolicy()
- && !storagePolicy()->updateFile( job->destinationFileName(), data );
- reportResult( job, error );
+ DownloadQueueSet * result = 0;
+ QMap<DownloadPolicyKey, DownloadQueueSet*>::iterator pos = m_queueSets.begin();
+ QMap<DownloadPolicyKey, DownloadQueueSet*>::iterator const end = m_queueSets.end();
+ for (; pos != end; ++pos ) {
+ if ( pos.key().matches( hostName )) {
+ result = pos.value();
+ break;
+ }
+ }
+ // FIXME:
+ if ( !result ) {
+ Q_ASSERT( !m_queueSets.isEmpty() );
+ result = m_queueSets.begin().value();
+ }
+ return result;
}
+
#include "HttpDownloadManager.moc"
--- trunk/KDE/kdeedu/marble/src/lib/HttpDownloadManager.h #1029977:1029978
@@ -19,6 +19,7 @@
#define MARBLE_HTTPDOWNLOADMANAGER_H
+#include <QtCore/QMap>
#include <QtCore/QString>
#include <QtCore/QUrl>
#include <QtCore/QList>
@@ -26,6 +27,9 @@
#include <QtCore/QStack>
#include <QtCore/QQueue>
+#include "DownloadPolicy.h"
+#include "DownloadQueueSet.h"
+
#include "marble_export.h"
class QTimer;
@@ -84,6 +88,7 @@
* Switches loading on/off, useful for offline mode.
*/
void setDownloadEnabled( const bool enable );
+ void addDownloadPolicy( const DownloadPolicy& );
StoragePolicy* storagePolicy() const;
@@ -98,12 +103,7 @@
*/
void addJob( const QUrl& sourceUrl, const QString& destFilename, const QString &id );
- /**
- * Removes the @p job from the manager.
- */
- void removeJob( HttpJob *job );
-
Q_SIGNALS:
void downloadComplete( QString, QString );
@@ -128,41 +128,30 @@
private Q_SLOTS:
- void activateJobs();
- void reportResult( Marble::HttpJob *job, int id );
+ void finishJob( const QByteArray& data, const QString& destinationFileName,
+ const QString& id );
void requeue();
- void jobRedirected( HttpJob *job, QUrl newLocation );
- void jobDataReceived( HttpJob *job, QByteArray data );
+ void startRetryTimer();
private:
Q_DISABLE_COPY( HttpDownloadManager )
- // Check whether the job gets processed already or whether it got blacklisted
- bool acceptJob( HttpJob *job );
HttpJob *createJob( const QUrl& sourceUrl, const QString& destFileName,
const QString &id );
+ DownloadQueueSet *findQueues( const QString& hostName );
+ DownloadQueueSet const *findQueues( const QString& hostName ) const;
- /**
- * Helper method for activateJobs(), also used in handling of redirections,
- * where the new job should be activated imediately.
- */
- void activateJob( HttpJob * const job );
-
- /**
- * Helper method for the public addJob methods which contains shared code.
- */
- void addJob( HttpJob* );
-
bool m_downloadEnabled;
QTimer *m_requeueTimer;
- QStack<HttpJob*> m_jobQueue;
- QQueue<HttpJob*> m_waitingQueue;
- QList<HttpJob*> m_activatedJobList;
- /// Contains the blacklisted source urls
- QSet<QString> m_jobBlackList;
+ /**
+ * Contains per download policy a queue set containing of
+ * - a queue where jobs are waiting for being activated (=downloaded)
+ * - a queue containing currently being downloaded
+ * - a queue for retries of failed downloads
+ */
+ QMap<DownloadPolicyKey, DownloadQueueSet *> m_queueSets;
- int m_activatedJobsLimit;
int m_jobQueueLimit;
QUrl m_serverUrl;
More information about the Marble-commits
mailing list