From f8e8414d544f1227e86099146bba903c9082d09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Mon, 2 Sep 2013 00:25:40 +0200 Subject: Speed up the downloads. Massively. --- logic/net/DownloadJob.cpp | 90 +++++++++++++++++------ logic/net/DownloadJob.h | 111 +++++++++++++++++++++++----- logic/net/JobQueue.h | 180 ---------------------------------------------- logic/net/NetWorker.cpp | 20 +++++- logic/net/NetWorker.h | 17 ++++- 5 files changed, 194 insertions(+), 224 deletions(-) delete mode 100644 logic/net/JobQueue.h (limited to 'logic/net') diff --git a/logic/net/DownloadJob.cpp b/logic/net/DownloadJob.cpp index 6d87a132..5c8ed4b9 100644 --- a/logic/net/DownloadJob.cpp +++ b/logic/net/DownloadJob.cpp @@ -2,9 +2,7 @@ #include "pathutils.h" #include "NetWorker.h" -DownloadJob::DownloadJob (QUrl url, - QString target_path, - QString expected_md5 ) +Download::Download (QUrl url, QString target_path, QString expected_md5 ) :Job() { m_url = url; @@ -17,14 +15,7 @@ DownloadJob::DownloadJob (QUrl url, m_opened_for_saving = false; } -JobPtr DownloadJob::create (QUrl url, - QString target_path, - QString expected_md5 ) -{ - return JobPtr ( new DownloadJob ( url, target_path, expected_md5 ) ); -} - -void DownloadJob::start() +void Download::start() { if ( m_save_to_file ) { @@ -40,7 +31,7 @@ void DownloadJob::start() if ( m_check_md5 && hash == m_expected_md5 ) { qDebug() << "Skipping " << m_url.toString() << ": md5 match."; - emit finish(); + emit succeeded(index_within_job); return; } else @@ -50,7 +41,7 @@ void DownloadJob::start() } if(!ensureFilePathExists(filename)) { - emit fail(); + emit failed(index_within_job); return; } } @@ -58,7 +49,7 @@ void DownloadJob::start() QNetworkRequest request ( m_url ); request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1()); - auto &worker = NetWorker::spawn(); + auto &worker = NetWorker::qnam(); QNetworkReply * rep = worker.get ( request ); m_reply = QSharedPointer ( rep, &QObject::deleteLater ); @@ -68,19 +59,19 @@ void DownloadJob::start() connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); } -void DownloadJob::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) +void Download::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) { - emit progress ( bytesReceived, bytesTotal ); + emit progress (index_within_job, bytesReceived, bytesTotal ); } -void DownloadJob::downloadError ( QNetworkReply::NetworkError error ) +void Download::downloadError ( QNetworkReply::NetworkError error ) { // error happened during download. // TODO: log the reason why m_status = Job_Failed; } -void DownloadJob::downloadFinished() +void Download::downloadFinished() { // if the download succeeded if ( m_status != Job_Failed ) @@ -99,7 +90,7 @@ void DownloadJob::downloadFinished() //TODO: check md5 here! m_reply.clear(); - emit finish(); + emit succeeded(index_within_job); return; } // else the download failed @@ -111,12 +102,12 @@ void DownloadJob::downloadFinished() m_output_file.remove(); } m_reply.clear(); - emit fail(); + emit failed(index_within_job); return; } } -void DownloadJob::downloadReadyRead() +void Download::downloadReadyRead() { if( m_save_to_file ) { @@ -128,7 +119,7 @@ void DownloadJob::downloadReadyRead() * Can't open the file... the job failed */ m_reply->abort(); - emit fail(); + emit failed(index_within_job); return; } m_opened_for_saving = true; @@ -136,3 +127,58 @@ void DownloadJob::downloadReadyRead() m_output_file.write ( m_reply->readAll() ); } } + +DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expected_md5 ) +{ + DownloadPtr ptr (new Download(url, rel_target_path, expected_md5)); + ptr->index_within_job = downloads.size(); + downloads.append(ptr); + return ptr; +} + +void DownloadJob::partSucceeded ( int index ) +{ + num_succeeded++; + if(num_failed + num_succeeded == downloads.size()) + { + if(num_failed) + { + emit failed(); + } + else + { + emit succeeded(); + } + } +} + +void DownloadJob::partFailed ( int index ) +{ + num_failed++; + if(num_failed + num_succeeded == downloads.size()) + { + if(num_failed) + { + emit failed(); + } + else + { + emit succeeded(); + } + } +} + +void DownloadJob::partProgress ( int index, qint64 bytesReceived, qint64 bytesTotal ) +{ + // PROGRESS? DENIED! +} + + +void DownloadJob::start() +{ + for(auto iter: downloads) + { + connect(iter.data(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int))); + iter->start(); + } +} diff --git a/logic/net/DownloadJob.h b/logic/net/DownloadJob.h index 94abb9af..adeef646 100644 --- a/logic/net/DownloadJob.h +++ b/logic/net/DownloadJob.h @@ -1,28 +1,37 @@ #pragma once -#include "JobQueue.h" #include -/** - * A single file for the downloader/cache to process. - */ -class DownloadJob : public Job +class DownloadJob; +class Download; +typedef QSharedPointer DownloadJobPtr; +typedef QSharedPointer DownloadPtr; + +enum JobStatus +{ + Job_NotStarted, + Job_InProgress, + Job_Finished, + Job_Failed +}; + +class Job : public QObject { Q_OBJECT +protected: + explicit Job(): QObject(0){}; public: - DownloadJob(QUrl url, - QString rel_target_path = QString(), - QString expected_md5 = QString() - ); - static JobPtr create(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()); + virtual ~Job() {}; + public slots: - virtual void start(); - -private slots: - void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);; - void downloadError(QNetworkReply::NetworkError error); - void downloadFinished(); - void downloadReadyRead(); - + virtual void start() = 0; +}; + +class Download: public Job +{ + friend class DownloadJob; + Q_OBJECT +protected: + Download(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()); public: /// the network reply QSharedPointer m_reply; @@ -46,6 +55,72 @@ public: /// if not saving to file, downloaded data is placed here QByteArray m_data; + int currentProgress = 0; + int totalProgress = 0; + /// The file's status JobStatus m_status; + + int index_within_job = 0; +signals: + void started(int index); + void progress(int index, qint64 current, qint64 total); + void succeeded(int index); + void failed(int index); +public slots: + virtual void start(); + +private slots: + void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);; + void downloadError(QNetworkReply::NetworkError error); + void downloadFinished(); + void downloadReadyRead(); +}; + +/** + * A single file for the downloader/cache to process. + */ +class DownloadJob : public Job +{ + Q_OBJECT +public: + explicit DownloadJob() + :Job(){}; + explicit DownloadJob(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()) + :Job() + { + add(url, rel_target_path, expected_md5); + }; + + DownloadPtr add(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()); + DownloadPtr operator[](int index) + { + return downloads[index]; + }; + DownloadPtr first() + { + if(downloads.size()) + return downloads[0]; + return DownloadPtr(); + } + int size() const + { + return downloads.size(); + } +signals: + void started(); + void progress(qint64 current, qint64 total); + void succeeded(); + void failed(); +public slots: + virtual void start(); +private slots: + void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal);; + void partSucceeded(int index); + void partFailed(int index); +private: + QList downloads; + int num_succeeded = 0; + int num_failed = 0; }; + diff --git a/logic/net/JobQueue.h b/logic/net/JobQueue.h deleted file mode 100644 index b6895f60..00000000 --- a/logic/net/JobQueue.h +++ /dev/null @@ -1,180 +0,0 @@ -#pragma once -#include -#include "libutil_config.h" - -enum JobStatus -{ - Job_NotStarted, - Job_InProgress, - Job_Finished, - Job_Failed -}; - -class JobList; - -class Job : public QObject -{ - Q_OBJECT -protected: - explicit Job(): QObject(0){}; -public: - virtual ~Job() {}; -signals: - void finish(); - void fail(); - void progress(qint64 current, qint64 total); -public slots: - virtual void start() = 0; -}; -typedef QSharedPointer JobPtr; - -/** - * A list of jobs, to be processed one by one. - */ -class JobList : public QObject -{ - friend class JobListQueue; - Q_OBJECT -public: - - JobList() : QObject(0) - { - m_status = Job_NotStarted; - current_job_idx = 0; - } - JobStatus getStatus() - { - return m_status; - } - void add(JobPtr dlable) - { - if(m_status == Job_NotStarted) - m_jobs.append(dlable); - //else there's a bug. TODO: catch the bugs - } - JobPtr getFirstJob() - { - if(m_jobs.size()) - return m_jobs[0]; - else - return JobPtr(); - } - void start() - { - current_job_idx = 0; - auto job = m_jobs[current_job_idx]; - - connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64))); - connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished())); - connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed())); - job->start(); - emit started(); - } -private slots: - void currentJobFinished() - { - if(current_job_idx == m_jobs.size() - 1) - { - m_status = Job_Finished; - emit finished(); - } - else - { - current_job_idx++; - auto job = m_jobs[current_job_idx]; - connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64))); - connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished())); - connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed())); - job->start(); - } - } - void currentJobFailed() - { - m_status = Job_Failed; - emit failed(); - } - void currentJobProgress(qint64 current, qint64 total) - { - if(!total) - return; - - int total_jobs = m_jobs.size(); - - if(!total_jobs) - return; - - float job_chunk = 1000.0 / float(total_jobs); - float cur = current; - float tot = total; - float last_chunk = (cur / tot) * job_chunk; - - float list_total = job_chunk * current_job_idx + last_chunk; - emit progress(qint64(list_total), 1000LL); - } -private: - QVector m_jobs; - /// The overall status of this job list - JobStatus m_status; - int current_job_idx; -signals: - void progress(qint64 current, qint64 total); - void started(); - void finished(); - void failed(); -}; -typedef QSharedPointer JobListPtr; - - -/** - * A queue of job lists! The job lists fail or finish as units. - */ -class JobListQueue : public QObject -{ - Q_OBJECT -public: - JobListQueue(QObject *p = 0): - QObject(p), - currentIndex(0), - is_running(false){} - - void enqueue(JobListPtr job) - { - jobs.enqueue(job); - - // finish or fail, we should catch that and start the next one - connect(job.data(),SIGNAL(finished()), SLOT(startNextJob())); - connect(job.data(),SIGNAL(failed()), SLOT(startNextJob())); - - if(!is_running) - { - QTimer::singleShot(0, this, SLOT(startNextJob())); - } - } - -private slots: - void startNextJob() - { - if (jobs.isEmpty()) - { - currentJobList.clear(); - currentIndex = 0; - is_running = false; - emit finishedAllJobs(); - return; - } - - currentJobList = jobs.dequeue(); - is_running = true; - currentIndex = 0; - currentJobList->start(); - } - -signals: - void finishedAllJobs(); - -private: - JobListPtr currentJobList; - QQueue jobs; - unsigned currentIndex; - bool is_running; -}; diff --git a/logic/net/NetWorker.cpp b/logic/net/NetWorker.cpp index 1eef13d9..c5943348 100644 --- a/logic/net/NetWorker.cpp +++ b/logic/net/NetWorker.cpp @@ -1,7 +1,25 @@ #include "NetWorker.h" #include -NetWorker& NetWorker::spawn() +class NetWorker::Private +{ +public: + QNetworkAccessManager manager; +}; + +NetWorker::NetWorker ( QObject* parent ) : QObject ( parent ) +{ + d = new Private(); +} + +QNetworkAccessManager& NetWorker::qnam() +{ + auto & w = worker(); + return w.d->manager; +} + + +NetWorker& NetWorker::worker() { static QThreadStorage storage; if (!storage.hasLocalData()) diff --git a/logic/net/NetWorker.h b/logic/net/NetWorker.h index 98374e3b..cf7e72e1 100644 --- a/logic/net/NetWorker.h +++ b/logic/net/NetWorker.h @@ -9,12 +9,23 @@ 'OOOOOY _.' '""""'' */ - #pragma once + #include -class NetWorker : public QNetworkAccessManager +#include + +class NetWorker : public QObject { Q_OBJECT public: - static NetWorker &spawn(); + // for high level access to the sevices (preferred) + static NetWorker &worker(); + // for low-level access to the network manager object + static QNetworkAccessManager &qnam(); +public: + +private: + explicit NetWorker ( QObject* parent = 0 ); + class Private; + Private * d; }; \ No newline at end of file -- cgit v1.2.3