Speed up the downloads. Massively.

5 files changed, 194 insertions, 224 deletions

diff --git a/logic/net/DownloadJob.cpp b/logic/net/DownloadJob.cpp
index 6d87a132..5c8ed4b9 100644
--- a/logic/net/DownloadJob.cpp
+++ b/logic/net/DownloadJob.cpp
@@ -2,9 +2,7 @@
 #include "pathutils.h"
 #include "NetWorker.h"
 
-DownloadJob::DownloadJob (QUrl url,
-						  QString target_path,
-						  QString expected_md5 )
+Download::Download (QUrl url, QString target_path, QString expected_md5 )
 	:Job()
 {
 	m_url = url;
@@ -17,14 +15,7 @@ DownloadJob::DownloadJob (QUrl url,
 	m_opened_for_saving = false;
 }
 
-JobPtr DownloadJob::create (QUrl url,
-							QString target_path,
-							QString expected_md5 )
-{
-	return JobPtr ( new DownloadJob ( url, target_path, expected_md5 ) );
-}
-
-void DownloadJob::start()
+void Download::start()
 {
 	if ( m_save_to_file )
 	{
@@ -40,7 +31,7 @@ void DownloadJob::start()
 			if ( m_check_md5 && hash == m_expected_md5 )
 			{
 				qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
-				emit finish();
+				emit succeeded(index_within_job);
 				return;
 			}
 			else
@@ -50,7 +41,7 @@ void DownloadJob::start()
 		}
 		if(!ensureFilePathExists(filename))
 		{
-			emit fail();
+			emit failed(index_within_job);
 			return;
 		}
 	}
@@ -58,7 +49,7 @@ void DownloadJob::start()
 	QNetworkRequest request ( m_url );
 	request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1()); 
 	
-	auto &worker = NetWorker::spawn();
+	auto &worker = NetWorker::qnam();
 	QNetworkReply * rep = worker.get ( request );
 	
 	m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
@@ -68,19 +59,19 @@ void DownloadJob::start()
 	connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
 }
 
-void DownloadJob::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
+void Download::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
 {
-	emit progress ( bytesReceived, bytesTotal );
+	emit progress (index_within_job, bytesReceived, bytesTotal );
 }
 
-void DownloadJob::downloadError ( QNetworkReply::NetworkError error )
+void Download::downloadError ( QNetworkReply::NetworkError error )
 {
 	// error happened during download.
 	// TODO: log the reason why
 	m_status = Job_Failed;
 }
 
-void DownloadJob::downloadFinished()
+void Download::downloadFinished()
 {
 	// if the download succeeded
 	if ( m_status != Job_Failed )
@@ -99,7 +90,7 @@ void DownloadJob::downloadFinished()
 
 		//TODO: check md5 here!
 		m_reply.clear();
-		emit finish();
+		emit succeeded(index_within_job);
 		return;
 	}
 	// else the download failed
@@ -111,12 +102,12 @@ void DownloadJob::downloadFinished()
 			m_output_file.remove();
 		}
 		m_reply.clear();
-		emit fail();
+		emit failed(index_within_job);
 		return;
 	}
 }
 
-void DownloadJob::downloadReadyRead()
+void Download::downloadReadyRead()
 {
 	if( m_save_to_file )
 	{
@@ -128,7 +119,7 @@ void DownloadJob::downloadReadyRead()
 				* Can't open the file... the job failed
 				*/
 				m_reply->abort();
-				emit fail();
+				emit failed(index_within_job);
 				return;
 			}
 			m_opened_for_saving = true;
@@ -136,3 +127,58 @@ void DownloadJob::downloadReadyRead()
 		m_output_file.write ( m_reply->readAll() );
 	}
 }
+
+DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expected_md5 )
+{
+	DownloadPtr ptr (new Download(url, rel_target_path, expected_md5));
+	ptr->index_within_job = downloads.size();
+	downloads.append(ptr);
+	return ptr;
+}
+
+void DownloadJob::partSucceeded ( int index )
+{
+	num_succeeded++;
+	if(num_failed + num_succeeded == downloads.size())
+	{
+		if(num_failed)
+		{
+			emit failed();
+		}
+		else
+		{
+			emit succeeded();
+		}
+	}
+}
+
+void DownloadJob::partFailed ( int index )
+{
+	num_failed++;
+	if(num_failed + num_succeeded == downloads.size())
+	{
+		if(num_failed)
+		{
+			emit failed();
+		}
+		else
+		{
+			emit succeeded();
+		}
+	}
+}
+
+void DownloadJob::partProgress ( int index, qint64 bytesReceived, qint64 bytesTotal )
+{
+	// PROGRESS? DENIED!
+}
+
+
+void DownloadJob::start()
+{
+	for(auto iter: downloads)
+	{
+		connect(iter.data(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));
+		iter->start();
+	}
+}
diff --git a/logic/net/DownloadJob.h b/logic/net/DownloadJob.h
index 94abb9af..adeef646 100644
--- a/logic/net/DownloadJob.h
+++ b/logic/net/DownloadJob.h
@@ -1,28 +1,37 @@
 #pragma once
-#include "JobQueue.h"
 #include <QtNetwork>
 
-/**
- * A single file for the downloader/cache to process.
- */
-class DownloadJob : public Job
+class DownloadJob;
+class Download;
+typedef QSharedPointer<DownloadJob> DownloadJobPtr;
+typedef QSharedPointer<Download> DownloadPtr;
+
+enum JobStatus
+{
+	Job_NotStarted,
+	Job_InProgress,
+	Job_Finished,
+	Job_Failed
+};
+
+class Job : public QObject
 {
 	Q_OBJECT
+protected:
+	explicit Job(): QObject(0){};
 public:
-	DownloadJob(QUrl url,
-				QString rel_target_path = QString(),
-				QString expected_md5 = QString()
-			   );
-	static JobPtr create(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
+	virtual ~Job() {};
+
 public slots:
-	virtual void start();
-	
-private slots:
-	void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);;
-	void downloadError(QNetworkReply::NetworkError error);
-	void downloadFinished();
-	void downloadReadyRead();
-	
+	virtual void start() = 0;
+};
+
+class Download: public Job
+{
+	friend class DownloadJob;
+	Q_OBJECT
+protected:
+	Download(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
 public:
 	/// the network reply
 	QSharedPointer<QNetworkReply> m_reply;
@@ -46,6 +55,72 @@ public:
 	/// if not saving to file, downloaded data is placed here
 	QByteArray m_data;
 	
+	int currentProgress = 0;
+	int totalProgress = 0;
+	
 	/// The file's status
 	JobStatus m_status;
+	
+	int index_within_job = 0;
+signals:
+	void started(int index);
+	void progress(int index, qint64 current, qint64 total);
+	void succeeded(int index);
+	void failed(int index);
+public slots:
+	virtual void start();
+	
+private slots:
+	void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);;
+	void downloadError(QNetworkReply::NetworkError error);
+	void downloadFinished();
+	void downloadReadyRead();
+};
+
+/**
+ * A single file for the downloader/cache to process.
+ */
+class DownloadJob : public Job
+{
+	Q_OBJECT
+public:
+	explicit DownloadJob()
+		:Job(){};
+	explicit DownloadJob(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString())
+		:Job()
+	{
+		add(url, rel_target_path, expected_md5);
+	};
+	
+	DownloadPtr add(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
+	DownloadPtr operator[](int index)
+	{
+		return downloads[index];
+	};
+	DownloadPtr first()
+	{
+		if(downloads.size())
+			return downloads[0];
+		return DownloadPtr();
+	}
+	int size() const
+	{
+		return downloads.size();
+	}
+signals:
+	void started();
+	void progress(qint64 current, qint64 total);
+	void succeeded();
+	void failed();
+public slots:
+	virtual void start();
+private slots:
+	void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal);;
+	void partSucceeded(int index);
+	void partFailed(int index);
+private:
+	QList<DownloadPtr> downloads;
+	int num_succeeded = 0;
+	int num_failed = 0;
 };
+
diff --git a/logic/net/JobQueue.h b/logic/net/JobQueue.h
deleted file mode 100644
index b6895f60..00000000
--- a/logic/net/JobQueue.h
+++ /dev/null
@@ -1,180 +0,0 @@
-#pragma once
-#include <QtCore>
-#include "libutil_config.h"
-
-enum JobStatus
-{
-	Job_NotStarted,
-	Job_InProgress,
-	Job_Finished,
-	Job_Failed
-};
-
-class JobList;
-
-class Job : public QObject
-{
-	Q_OBJECT
-protected:
-	explicit Job(): QObject(0){};
-public:
-	virtual ~Job() {};
-signals:
-	void finish();
-	void fail();
-	void progress(qint64 current, qint64 total);
-public slots:
-	virtual void start() = 0;
-};
-typedef QSharedPointer<Job> JobPtr;
-
-/**
- * A list of jobs, to be processed one by one.
- */
-class JobList : public QObject
-{
-	friend class JobListQueue;
-	Q_OBJECT
-public:
-
-	JobList() : QObject(0)
-	{
-		m_status = Job_NotStarted;
-		current_job_idx = 0;
-	}
-	JobStatus getStatus()
-	{
-		return m_status;
-	}
-	void add(JobPtr dlable)
-	{
-		if(m_status == Job_NotStarted)
-			m_jobs.append(dlable);
-		//else there's a bug. TODO: catch the bugs
-	}
-	JobPtr getFirstJob()
-	{
-		if(m_jobs.size())
-			return m_jobs[0];
-		else
-			return JobPtr();
-	}
-	void start()
-	{
-		current_job_idx = 0;
-		auto job = m_jobs[current_job_idx];
-		
-		connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64)));
-		connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished()));
-		connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed()));
-		job->start();
-		emit started();
-	}
-private slots:
-	void currentJobFinished()
-	{
-		if(current_job_idx == m_jobs.size() - 1)
-		{
-			m_status = Job_Finished;
-			emit finished();
-		}
-		else
-		{
-			current_job_idx++;
-			auto job = m_jobs[current_job_idx];
-			connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64)));
-			connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished()));
-			connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed()));
-			job->start();
-		}
-	}
-	void currentJobFailed()
-	{
-		m_status = Job_Failed;
-		emit failed();
-	}
-	void currentJobProgress(qint64 current, qint64 total)
-	{
-		if(!total)
-			return;
-		
-		int total_jobs = m_jobs.size();
-		
-		if(!total_jobs)
-			return;
-		
-		float job_chunk = 1000.0 / float(total_jobs);
-		float cur = current;
-		float tot = total;
-		float last_chunk = (cur / tot) * job_chunk;
-		
-		float list_total = job_chunk * current_job_idx + last_chunk;
-		emit progress(qint64(list_total), 1000LL);
-	}
-private:
-	QVector<JobPtr> m_jobs;
-	/// The overall status of this job list
-	JobStatus m_status;
-	int current_job_idx;
-signals:
-	void progress(qint64 current, qint64 total);
-	void started();
-	void finished();
-	void failed();
-};
-typedef QSharedPointer<JobList> JobListPtr;
-
-
-/**
- * A queue of job lists! The job lists fail or finish as units.
- */
-class JobListQueue : public QObject
-{
-	Q_OBJECT
-public:
-	JobListQueue(QObject *p = 0):
-		QObject(p),
-		currentIndex(0),
-		is_running(false){}
-	
-	void enqueue(JobListPtr job)
-	{
-		jobs.enqueue(job);
-		
-		// finish or fail, we should catch that and start the next one
-		connect(job.data(),SIGNAL(finished()), SLOT(startNextJob()));
-		connect(job.data(),SIGNAL(failed()), SLOT(startNextJob()));
-		
-		if(!is_running)
-		{
-			QTimer::singleShot(0, this, SLOT(startNextJob()));
-		}
-	}
-	
-private slots:
-	void startNextJob()
-	{
-		if (jobs.isEmpty())
-		{
-			currentJobList.clear();
-			currentIndex = 0;
-			is_running = false;
-			emit finishedAllJobs();
-			return;
-		}
-		
-		currentJobList = jobs.dequeue();
-		is_running = true;
-		currentIndex = 0;
-		currentJobList->start();
-	}
-	
-signals:
-	void finishedAllJobs();
-	
-private:
-	JobListPtr currentJobList;
-	QQueue<JobListPtr> jobs;
-	unsigned currentIndex;
-	bool is_running;
-};
diff --git a/logic/net/NetWorker.cpp b/logic/net/NetWorker.cpp
index 1eef13d9..c5943348 100644
--- a/logic/net/NetWorker.cpp
+++ b/logic/net/NetWorker.cpp
@@ -1,7 +1,25 @@
 #include "NetWorker.h"
 #include <QThreadStorage>
 
-NetWorker& NetWorker::spawn()
+class NetWorker::Private
+{
+public:
+	QNetworkAccessManager manager;
+};
+
+NetWorker::NetWorker ( QObject* parent ) : QObject ( parent )
+{
+	d = new Private();
+}
+
+QNetworkAccessManager& NetWorker::qnam()
+{
+	auto & w = worker();
+	return w.d->manager;
+}
+
+
+NetWorker& NetWorker::worker()
 {
 	static QThreadStorage<NetWorker *> storage;
 	if (!storage.hasLocalData())
diff --git a/logic/net/NetWorker.h b/logic/net/NetWorker.h
index 98374e3b..cf7e72e1 100644
--- a/logic/net/NetWorker.h
+++ b/logic/net/NetWorker.h
@@ -9,12 +9,23 @@
       'OOOOOY  _.'
         '""""''
 */
-
 #pragma once
+
 #include <QNetworkAccessManager>
-class NetWorker : public QNetworkAccessManager
+#include <QUrl>
+
+class NetWorker : public QObject
 {
 	Q_OBJECT
 public:
-	static NetWorker &spawn();
+	// for high level access to the sevices (preferred)
+	static NetWorker &worker();
+	// for low-level access to the network manager object
+	static QNetworkAccessManager &qnam();
+public:
+	
+private:
+	explicit NetWorker ( QObject* parent = 0 );
+	class Private;
+	Private * d;
 };
\ No newline at end of file