summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPetr Mrázek <peterix@gmail.com>2013-07-06 00:55:54 +0200
committerPetr Mrázek <peterix@gmail.com>2013-07-06 00:55:54 +0200
commit84298d621dc1af8dc07a2b959b2225a1df230708 (patch)
tree7985766c3ba6aa26c4b6bd542313691c4edeede0
parent030e03e134d81fea91dfa11036365b06156101a3 (diff)
downloadMultiMC-84298d621dc1af8dc07a2b959b2225a1df230708.tar
MultiMC-84298d621dc1af8dc07a2b959b2225a1df230708.tar.gz
MultiMC-84298d621dc1af8dc07a2b959b2225a1df230708.tar.lz
MultiMC-84298d621dc1af8dc07a2b959b2225a1df230708.tar.xz
MultiMC-84298d621dc1af8dc07a2b959b2225a1df230708.zip
Restructure the downloader into a generic task list/queue.
Yay for bringing paper and pencil on long train rides. Nothing to do but design ... and possibly chat with random strangers ;)
-rw-r--r--asset_test.cpp384
1 files changed, 218 insertions, 166 deletions
diff --git a/asset_test.cpp b/asset_test.cpp
index 7a8687cb..ca4ef506 100644
--- a/asset_test.cpp
+++ b/asset_test.cpp
@@ -3,183 +3,231 @@
#include <QDebug>
#include <QtXml/QtXml>
-enum DlStatus
+enum JobStatus
{
- Dl_NotStarted,
- Dl_InProgress,
- Dl_Finished,
- Dl_Failed
+ Job_NotStarted,
+ Job_InProgress,
+ Job_Finished,
+ Job_Failed
};
-/**
- * A single file for the downloader/cache to process.
- */
-struct Downloadable
+class JobList;
+
+class Job : public QObject
{
- Downloadable(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString())
- :m_url(url), m_rel_target_path(rel_target_path), m_expected_md5(expected_md5)
- {
- m_check_md5 = m_expected_md5.size();
- m_save_to_file = m_rel_target_path.size();
- status = Dl_NotStarted;
- };
- /// source URL
- QUrl m_url;
-
- /// if true, check the md5sum against a provided md5sum
- /// also, if a file exists, perform an md5sum first and don't download only if they don't match
- bool m_check_md5;
- /// the expected md5 checksum
- QString m_expected_md5;
-
- /// save to file?
- bool m_save_to_file;
- /// if saving to file, use the one specified in this string
- QString m_rel_target_path;
- /// if not saving to file, downloaded data is placed here
- QByteArray data;
- /// The file's status
- DlStatus status;
+ Q_OBJECT
+protected:
+ explicit Job(): QObject(0){};
+public:
+ virtual ~Job() {};
+signals:
+ void finish();
+ void fail();
+ void progress(qint64 current, qint64 total);
+public slots:
+ virtual void start() = 0;
};
-typedef QSharedPointer<Downloadable> DownloadablePtr;
-
-class Downloader;
+typedef QSharedPointer<Job> JobPtr;
/**
- * A downloader job, which can have multiple files
- * the user of the downloader is responsible for creating it
- * and connecting its signals to his own slots.
+ * A list of jobs, to be processed one by one.
*/
-class DLJob : public QObject
+class JobList : public QObject
{
- friend class Downloader;
+ friend class JobListQueue;
Q_OBJECT
public:
- DLJob() : QObject(0) {}
- DLJob (QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString() ) : QObject(0)
+ JobList() : QObject(0)
{
- m_status = Dl_NotStarted;
- append(url, rel_target_path, expected_md5);
+ m_status = Job_NotStarted;
+ current_job_idx = 0;
}
- DlStatus getStatus()
+ JobStatus getStatus()
{
return m_status;
}
- void append (QUrl url, QString target = QString(), QString md5 = QString())
+ void add(JobPtr dlable)
{
- Downloadable * dlable = new Downloadable(url, target, md5);
- m_downloads.append(DownloadablePtr(dlable));
+ if(m_status == Job_NotStarted)
+ m_jobs.append(dlable);
+ //else there's a bug. TODO: catch the bugs
}
- QByteArray getFirstFileData()
+ JobPtr getFirstJob()
{
- if(!m_downloads.size())
- return QByteArray();
- else return m_downloads[0]->data;
+ if(m_jobs.size())
+ return m_jobs[0];
+ else
+ return JobPtr();
}
- void add(DownloadablePtr dlable)
+ void start()
{
- m_downloads.append(dlable);
+ current_job_idx = 0;
+ auto job = m_jobs[current_job_idx];
+
+ connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64)));
+ connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished()));
+ connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed()));
+ job->start();
+ emit started();
}
-private:
- void emitStart()
+private slots:
+ void currentJobFinished()
{
- m_status = Dl_InProgress;
- emit started();
+ if(current_job_idx == m_jobs.size() - 1)
+ {
+ m_status = Job_Finished;
+ emit finished();
+ }
+ else
+ {
+ current_job_idx++;
+ auto job = m_jobs[current_job_idx];
+ connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64)));
+ connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished()));
+ connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed()));
+ job->start();
+ }
}
- void emitFail()
+ void currentJobFailed()
{
- m_status = Dl_Failed;
+ m_status = Job_Failed;
emit failed();
}
- void emitFinish()
+ void currentJobProgress(qint64 current, qint64 total)
{
- m_status = Dl_Finished;
- emit finished();
+ if(!total)
+ return;
+
+ int total_jobs = m_jobs.size();
+
+ if(!total_jobs)
+ return;
+
+ float job_chunk = 1000.0 / float(total_jobs);
+ float cur = current;
+ float tot = total;
+ float last_chunk = (cur / tot) * job_chunk;
+
+ float list_total = job_chunk * current_job_idx + last_chunk;
+ emit progress(qint64(list_total), 1000LL);
}
private:
- QVector<DownloadablePtr> m_downloads;
- /// The job's status
- DlStatus m_status;
+ QVector<JobPtr> m_jobs;
+ /// The overall status of this job list
+ JobStatus m_status;
+ int current_job_idx;
signals:
+ void progress(qint64 current, qint64 total);
void started();
void finished();
void failed();
};
-typedef QSharedPointer<DLJob> DLJobPtr;
+typedef QSharedPointer<JobList> JobListPtr;
+
/**
- * The downloader itself. Make one, use it.
- * User is responsible for keeping it around until all the jobs either finish or fail.
+ * A queue of job lists! The job lists fail or finish as units.
*/
-class Downloader : public QObject
+class JobListQueue : public QObject
{
Q_OBJECT
public:
- Downloader(QObject *p = 0):
+ JobListQueue(QObject *p = 0):
QObject(p),
- nam(new QNetworkAccessManager(this)),
- currentReply(0),
- currentIndex(0){}
+ nam(new QNetworkAccessManager()),
+ currentIndex(0),
+ is_running(false){}
- void enqueue(DLJobPtr job)
+ void enqueue(JobListPtr job)
{
- if(jobs.empty())
+ jobs.enqueue(job);
+
+ // finish or fail, we should catch that and start the next one
+ connect(job.data(),SIGNAL(finished()), SLOT(startNextJob()));
+ connect(job.data(),SIGNAL(failed()), SLOT(startNextJob()));
+
+ if(!is_running)
{
- qDebug() << "NEXT JOB!";
QTimer::singleShot(0, this, SLOT(startNextJob()));
}
- jobs.enqueue(job);
}
- private slots:
+private slots:
void startNextJob()
{
if (jobs.isEmpty())
{
- currentJob.clear();
+ currentJobList.clear();
currentIndex = 0;
+ is_running = false;
emit finishedAllJobs();
return;
}
- currentJob = jobs.dequeue();
+ currentJobList = jobs.dequeue();
+ is_running = true;
currentIndex = 0;
- currentJob->emitStart();
-
- startNextDownload();
+ currentJobList->start();
}
- void startNextDownload()
+signals:
+ void finishedAllJobs();
+
+private:
+ JobListPtr currentJobList;
+ QQueue<JobListPtr> jobs;
+ QSharedPointer<QNetworkAccessManager> nam;
+ unsigned currentIndex;
+ bool is_running;
+};
+
+/**
+ * A single file for the downloader/cache to process.
+ */
+class DownloadJob : public Job
+{
+ friend class Downloader;
+ Q_OBJECT
+private:
+ DownloadJob(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString())
+ :Job()
{
- // we finished the current job. Good job.
- if(currentIndex >= currentJob->m_downloads.size())
- {
- currentJob->emitFinish();
- qDebug() << "NEXT JOB!";
- QTimer::singleShot(0, this, SLOT(startNextJob()));
- return;
- }
+ m_url = url;
+ m_rel_target_path = rel_target_path;
+ m_expected_md5 = expected_md5;
- DownloadablePtr dlable = currentJob->m_downloads[currentIndex];
- if(dlable->m_save_to_file)
+ m_check_md5 = m_expected_md5.size();
+ m_save_to_file = m_rel_target_path.size();
+ m_status = Job_NotStarted;
+ };
+public:
+ static JobPtr create(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString())
+ {
+ return JobPtr(new DownloadJob(url, rel_target_path, expected_md5));
+ }
+public slots:
+ virtual void start()
+ {
+ m_manager.reset(new QNetworkAccessManager());
+ if(m_save_to_file)
{
- QString filename = dlable->m_rel_target_path;
- currentOutput.setFileName(filename);
+ QString filename = m_rel_target_path;
+ m_output_file.setFileName(filename);
// if there already is a file and md5 checking is in effect
- if(currentOutput.exists() && dlable->m_check_md5)
+ if(m_output_file.exists() && m_check_md5)
{
// and it can be opened
- if(currentOutput.open(QIODevice::ReadOnly))
+ if(m_output_file.open(QIODevice::ReadOnly))
{
// check the md5 against the expected one
- QString hash = QCryptographicHash::hash(currentOutput.readAll(), QCryptographicHash::Md5).toHex().constData();
- currentOutput.close();
+ QString hash = QCryptographicHash::hash(m_output_file.readAll(), QCryptographicHash::Md5).toHex().constData();
+ m_output_file.close();
// skip this file if they match
- if(hash == dlable->m_expected_md5)
+ if(hash == m_expected_md5)
{
- currentIndex++;
- QTimer::singleShot(0, this, SLOT(startNextDownload()));
+ qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
+ emit finish();
return;
}
}
@@ -189,106 +237,111 @@ public:
if(!dir.mkpath(a.path()))
{
/*
- * TODO: error when making the folder structure
+ * error when making the folder structure
*/
- currentJob->emitFail();
- currentJob.clear();
- currentIndex = 0;
- QTimer::singleShot(0, this, SLOT(startNextJob()));
- qDebug() << "NEXT JOB!";
+ emit fail();
return;
}
- if (!currentOutput.open(QIODevice::WriteOnly))
+ if (!m_output_file.open(QIODevice::WriteOnly))
{
/*
- * TODO: Can't open the file... the job failed
+ * Can't open the file... the job failed
*/
- currentJob->emitFail();
- currentJob.clear();
- currentIndex = 0;
- QTimer::singleShot(0, this, SLOT(startNextJob()));
- qDebug() << "NEXT JOB!";
+ emit fail();
return;
}
}
-
- QNetworkRequest request(dlable->m_url);
- QNetworkReply * rep = nam->get(request);
- currentReply = QSharedPointer<QNetworkReply>(rep, &QObject::deleteLater);
+ qDebug() << "Downloading " << m_url.toString();
+ QNetworkRequest request(m_url);
+ QNetworkReply * rep = m_manager->get(request);
+ m_reply = QSharedPointer<QNetworkReply>(rep, &QObject::deleteLater);
connect(rep, SIGNAL(downloadProgress(qint64,qint64)), SLOT(downloadProgress(qint64,qint64)));
connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(downloadError(QNetworkReply::NetworkError)));
connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead()));
- }
+ };
+private slots:
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
{
- // eventually, progress bars. yeah.
+ emit progress(bytesReceived, bytesTotal);
};
void downloadError(QNetworkReply::NetworkError error)
{
- // error happened during download. :<
- DownloadablePtr dlable = currentJob->m_downloads[currentIndex];
- //TODO: log the reason why
- dlable->status = Dl_Failed;
+ // error happened during download.
+ // TODO: log the reason why
+ m_status = Job_Failed;
}
void downloadFinished()
{
- DownloadablePtr dlable = currentJob->m_downloads[currentIndex];
// if the download succeeded
- if(dlable->status != Dl_Failed)
+ if(m_status != Job_Failed)
{
// nothing went wrong...
- dlable->status = Dl_Finished;
+ m_status = Job_Finished;
// save the data to the downloadable if we aren't saving to file
- if(!dlable->m_save_to_file)
+ if(!m_save_to_file)
{
- dlable->data = currentReply->readAll();
+ m_data = m_reply->readAll();
}
else
{
- currentOutput.close();
+ m_output_file.close();
}
//TODO: check md5 here!
-
- // continue with the next download, if any
- currentIndex ++;
- QTimer::singleShot(0, this, SLOT(startNextDownload()));
+ m_reply.clear();
+ emit finish();
return;
}
// else the download failed
else
{
- if(dlable->m_save_to_file)
+ if(m_save_to_file)
{
- currentOutput.close();
- currentOutput.remove();
+ m_output_file.close();
+ m_output_file.remove();
}
+ m_reply.clear();
+ emit fail();
+ return;
}
}
void downloadReadyRead()
{
- DownloadablePtr dlable = currentJob->m_downloads[currentIndex];
- if(dlable->m_save_to_file)
+ if(m_save_to_file)
{
- currentOutput.write(currentReply->readAll());
+ m_output_file.write( m_reply->readAll());
}
};
-
-signals:
- void finishedAllJobs();
-public slots:
+public:
+ /// the associated network manager
+ QSharedPointer<QNetworkAccessManager> m_manager;
+ /// the network reply
+ QSharedPointer<QNetworkReply> m_reply;
+ /// source URL
+ QUrl m_url;
-private:
- QSharedPointer<QNetworkAccessManager> nam;
- DLJobPtr currentJob;
- QSharedPointer<QNetworkReply> currentReply;
- QQueue<DLJobPtr> jobs;
- QFile currentOutput;
- unsigned currentIndex;
+ /// if true, check the md5sum against a provided md5sum
+ /// also, if a file exists, perform an md5sum first and don't download only if they don't match
+ bool m_check_md5;
+ /// the expected md5 checksum
+ QString m_expected_md5;
+
+ /// save to file?
+ bool m_save_to_file;
+ /// if saving to file, use the one specified in this string
+ QString m_rel_target_path;
+ /// this is the output file, if any
+ QFile m_output_file;
+ /// if not saving to file, downloaded data is placed here
+ QByteArray m_data;
+
+
+ /// The file's status
+ JobStatus m_status;
};
inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
@@ -311,7 +364,9 @@ public slots:
void fetchFinished()
{
- QByteArray ba = index_job->getFirstFileData();
+ JobPtr firstJob = index_job->getFirstJob();
+ auto DlJob = firstJob.dynamicCast<DownloadJob>();
+ QByteArray ba = DlJob->m_data;
QString xmlErrorMsg;
QDomDocument doc;
@@ -323,7 +378,7 @@ public slots:
QRegExp etag_match(".*([a-f0-9]{32}).*");
QDomNodeList contents = doc.elementsByTagName("Contents");
- DLJob *job = new DLJob();
+ JobList *job = new JobList();
connect(job, SIGNAL(finished()), SLOT(filesFinished()));
for (int i = 0; i < contents.length(); i++)
@@ -353,11 +408,7 @@ public slots:
QString trimmedEtag = etagStr.remove('"');
QString prefix("http://s3.amazonaws.com/Minecraft.Resources/");
QString fprefix("assets/");
- Downloadable * d = new Downloadable(QUrl(prefix + keyStr),fprefix + keyStr, trimmedEtag);
-
- job->add(DownloadablePtr(d));
-
- //qDebug() << keyStr << " " << lastModStr << " " << etagStr << sizeStr;
+ job->add(DownloadJob::create(QUrl(prefix + keyStr),fprefix + keyStr, trimmedEtag));
}
files_job.reset(job);
dl.enqueue(files_job);
@@ -369,15 +420,16 @@ public slots:
public:
void start()
{
- DLJob *job = new DLJob(QUrl("http://s3.amazonaws.com/Minecraft.Resources/"));
+ JobList *job = new JobList();
+ job->add(DownloadJob::create(QUrl("http://s3.amazonaws.com/Minecraft.Resources/")));
connect(job, SIGNAL(finished()), SLOT(fetchFinished()));
connect(job, SIGNAL(started()), SLOT(fetchStarted()));
index_job.reset(job);
dl.enqueue(index_job);
}
- Downloader dl;
- DLJobPtr index_job;
- DLJobPtr files_job;
+ JobListQueue dl;
+ JobListPtr index_job;
+ JobListPtr files_job;
};
int main(int argc, char *argv[])