From 84298d621dc1af8dc07a2b959b2225a1df230708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sat, 6 Jul 2013 00:55:54 +0200 Subject: Restructure the downloader into a generic task list/queue. Yay for bringing paper and pencil on long train rides. Nothing to do but design ... and possibly chat with random strangers ;) --- asset_test.cpp | 384 ++++++++++++++++++++++++++++++++------------------------- 1 file changed, 218 insertions(+), 166 deletions(-) (limited to 'asset_test.cpp') diff --git a/asset_test.cpp b/asset_test.cpp index 7a8687cb..ca4ef506 100644 --- a/asset_test.cpp +++ b/asset_test.cpp @@ -3,183 +3,231 @@ #include #include -enum DlStatus +enum JobStatus { - Dl_NotStarted, - Dl_InProgress, - Dl_Finished, - Dl_Failed + Job_NotStarted, + Job_InProgress, + Job_Finished, + Job_Failed }; -/** - * A single file for the downloader/cache to process. - */ -struct Downloadable +class JobList; + +class Job : public QObject { - Downloadable(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()) - :m_url(url), m_rel_target_path(rel_target_path), m_expected_md5(expected_md5) - { - m_check_md5 = m_expected_md5.size(); - m_save_to_file = m_rel_target_path.size(); - status = Dl_NotStarted; - }; - /// source URL - QUrl m_url; - - /// if true, check the md5sum against a provided md5sum - /// also, if a file exists, perform an md5sum first and don't download only if they don't match - bool m_check_md5; - /// the expected md5 checksum - QString m_expected_md5; - - /// save to file? - bool m_save_to_file; - /// if saving to file, use the one specified in this string - QString m_rel_target_path; - /// if not saving to file, downloaded data is placed here - QByteArray data; - /// The file's status - DlStatus status; + Q_OBJECT +protected: + explicit Job(): QObject(0){}; +public: + virtual ~Job() {}; +signals: + void finish(); + void fail(); + void progress(qint64 current, qint64 total); +public slots: + virtual void start() = 0; }; -typedef QSharedPointer DownloadablePtr; - -class Downloader; +typedef QSharedPointer JobPtr; /** - * A downloader job, which can have multiple files - * the user of the downloader is responsible for creating it - * and connecting its signals to his own slots. + * A list of jobs, to be processed one by one. */ -class DLJob : public QObject +class JobList : public QObject { - friend class Downloader; + friend class JobListQueue; Q_OBJECT public: - DLJob() : QObject(0) {} - DLJob (QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString() ) : QObject(0) + JobList() : QObject(0) { - m_status = Dl_NotStarted; - append(url, rel_target_path, expected_md5); + m_status = Job_NotStarted; + current_job_idx = 0; } - DlStatus getStatus() + JobStatus getStatus() { return m_status; } - void append (QUrl url, QString target = QString(), QString md5 = QString()) + void add(JobPtr dlable) { - Downloadable * dlable = new Downloadable(url, target, md5); - m_downloads.append(DownloadablePtr(dlable)); + if(m_status == Job_NotStarted) + m_jobs.append(dlable); + //else there's a bug. TODO: catch the bugs } - QByteArray getFirstFileData() + JobPtr getFirstJob() { - if(!m_downloads.size()) - return QByteArray(); - else return m_downloads[0]->data; + if(m_jobs.size()) + return m_jobs[0]; + else + return JobPtr(); } - void add(DownloadablePtr dlable) + void start() { - m_downloads.append(dlable); + current_job_idx = 0; + auto job = m_jobs[current_job_idx]; + + connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64))); + connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished())); + connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed())); + job->start(); + emit started(); } -private: - void emitStart() +private slots: + void currentJobFinished() { - m_status = Dl_InProgress; - emit started(); + if(current_job_idx == m_jobs.size() - 1) + { + m_status = Job_Finished; + emit finished(); + } + else + { + current_job_idx++; + auto job = m_jobs[current_job_idx]; + connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64))); + connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished())); + connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed())); + job->start(); + } } - void emitFail() + void currentJobFailed() { - m_status = Dl_Failed; + m_status = Job_Failed; emit failed(); } - void emitFinish() + void currentJobProgress(qint64 current, qint64 total) { - m_status = Dl_Finished; - emit finished(); + if(!total) + return; + + int total_jobs = m_jobs.size(); + + if(!total_jobs) + return; + + float job_chunk = 1000.0 / float(total_jobs); + float cur = current; + float tot = total; + float last_chunk = (cur / tot) * job_chunk; + + float list_total = job_chunk * current_job_idx + last_chunk; + emit progress(qint64(list_total), 1000LL); } private: - QVector m_downloads; - /// The job's status - DlStatus m_status; + QVector m_jobs; + /// The overall status of this job list + JobStatus m_status; + int current_job_idx; signals: + void progress(qint64 current, qint64 total); void started(); void finished(); void failed(); }; -typedef QSharedPointer DLJobPtr; +typedef QSharedPointer JobListPtr; + /** - * The downloader itself. Make one, use it. - * User is responsible for keeping it around until all the jobs either finish or fail. + * A queue of job lists! The job lists fail or finish as units. */ -class Downloader : public QObject +class JobListQueue : public QObject { Q_OBJECT public: - Downloader(QObject *p = 0): + JobListQueue(QObject *p = 0): QObject(p), - nam(new QNetworkAccessManager(this)), - currentReply(0), - currentIndex(0){} + nam(new QNetworkAccessManager()), + currentIndex(0), + is_running(false){} - void enqueue(DLJobPtr job) + void enqueue(JobListPtr job) { - if(jobs.empty()) + jobs.enqueue(job); + + // finish or fail, we should catch that and start the next one + connect(job.data(),SIGNAL(finished()), SLOT(startNextJob())); + connect(job.data(),SIGNAL(failed()), SLOT(startNextJob())); + + if(!is_running) { - qDebug() << "NEXT JOB!"; QTimer::singleShot(0, this, SLOT(startNextJob())); } - jobs.enqueue(job); } - private slots: +private slots: void startNextJob() { if (jobs.isEmpty()) { - currentJob.clear(); + currentJobList.clear(); currentIndex = 0; + is_running = false; emit finishedAllJobs(); return; } - currentJob = jobs.dequeue(); + currentJobList = jobs.dequeue(); + is_running = true; currentIndex = 0; - currentJob->emitStart(); - - startNextDownload(); + currentJobList->start(); } - void startNextDownload() +signals: + void finishedAllJobs(); + +private: + JobListPtr currentJobList; + QQueue jobs; + QSharedPointer nam; + unsigned currentIndex; + bool is_running; +}; + +/** + * A single file for the downloader/cache to process. + */ +class DownloadJob : public Job +{ + friend class Downloader; + Q_OBJECT +private: + DownloadJob(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()) + :Job() { - // we finished the current job. Good job. - if(currentIndex >= currentJob->m_downloads.size()) - { - currentJob->emitFinish(); - qDebug() << "NEXT JOB!"; - QTimer::singleShot(0, this, SLOT(startNextJob())); - return; - } + m_url = url; + m_rel_target_path = rel_target_path; + m_expected_md5 = expected_md5; - DownloadablePtr dlable = currentJob->m_downloads[currentIndex]; - if(dlable->m_save_to_file) + m_check_md5 = m_expected_md5.size(); + m_save_to_file = m_rel_target_path.size(); + m_status = Job_NotStarted; + }; +public: + static JobPtr create(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()) + { + return JobPtr(new DownloadJob(url, rel_target_path, expected_md5)); + } +public slots: + virtual void start() + { + m_manager.reset(new QNetworkAccessManager()); + if(m_save_to_file) { - QString filename = dlable->m_rel_target_path; - currentOutput.setFileName(filename); + QString filename = m_rel_target_path; + m_output_file.setFileName(filename); // if there already is a file and md5 checking is in effect - if(currentOutput.exists() && dlable->m_check_md5) + if(m_output_file.exists() && m_check_md5) { // and it can be opened - if(currentOutput.open(QIODevice::ReadOnly)) + if(m_output_file.open(QIODevice::ReadOnly)) { // check the md5 against the expected one - QString hash = QCryptographicHash::hash(currentOutput.readAll(), QCryptographicHash::Md5).toHex().constData(); - currentOutput.close(); + QString hash = QCryptographicHash::hash(m_output_file.readAll(), QCryptographicHash::Md5).toHex().constData(); + m_output_file.close(); // skip this file if they match - if(hash == dlable->m_expected_md5) + if(hash == m_expected_md5) { - currentIndex++; - QTimer::singleShot(0, this, SLOT(startNextDownload())); + qDebug() << "Skipping " << m_url.toString() << ": md5 match."; + emit finish(); return; } } @@ -189,106 +237,111 @@ public: if(!dir.mkpath(a.path())) { /* - * TODO: error when making the folder structure + * error when making the folder structure */ - currentJob->emitFail(); - currentJob.clear(); - currentIndex = 0; - QTimer::singleShot(0, this, SLOT(startNextJob())); - qDebug() << "NEXT JOB!"; + emit fail(); return; } - if (!currentOutput.open(QIODevice::WriteOnly)) + if (!m_output_file.open(QIODevice::WriteOnly)) { /* - * TODO: Can't open the file... the job failed + * Can't open the file... the job failed */ - currentJob->emitFail(); - currentJob.clear(); - currentIndex = 0; - QTimer::singleShot(0, this, SLOT(startNextJob())); - qDebug() << "NEXT JOB!"; + emit fail(); return; } } - - QNetworkRequest request(dlable->m_url); - QNetworkReply * rep = nam->get(request); - currentReply = QSharedPointer(rep, &QObject::deleteLater); + qDebug() << "Downloading " << m_url.toString(); + QNetworkRequest request(m_url); + QNetworkReply * rep = m_manager->get(request); + m_reply = QSharedPointer(rep, &QObject::deleteLater); connect(rep, SIGNAL(downloadProgress(qint64,qint64)), SLOT(downloadProgress(qint64,qint64))); connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(downloadError(QNetworkReply::NetworkError))); connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); - } + }; +private slots: void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) { - // eventually, progress bars. yeah. + emit progress(bytesReceived, bytesTotal); }; void downloadError(QNetworkReply::NetworkError error) { - // error happened during download. :< - DownloadablePtr dlable = currentJob->m_downloads[currentIndex]; - //TODO: log the reason why - dlable->status = Dl_Failed; + // error happened during download. + // TODO: log the reason why + m_status = Job_Failed; } void downloadFinished() { - DownloadablePtr dlable = currentJob->m_downloads[currentIndex]; // if the download succeeded - if(dlable->status != Dl_Failed) + if(m_status != Job_Failed) { // nothing went wrong... - dlable->status = Dl_Finished; + m_status = Job_Finished; // save the data to the downloadable if we aren't saving to file - if(!dlable->m_save_to_file) + if(!m_save_to_file) { - dlable->data = currentReply->readAll(); + m_data = m_reply->readAll(); } else { - currentOutput.close(); + m_output_file.close(); } //TODO: check md5 here! - - // continue with the next download, if any - currentIndex ++; - QTimer::singleShot(0, this, SLOT(startNextDownload())); + m_reply.clear(); + emit finish(); return; } // else the download failed else { - if(dlable->m_save_to_file) + if(m_save_to_file) { - currentOutput.close(); - currentOutput.remove(); + m_output_file.close(); + m_output_file.remove(); } + m_reply.clear(); + emit fail(); + return; } } void downloadReadyRead() { - DownloadablePtr dlable = currentJob->m_downloads[currentIndex]; - if(dlable->m_save_to_file) + if(m_save_to_file) { - currentOutput.write(currentReply->readAll()); + m_output_file.write( m_reply->readAll()); } }; - -signals: - void finishedAllJobs(); -public slots: +public: + /// the associated network manager + QSharedPointer m_manager; + /// the network reply + QSharedPointer m_reply; + /// source URL + QUrl m_url; -private: - QSharedPointer nam; - DLJobPtr currentJob; - QSharedPointer currentReply; - QQueue jobs; - QFile currentOutput; - unsigned currentIndex; + /// if true, check the md5sum against a provided md5sum + /// also, if a file exists, perform an md5sum first and don't download only if they don't match + bool m_check_md5; + /// the expected md5 checksum + QString m_expected_md5; + + /// save to file? + bool m_save_to_file; + /// if saving to file, use the one specified in this string + QString m_rel_target_path; + /// this is the output file, if any + QFile m_output_file; + /// if not saving to file, downloaded data is placed here + QByteArray m_data; + + + /// The file's status + JobStatus m_status; }; inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname) @@ -311,7 +364,9 @@ public slots: void fetchFinished() { - QByteArray ba = index_job->getFirstFileData(); + JobPtr firstJob = index_job->getFirstJob(); + auto DlJob = firstJob.dynamicCast(); + QByteArray ba = DlJob->m_data; QString xmlErrorMsg; QDomDocument doc; @@ -323,7 +378,7 @@ public slots: QRegExp etag_match(".*([a-f0-9]{32}).*"); QDomNodeList contents = doc.elementsByTagName("Contents"); - DLJob *job = new DLJob(); + JobList *job = new JobList(); connect(job, SIGNAL(finished()), SLOT(filesFinished())); for (int i = 0; i < contents.length(); i++) @@ -353,11 +408,7 @@ public slots: QString trimmedEtag = etagStr.remove('"'); QString prefix("http://s3.amazonaws.com/Minecraft.Resources/"); QString fprefix("assets/"); - Downloadable * d = new Downloadable(QUrl(prefix + keyStr),fprefix + keyStr, trimmedEtag); - - job->add(DownloadablePtr(d)); - - //qDebug() << keyStr << " " << lastModStr << " " << etagStr << sizeStr; + job->add(DownloadJob::create(QUrl(prefix + keyStr),fprefix + keyStr, trimmedEtag)); } files_job.reset(job); dl.enqueue(files_job); @@ -369,15 +420,16 @@ public slots: public: void start() { - DLJob *job = new DLJob(QUrl("http://s3.amazonaws.com/Minecraft.Resources/")); + JobList *job = new JobList(); + job->add(DownloadJob::create(QUrl("http://s3.amazonaws.com/Minecraft.Resources/"))); connect(job, SIGNAL(finished()), SLOT(fetchFinished())); connect(job, SIGNAL(started()), SLOT(fetchStarted())); index_job.reset(job); dl.enqueue(index_job); } - Downloader dl; - DLJobPtr index_job; - DLJobPtr files_job; + JobListQueue dl; + JobListPtr index_job; + JobListPtr files_job; }; int main(int argc, char *argv[]) -- cgit v1.2.3