summaryrefslogtreecommitdiffstats
path: root/libutil
diff options
context:
space:
mode:
authorPetr Mrázek <peterix@gmail.com>2013-07-06 01:50:07 +0200
committerPetr Mrázek <peterix@gmail.com>2013-07-06 01:50:07 +0200
commit5c07e1b042ecd22dfcd3ea8cf656c99549cc984b (patch)
tree40e4f5277fb94488e8ad595790ada8daa88ba0c5 /libutil
parent84298d621dc1af8dc07a2b959b2225a1df230708 (diff)
downloadMultiMC-5c07e1b042ecd22dfcd3ea8cf656c99549cc984b.tar
MultiMC-5c07e1b042ecd22dfcd3ea8cf656c99549cc984b.tar.gz
MultiMC-5c07e1b042ecd22dfcd3ea8cf656c99549cc984b.tar.lz
MultiMC-5c07e1b042ecd22dfcd3ea8cf656c99549cc984b.tar.xz
MultiMC-5c07e1b042ecd22dfcd3ea8cf656c99549cc984b.zip
Move job queue to libutil
Diffstat (limited to 'libutil')
-rw-r--r--libutil/CMakeLists.txt3
-rw-r--r--libutil/include/dlqueue.h49
-rw-r--r--libutil/include/jobqueue.h179
-rw-r--r--libutil/src/dlqueue.cpp128
4 files changed, 359 insertions, 0 deletions
diff --git a/libutil/CMakeLists.txt b/libutil/CMakeLists.txt
index caafc756..864c7714 100644
--- a/libutil/CMakeLists.txt
+++ b/libutil/CMakeLists.txt
@@ -32,6 +32,8 @@ include/osutils.h
include/userutils.h
include/cmdutils.h
include/netutils.h
+include/jobqueue.h
+include/dlqueue.h
)
SET(LIBUTIL_SOURCES
@@ -40,6 +42,7 @@ src/osutils.cpp
src/userutils.cpp
src/cmdutils.cpp
src/netutils.cpp
+src/dlqueue.cpp
)
# Set the include dir path.
diff --git a/libutil/include/dlqueue.h b/libutil/include/dlqueue.h
new file mode 100644
index 00000000..9041e762
--- /dev/null
+++ b/libutil/include/dlqueue.h
@@ -0,0 +1,49 @@
+#pragma once
+#include "jobqueue.h"
+#include <QtNetwork>
+
+/**
+ * A single file for the downloader/cache to process.
+ */
+class DownloadJob : public Job
+{
+ Q_OBJECT
+public:
+ DownloadJob(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
+ static JobPtr create(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
+
+public slots:
+ virtual void start();
+
+private slots:
+ void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);;
+ void downloadError(QNetworkReply::NetworkError error);
+ void downloadFinished();
+ void downloadReadyRead();
+
+public:
+ /// the associated network manager
+ QSharedPointer<QNetworkAccessManager> m_manager;
+ /// the network reply
+ QSharedPointer<QNetworkReply> m_reply;
+ /// source URL
+ QUrl m_url;
+
+ /// if true, check the md5sum against a provided md5sum
+ /// also, if a file exists, perform an md5sum first and don't download only if they don't match
+ bool m_check_md5;
+ /// the expected md5 checksum
+ QString m_expected_md5;
+
+ /// save to file?
+ bool m_save_to_file;
+ /// if saving to file, use the one specified in this string
+ QString m_rel_target_path;
+ /// this is the output file, if any
+ QFile m_output_file;
+ /// if not saving to file, downloaded data is placed here
+ QByteArray m_data;
+
+ /// The file's status
+ JobStatus m_status;
+};
diff --git a/libutil/include/jobqueue.h b/libutil/include/jobqueue.h
new file mode 100644
index 00000000..061686f6
--- /dev/null
+++ b/libutil/include/jobqueue.h
@@ -0,0 +1,179 @@
+#pragma once
+#include <QtCore>
+
+enum JobStatus
+{
+ Job_NotStarted,
+ Job_InProgress,
+ Job_Finished,
+ Job_Failed
+};
+
+class JobList;
+
+class Job : public QObject
+{
+ Q_OBJECT
+protected:
+ explicit Job(): QObject(0){};
+public:
+ virtual ~Job() {};
+signals:
+ void finish();
+ void fail();
+ void progress(qint64 current, qint64 total);
+public slots:
+ virtual void start() = 0;
+};
+typedef QSharedPointer<Job> JobPtr;
+
+/**
+ * A list of jobs, to be processed one by one.
+ */
+class JobList : public QObject
+{
+ friend class JobListQueue;
+ Q_OBJECT
+public:
+
+ JobList() : QObject(0)
+ {
+ m_status = Job_NotStarted;
+ current_job_idx = 0;
+ }
+ JobStatus getStatus()
+ {
+ return m_status;
+ }
+ void add(JobPtr dlable)
+ {
+ if(m_status == Job_NotStarted)
+ m_jobs.append(dlable);
+ //else there's a bug. TODO: catch the bugs
+ }
+ JobPtr getFirstJob()
+ {
+ if(m_jobs.size())
+ return m_jobs[0];
+ else
+ return JobPtr();
+ }
+ void start()
+ {
+ current_job_idx = 0;
+ auto job = m_jobs[current_job_idx];
+
+ connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64)));
+ connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished()));
+ connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed()));
+ job->start();
+ emit started();
+ }
+private slots:
+ void currentJobFinished()
+ {
+ if(current_job_idx == m_jobs.size() - 1)
+ {
+ m_status = Job_Finished;
+ emit finished();
+ }
+ else
+ {
+ current_job_idx++;
+ auto job = m_jobs[current_job_idx];
+ connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64)));
+ connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished()));
+ connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed()));
+ job->start();
+ }
+ }
+ void currentJobFailed()
+ {
+ m_status = Job_Failed;
+ emit failed();
+ }
+ void currentJobProgress(qint64 current, qint64 total)
+ {
+ if(!total)
+ return;
+
+ int total_jobs = m_jobs.size();
+
+ if(!total_jobs)
+ return;
+
+ float job_chunk = 1000.0 / float(total_jobs);
+ float cur = current;
+ float tot = total;
+ float last_chunk = (cur / tot) * job_chunk;
+
+ float list_total = job_chunk * current_job_idx + last_chunk;
+ emit progress(qint64(list_total), 1000LL);
+ }
+private:
+ QVector<JobPtr> m_jobs;
+ /// The overall status of this job list
+ JobStatus m_status;
+ int current_job_idx;
+signals:
+ void progress(qint64 current, qint64 total);
+ void started();
+ void finished();
+ void failed();
+};
+typedef QSharedPointer<JobList> JobListPtr;
+
+
+/**
+ * A queue of job lists! The job lists fail or finish as units.
+ */
+class JobListQueue : public QObject
+{
+ Q_OBJECT
+public:
+ JobListQueue(QObject *p = 0):
+ QObject(p),
+ currentIndex(0),
+ is_running(false){}
+
+ void enqueue(JobListPtr job)
+ {
+ jobs.enqueue(job);
+
+ // finish or fail, we should catch that and start the next one
+ connect(job.data(),SIGNAL(finished()), SLOT(startNextJob()));
+ connect(job.data(),SIGNAL(failed()), SLOT(startNextJob()));
+
+ if(!is_running)
+ {
+ QTimer::singleShot(0, this, SLOT(startNextJob()));
+ }
+ }
+
+private slots:
+ void startNextJob()
+ {
+ if (jobs.isEmpty())
+ {
+ currentJobList.clear();
+ currentIndex = 0;
+ is_running = false;
+ emit finishedAllJobs();
+ return;
+ }
+
+ currentJobList = jobs.dequeue();
+ is_running = true;
+ currentIndex = 0;
+ currentJobList->start();
+ }
+
+signals:
+ void finishedAllJobs();
+
+private:
+ JobListPtr currentJobList;
+ QQueue<JobListPtr> jobs;
+ unsigned currentIndex;
+ bool is_running;
+};
diff --git a/libutil/src/dlqueue.cpp b/libutil/src/dlqueue.cpp
new file mode 100644
index 00000000..dfc51f36
--- /dev/null
+++ b/libutil/src/dlqueue.cpp
@@ -0,0 +1,128 @@
+#include "include/dlqueue.h"
+
+DownloadJob::DownloadJob ( QUrl url, QString rel_target_path, QString expected_md5 )
+ :Job()
+{
+ m_url = url;
+ m_rel_target_path = rel_target_path;
+ m_expected_md5 = expected_md5;
+
+ m_check_md5 = m_expected_md5.size();
+ m_save_to_file = m_rel_target_path.size();
+ m_status = Job_NotStarted;
+}
+
+JobPtr DownloadJob::create ( QUrl url, QString rel_target_path, QString expected_md5 )
+{
+ return JobPtr ( new DownloadJob ( url, rel_target_path, expected_md5 ) );
+}
+
+void DownloadJob::start()
+{
+ m_manager.reset ( new QNetworkAccessManager() );
+ if ( m_save_to_file )
+ {
+ QString filename = m_rel_target_path;
+ m_output_file.setFileName ( filename );
+ // if there already is a file and md5 checking is in effect
+ if ( m_output_file.exists() && m_check_md5 )
+ {
+ // and it can be opened
+ if ( m_output_file.open ( QIODevice::ReadOnly ) )
+ {
+ // check the md5 against the expected one
+ QString hash = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
+ m_output_file.close();
+ // skip this file if they match
+ if ( hash == m_expected_md5 )
+ {
+ qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
+ emit finish();
+ return;
+ }
+ }
+ }
+ QFileInfo a ( filename );
+ QDir dir;
+ if ( !dir.mkpath ( a.path() ) )
+ {
+ /*
+ * error when making the folder structure
+ */
+ emit fail();
+ return;
+ }
+ if ( !m_output_file.open ( QIODevice::WriteOnly ) )
+ {
+ /*
+ * Can't open the file... the job failed
+ */
+ emit fail();
+ return;
+ }
+ }
+ qDebug() << "Downloading " << m_url.toString();
+ QNetworkRequest request ( m_url );
+ QNetworkReply * rep = m_manager->get ( request );
+ m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
+ connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
+ connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
+ connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
+ connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
+}
+
+void DownloadJob::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
+{
+ emit progress ( bytesReceived, bytesTotal );
+}
+
+void DownloadJob::downloadError ( QNetworkReply::NetworkError error )
+{
+ // error happened during download.
+ // TODO: log the reason why
+ m_status = Job_Failed;
+}
+
+void DownloadJob::downloadFinished()
+{
+ // if the download succeeded
+ if ( m_status != Job_Failed )
+ {
+ // nothing went wrong...
+ m_status = Job_Finished;
+ // save the data to the downloadable if we aren't saving to file
+ if ( !m_save_to_file )
+ {
+ m_data = m_reply->readAll();
+ }
+ else
+ {
+ m_output_file.close();
+ }
+
+ //TODO: check md5 here!
+ m_reply.clear();
+ emit finish();
+ return;
+ }
+ // else the download failed
+ else
+ {
+ if ( m_save_to_file )
+ {
+ m_output_file.close();
+ m_output_file.remove();
+ }
+ m_reply.clear();
+ emit fail();
+ return;
+ }
+}
+
+void DownloadJob::downloadReadyRead()
+{
+ if ( m_save_to_file )
+ {
+ m_output_file.write ( m_reply->readAll() );
+ }
+}