summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPetr Mrázek <peterix@gmail.com>2013-09-08 02:15:20 +0200
committerPetr Mrázek <peterix@gmail.com>2013-09-08 02:15:20 +0200
commit6bea4ec988b7caeac63353fb9d2a354f2fd47dad (patch)
tree77dc597c82ff3952d9f34f4ca13da1c36ca89bd8
parent6892c11e9f287dcfb1e698f8f46233a01fb7abb6 (diff)
downloadMultiMC-6bea4ec988b7caeac63353fb9d2a354f2fd47dad.tar
MultiMC-6bea4ec988b7caeac63353fb9d2a354f2fd47dad.tar.gz
MultiMC-6bea4ec988b7caeac63353fb9d2a354f2fd47dad.tar.lz
MultiMC-6bea4ec988b7caeac63353fb9d2a354f2fd47dad.tar.xz
MultiMC-6bea4ec988b7caeac63353fb9d2a354f2fd47dad.zip
Use HttpMetaCache to minimize network use.
-rw-r--r--CMakeLists.txt7
-rw-r--r--MultiMC.cpp21
-rw-r--r--MultiMC.h8
-rw-r--r--logic/LegacyUpdate.cpp4
-rw-r--r--logic/OneSixAssets.cpp29
-rw-r--r--logic/OneSixUpdate.cpp20
-rw-r--r--logic/net/ByteArrayDownload.cpp62
-rw-r--r--logic/net/ByteArrayDownload.h24
-rw-r--r--logic/net/CacheDownload.cpp122
-rw-r--r--logic/net/CacheDownload.h34
-rw-r--r--logic/net/Download.h54
-rw-r--r--logic/net/DownloadJob.cpp146
-rw-r--r--logic/net/DownloadJob.h95
-rw-r--r--logic/net/FileDownload.cpp111
-rw-r--r--logic/net/FileDownload.h35
-rw-r--r--logic/net/HttpMetaCache.cpp157
-rw-r--r--logic/net/HttpMetaCache.h33
17 files changed, 693 insertions, 269 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 27f8eaba..cbc3d842 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -184,6 +184,10 @@ logic/ModList.h
logic/InstanceLauncher.h
# network stuffs
+logic/net/Download.h
+logic/net/FileDownload.h
+logic/net/ByteArrayDownload.h
+logic/net/CacheDownload.h
logic/net/DownloadJob.h
logic/net/HttpMetaCache.h
@@ -252,6 +256,9 @@ logic/ModList.cpp
logic/InstanceLauncher.cpp
# network stuffs - to be moved into a depend lib ~_~
+logic/net/FileDownload.cpp
+logic/net/ByteArrayDownload.cpp
+logic/net/CacheDownload.cpp
logic/net/DownloadJob.cpp
logic/net/HttpMetaCache.cpp
diff --git a/MultiMC.cpp b/MultiMC.cpp
index 88eb3e62..f212d830 100644
--- a/MultiMC.cpp
+++ b/MultiMC.cpp
@@ -8,6 +8,7 @@
#include "logic/lists/InstanceList.h"
#include "logic/lists/IconList.h"
#include "logic/InstanceLauncher.h"
+#include "logic/net/HttpMetaCache.h"
#include "pathutils.h"
@@ -112,11 +113,16 @@ MultiMC::MultiMC ( int& argc, char** argv )
// load settings
initGlobalSettings();
+
// and instances
m_instances = new InstanceList(m_settings->get("InstanceDir").toString(),this);
std::cout << "Loading Instances..." << std::endl;
m_instances->loadList();
- // network manager
+
+ // init the http meta cache
+ initHttpMetaCache();
+
+ // create the global network manager
m_qnam = new QNetworkAccessManager(this);
// Register meta types.
@@ -137,11 +143,12 @@ MultiMC::MultiMC ( int& argc, char** argv )
MultiMC::~MultiMC()
{
delete m_settings;
+ delete m_metacache;
}
void MultiMC::initGlobalSettings()
{
- m_settings = new INISettingsObject(applicationDirPath() + "/multimc.cfg", this);
+ m_settings = new INISettingsObject("multimc.cfg", this);
// Updates
m_settings->registerSetting(new Setting("UseDevBuilds", false));
m_settings->registerSetting(new Setting("AutoUpdate", true));
@@ -189,6 +196,16 @@ void MultiMC::initGlobalSettings()
m_settings->registerSetting(new Setting("TheCat", false));
}
+void MultiMC::initHttpMetaCache()
+{
+ m_metacache = new HttpMetaCache("metacache");
+ m_metacache->addBase("assets", QDir("assets").absolutePath());
+ m_metacache->addBase("versions", QDir("versions").absolutePath());
+ m_metacache->addBase("libraries", QDir("libraries").absolutePath());
+ m_metacache->Load();
+}
+
+
IconList* MultiMC::icons()
{
if ( !m_icons )
diff --git a/MultiMC.h b/MultiMC.h
index 99d90b99..0c8b673b 100644
--- a/MultiMC.h
+++ b/MultiMC.h
@@ -4,6 +4,7 @@
#include "MultiMCVersion.h"
#include "config.h"
+class HttpMetaCache;
class SettingsObject;
class InstanceList;
class IconList;
@@ -55,14 +56,21 @@ public:
{
return m_qnam;
}
+
+ HttpMetaCache * metacache()
+ {
+ return m_metacache;
+ }
private:
void initGlobalSettings();
+ void initHttpMetaCache();
private:
SettingsObject * m_settings = nullptr;
InstanceList * m_instances = nullptr;
IconList * m_icons = nullptr;
QNetworkAccessManager * m_qnam = nullptr;
+ HttpMetaCache * m_metacache = nullptr;
Status m_status = MultiMC::Failed;
MultiMCVersion m_version = {VERSION_MAJOR, VERSION_MINOR, VERSION_REVISION, VERSION_BUILD};
}; \ No newline at end of file
diff --git a/logic/LegacyUpdate.cpp b/logic/LegacyUpdate.cpp
index 626ad1e0..c75a0011 100644
--- a/logic/LegacyUpdate.cpp
+++ b/logic/LegacyUpdate.cpp
@@ -227,7 +227,9 @@ void LegacyUpdate::jarStart()
QString intended_version_id = inst->intendedVersionId();
urlstr += intended_version_id + "/" + intended_version_id + ".jar";
- legacyDownloadJob.reset(new DownloadJob(QUrl(urlstr), inst->defaultBaseJar()));
+ auto dljob = new DownloadJob("Minecraft.jar for version " + intended_version_id);
+ dljob->add(QUrl(urlstr), inst->defaultBaseJar());
+ legacyDownloadJob.reset();
connect(legacyDownloadJob.data(), SIGNAL(finished()), SLOT(jarFinished()));
connect(legacyDownloadJob.data(), SIGNAL(failed()), SLOT(jarFailed()));
connect(legacyDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64)));
diff --git a/logic/OneSixAssets.cpp b/logic/OneSixAssets.cpp
index c65ee607..5bdd29d7 100644
--- a/logic/OneSixAssets.cpp
+++ b/logic/OneSixAssets.cpp
@@ -3,6 +3,8 @@
#include <QtXml/QtXml>
#include "OneSixAssets.h"
#include "net/DownloadJob.h"
+#include "net/HttpMetaCache.h"
+#include "MultiMC.h"
inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
{
@@ -65,7 +67,7 @@ void OneSixAssets::fetchXMLFinished()
nuke_whitelist.clear();
auto firstJob = index_job->first();
- QByteArray ba = firstJob->m_data;
+ QByteArray ba = firstJob.dynamicCast<ByteArrayDownload>()->m_data;
QString xmlErrorMsg;
QDomDocument doc;
@@ -76,10 +78,12 @@ void OneSixAssets::fetchXMLFinished()
//QRegExp etag_match(".*([a-f0-9]{32}).*");
QDomNodeList contents = doc.elementsByTagName ( "Contents" );
- DownloadJob *job = new DownloadJob();
+ DownloadJob *job = new DownloadJob("Assets");
connect ( job, SIGNAL(succeeded()), SLOT(downloadFinished()) );
connect ( job, SIGNAL(failed()), SIGNAL(failed()) );
+ auto metacache = MMC->metacache();
+
for ( int i = 0; i < contents.length(); i++ )
{
QDomElement element = contents.at ( i ).toElement();
@@ -104,22 +108,12 @@ void OneSixAssets::fetchXMLFinished()
if ( sizeStr == "0" )
continue;
- QString filename = fprefix + keyStr;
- QFile check_file ( filename );
- QString client_etag = "nonsense";
- // if there already is a file and md5 checking is in effect and it can be opened
- if ( check_file.exists() && check_file.open ( QIODevice::ReadOnly ) )
- {
- // check the md5 against the expected one
- client_etag = QCryptographicHash::hash ( check_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
- check_file.close();
- }
-
- QString trimmedEtag = etagStr.remove ( '"' );
nuke_whitelist.append ( keyStr );
- if(trimmedEtag != client_etag)
+
+ auto entry = metacache->resolveEntry("assets", keyStr, etagStr);
+ if(entry->stale)
{
- job->add ( QUrl ( prefix + keyStr ), filename );
+ job->add(QUrl(prefix + keyStr), entry);
}
}
if(job->size())
@@ -135,7 +129,8 @@ void OneSixAssets::fetchXMLFinished()
}
void OneSixAssets::start()
{
- DownloadJob * job = new DownloadJob(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" ));
+ auto job = new DownloadJob("Assets index");
+ job->add(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" ));
connect ( job, SIGNAL(succeeded()), SLOT ( fetchXMLFinished() ) );
index_job.reset ( job );
job->start();
diff --git a/logic/OneSixUpdate.cpp b/logic/OneSixUpdate.cpp
index 428d6ef7..ce71bde0 100644
--- a/logic/OneSixUpdate.cpp
+++ b/logic/OneSixUpdate.cpp
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+#include "MultiMC.h"
#include "OneSixUpdate.h"
#include <QtNetwork>
@@ -72,7 +72,9 @@ void OneSixUpdate::versionFileStart()
QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/");
urlstr += targetVersion->descriptor + "/" + targetVersion->descriptor + ".json";
- specificVersionDownloadJob.reset(new DownloadJob(QUrl(urlstr)));
+ auto job = new DownloadJob("Version index");
+ job->add(QUrl(urlstr));
+ specificVersionDownloadJob.reset(job);
connect(specificVersionDownloadJob.data(), SIGNAL(succeeded()), SLOT(versionFileFinished()));
connect(specificVersionDownloadJob.data(), SIGNAL(failed()), SLOT(versionFileFailed()));
connect(specificVersionDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64)));
@@ -92,7 +94,7 @@ void OneSixUpdate::versionFileFinished()
// FIXME: detect errors here, download to a temp file, swap
QFile vfile1 (version1);
vfile1.open(QIODevice::Truncate | QIODevice::WriteOnly );
- vfile1.write(DlJob->m_data);
+ vfile1.write(DlJob.dynamicCast<ByteArrayDownload>()->m_data);
vfile1.close();
}
@@ -134,16 +136,22 @@ void OneSixUpdate::jarlibStart()
QString targetstr ("versions/");
targetstr += version->id + "/" + version->id + ".jar";
- jarlibDownloadJob.reset(new DownloadJob(QUrl(urlstr), targetstr));
+ auto job = new DownloadJob("Libraries for instance " + inst->name());
+ job->add(QUrl(urlstr), targetstr);
+ jarlibDownloadJob.reset(job);
auto libs = version->getActiveNativeLibs();
libs.append(version->getActiveNormalLibs());
+ auto metacache = MMC->metacache();
for(auto lib: libs)
{
QString download_path = lib->downloadPath();
- QString storage_path = "libraries/" + lib->storagePath();
- jarlibDownloadJob->add(download_path, storage_path);
+ auto entry = metacache->resolveEntry("libraries", lib->storagePath());
+ if(entry->stale)
+ {
+ jarlibDownloadJob->add(download_path, entry);
+ }
}
connect(jarlibDownloadJob.data(), SIGNAL(succeeded()), SLOT(jarlibFinished()));
connect(jarlibDownloadJob.data(), SIGNAL(failed()), SLOT(jarlibFailed()));
diff --git a/logic/net/ByteArrayDownload.cpp b/logic/net/ByteArrayDownload.cpp
new file mode 100644
index 00000000..6ae3f121
--- /dev/null
+++ b/logic/net/ByteArrayDownload.cpp
@@ -0,0 +1,62 @@
+#include "ByteArrayDownload.h"
+#include "MultiMC.h"
+#include <QDebug>
+
+ByteArrayDownload::ByteArrayDownload ( QUrl url )
+ : Download()
+{
+ m_url = url;
+ m_status = Job_NotStarted;
+}
+
+void ByteArrayDownload::start()
+{
+ qDebug() << "Downloading " << m_url.toString();
+ QNetworkRequest request ( m_url );
+ auto worker = MMC->qnam();
+ QNetworkReply * rep = worker->get ( request );
+
+ m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
+ connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
+ connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
+ connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
+ connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
+}
+
+void ByteArrayDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
+{
+ emit progress (index_within_job, bytesReceived, bytesTotal );
+}
+
+void ByteArrayDownload::downloadError ( QNetworkReply::NetworkError error )
+{
+ // error happened during download.
+ // TODO: log the reason why
+ m_status = Job_Failed;
+}
+
+void ByteArrayDownload::downloadFinished()
+{
+ // if the download succeeded
+ if ( m_status != Job_Failed )
+ {
+ // nothing went wrong...
+ m_status = Job_Finished;
+ m_data = m_reply->readAll();
+ m_reply.clear();
+ emit succeeded(index_within_job);
+ return;
+ }
+ // else the download failed
+ else
+ {
+ m_reply.clear();
+ emit failed(index_within_job);
+ return;
+ }
+}
+
+void ByteArrayDownload::downloadReadyRead()
+{
+ // ~_~
+}
diff --git a/logic/net/ByteArrayDownload.h b/logic/net/ByteArrayDownload.h
new file mode 100644
index 00000000..428b21db
--- /dev/null
+++ b/logic/net/ByteArrayDownload.h
@@ -0,0 +1,24 @@
+#pragma once
+#include "Download.h"
+
+class ByteArrayDownload: public Download
+{
+ Q_OBJECT
+public:
+ ByteArrayDownload(QUrl url);
+
+public:
+ /// if not saving to file, downloaded data is placed here
+ QByteArray m_data;
+
+public slots:
+ virtual void start();
+
+protected slots:
+ void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
+ void downloadError(QNetworkReply::NetworkError error);
+ void downloadFinished();
+ void downloadReadyRead();
+};
+
+typedef QSharedPointer<ByteArrayDownload> ByteArrayDownloadPtr;
diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp
new file mode 100644
index 00000000..c0074574
--- /dev/null
+++ b/logic/net/CacheDownload.cpp
@@ -0,0 +1,122 @@
+#include "MultiMC.h"
+#include "CacheDownload.h"
+#include <pathutils.h>
+
+#include <QCryptographicHash>
+#include <QFileInfo>
+#include <QDateTime>
+#include <QDebug>
+
+CacheDownload::CacheDownload (QUrl url, MetaEntryPtr entry )
+ :Download(), md5sum(QCryptographicHash::Md5)
+{
+ m_url = url;
+ m_entry = entry;
+ m_target_path = entry->getFullPath();
+ m_status = Job_NotStarted;
+ m_opened_for_saving = false;
+}
+
+void CacheDownload::start()
+{
+ if(!m_entry->stale)
+ {
+ emit succeeded(index_within_job);
+ return;
+ }
+ m_output_file.setFileName(m_target_path);
+ // if there already is a file and md5 checking is in effect and it can be opened
+ if(!ensureFilePathExists(m_target_path))
+ {
+ emit failed(index_within_job);
+ return;
+ }
+ qDebug() << "Downloading " << m_url.toString();
+ QNetworkRequest request ( m_url );
+ request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1());
+
+ auto worker = MMC->qnam();
+ QNetworkReply * rep = worker->get ( request );
+
+ m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
+ connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
+ connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
+ connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
+ connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
+}
+
+void CacheDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
+{
+ emit progress (index_within_job, bytesReceived, bytesTotal );
+}
+
+void CacheDownload::downloadError ( QNetworkReply::NetworkError error )
+{
+ // error happened during download.
+ // TODO: log the reason why
+ m_status = Job_Failed;
+}
+void CacheDownload::downloadFinished()
+{
+ // if the download succeeded
+ if ( m_status != Job_Failed )
+ {
+
+ // nothing went wrong...
+ m_status = Job_Finished;
+ if(m_opened_for_saving)
+ {
+ // save the data to the downloadable if we aren't saving to file
+ m_output_file.close();
+ m_entry->md5sum = md5sum.result().toHex().constData();
+ }
+ else
+ {
+ if ( m_output_file.open ( QIODevice::ReadOnly ) )
+ {
+ m_entry->md5sum = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
+ m_output_file.close();
+ }
+ }
+ QFileInfo output_file_info(m_target_path);
+
+
+ m_entry->etag = m_reply->rawHeader("ETag").constData();
+ m_entry->last_changed_timestamp = output_file_info.lastModified().toUTC().toMSecsSinceEpoch();
+ m_entry->stale = false;
+ MMC->metacache()->updateEntry(m_entry);
+
+ m_reply.clear();
+ emit succeeded(index_within_job);
+ return;
+ }
+ // else the download failed
+ else
+ {
+ m_output_file.close();
+ m_output_file.remove();
+ m_reply.clear();
+ emit failed(index_within_job);
+ return;
+ }
+}
+
+void CacheDownload::downloadReadyRead()
+{
+ if(!m_opened_for_saving)
+ {
+ if ( !m_output_file.open ( QIODevice::WriteOnly ) )
+ {
+ /*
+ * Can't open the file... the job failed
+ */
+ m_reply->abort();
+ emit failed(index_within_job);
+ return;
+ }
+ m_opened_for_saving = true;
+ }
+ QByteArray ba = m_reply->readAll();
+ md5sum.addData(ba);
+ m_output_file.write ( ba );
+}
diff --git a/logic/net/CacheDownload.h b/logic/net/CacheDownload.h
new file mode 100644
index 00000000..f95dabd5
--- /dev/null
+++ b/logic/net/CacheDownload.h
@@ -0,0 +1,34 @@
+#pragma once
+
+#include "Download.h"
+#include "HttpMetaCache.h"
+#include <QFile>
+#include <qcryptographichash.h>
+
+class CacheDownload : public Download
+{
+ Q_OBJECT
+public:
+ MetaEntryPtr m_entry;
+ /// is the saving file already open?
+ bool m_opened_for_saving;
+ /// if saving to file, use the one specified in this string
+ QString m_target_path;
+ /// this is the output file, if any
+ QFile m_output_file;
+ /// the hash-as-you-download
+ QCryptographicHash md5sum;
+public:
+ explicit CacheDownload(QUrl url, MetaEntryPtr entry);
+
+protected slots:
+ virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
+ virtual void downloadError(QNetworkReply::NetworkError error);
+ virtual void downloadFinished();
+ virtual void downloadReadyRead();
+
+public slots:
+ virtual void start();
+};
+
+typedef QSharedPointer<CacheDownload> CacheDownloadPtr;
diff --git a/logic/net/Download.h b/logic/net/Download.h
new file mode 100644
index 00000000..91f09dec
--- /dev/null
+++ b/logic/net/Download.h
@@ -0,0 +1,54 @@
+#pragma once
+
+#include <QObject>
+#include <QUrl>
+#include <QSharedPointer>
+#include <QNetworkReply>
+
+
+enum JobStatus
+{
+ Job_NotStarted,
+ Job_InProgress,
+ Job_Finished,
+ Job_Failed
+};
+
+class Download : public QObject
+{
+ Q_OBJECT
+protected:
+ explicit Download(): QObject(0){};
+public:
+ virtual ~Download() {};
+
+public:
+ /// the network reply
+ QSharedPointer<QNetworkReply> m_reply;
+
+ /// source URL
+ QUrl m_url;
+
+ /// The file's status
+ JobStatus m_status;
+
+ /// index within the parent job
+ int index_within_job = 0;
+
+signals:
+ void started(int index);
+ void progress(int index, qint64 current, qint64 total);
+ void succeeded(int index);
+ void failed(int index);
+
+protected slots:
+ virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0;
+ virtual void downloadError(QNetworkReply::NetworkError error) = 0;
+ virtual void downloadFinished() = 0;
+ virtual void downloadReadyRead() = 0;
+
+public slots:
+ virtual void start() = 0;
+};
+
+typedef QSharedPointer<Download> DownloadPtr;
diff --git a/logic/net/DownloadJob.cpp b/logic/net/DownloadJob.cpp
index cad9ae72..cbc2c5fe 100644
--- a/logic/net/DownloadJob.cpp
+++ b/logic/net/DownloadJob.cpp
@@ -1,136 +1,29 @@
#include "DownloadJob.h"
#include "pathutils.h"
#include "MultiMC.h"
+#include "FileDownload.h"
+#include "ByteArrayDownload.h"
+#include "CacheDownload.h"
-Download::Download (QUrl url, QString target_path, QString expected_md5 )
- :Job()
+ByteArrayDownloadPtr DownloadJob::add ( QUrl url )
{
- m_url = url;
- m_target_path = target_path;
- m_expected_md5 = expected_md5;
-
- m_check_md5 = m_expected_md5.size();
- m_save_to_file = m_target_path.size();
- m_status = Job_NotStarted;
- m_opened_for_saving = false;
-}
-
-void Download::start()
-{
- if ( m_save_to_file )
- {
- QString filename = m_target_path;
- m_output_file.setFileName ( filename );
- // if there already is a file and md5 checking is in effect and it can be opened
- if ( m_output_file.exists() && m_output_file.open ( QIODevice::ReadOnly ) )
- {
- // check the md5 against the expected one
- QString hash = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
- m_output_file.close();
- // skip this file if they match
- if ( m_check_md5 && hash == m_expected_md5 )
- {
- qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
- emit succeeded(index_within_job);
- return;
- }
- else
- {
- m_expected_md5 = hash;
- }
- }
- if(!ensureFilePathExists(filename))
- {
- emit failed(index_within_job);
- return;
- }
- }
- qDebug() << "Downloading " << m_url.toString();
- QNetworkRequest request ( m_url );
- request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1());
-
- auto worker = MMC->qnam();
- QNetworkReply * rep = worker->get ( request );
-
- m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
- connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
- connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
- connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
- connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
-}
-
-void Download::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
-{
- emit progress (index_within_job, bytesReceived, bytesTotal );
-}
-
-void Download::downloadError ( QNetworkReply::NetworkError error )
-{
- // error happened during download.
- // TODO: log the reason why
- m_status = Job_Failed;
-}
-
-void Download::downloadFinished()
-{
- // if the download succeeded
- if ( m_status != Job_Failed )
- {
- // nothing went wrong...
- m_status = Job_Finished;
- // save the data to the downloadable if we aren't saving to file
- if ( !m_save_to_file )
- {
- m_data = m_reply->readAll();
- }
- else
- {
- m_output_file.close();
- }
-
- //TODO: check md5 here!
- m_reply.clear();
- emit succeeded(index_within_job);
- return;
- }
- // else the download failed
- else
- {
- if ( m_save_to_file )
- {
- m_output_file.close();
- m_output_file.remove();
- }
- m_reply.clear();
- emit failed(index_within_job);
- return;
- }
+ ByteArrayDownloadPtr ptr (new ByteArrayDownload(url));
+ ptr->index_within_job = downloads.size();
+ downloads.append(ptr);
+ return ptr;
}
-void Download::downloadReadyRead()
+FileDownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path)
{
- if( m_save_to_file )
- {
- if(!m_opened_for_saving)
- {
- if ( !m_output_file.open ( QIODevice::WriteOnly ) )
- {
- /*
- * Can't open the file... the job failed
- */
- m_reply->abort();
- emit failed(index_within_job);
- return;
- }
- m_opened_for_saving = true;
- }
- m_output_file.write ( m_reply->readAll() );
- }
+ FileDownloadPtr ptr (new FileDownload(url, rel_target_path));
+ ptr->index_within_job = downloads.size();
+ downloads.append(ptr);
+ return ptr;
}
-DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expected_md5 )
+CacheDownloadPtr DownloadJob::add ( QUrl url, MetaEntryPtr entry)
{
- DownloadPtr ptr (new Download(url, rel_target_path, expected_md5));
+ CacheDownloadPtr ptr (new CacheDownload(url, entry));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
return ptr;
@@ -139,16 +32,17 @@ DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expect
void DownloadJob::partSucceeded ( int index )
{
num_succeeded++;
+ qDebug() << m_job_name.toLocal8Bit() << " progress: " << num_succeeded << "/" << downloads.size();
if(num_failed + num_succeeded == downloads.size())
{
if(num_failed)
{
- qDebug() << "Download JOB failed: " << this;
+ qDebug() << m_job_name.toLocal8Bit() << " failed.";
emit failed();
}
else
{
- qDebug() << "Download JOB succeeded: " << this;
+ qDebug() << m_job_name.toLocal8Bit() << " succeeded.";
emit succeeded();
}
}
@@ -159,7 +53,7 @@ void DownloadJob::partFailed ( int index )
num_failed++;
if(num_failed + num_succeeded == downloads.size())
{
- qDebug() << "Download JOB failed: " << this;
+ qDebug() << m_job_name.toLocal8Bit() << " failed.";
emit failed();
}
}
@@ -172,7 +66,7 @@ void DownloadJob::partProgress ( int index, qint64 bytesReceived, qint64 bytesTo
void DownloadJob::start()
{
- qDebug() << "Download JOB started: " << this;
+ qDebug() << m_job_name.toLocal8Bit() << " started.";
for(auto iter: downloads)
{
connect(iter.data(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));
diff --git a/logic/net/DownloadJob.h b/logic/net/DownloadJob.h
index adeef646..71466282 100644
--- a/logic/net/DownloadJob.h
+++ b/logic/net/DownloadJob.h
@@ -1,98 +1,28 @@
#pragma once
#include <QtNetwork>
+#include "Download.h"
+#include "ByteArrayDownload.h"
+#include "FileDownload.h"
+#include "CacheDownload.h"
+#include "HttpMetaCache.h"
class DownloadJob;
-class Download;
typedef QSharedPointer<DownloadJob> DownloadJobPtr;
-typedef QSharedPointer<Download> DownloadPtr;
-
-enum JobStatus
-{
- Job_NotStarted,
- Job_InProgress,
- Job_Finished,
- Job_Failed
-};
-
-class Job : public QObject
-{
- Q_OBJECT
-protected:
- explicit Job(): QObject(0){};
-public:
- virtual ~Job() {};
-
-public slots:
- virtual void start() = 0;
-};
-
-class Download: public Job
-{
- friend class DownloadJob;
- Q_OBJECT
-protected:
- Download(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
-public:
- /// the network reply
- QSharedPointer<QNetworkReply> m_reply;
- /// source URL
- QUrl m_url;
-
- /// if true, check the md5sum against a provided md5sum
- /// also, if a file exists, perform an md5sum first and don't download only if they don't match
- bool m_check_md5;
- /// the expected md5 checksum
- QString m_expected_md5;
-
- /// save to file?
- bool m_save_to_file;
- /// is the saving file already open?
- bool m_opened_for_saving;
- /// if saving to file, use the one specified in this string
- QString m_target_path;
- /// this is the output file, if any
- QFile m_output_file;
- /// if not saving to file, downloaded data is placed here
- QByteArray m_data;
-
- int currentProgress = 0;
- int totalProgress = 0;
-
- /// The file's status
- JobStatus m_status;
-
- int index_within_job = 0;
-signals:
- void started(int index);
- void progress(int index, qint64 current, qint64 total);
- void succeeded(int index);
- void failed(int index);
-public slots:
- virtual void start();
-
-private slots:
- void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);;
- void downloadError(QNetworkReply::NetworkError error);
- void downloadFinished();
- void downloadReadyRead();
-};
/**
* A single file for the downloader/cache to process.
*/
-class DownloadJob : public Job
+class DownloadJob : public QObject
{
Q_OBJECT
public:
- explicit DownloadJob()
- :Job(){};
- explicit DownloadJob(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString())
- :Job()
- {
- add(url, rel_target_path, expected_md5);
- };
+ explicit DownloadJob(QString job_name)
+ :QObject(), m_job_name(job_name){};
+
+ ByteArrayDownloadPtr add(QUrl url);
+ FileDownloadPtr add(QUrl url, QString rel_target_path);
+ CacheDownloadPtr add(QUrl url, MetaEntryPtr entry);
- DownloadPtr add(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
DownloadPtr operator[](int index)
{
return downloads[index];
@@ -119,6 +49,7 @@ private slots:
void partSucceeded(int index);
void partFailed(int index);
private:
+ QString m_job_name;
QList<DownloadPtr> downloads;
int num_succeeded = 0;
int num_failed = 0;
diff --git a/logic/net/FileDownload.cpp b/logic/net/FileDownload.cpp
new file mode 100644
index 00000000..aad4a991
--- /dev/null
+++ b/logic/net/FileDownload.cpp
@@ -0,0 +1,111 @@
+#include "MultiMC.h"
+#include "FileDownload.h"
+#include <pathutils.h>
+#include <QCryptographicHash>
+#include <QDebug>
+
+
+FileDownload::FileDownload ( QUrl url, QString target_path )
+ :Download()
+{
+ m_url = url;
+ m_target_path = target_path;
+ m_check_md5 = false;
+ m_status = Job_NotStarted;
+ m_opened_for_saving = false;
+}
+
+void FileDownload::start()
+{
+ QString filename = m_target_path;
+ m_output_file.setFileName ( filename );
+ // if there already is a file and md5 checking is in effect and it can be opened
+ if ( m_output_file.exists() && m_output_file.open ( QIODevice::ReadOnly ) )
+ {
+ // check the md5 against the expected one
+ QString hash = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
+ m_output_file.close();
+ // skip this file if they match
+ if ( m_check_md5 && hash == m_expected_md5 )
+ {
+ qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
+ emit succeeded(index_within_job);
+ return;
+ }
+ else
+ {
+ m_expected_md5 = hash;
+ }
+ }
+ if(!ensureFilePathExists(filename))
+ {
+ emit failed(index_within_job);
+ return;
+ }
+
+ qDebug() << "Downloading " << m_url.toString();
+ QNetworkRequest request ( m_url );
+ request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1());
+
+ auto worker = MMC->qnam();
+ QNetworkReply * rep = worker->get ( request );
+
+ m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
+ connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
+ connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
+ connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
+ connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
+}
+
+void FileDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
+{
+ emit progress (index_within_job, bytesReceived, bytesTotal );
+}
+
+void FileDownload::downloadError ( QNetworkReply::NetworkError error )
+{
+ // error happened during download.
+ // TODO: log the reason why
+ m_status = Job_Failed;
+}
+
+void FileDownload::downloadFinished()
+{
+ // if the download succeeded
+ if ( m_status != Job_Failed )
+ {
+ // nothing went wrong...
+ m_status = Job_Finished;
+ m_output_file.close();
+
+ m_reply.clear();
+ emit succeeded(index_within_job);
+ return;
+ }
+ // else the download failed
+ else
+ {
+ m_output_file.close();
+ m_reply.clear();
+ emit failed(index_within_job);
+ return;
+ }
+}
+
+void FileDownload::downloadReadyRead()
+{
+ if(!m_opened_for_saving)
+ {
+ if ( !m_output_file.open ( QIODevice::WriteOnly ) )
+ {
+ /*
+ * Can't open the file... the job failed
+ */
+ m_reply->abort();
+ emit failed(index_within_job);
+ return;
+ }
+ m_opened_for_saving = true;
+ }
+ m_output_file.write ( m_reply->readAll() );
+}
diff --git a/logic/net/FileDownload.h b/logic/net/FileDownload.h
new file mode 100644
index 00000000..190bee58
--- /dev/null
+++ b/logic/net/FileDownload.h
@@ -0,0 +1,35 @@
+#pragma once
+
+#include "Download.h"
+#include <QFile>
+
+class FileDownload : public Download
+{
+ Q_OBJECT
+public:
+ /// if true, check the md5sum against a provided md5sum
+ /// also, if a file exists, perform an md5sum first and don't download only if they don't match
+ bool m_check_md5;
+ /// the expected md5 checksum
+ QString m_expected_md5;
+ /// is the saving file already open?
+ bool m_opened_for_saving;
+ /// if saving to file, use the one specified in this string
+ QString m_target_path;
+ /// this is the output file, if any
+ QFile m_output_file;
+
+public:
+ explicit FileDownload(QUrl url, QString target_path);
+
+protected slots:
+ virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
+ virtual void downloadError(QNetworkReply::NetworkError error);
+ virtual void downloadFinished();
+ virtual void downloadReadyRead();
+
+public slots:
+ virtual void start();
+};
+
+typedef QSharedPointer<FileDownload> FileDownloadPtr;
diff --git a/logic/net/HttpMetaCache.cpp b/logic/net/HttpMetaCache.cpp
index 87741dc9..50a1136e 100644
--- a/logic/net/HttpMetaCache.cpp
+++ b/logic/net/HttpMetaCache.cpp
@@ -1,38 +1,136 @@
+#include "MultiMC.h"
#include "HttpMetaCache.h"
#include <pathutils.h>
+
+#include <QFileInfo>
#include <QFile>
-#include <qjsondocument.h>
-#include <qjsonarray.h>
-#include <qjsonobject.h>
-#include <qfileinfo.h>
-#include <qtemporaryfile.h>
-#include <qsavefile.h>
+#include <QTemporaryFile>
+#include <QSaveFile>
+#include <QDateTime>
+#include <QCryptographicHash>
+
+#include <QDebug>
+
+#include <QJsonDocument>
+#include <QJsonArray>
+#include <QJsonObject>
+
+QString MetaEntry::getFullPath()
+{
+ return PathCombine(MMC->metacache()->getBasePath(base), path);
+}
+
HttpMetaCache::HttpMetaCache(QString path)
+ :QObject()
{
m_index_file = path;
+ saveBatchingTimer.setSingleShot(true);
+ saveBatchingTimer.setTimerType(Qt::VeryCoarseTimer);
+ connect(&saveBatchingTimer,SIGNAL(timeout()),SLOT(SaveNow()));
}
+
HttpMetaCache::~HttpMetaCache()
{
- Save();
+ saveBatchingTimer.stop();
+ SaveNow();
}
-void HttpMetaCache::addEntry ( QString base, QString resource_path, QString etag )
+void HttpMetaCache::SaveEventually()
+{
+ saveBatchingTimer.stop();
+ saveBatchingTimer.start(30000);
+}
+
+MetaEntryPtr HttpMetaCache::getEntry ( QString base, QString resource_path )
{
// no base. no base path. can't store
if(!m_entries.contains(base))
- return;
- QString real_path = PathCombine(m_entries[base].base_path, resource_path);
+ {
+ // TODO: log problem
+ return MetaEntryPtr();
+ }
+ EntryMap & map = m_entries[base];
+ if(map.entry_list.contains(resource_path))
+ {
+ return map.entry_list[resource_path];
+ }
+ return MetaEntryPtr();
+}
+
+MetaEntryPtr HttpMetaCache::resolveEntry ( QString base, QString resource_path, QString expected_etag )
+{
+ auto entry = getEntry(base, resource_path);
+ // it's not present? generate a default stale entry
+ if(!entry)
+ {
+ return staleEntry(base, resource_path);
+ }
+
+ auto & selected_base = m_entries[base];
+ QString real_path = PathCombine(selected_base.base_path, resource_path);
QFileInfo finfo(real_path);
- // just ignore it, it's garbage if it's not a proper file
+ // is the file really there? if not -> stale
if(!finfo.isFile() || !finfo.isReadable())
{
- // TODO: log problem
- return;
+ // if the file doesn't exist, we disown the entry
+ selected_base.entry_list.remove(resource_path);
+ return staleEntry(base, resource_path);
+ }
+
+ if(!expected_etag.isEmpty() && expected_etag != entry->etag)
+ {
+ // if the etag doesn't match expected, we disown the entry
+ selected_base.entry_list.remove(resource_path);
+ return staleEntry(base, resource_path);
}
- Save();
+ // if the file changed, check md5sum
+ qint64 file_last_changed = finfo.lastModified().toUTC().toMSecsSinceEpoch();
+ if(file_last_changed != entry->last_changed_timestamp)
+ {
+ QFile input(real_path);
+ input.open(QIODevice::ReadOnly);
+ QString md5sum = QCryptographicHash::hash(input.readAll(), QCryptographicHash::Md5).toHex().constData();
+ if(entry->md5sum != md5sum)
+ {
+ selected_base.entry_list.remove(resource_path);
+ return staleEntry(base, resource_path);
+ }
+ // md5sums matched... keep entry and save the new state to file
+ entry->last_changed_timestamp = file_last_changed;
+ SaveEventually();
+ }
+
+ // entry passed all the checks we cared about.
+ return entry;
+}
+
+bool HttpMetaCache::updateEntry ( MetaEntryPtr stale_entry )
+{
+ if(!m_entries.contains(stale_entry->base))
+ {
+ qDebug() << "Cannot add entry with unknown base: " << stale_entry->base.toLocal8Bit();
+ return false;
+ }
+ if(stale_entry->stale)
+ {
+ qDebug() << "Cannot add stale entry: " << stale_entry->getFullPath().toLocal8Bit();
+ return false;
+ }
+ m_entries[stale_entry->base].entry_list[stale_entry->path] = stale_entry;
+ SaveEventually();
+ return true;
+}
+
+MetaEntryPtr HttpMetaCache::staleEntry(QString base, QString resource_path)
+{
+ auto foo = new MetaEntry;
+ foo->base = base;
+ foo->path = resource_path;
+ foo->stale = true;
+ return MetaEntryPtr(foo);
}
void HttpMetaCache::addBase ( QString base, QString base_root )
@@ -46,6 +144,16 @@ void HttpMetaCache::addBase ( QString base, QString base_root )
m_entries[base] = foo;
}
+QString HttpMetaCache::getBasePath ( QString base )
+{
+ if(m_entries.contains(base))
+ {
+ return m_entries[base].base_path;
+ }
+ return QString();
+}
+
+
void HttpMetaCache::Load()
{
QFile index(m_index_file);
@@ -65,12 +173,12 @@ void HttpMetaCache::Load()
// read the entry array
auto entries_val =root.value("entries");
- if(!version_val.isArray())
+ if(!entries_val.isArray())
return;
- QJsonArray array = json.array();
+ QJsonArray array = entries_val.toArray();
for(auto element: array)
{
- if(!element.isObject());
+ if(!element.isObject())
return;
auto element_obj = element.toObject();
QString base = element_obj.value("base").toString();
@@ -83,11 +191,13 @@ void HttpMetaCache::Load()
foo->md5sum = element_obj.value("md5sum").toString();
foo->etag = element_obj.value("etag").toString();
foo->last_changed_timestamp = element_obj.value("last_changed_timestamp").toDouble();
+ // presumed innocent until closer examination
+ foo->stale = false;
entrymap.entry_list[path] = MetaEntryPtr( foo );
}
}
-void HttpMetaCache::Save()
+void HttpMetaCache::SaveNow()
{
QSaveFile tfile(m_index_file);
if(!tfile.open(QIODevice::WriteOnly | QIODevice::Truncate))
@@ -118,14 +228,3 @@ void HttpMetaCache::Save()
return;
tfile.commit();
}
-
-
-MetaEntryPtr HttpMetaCache::getEntryForResource ( QString base, QString resource_path )
-{
- if(!m_entries.contains(base))
- return MetaEntryPtr();
- auto & entrymap = m_entries[base];
- if(!entrymap.entry_list.contains(resource_path))
- return MetaEntryPtr();
- return entrymap.entry_list[resource_path];
-}
diff --git a/logic/net/HttpMetaCache.h b/logic/net/HttpMetaCache.h
index 161483ad..fac6bec3 100644
--- a/logic/net/HttpMetaCache.h
+++ b/logic/net/HttpMetaCache.h
@@ -2,6 +2,7 @@
#include <QString>
#include <QSharedPointer>
#include <QMap>
+#include <qtimer.h>
struct MetaEntry
{
@@ -9,23 +10,42 @@ struct MetaEntry
QString path;
QString md5sum;
QString etag;
- quint64 last_changed_timestamp = 0;
+ qint64 last_changed_timestamp = 0;
+ bool stale = true;
+ QString getFullPath();
};
typedef QSharedPointer<MetaEntry> MetaEntryPtr;
-class HttpMetaCache
+class HttpMetaCache : public QObject
{
+ Q_OBJECT
public:
// supply path to the cache index file
HttpMetaCache(QString path);
~HttpMetaCache();
- MetaEntryPtr getEntryForResource(QString base, QString resource_path);
- void addEntry(QString base, QString resource_path, QString etag);
+
+ // get the entry solely from the cache
+ // you probably don't want this, unless you have some specific caching needs.
+ MetaEntryPtr getEntry(QString base, QString resource_path);
+
+ // get the entry from cache and verify that it isn't stale (within reason)
+ MetaEntryPtr resolveEntry(QString base, QString resource_path, QString expected_etag = QString());
+
+ // add a previously resolved stale entry
+ bool updateEntry(MetaEntryPtr stale_entry);
+
void addBase(QString base, QString base_root);
-private:
- void Save();
+
+ // (re)start a timer that calls SaveNow later.
+ void SaveEventually();
void Load();
+ QString getBasePath ( QString base );
+public slots:
+ void SaveNow();
+private:
+ // create a new stale entry, given the parameters
+ MetaEntryPtr staleEntry(QString base, QString resource_path);
struct EntryMap
{
QString base_path;
@@ -33,4 +53,5 @@ private:
};
QMap<QString, EntryMap> m_entries;
QString m_index_file;
+ QTimer saveBatchingTimer;
}; \ No newline at end of file