From 6bea4ec988b7caeac63353fb9d2a354f2fd47dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sun, 8 Sep 2013 02:15:20 +0200 Subject: Use HttpMetaCache to minimize network use. --- logic/net/CacheDownload.cpp | 122 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 logic/net/CacheDownload.cpp (limited to 'logic/net/CacheDownload.cpp') diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp new file mode 100644 index 00000000..c0074574 --- /dev/null +++ b/logic/net/CacheDownload.cpp @@ -0,0 +1,122 @@ +#include "MultiMC.h" +#include "CacheDownload.h" +#include + +#include +#include +#include +#include + +CacheDownload::CacheDownload (QUrl url, MetaEntryPtr entry ) + :Download(), md5sum(QCryptographicHash::Md5) +{ + m_url = url; + m_entry = entry; + m_target_path = entry->getFullPath(); + m_status = Job_NotStarted; + m_opened_for_saving = false; +} + +void CacheDownload::start() +{ + if(!m_entry->stale) + { + emit succeeded(index_within_job); + return; + } + m_output_file.setFileName(m_target_path); + // if there already is a file and md5 checking is in effect and it can be opened + if(!ensureFilePathExists(m_target_path)) + { + emit failed(index_within_job); + return; + } + qDebug() << "Downloading " << m_url.toString(); + QNetworkRequest request ( m_url ); + request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); + + auto worker = MMC->qnam(); + QNetworkReply * rep = worker->get ( request ); + + m_reply = QSharedPointer ( rep, &QObject::deleteLater ); + connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) ); + connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) ); + connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) ); + connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); +} + +void CacheDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) +{ + emit progress (index_within_job, bytesReceived, bytesTotal ); +} + +void CacheDownload::downloadError ( QNetworkReply::NetworkError error ) +{ + // error happened during download. + // TODO: log the reason why + m_status = Job_Failed; +} +void CacheDownload::downloadFinished() +{ + // if the download succeeded + if ( m_status != Job_Failed ) + { + + // nothing went wrong... + m_status = Job_Finished; + if(m_opened_for_saving) + { + // save the data to the downloadable if we aren't saving to file + m_output_file.close(); + m_entry->md5sum = md5sum.result().toHex().constData(); + } + else + { + if ( m_output_file.open ( QIODevice::ReadOnly ) ) + { + m_entry->md5sum = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData(); + m_output_file.close(); + } + } + QFileInfo output_file_info(m_target_path); + + + m_entry->etag = m_reply->rawHeader("ETag").constData(); + m_entry->last_changed_timestamp = output_file_info.lastModified().toUTC().toMSecsSinceEpoch(); + m_entry->stale = false; + MMC->metacache()->updateEntry(m_entry); + + m_reply.clear(); + emit succeeded(index_within_job); + return; + } + // else the download failed + else + { + m_output_file.close(); + m_output_file.remove(); + m_reply.clear(); + emit failed(index_within_job); + return; + } +} + +void CacheDownload::downloadReadyRead() +{ + if(!m_opened_for_saving) + { + if ( !m_output_file.open ( QIODevice::WriteOnly ) ) + { + /* + * Can't open the file... the job failed + */ + m_reply->abort(); + emit failed(index_within_job); + return; + } + m_opened_for_saving = true; + } + QByteArray ba = m_reply->readAll(); + md5sum.addData(ba); + m_output_file.write ( ba ); +} -- cgit v1.2.3 From 9d03a9c1e3b9c24a4146adedb2971591d23b037a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sun, 22 Sep 2013 14:00:37 +0200 Subject: Cache forge version list (it's huge) --- logic/net/CacheDownload.cpp | 63 ++++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 29 deletions(-) (limited to 'logic/net/CacheDownload.cpp') diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp index c0074574..dc2e0448 100644 --- a/logic/net/CacheDownload.cpp +++ b/logic/net/CacheDownload.cpp @@ -7,8 +7,8 @@ #include #include -CacheDownload::CacheDownload (QUrl url, MetaEntryPtr entry ) - :Download(), md5sum(QCryptographicHash::Md5) +CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry) + : Download(), md5sum(QCryptographicHash::Md5) { m_url = url; m_entry = entry; @@ -19,38 +19,40 @@ CacheDownload::CacheDownload (QUrl url, MetaEntryPtr entry ) void CacheDownload::start() { - if(!m_entry->stale) + if (!m_entry->stale) { emit succeeded(index_within_job); return; } m_output_file.setFileName(m_target_path); // if there already is a file and md5 checking is in effect and it can be opened - if(!ensureFilePathExists(m_target_path)) + if (!ensureFilePathExists(m_target_path)) { emit failed(index_within_job); return; } qDebug() << "Downloading " << m_url.toString(); - QNetworkRequest request ( m_url ); - request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); - + QNetworkRequest request(m_url); + request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); + auto worker = MMC->qnam(); - QNetworkReply * rep = worker->get ( request ); - - m_reply = QSharedPointer ( rep, &QObject::deleteLater ); - connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) ); - connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) ); - connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) ); - connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); + QNetworkReply *rep = worker->get(request); + + m_reply = QSharedPointer(rep, &QObject::deleteLater); + connect(rep, SIGNAL(downloadProgress(qint64, qint64)), + SLOT(downloadProgress(qint64, qint64))); + connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); + connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), + SLOT(downloadError(QNetworkReply::NetworkError))); + connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); } -void CacheDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) +void CacheDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) { - emit progress (index_within_job, bytesReceived, bytesTotal ); + emit progress(index_within_job, bytesReceived, bytesTotal); } -void CacheDownload::downloadError ( QNetworkReply::NetworkError error ) +void CacheDownload::downloadError(QNetworkReply::NetworkError error) { // error happened during download. // TODO: log the reason why @@ -59,12 +61,12 @@ void CacheDownload::downloadError ( QNetworkReply::NetworkError error ) void CacheDownload::downloadFinished() { // if the download succeeded - if ( m_status != Job_Failed ) + if (m_status != Job_Failed) { - + // nothing went wrong... m_status = Job_Finished; - if(m_opened_for_saving) + if (m_opened_for_saving) { // save the data to the downloadable if we aren't saving to file m_output_file.close(); @@ -72,20 +74,23 @@ void CacheDownload::downloadFinished() } else { - if ( m_output_file.open ( QIODevice::ReadOnly ) ) + if (m_output_file.open(QIODevice::ReadOnly)) { - m_entry->md5sum = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData(); + m_entry->md5sum = + QCryptographicHash::hash(m_output_file.readAll(), QCryptographicHash::Md5) + .toHex() + .constData(); m_output_file.close(); } } QFileInfo output_file_info(m_target_path); - - + m_entry->etag = m_reply->rawHeader("ETag").constData(); - m_entry->last_changed_timestamp = output_file_info.lastModified().toUTC().toMSecsSinceEpoch(); + m_entry->last_changed_timestamp = + output_file_info.lastModified().toUTC().toMSecsSinceEpoch(); m_entry->stale = false; MMC->metacache()->updateEntry(m_entry); - + m_reply.clear(); emit succeeded(index_within_job); return; @@ -103,9 +108,9 @@ void CacheDownload::downloadFinished() void CacheDownload::downloadReadyRead() { - if(!m_opened_for_saving) + if (!m_opened_for_saving) { - if ( !m_output_file.open ( QIODevice::WriteOnly ) ) + if (!m_output_file.open(QIODevice::WriteOnly)) { /* * Can't open the file... the job failed @@ -118,5 +123,5 @@ void CacheDownload::downloadReadyRead() } QByteArray ba = m_reply->readAll(); md5sum.addData(ba); - m_output_file.write ( ba ); + m_output_file.write(ba); } -- cgit v1.2.3 From eba9b3d759dbf6e402e91ab897059f1d274aef90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Wed, 2 Oct 2013 23:35:45 +0200 Subject: Add user agent header to most MultiMC download requests. --- logic/net/CacheDownload.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'logic/net/CacheDownload.cpp') diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp index dc2e0448..9fc1f127 100644 --- a/logic/net/CacheDownload.cpp +++ b/logic/net/CacheDownload.cpp @@ -34,6 +34,7 @@ void CacheDownload::start() qDebug() << "Downloading " << m_url.toString(); QNetworkRequest request(m_url); request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); + request.setHeader(QNetworkRequest::UserAgentHeader,"MultiMC/5.0 (Cached)"); auto worker = MMC->qnam(); QNetworkReply *rep = worker->get(request); -- cgit v1.2.3 From f83119ce7ec3d11a903901b8eff762d2b0a9f635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sun, 6 Oct 2013 01:13:40 +0200 Subject: Added file logger --- logic/net/CacheDownload.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'logic/net/CacheDownload.cpp') diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp index 9fc1f127..0e653e05 100644 --- a/logic/net/CacheDownload.cpp +++ b/logic/net/CacheDownload.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry) : Download(), md5sum(QCryptographicHash::Md5) @@ -31,7 +31,7 @@ void CacheDownload::start() emit failed(index_within_job); return; } - qDebug() << "Downloading " << m_url.toString(); + QLOG_INFO() << "Downloading " << m_url.toString(); QNetworkRequest request(m_url); request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); request.setHeader(QNetworkRequest::UserAgentHeader,"MultiMC/5.0 (Cached)"); @@ -39,7 +39,7 @@ void CacheDownload::start() auto worker = MMC->qnam(); QNetworkReply *rep = worker->get(request); - m_reply = QSharedPointer(rep, &QObject::deleteLater); + m_reply = std::shared_ptr(rep); connect(rep, SIGNAL(downloadProgress(qint64, qint64)), SLOT(downloadProgress(qint64, qint64))); connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); @@ -92,7 +92,7 @@ void CacheDownload::downloadFinished() m_entry->stale = false; MMC->metacache()->updateEntry(m_entry); - m_reply.clear(); + m_reply.reset(); emit succeeded(index_within_job); return; } @@ -101,7 +101,7 @@ void CacheDownload::downloadFinished() { m_output_file.close(); m_output_file.remove(); - m_reply.clear(); + m_reply.reset(); emit failed(index_within_job); return; } -- cgit v1.2.3 From 651bed91a0d6aff551fd6880a7cdbca403064c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sun, 6 Oct 2013 03:47:41 +0200 Subject: Log failure reasons of cache downloads --- logic/net/CacheDownload.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'logic/net/CacheDownload.cpp') diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp index 0e653e05..6a76a4ae 100644 --- a/logic/net/CacheDownload.cpp +++ b/logic/net/CacheDownload.cpp @@ -56,7 +56,7 @@ void CacheDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) void CacheDownload::downloadError(QNetworkReply::NetworkError error) { // error happened during download. - // TODO: log the reason why + QLOG_ERROR() << "Failed" << m_url.toString() << "with reason" << error; m_status = Job_Failed; } void CacheDownload::downloadFinished() -- cgit v1.2.3 From 205570be32b5cbd40eeb2b7e2d8d4fe116b07f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Fri, 18 Oct 2013 01:00:46 +0200 Subject: Support version format 9, fix version-related segfault, (maybe) fix forge lists. --- logic/net/CacheDownload.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'logic/net/CacheDownload.cpp') diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp index 6a76a4ae..309eb345 100644 --- a/logic/net/CacheDownload.cpp +++ b/logic/net/CacheDownload.cpp @@ -33,7 +33,11 @@ void CacheDownload::start() } QLOG_INFO() << "Downloading " << m_url.toString(); QNetworkRequest request(m_url); - request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); + if(m_entry->remote_changed_timestamp.size()) + request.setRawHeader(QString("If-Modified-Since").toLatin1(), m_entry->remote_changed_timestamp.toLatin1()); + if(m_entry->etag.size()) + request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); + request.setHeader(QNetworkRequest::UserAgentHeader,"MultiMC/5.0 (Cached)"); auto worker = MMC->qnam(); @@ -87,7 +91,11 @@ void CacheDownload::downloadFinished() QFileInfo output_file_info(m_target_path); m_entry->etag = m_reply->rawHeader("ETag").constData(); - m_entry->last_changed_timestamp = + if(m_reply->hasRawHeader("Last-Modified")) + { + m_entry->remote_changed_timestamp = m_reply->rawHeader("Last-Modified").constData(); + } + m_entry->local_changed_timestamp = output_file_info.lastModified().toUTC().toMSecsSinceEpoch(); m_entry->stale = false; MMC->metacache()->updateEntry(m_entry); -- cgit v1.2.3