From a1abbd9e05c80584d831b1d12c27c5f7d731cece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sat, 28 May 2016 19:54:17 +0200 Subject: NOISSUE refactor *Download into more, smaller pieces * Download is now Download. * Download uses Sink subclasses to process various events. * Validators can be used to further customize the Sink behaviour. --- api/logic/net/ByteArrayDownload.cpp | 105 ------------------- api/logic/net/ByteArrayDownload.h | 48 --------- api/logic/net/ByteArraySink.h | 57 +++++++++++ api/logic/net/CacheDownload.cpp | 192 ---------------------------------- api/logic/net/CacheDownload.h | 63 ------------ api/logic/net/ChecksumValidator.h | 57 +++++++++++ api/logic/net/Download.cpp | 199 ++++++++++++++++++++++++++++++++++++ api/logic/net/Download.h | 66 ++++++++++++ api/logic/net/FileSink.cpp | 99 ++++++++++++++++++ api/logic/net/FileSink.h | 27 +++++ api/logic/net/MD5EtagDownload.cpp | 155 ---------------------------- api/logic/net/MD5EtagDownload.h | 52 ---------- api/logic/net/MetaCacheSink.cpp | 59 +++++++++++ api/logic/net/MetaCacheSink.h | 21 ++++ api/logic/net/NetAction.h | 3 - api/logic/net/NetJob.cpp | 4 +- api/logic/net/NetJob.h | 4 +- api/logic/net/Sink.h | 70 +++++++++++++ api/logic/net/Validator.h | 20 ++++ 19 files changed, 677 insertions(+), 624 deletions(-) delete mode 100644 api/logic/net/ByteArrayDownload.cpp delete mode 100644 api/logic/net/ByteArrayDownload.h create mode 100644 api/logic/net/ByteArraySink.h delete mode 100644 api/logic/net/CacheDownload.cpp delete mode 100644 api/logic/net/CacheDownload.h create mode 100644 api/logic/net/ChecksumValidator.h create mode 100644 api/logic/net/Download.cpp create mode 100644 api/logic/net/Download.h create mode 100644 api/logic/net/FileSink.cpp create mode 100644 api/logic/net/FileSink.h delete mode 100644 api/logic/net/MD5EtagDownload.cpp delete mode 100644 api/logic/net/MD5EtagDownload.h create mode 100644 api/logic/net/MetaCacheSink.cpp create mode 100644 api/logic/net/MetaCacheSink.h create mode 100644 api/logic/net/Sink.h create mode 100644 api/logic/net/Validator.h (limited to 'api/logic/net') diff --git a/api/logic/net/ByteArrayDownload.cpp b/api/logic/net/ByteArrayDownload.cpp deleted file mode 100644 index 21990eeb..00000000 --- a/api/logic/net/ByteArrayDownload.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/* Copyright 2013-2015 MultiMC Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ByteArrayDownload.h" -#include "Env.h" -#include - -ByteArrayDownload::ByteArrayDownload(QUrl url) : NetAction() -{ - m_url = url; - m_status = Job_NotStarted; -} - -void ByteArrayDownload::start() -{ - qDebug() << "Downloading " << m_url.toString(); - QNetworkRequest request(m_url); - request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)"); - auto worker = ENV.qnam(); - QNetworkReply *rep = worker->get(request); - - m_reply.reset(rep); - connect(rep, SIGNAL(downloadProgress(qint64, qint64)), - SLOT(downloadProgress(qint64, qint64))); - connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); - connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), - SLOT(downloadError(QNetworkReply::NetworkError))); - connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); -} - -void ByteArrayDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) -{ - m_total_progress = bytesTotal; - m_progress = bytesReceived; - emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal); -} - -void ByteArrayDownload::downloadError(QNetworkReply::NetworkError error) -{ - // error happened during download. - qCritical() << "Error getting URL:" << m_url.toString().toLocal8Bit() - << "Network error: " << error; - m_status = Job_Failed; - m_errorString = m_reply->errorString(); -} - -void ByteArrayDownload::downloadFinished() -{ - QVariant redirect = m_reply->header(QNetworkRequest::LocationHeader); - QString redirectURL; - if(redirect.isValid()) - { - redirectURL = redirect.toString(); - } - // FIXME: This is a hack for https://bugreports.qt-project.org/browse/QTBUG-41061 - else if(m_reply->hasRawHeader("Location")) - { - auto data = m_reply->rawHeader("Location"); - if(data.size() > 2 && data[0] == '/' && data[1] == '/') - redirectURL = m_reply->url().scheme() + ":" + data; - } - if (!redirectURL.isEmpty()) - { - m_url = QUrl(redirect.toString()); - qDebug() << "Following redirect to " << m_url.toString(); - start(); - return; - } - - // if the download succeeded - if (m_status != Job_Failed) - { - // nothing went wrong... - m_status = Job_Finished; - m_data = m_reply->readAll(); - m_content_type = m_reply->header(QNetworkRequest::ContentTypeHeader).toString(); - m_reply.reset(); - emit succeeded(m_index_within_job); - return; - } - // else the download failed - else - { - m_reply.reset(); - emit failed(m_index_within_job); - return; - } -} - -void ByteArrayDownload::downloadReadyRead() -{ - // ~_~ -} diff --git a/api/logic/net/ByteArrayDownload.h b/api/logic/net/ByteArrayDownload.h deleted file mode 100644 index e2fc2911..00000000 --- a/api/logic/net/ByteArrayDownload.h +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright 2013-2015 MultiMC Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include "NetAction.h" - -#include "multimc_logic_export.h" - -typedef std::shared_ptr ByteArrayDownloadPtr; -class MULTIMC_LOGIC_EXPORT ByteArrayDownload : public NetAction -{ - Q_OBJECT -public: - ByteArrayDownload(QUrl url); - static ByteArrayDownloadPtr make(QUrl url) - { - return ByteArrayDownloadPtr(new ByteArrayDownload(url)); - } - virtual ~ByteArrayDownload() {}; -public: - /// if not saving to file, downloaded data is placed here - QByteArray m_data; - - QString m_errorString; - -public -slots: - virtual void start(); - -protected -slots: - void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); - void downloadError(QNetworkReply::NetworkError error); - void downloadFinished(); - void downloadReadyRead(); -}; diff --git a/api/logic/net/ByteArraySink.h b/api/logic/net/ByteArraySink.h new file mode 100644 index 00000000..3deef1ed --- /dev/null +++ b/api/logic/net/ByteArraySink.h @@ -0,0 +1,57 @@ +#pragma once + +#include "Sink.h" + +namespace Net { +/* + * Sink object for downloads that uses an external QByteArray it doesn't own as a target. + */ +class ByteArraySink : public Sink +{ +public: + ByteArraySink(QByteArray *output) + :m_output(output) + { + // nil + }; + + virtual ~ByteArraySink() + { + // nil + } + +public: + JobStatus init(QNetworkRequest & request) override + { + m_output->clear(); + if(initAllValidators(request)) + return Job_InProgress; + return Job_Failed; + }; + + JobStatus write(QByteArray & data) override + { + m_output->append(data); + if(writeAllValidators(data)) + return Job_InProgress; + return Job_Failed; + } + + JobStatus abort() override + { + m_output->clear(); + failAllValidators(); + return Job_Failed; + } + + JobStatus finalize(QNetworkReply &reply) override + { + if(finalizeAllValidators(reply)) + return Job_Finished; + return Job_Failed; + } + +private: + QByteArray * m_output; +}; +} diff --git a/api/logic/net/CacheDownload.cpp b/api/logic/net/CacheDownload.cpp deleted file mode 100644 index 1ac55180..00000000 --- a/api/logic/net/CacheDownload.cpp +++ /dev/null @@ -1,192 +0,0 @@ -/* Copyright 2013-2015 MultiMC Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "CacheDownload.h" - -#include -#include -#include -#include -#include "Env.h" -#include - -CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry) - : NetAction(), md5sum(QCryptographicHash::Md5) -{ - m_url = url; - m_entry = entry; - m_target_path = entry->getFullPath(); - m_status = Job_NotStarted; -} - -void CacheDownload::start() -{ - m_status = Job_InProgress; - if (!m_entry->isStale()) - { - m_status = Job_Finished; - emit succeeded(m_index_within_job); - return; - } - // create a new save file - m_output_file.reset(new QSaveFile(m_target_path)); - - // if there already is a file and md5 checking is in effect and it can be opened - if (!FS::ensureFilePathExists(m_target_path)) - { - qCritical() << "Could not create folder for " + m_target_path; - m_status = Job_Failed; - emit failed(m_index_within_job); - return; - } - if (!m_output_file->open(QIODevice::WriteOnly)) - { - qCritical() << "Could not open " + m_target_path + " for writing"; - m_status = Job_Failed; - emit failed(m_index_within_job); - return; - } - qDebug() << "Downloading " << m_url.toString(); - QNetworkRequest request(m_url); - - // check file consistency first. - QFile current(m_target_path); - if(current.exists() && current.size() != 0) - { - if (m_entry->getRemoteChangedTimestamp().size()) - request.setRawHeader(QString("If-Modified-Since").toLatin1(), - m_entry->getRemoteChangedTimestamp().toLatin1()); - if (m_entry->getETag().size()) - request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->getETag().toLatin1()); - } - - request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Cached)"); - - auto worker = ENV.qnam(); - QNetworkReply *rep = worker->get(request); - - m_reply.reset(rep); - connect(rep, SIGNAL(downloadProgress(qint64, qint64)), - SLOT(downloadProgress(qint64, qint64))); - connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); - connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), - SLOT(downloadError(QNetworkReply::NetworkError))); - connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); -} - -void CacheDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) -{ - m_total_progress = bytesTotal; - m_progress = bytesReceived; - emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal); -} - -void CacheDownload::downloadError(QNetworkReply::NetworkError error) -{ - // error happened during download. - qCritical() << "Failed " << m_url.toString() << " with reason " << error; - m_status = Job_Failed; -} -void CacheDownload::downloadFinished() -{ - QVariant redirect = m_reply->header(QNetworkRequest::LocationHeader); - QString redirectURL; - if(redirect.isValid()) - { - redirectURL = redirect.toString(); - } - // FIXME: This is a hack for https://bugreports.qt-project.org/browse/QTBUG-41061 - else if(m_reply->hasRawHeader("Location")) - { - auto data = m_reply->rawHeader("Location"); - if(data.size() > 2 && data[0] == '/' && data[1] == '/') - redirectURL = m_reply->url().scheme() + ":" + data; - } - if (!redirectURL.isEmpty()) - { - m_url = QUrl(redirect.toString()); - qDebug() << "Following redirect to " << m_url.toString(); - start(); - return; - } - - // if the download succeeded - if (m_status == Job_Failed) - { - m_output_file->cancelWriting(); - m_reply.reset(); - emit failed(m_index_within_job); - return; - } - - // if we wrote any data to the save file, we try to commit the data to the real file. - if (wroteAnyData) - { - // nothing went wrong... - if (m_output_file->commit()) - { - m_status = Job_Finished; - m_entry->setMD5Sum(md5sum.result().toHex().constData()); - } - else - { - qCritical() << "Failed to commit changes to " << m_target_path; - m_output_file->cancelWriting(); - m_reply.reset(); - m_status = Job_Failed; - emit failed(m_index_within_job); - return; - } - } - else - { - m_status = Job_Finished; - } - - // then get rid of the save file - m_output_file.reset(); - - QFileInfo output_file_info(m_target_path); - - m_entry->setETag(m_reply->rawHeader("ETag").constData()); - if (m_reply->hasRawHeader("Last-Modified")) - { - m_entry->setRemoteChangedTimestamp(m_reply->rawHeader("Last-Modified").constData()); - } - m_entry->setLocalChangedTimestamp(output_file_info.lastModified().toUTC().toMSecsSinceEpoch()); - m_entry->setStale(false); - ENV.metacache()->updateEntry(m_entry); - - m_reply.reset(); - emit succeeded(m_index_within_job); - return; -} - -void CacheDownload::downloadReadyRead() -{ - QByteArray ba = m_reply->readAll(); - md5sum.addData(ba); - if (m_output_file->write(ba) != ba.size()) - { - qCritical() << "Failed writing into " + m_target_path; - m_status = Job_Failed; - m_output_file->cancelWriting(); - m_output_file.reset(); - emit failed(m_index_within_job); - wroteAnyData = false; - return; - } - wroteAnyData = true; -} diff --git a/api/logic/net/CacheDownload.h b/api/logic/net/CacheDownload.h deleted file mode 100644 index d83b2a0f..00000000 --- a/api/logic/net/CacheDownload.h +++ /dev/null @@ -1,63 +0,0 @@ -/* Copyright 2013-2015 MultiMC Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "NetAction.h" -#include "HttpMetaCache.h" -#include -#include - -#include "multimc_logic_export.h" - -typedef std::shared_ptr CacheDownloadPtr; -class MULTIMC_LOGIC_EXPORT CacheDownload : public NetAction -{ - Q_OBJECT -private: - MetaEntryPtr m_entry; - /// if saving to file, use the one specified in this string - QString m_target_path; - - /// this is the output file, if any - std::unique_ptr m_output_file; - - /// the hash-as-you-download - QCryptographicHash md5sum; - - bool wroteAnyData = false; - -public: - explicit CacheDownload(QUrl url, MetaEntryPtr entry); - static CacheDownloadPtr make(QUrl url, MetaEntryPtr entry) - { - return CacheDownloadPtr(new CacheDownload(url, entry)); - } - virtual ~CacheDownload(){}; - QString getTargetFilepath() - { - return m_target_path; - } -protected -slots: - virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); - virtual void downloadError(QNetworkReply::NetworkError error); - virtual void downloadFinished(); - virtual void downloadReadyRead(); - -public -slots: - virtual void start(); -}; diff --git a/api/logic/net/ChecksumValidator.h b/api/logic/net/ChecksumValidator.h new file mode 100644 index 00000000..3cd25bc3 --- /dev/null +++ b/api/logic/net/ChecksumValidator.h @@ -0,0 +1,57 @@ +#pragma once + +#include "Validator.h" +#include +#include +#include + +namespace Net { +class ChecksumValidator: public Validator +{ +public: /* con/des */ + ChecksumValidator(QCryptographicHash::Algorithm algorithm, QByteArray expected = QByteArray()) + :m_checksum(algorithm), m_expected(expected) + { + }; + virtual ~ChecksumValidator() {}; + +public: /* methods */ + bool init(QNetworkRequest &) override + { + m_checksum.reset(); + return true; + } + bool write(QByteArray & data) override + { + m_checksum.addData(data); + this->data.append(data); + return true; + } + bool abort() override + { + return true; + } + bool validate(QNetworkReply &) override + { + if(m_expected.size() && m_expected != hash()) + { + qWarning() << "Checksum mismatch, download is bad."; + return false; + } + return true; + } + QByteArray hash() + { + return m_checksum.result(); + } + void setExpected(QByteArray expected) + { + m_expected = expected; + } + +private: /* data */ + QByteArray data; + QCryptographicHash m_checksum; + QByteArray m_expected; +}; +} \ No newline at end of file diff --git a/api/logic/net/Download.cpp b/api/logic/net/Download.cpp new file mode 100644 index 00000000..70fe7608 --- /dev/null +++ b/api/logic/net/Download.cpp @@ -0,0 +1,199 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Download.h" + +#include +#include +#include +#include "Env.h" +#include +#include "ChecksumValidator.h" +#include "MetaCacheSink.h" +#include "ByteArraySink.h" + +namespace Net { + +Download::Download():NetAction() +{ + m_status = Job_NotStarted; +} + +Download::Ptr Download::makeCached(QUrl url, MetaEntryPtr entry) +{ + Download * dl = new Download(); + dl->m_url = url; + auto md5Node = new ChecksumValidator(QCryptographicHash::Md5); + auto cachedNode = new MetaCacheSink(entry, md5Node); + dl->m_sink.reset(cachedNode); + dl->m_target_path = entry->getFullPath(); + return std::shared_ptr(dl); +} + +Download::Ptr Download::makeByteArray(QUrl url, QByteArray *output) +{ + Download * dl = new Download(); + dl->m_url = url; + dl->m_sink.reset(new ByteArraySink(output)); + return std::shared_ptr(dl); +} + +Download::Ptr Download::makeFile(QUrl url, QString path) +{ + Download * dl = new Download(); + dl->m_url = url; + dl->m_sink.reset(new FileSink(path)); + return std::shared_ptr(dl); +} + +void Download::addValidator(Validator * v) +{ + m_sink->addValidator(v); +} + +void Download::start() +{ + QNetworkRequest request(m_url); + m_status = m_sink->init(request); + switch(m_status) + { + case Job_Finished: + emit succeeded(m_index_within_job); + qDebug() << "Download cache hit " << m_url.toString(); + return; + case Job_InProgress: + qDebug() << "Downloading " << m_url.toString(); + break; + case Job_NotStarted: + case Job_Failed: + emit failed(m_index_within_job); + return; + } + + request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0"); + + auto worker = ENV.qnam(); + QNetworkReply *rep = worker->get(request); + + m_reply.reset(rep); + connect(rep, SIGNAL(downloadProgress(qint64, qint64)), SLOT(downloadProgress(qint64, qint64))); + connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); + connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(downloadError(QNetworkReply::NetworkError))); + connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); +} + +void Download::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) +{ + m_total_progress = bytesTotal; + m_progress = bytesReceived; + emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal); +} + +void Download::downloadError(QNetworkReply::NetworkError error) +{ + // error happened during download. + qCritical() << "Failed " << m_url.toString() << " with reason " << error; + m_status = Job_Failed; +} + +bool Download::handleRedirect() +{ + QVariant redirect = m_reply->header(QNetworkRequest::LocationHeader); + QString redirectURL; + if(redirect.isValid()) + { + redirectURL = redirect.toString(); + } + // FIXME: This is a hack for https://bugreports.qt-project.org/browse/QTBUG-41061 + else if(m_reply->hasRawHeader("Location")) + { + auto data = m_reply->rawHeader("Location"); + if(data.size() > 2 && data[0] == '/' && data[1] == '/') + { + redirectURL = m_reply->url().scheme() + ":" + data; + } + } + if (!redirectURL.isEmpty()) + { + m_url = QUrl(redirect.toString()); + qDebug() << "Following redirect to " << m_url.toString(); + start(); + return true; + } + return false; +} + + +void Download::downloadFinished() +{ + // handle HTTP redirection first + if(handleRedirect()) + { + qDebug() << "Download redirected:" << m_url.toString(); + return; + } + + // if the download failed before this point ... + if (m_status == Job_Failed) + { + qDebug() << "Download failed in previous step:" << m_url.toString(); + m_sink->abort(); + m_reply.reset(); + emit failed(m_index_within_job); + return; + } + + // make sure we got all the remaining data, if any + auto data = m_reply->readAll(); + if(data.size()) + { + qDebug() << "Writing extra" << data.size() << "bytes to" << m_target_path; + m_status = m_sink->write(data); + } + + // otherwise, finalize the whole graph + m_status = m_sink->finalize(*m_reply.get()); + if (m_status != Job_Finished) + { + qDebug() << "Download failed to finalize:" << m_url.toString(); + m_sink->abort(); + m_reply.reset(); + emit failed(m_index_within_job); + return; + } + m_reply.reset(); + qDebug() << "Download succeeded:" << m_url.toString(); + emit succeeded(m_index_within_job); +} + +void Download::downloadReadyRead() +{ + if(m_status == Job_InProgress) + { + auto data = m_reply->readAll(); + m_status = m_sink->write(data); + if(m_status == Job_Failed) + { + qCritical() << "Failed to process response chunk for " << m_target_path; + } + // qDebug() << "Download" << m_url.toString() << "gained" << data.size() << "bytes"; + } + else + { + qCritical() << "Cannot write to " << m_target_path << ", illegal status" << m_status; + } +} + +} diff --git a/api/logic/net/Download.h b/api/logic/net/Download.h new file mode 100644 index 00000000..8e38dfc9 --- /dev/null +++ b/api/logic/net/Download.h @@ -0,0 +1,66 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "NetAction.h" +#include "HttpMetaCache.h" +#include "Validator.h" +#include "Sink.h" + +#include "multimc_logic_export.h" +namespace Net { +class MULTIMC_LOGIC_EXPORT Download : public NetAction +{ + Q_OBJECT + +public: /* types */ + typedef std::shared_ptr Ptr; + +protected: /* con/des */ + explicit Download(); +public: + virtual ~Download(){}; + static Download::Ptr makeCached(QUrl url, MetaEntryPtr entry); + static Download::Ptr makeByteArray(QUrl url, QByteArray *output); + static Download::Ptr makeFile(QUrl url, QString path); + +public: /* methods */ + // FIXME: remove this + QString getTargetFilepath() + { + return m_target_path; + } + // FIXME: remove this + void addValidator(Validator * v); + +private: /* methods */ + bool handleRedirect(); + +protected slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); + virtual void downloadError(QNetworkReply::NetworkError error); + virtual void downloadFinished(); + virtual void downloadReadyRead(); + +public slots: + virtual void start(); + +private: /* data */ + // FIXME: remove this, it has no business being here. + QString m_target_path; + std::unique_ptr m_sink; +}; +} \ No newline at end of file diff --git a/api/logic/net/FileSink.cpp b/api/logic/net/FileSink.cpp new file mode 100644 index 00000000..b4c418a5 --- /dev/null +++ b/api/logic/net/FileSink.cpp @@ -0,0 +1,99 @@ +#include "FileSink.h" +#include +#include +#include "Env.h" +#include "FileSystem.h" + +namespace Net { + +FileSink::FileSink(QString filename) + :m_filename(filename) +{ + // nil +}; + +FileSink::~FileSink() +{ + // nil +}; + +JobStatus FileSink::init(QNetworkRequest& request) +{ + auto result = initCache(request); + if(result != Job_InProgress) + { + return result; + } + // create a new save file and open it for writing + if (!FS::ensureFilePathExists(m_filename)) + { + qCritical() << "Could not create folder for " + m_filename; + return Job_Failed; + } + m_output_file.reset(new QSaveFile(m_filename)); + if (!m_output_file->open(QIODevice::WriteOnly)) + { + qCritical() << "Could not open " + m_filename + " for writing"; + return Job_Failed; + } + + if(initAllValidators(request)) + return Job_InProgress; + return Job_Failed; +} + +JobStatus FileSink::initCache(QNetworkRequest &) +{ + return Job_InProgress; +} + +JobStatus FileSink::write(QByteArray& data) +{ + if (!writeAllValidators(data) || m_output_file->write(data) != data.size()) + { + qCritical() << "Failed writing into " + m_filename; + m_output_file->cancelWriting(); + m_output_file.reset(); + wroteAnyData = false; + return Job_Failed; + } + wroteAnyData = true; + return Job_InProgress; +} + +JobStatus FileSink::abort() +{ + m_output_file->cancelWriting(); + failAllValidators(); + return Job_Failed; +} + +JobStatus FileSink::finalize(QNetworkReply& reply) +{ + // if we wrote any data to the save file, we try to commit the data to the real file. + if (wroteAnyData) + { + // ask validators for data consistency + // we only do this for actual downloads, not 'your data is still the same' cache hits + if(!finalizeAllValidators(reply)) + return Job_Failed; + // nothing went wrong... + if (!m_output_file->commit()) + { + qCritical() << "Failed to commit changes to " << m_filename; + m_output_file->cancelWriting(); + return Job_Failed; + } + } + // then get rid of the save file + m_output_file.reset(); + + return finalizeCache(reply); +} + +JobStatus FileSink::finalizeCache(QNetworkReply &) +{ + return Job_Finished; +} + +} diff --git a/api/logic/net/FileSink.h b/api/logic/net/FileSink.h new file mode 100644 index 00000000..7adc9c47 --- /dev/null +++ b/api/logic/net/FileSink.h @@ -0,0 +1,27 @@ +#pragma once +#include "Sink.h" +#include + +namespace Net { +class FileSink : public Sink +{ +public: /* con/des */ + FileSink(QString filename); + virtual ~FileSink(); + +public: /* methods */ + JobStatus init(QNetworkRequest & request) override; + JobStatus write(QByteArray & data) override; + JobStatus abort() override; + JobStatus finalize(QNetworkReply & reply) override; + +protected: /* methods */ + virtual JobStatus initCache(QNetworkRequest &); + virtual JobStatus finalizeCache(QNetworkReply &reply); + +protected: /* data */ + QString m_filename; + bool wroteAnyData = false; + std::unique_ptr m_output_file; +}; +} diff --git a/api/logic/net/MD5EtagDownload.cpp b/api/logic/net/MD5EtagDownload.cpp deleted file mode 100644 index 3b4d5dcd..00000000 --- a/api/logic/net/MD5EtagDownload.cpp +++ /dev/null @@ -1,155 +0,0 @@ -/* Copyright 2013-2015 MultiMC Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "Env.h" -#include "MD5EtagDownload.h" -#include -#include -#include - -MD5EtagDownload::MD5EtagDownload(QUrl url, QString target_path) : NetAction() -{ - m_url = url; - m_target_path = target_path; - m_status = Job_NotStarted; -} - -void MD5EtagDownload::start() -{ - QString filename = m_target_path; - m_output_file.setFileName(filename); - // if there already is a file and md5 checking is in effect and it can be opened - if (m_output_file.exists() && m_output_file.open(QIODevice::ReadOnly)) - { - // get the md5 of the local file. - m_local_md5 = - QCryptographicHash::hash(m_output_file.readAll(), QCryptographicHash::Md5) - .toHex() - .constData(); - m_output_file.close(); - // if we are expecting some md5sum, compare it with the local one - if (!m_expected_md5.isEmpty()) - { - // skip if they match - if(m_local_md5 == m_expected_md5) - { - qDebug() << "Skipping " << m_url.toString() << ": md5 match."; - emit succeeded(m_index_within_job); - return; - } - } - else - { - // no expected md5. we use the local md5sum as an ETag - } - } - if (!FS::ensureFilePathExists(filename)) - { - emit failed(m_index_within_job); - return; - } - - QNetworkRequest request(m_url); - - qDebug() << "Downloading " << m_url.toString() << " local MD5: " << m_local_md5; - - if(!m_local_md5.isEmpty()) - { - request.setRawHeader(QString("If-None-Match").toLatin1(), m_local_md5.toLatin1()); - } - if(!m_expected_md5.isEmpty()) - qDebug() << "Expecting " << m_expected_md5; - - request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)"); - - // Go ahead and try to open the file. - // If we don't do this, empty files won't be created, which breaks the updater. - // Plus, this way, we don't end up starting a download for a file we can't open. - if (!m_output_file.open(QIODevice::WriteOnly)) - { - emit failed(m_index_within_job); - return; - } - - auto worker = ENV.qnam(); - QNetworkReply *rep = worker->get(request); - - m_reply.reset(rep); - connect(rep, SIGNAL(downloadProgress(qint64, qint64)), - SLOT(downloadProgress(qint64, qint64))); - connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); - connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), - SLOT(downloadError(QNetworkReply::NetworkError))); - connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); -} - -void MD5EtagDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) -{ - m_total_progress = bytesTotal; - m_progress = bytesReceived; - emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal); -} - -void MD5EtagDownload::downloadError(QNetworkReply::NetworkError error) -{ - qCritical() << "Error" << error << ":" << m_reply->errorString() << "while downloading" - << m_reply->url(); - m_status = Job_Failed; -} - -void MD5EtagDownload::downloadFinished() -{ - // if the download succeeded - if (m_status != Job_Failed) - { - // nothing went wrong... - m_status = Job_Finished; - m_output_file.close(); - - // FIXME: compare with the real written data md5sum - // this is just an ETag - qDebug() << "Finished " << m_url.toString() << " got " << m_reply->rawHeader("ETag").constData(); - - m_reply.reset(); - emit succeeded(m_index_within_job); - return; - } - // else the download failed - else - { - m_output_file.close(); - m_output_file.remove(); - m_reply.reset(); - emit failed(m_index_within_job); - return; - } -} - -void MD5EtagDownload::downloadReadyRead() -{ - if (!m_output_file.isOpen()) - { - if (!m_output_file.open(QIODevice::WriteOnly)) - { - /* - * Can't open the file... the job failed - */ - m_reply->abort(); - emit failed(m_index_within_job); - return; - } - } - m_output_file.write(m_reply->readAll()); -} diff --git a/api/logic/net/MD5EtagDownload.h b/api/logic/net/MD5EtagDownload.h deleted file mode 100644 index cd1cb550..00000000 --- a/api/logic/net/MD5EtagDownload.h +++ /dev/null @@ -1,52 +0,0 @@ -/* Copyright 2013-2015 MultiMC Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "NetAction.h" -#include - -typedef std::shared_ptr Md5EtagDownloadPtr; -class MD5EtagDownload : public NetAction -{ - Q_OBJECT -public: - /// the expected md5 checksum. Only set from outside - QString m_expected_md5; - /// the md5 checksum of a file that already exists. - QString m_local_md5; - /// if saving to file, use the one specified in this string - QString m_target_path; - /// this is the output file, if any - QFile m_output_file; - -public: - explicit MD5EtagDownload(QUrl url, QString target_path); - static Md5EtagDownloadPtr make(QUrl url, QString target_path) - { - return Md5EtagDownloadPtr(new MD5EtagDownload(url, target_path)); - } - virtual ~MD5EtagDownload(){}; -protected -slots: - virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); - virtual void downloadError(QNetworkReply::NetworkError error); - virtual void downloadFinished(); - virtual void downloadReadyRead(); - -public -slots: - virtual void start(); -}; diff --git a/api/logic/net/MetaCacheSink.cpp b/api/logic/net/MetaCacheSink.cpp new file mode 100644 index 00000000..539b4bb3 --- /dev/null +++ b/api/logic/net/MetaCacheSink.cpp @@ -0,0 +1,59 @@ +#include "MetaCacheSink.h" +#include +#include +#include "Env.h" +#include "FileSystem.h" + +namespace Net { + +MetaCacheSink::MetaCacheSink(MetaEntryPtr entry, ChecksumValidator * md5sum) + :Net::FileSink(entry->getFullPath()), m_entry(entry), m_md5Node(md5sum) +{ + addValidator(md5sum); +}; + +MetaCacheSink::~MetaCacheSink() +{ + // nil +}; + +JobStatus MetaCacheSink::initCache(QNetworkRequest& request) +{ + if (!m_entry->isStale()) + { + return Job_Finished; + } + // check if file exists, if it does, use its information for the request + QFile current(m_filename); + if(current.exists() && current.size() != 0) + { + if (m_entry->getRemoteChangedTimestamp().size()) + { + request.setRawHeader(QString("If-Modified-Since").toLatin1(), m_entry->getRemoteChangedTimestamp().toLatin1()); + } + if (m_entry->getETag().size()) + { + request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->getETag().toLatin1()); + } + } + return Job_InProgress; +} + +JobStatus MetaCacheSink::finalizeCache(QNetworkReply & reply) +{ + QFileInfo output_file_info(m_filename); + if(wroteAnyData) + { + m_entry->setMD5Sum(m_md5Node->hash().toHex().constData()); + } + m_entry->setETag(reply.rawHeader("ETag").constData()); + if (reply.hasRawHeader("Last-Modified")) + { + m_entry->setRemoteChangedTimestamp(reply.rawHeader("Last-Modified").constData()); + } + m_entry->setLocalChangedTimestamp(output_file_info.lastModified().toUTC().toMSecsSinceEpoch()); + m_entry->setStale(false); + ENV.metacache()->updateEntry(m_entry); + return Job_Finished; +} +} diff --git a/api/logic/net/MetaCacheSink.h b/api/logic/net/MetaCacheSink.h new file mode 100644 index 00000000..98e9bba1 --- /dev/null +++ b/api/logic/net/MetaCacheSink.h @@ -0,0 +1,21 @@ +#pragma once +#include "FileSink.h" +#include "ChecksumValidator.h" +#include "net/HttpMetaCache.h" + +namespace Net { +class MetaCacheSink : public FileSink +{ +public: /* con/des */ + MetaCacheSink(MetaEntryPtr entry, ChecksumValidator * md5sum); + virtual ~MetaCacheSink(); + +protected: /* methods */ + JobStatus initCache(QNetworkRequest & request) override; + JobStatus finalizeCache(QNetworkReply & reply) override; + +private: /* data */ + MetaEntryPtr m_entry; + ChecksumValidator * m_md5Node; +}; +} diff --git a/api/logic/net/NetAction.h b/api/logic/net/NetAction.h index 3c395605..2b3f1b7b 100644 --- a/api/logic/net/NetAction.h +++ b/api/logic/net/NetAction.h @@ -59,9 +59,6 @@ public: /// the network reply unique_qobject_ptr m_reply; - /// the content of the content-type header - QString m_content_type; - /// source URL QUrl m_url; diff --git a/api/logic/net/NetJob.cpp b/api/logic/net/NetJob.cpp index 76c61c35..ca86242a 100644 --- a/api/logic/net/NetJob.cpp +++ b/api/logic/net/NetJob.cpp @@ -14,9 +14,7 @@ */ #include "NetJob.h" -#include "MD5EtagDownload.h" -#include "ByteArrayDownload.h" -#include "CacheDownload.h" +#include "Download.h" #include diff --git a/api/logic/net/NetJob.h b/api/logic/net/NetJob.h index 167fe176..3d16b21f 100644 --- a/api/logic/net/NetJob.h +++ b/api/logic/net/NetJob.h @@ -16,9 +16,7 @@ #pragma once #include #include "NetAction.h" -#include "ByteArrayDownload.h" -#include "MD5EtagDownload.h" -#include "CacheDownload.h" +#include "Download.h" #include "HttpMetaCache.h" #include "tasks/Task.h" #include "QObjectPtr.h" diff --git a/api/logic/net/Sink.h b/api/logic/net/Sink.h new file mode 100644 index 00000000..53f2b221 --- /dev/null +++ b/api/logic/net/Sink.h @@ -0,0 +1,70 @@ +#pragma once + +#include "net/NetAction.h" + +#include "multimc_logic_export.h" +#include "Validator.h" + +namespace Net { +class MULTIMC_LOGIC_EXPORT Sink +{ +public: /* con/des */ + Sink() {}; + virtual ~Sink() {}; + +public: /* methods */ + virtual JobStatus init(QNetworkRequest & request) = 0; + virtual JobStatus write(QByteArray & data) = 0; + virtual JobStatus abort() = 0; + virtual JobStatus finalize(QNetworkReply & reply) = 0; + + void addValidator(Validator * validator) + { + if(validator) + { + validators.push_back(std::shared_ptr(validator)); + } + } + +protected: /* methods */ + bool finalizeAllValidators(QNetworkReply & reply) + { + for(auto & validator: validators) + { + if(!validator->validate(reply)) + return false; + } + return true; + } + bool failAllValidators() + { + bool success = true; + for(auto & validator: validators) + { + success &= validator->abort(); + } + return success; + } + bool initAllValidators(QNetworkRequest & request) + { + for(auto & validator: validators) + { + if(!validator->init(request)) + return false; + } + return true; + } + bool writeAllValidators(QByteArray & data) + { + for(auto & validator: validators) + { + if(!validator->write(data)) + return false; + } + return true; + } + +protected: /* data */ + std::vector> validators; +}; +} diff --git a/api/logic/net/Validator.h b/api/logic/net/Validator.h new file mode 100644 index 00000000..a390ab28 --- /dev/null +++ b/api/logic/net/Validator.h @@ -0,0 +1,20 @@ +#pragma once + +#include "net/NetAction.h" + +#include "multimc_logic_export.h" + +namespace Net { +class MULTIMC_LOGIC_EXPORT Validator +{ +public: /* con/des */ + Validator() {}; + virtual ~Validator() {}; + +public: /* methods */ + virtual bool init(QNetworkRequest & request) = 0; + virtual bool write(QByteArray & data) = 0; + virtual bool abort() = 0; + virtual bool validate(QNetworkReply & reply) = 0; +}; +} \ No newline at end of file -- cgit v1.2.3