From 47e37635f50c09b4f9a9ee7699e3120bab3e4088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sun, 10 Apr 2016 04:29:29 +0200 Subject: NOISSUE split GUI stuff from logic library --- libraries/logic/net/ByteArrayDownload.cpp | 105 ++++++++++++ libraries/logic/net/ByteArrayDownload.h | 48 ++++++ libraries/logic/net/CacheDownload.cpp | 192 +++++++++++++++++++++ libraries/logic/net/CacheDownload.h | 63 +++++++ libraries/logic/net/HttpMetaCache.cpp | 273 ++++++++++++++++++++++++++++++ libraries/logic/net/HttpMetaCache.h | 125 ++++++++++++++ libraries/logic/net/MD5EtagDownload.cpp | 155 +++++++++++++++++ libraries/logic/net/MD5EtagDownload.h | 52 ++++++ libraries/logic/net/NetAction.h | 96 +++++++++++ libraries/logic/net/NetJob.cpp | 125 ++++++++++++++ libraries/logic/net/NetJob.h | 117 +++++++++++++ libraries/logic/net/PasteUpload.cpp | 99 +++++++++++ libraries/logic/net/PasteUpload.h | 50 ++++++ libraries/logic/net/URLConstants.cpp | 16 ++ libraries/logic/net/URLConstants.h | 40 +++++ 15 files changed, 1556 insertions(+) create mode 100644 libraries/logic/net/ByteArrayDownload.cpp create mode 100644 libraries/logic/net/ByteArrayDownload.h create mode 100644 libraries/logic/net/CacheDownload.cpp create mode 100644 libraries/logic/net/CacheDownload.h create mode 100644 libraries/logic/net/HttpMetaCache.cpp create mode 100644 libraries/logic/net/HttpMetaCache.h create mode 100644 libraries/logic/net/MD5EtagDownload.cpp create mode 100644 libraries/logic/net/MD5EtagDownload.h create mode 100644 libraries/logic/net/NetAction.h create mode 100644 libraries/logic/net/NetJob.cpp create mode 100644 libraries/logic/net/NetJob.h create mode 100644 libraries/logic/net/PasteUpload.cpp create mode 100644 libraries/logic/net/PasteUpload.h create mode 100644 libraries/logic/net/URLConstants.cpp create mode 100644 libraries/logic/net/URLConstants.h (limited to 'libraries/logic/net') diff --git a/libraries/logic/net/ByteArrayDownload.cpp b/libraries/logic/net/ByteArrayDownload.cpp new file mode 100644 index 00000000..21990eeb --- /dev/null +++ b/libraries/logic/net/ByteArrayDownload.cpp @@ -0,0 +1,105 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ByteArrayDownload.h" +#include "Env.h" +#include + +ByteArrayDownload::ByteArrayDownload(QUrl url) : NetAction() +{ + m_url = url; + m_status = Job_NotStarted; +} + +void ByteArrayDownload::start() +{ + qDebug() << "Downloading " << m_url.toString(); + QNetworkRequest request(m_url); + request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)"); + auto worker = ENV.qnam(); + QNetworkReply *rep = worker->get(request); + + m_reply.reset(rep); + connect(rep, SIGNAL(downloadProgress(qint64, qint64)), + SLOT(downloadProgress(qint64, qint64))); + connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); + connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), + SLOT(downloadError(QNetworkReply::NetworkError))); + connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); +} + +void ByteArrayDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) +{ + m_total_progress = bytesTotal; + m_progress = bytesReceived; + emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal); +} + +void ByteArrayDownload::downloadError(QNetworkReply::NetworkError error) +{ + // error happened during download. + qCritical() << "Error getting URL:" << m_url.toString().toLocal8Bit() + << "Network error: " << error; + m_status = Job_Failed; + m_errorString = m_reply->errorString(); +} + +void ByteArrayDownload::downloadFinished() +{ + QVariant redirect = m_reply->header(QNetworkRequest::LocationHeader); + QString redirectURL; + if(redirect.isValid()) + { + redirectURL = redirect.toString(); + } + // FIXME: This is a hack for https://bugreports.qt-project.org/browse/QTBUG-41061 + else if(m_reply->hasRawHeader("Location")) + { + auto data = m_reply->rawHeader("Location"); + if(data.size() > 2 && data[0] == '/' && data[1] == '/') + redirectURL = m_reply->url().scheme() + ":" + data; + } + if (!redirectURL.isEmpty()) + { + m_url = QUrl(redirect.toString()); + qDebug() << "Following redirect to " << m_url.toString(); + start(); + return; + } + + // if the download succeeded + if (m_status != Job_Failed) + { + // nothing went wrong... + m_status = Job_Finished; + m_data = m_reply->readAll(); + m_content_type = m_reply->header(QNetworkRequest::ContentTypeHeader).toString(); + m_reply.reset(); + emit succeeded(m_index_within_job); + return; + } + // else the download failed + else + { + m_reply.reset(); + emit failed(m_index_within_job); + return; + } +} + +void ByteArrayDownload::downloadReadyRead() +{ + // ~_~ +} diff --git a/libraries/logic/net/ByteArrayDownload.h b/libraries/logic/net/ByteArrayDownload.h new file mode 100644 index 00000000..e2fc2911 --- /dev/null +++ b/libraries/logic/net/ByteArrayDownload.h @@ -0,0 +1,48 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include "NetAction.h" + +#include "multimc_logic_export.h" + +typedef std::shared_ptr ByteArrayDownloadPtr; +class MULTIMC_LOGIC_EXPORT ByteArrayDownload : public NetAction +{ + Q_OBJECT +public: + ByteArrayDownload(QUrl url); + static ByteArrayDownloadPtr make(QUrl url) + { + return ByteArrayDownloadPtr(new ByteArrayDownload(url)); + } + virtual ~ByteArrayDownload() {}; +public: + /// if not saving to file, downloaded data is placed here + QByteArray m_data; + + QString m_errorString; + +public +slots: + virtual void start(); + +protected +slots: + void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); + void downloadError(QNetworkReply::NetworkError error); + void downloadFinished(); + void downloadReadyRead(); +}; diff --git a/libraries/logic/net/CacheDownload.cpp b/libraries/logic/net/CacheDownload.cpp new file mode 100644 index 00000000..1ac55180 --- /dev/null +++ b/libraries/logic/net/CacheDownload.cpp @@ -0,0 +1,192 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "CacheDownload.h" + +#include +#include +#include +#include +#include "Env.h" +#include + +CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry) + : NetAction(), md5sum(QCryptographicHash::Md5) +{ + m_url = url; + m_entry = entry; + m_target_path = entry->getFullPath(); + m_status = Job_NotStarted; +} + +void CacheDownload::start() +{ + m_status = Job_InProgress; + if (!m_entry->isStale()) + { + m_status = Job_Finished; + emit succeeded(m_index_within_job); + return; + } + // create a new save file + m_output_file.reset(new QSaveFile(m_target_path)); + + // if there already is a file and md5 checking is in effect and it can be opened + if (!FS::ensureFilePathExists(m_target_path)) + { + qCritical() << "Could not create folder for " + m_target_path; + m_status = Job_Failed; + emit failed(m_index_within_job); + return; + } + if (!m_output_file->open(QIODevice::WriteOnly)) + { + qCritical() << "Could not open " + m_target_path + " for writing"; + m_status = Job_Failed; + emit failed(m_index_within_job); + return; + } + qDebug() << "Downloading " << m_url.toString(); + QNetworkRequest request(m_url); + + // check file consistency first. + QFile current(m_target_path); + if(current.exists() && current.size() != 0) + { + if (m_entry->getRemoteChangedTimestamp().size()) + request.setRawHeader(QString("If-Modified-Since").toLatin1(), + m_entry->getRemoteChangedTimestamp().toLatin1()); + if (m_entry->getETag().size()) + request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->getETag().toLatin1()); + } + + request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Cached)"); + + auto worker = ENV.qnam(); + QNetworkReply *rep = worker->get(request); + + m_reply.reset(rep); + connect(rep, SIGNAL(downloadProgress(qint64, qint64)), + SLOT(downloadProgress(qint64, qint64))); + connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); + connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), + SLOT(downloadError(QNetworkReply::NetworkError))); + connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); +} + +void CacheDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) +{ + m_total_progress = bytesTotal; + m_progress = bytesReceived; + emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal); +} + +void CacheDownload::downloadError(QNetworkReply::NetworkError error) +{ + // error happened during download. + qCritical() << "Failed " << m_url.toString() << " with reason " << error; + m_status = Job_Failed; +} +void CacheDownload::downloadFinished() +{ + QVariant redirect = m_reply->header(QNetworkRequest::LocationHeader); + QString redirectURL; + if(redirect.isValid()) + { + redirectURL = redirect.toString(); + } + // FIXME: This is a hack for https://bugreports.qt-project.org/browse/QTBUG-41061 + else if(m_reply->hasRawHeader("Location")) + { + auto data = m_reply->rawHeader("Location"); + if(data.size() > 2 && data[0] == '/' && data[1] == '/') + redirectURL = m_reply->url().scheme() + ":" + data; + } + if (!redirectURL.isEmpty()) + { + m_url = QUrl(redirect.toString()); + qDebug() << "Following redirect to " << m_url.toString(); + start(); + return; + } + + // if the download succeeded + if (m_status == Job_Failed) + { + m_output_file->cancelWriting(); + m_reply.reset(); + emit failed(m_index_within_job); + return; + } + + // if we wrote any data to the save file, we try to commit the data to the real file. + if (wroteAnyData) + { + // nothing went wrong... + if (m_output_file->commit()) + { + m_status = Job_Finished; + m_entry->setMD5Sum(md5sum.result().toHex().constData()); + } + else + { + qCritical() << "Failed to commit changes to " << m_target_path; + m_output_file->cancelWriting(); + m_reply.reset(); + m_status = Job_Failed; + emit failed(m_index_within_job); + return; + } + } + else + { + m_status = Job_Finished; + } + + // then get rid of the save file + m_output_file.reset(); + + QFileInfo output_file_info(m_target_path); + + m_entry->setETag(m_reply->rawHeader("ETag").constData()); + if (m_reply->hasRawHeader("Last-Modified")) + { + m_entry->setRemoteChangedTimestamp(m_reply->rawHeader("Last-Modified").constData()); + } + m_entry->setLocalChangedTimestamp(output_file_info.lastModified().toUTC().toMSecsSinceEpoch()); + m_entry->setStale(false); + ENV.metacache()->updateEntry(m_entry); + + m_reply.reset(); + emit succeeded(m_index_within_job); + return; +} + +void CacheDownload::downloadReadyRead() +{ + QByteArray ba = m_reply->readAll(); + md5sum.addData(ba); + if (m_output_file->write(ba) != ba.size()) + { + qCritical() << "Failed writing into " + m_target_path; + m_status = Job_Failed; + m_output_file->cancelWriting(); + m_output_file.reset(); + emit failed(m_index_within_job); + wroteAnyData = false; + return; + } + wroteAnyData = true; +} diff --git a/libraries/logic/net/CacheDownload.h b/libraries/logic/net/CacheDownload.h new file mode 100644 index 00000000..d83b2a0f --- /dev/null +++ b/libraries/logic/net/CacheDownload.h @@ -0,0 +1,63 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "NetAction.h" +#include "HttpMetaCache.h" +#include +#include + +#include "multimc_logic_export.h" + +typedef std::shared_ptr CacheDownloadPtr; +class MULTIMC_LOGIC_EXPORT CacheDownload : public NetAction +{ + Q_OBJECT +private: + MetaEntryPtr m_entry; + /// if saving to file, use the one specified in this string + QString m_target_path; + + /// this is the output file, if any + std::unique_ptr m_output_file; + + /// the hash-as-you-download + QCryptographicHash md5sum; + + bool wroteAnyData = false; + +public: + explicit CacheDownload(QUrl url, MetaEntryPtr entry); + static CacheDownloadPtr make(QUrl url, MetaEntryPtr entry) + { + return CacheDownloadPtr(new CacheDownload(url, entry)); + } + virtual ~CacheDownload(){}; + QString getTargetFilepath() + { + return m_target_path; + } +protected +slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); + virtual void downloadError(QNetworkReply::NetworkError error); + virtual void downloadFinished(); + virtual void downloadReadyRead(); + +public +slots: + virtual void start(); +}; diff --git a/libraries/logic/net/HttpMetaCache.cpp b/libraries/logic/net/HttpMetaCache.cpp new file mode 100644 index 00000000..ea3e2834 --- /dev/null +++ b/libraries/logic/net/HttpMetaCache.cpp @@ -0,0 +1,273 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Env.h" +#include "HttpMetaCache.h" +#include "FileSystem.h" + +#include +#include +#include +#include + +#include + +#include +#include +#include + +QString MetaEntry::getFullPath() +{ + // FIXME: make local? + return FS::PathCombine(basePath, relativePath); +} + +HttpMetaCache::HttpMetaCache(QString path) : QObject() +{ + m_index_file = path; + saveBatchingTimer.setSingleShot(true); + saveBatchingTimer.setTimerType(Qt::VeryCoarseTimer); + connect(&saveBatchingTimer, SIGNAL(timeout()), SLOT(SaveNow())); +} + +HttpMetaCache::~HttpMetaCache() +{ + saveBatchingTimer.stop(); + SaveNow(); +} + +MetaEntryPtr HttpMetaCache::getEntry(QString base, QString resource_path) +{ + // no base. no base path. can't store + if (!m_entries.contains(base)) + { + // TODO: log problem + return MetaEntryPtr(); + } + EntryMap &map = m_entries[base]; + if (map.entry_list.contains(resource_path)) + { + return map.entry_list[resource_path]; + } + return MetaEntryPtr(); +} + +MetaEntryPtr HttpMetaCache::resolveEntry(QString base, QString resource_path, QString expected_etag) +{ + auto entry = getEntry(base, resource_path); + // it's not present? generate a default stale entry + if (!entry) + { + return staleEntry(base, resource_path); + } + + auto &selected_base = m_entries[base]; + QString real_path = FS::PathCombine(selected_base.base_path, resource_path); + QFileInfo finfo(real_path); + + // is the file really there? if not -> stale + if (!finfo.isFile() || !finfo.isReadable()) + { + // if the file doesn't exist, we disown the entry + selected_base.entry_list.remove(resource_path); + return staleEntry(base, resource_path); + } + + if (!expected_etag.isEmpty() && expected_etag != entry->etag) + { + // if the etag doesn't match expected, we disown the entry + selected_base.entry_list.remove(resource_path); + return staleEntry(base, resource_path); + } + + // if the file changed, check md5sum + qint64 file_last_changed = finfo.lastModified().toUTC().toMSecsSinceEpoch(); + if (file_last_changed != entry->local_changed_timestamp) + { + QFile input(real_path); + input.open(QIODevice::ReadOnly); + QString md5sum = QCryptographicHash::hash(input.readAll(), QCryptographicHash::Md5) + .toHex() + .constData(); + if (entry->md5sum != md5sum) + { + selected_base.entry_list.remove(resource_path); + return staleEntry(base, resource_path); + } + // md5sums matched... keep entry and save the new state to file + entry->local_changed_timestamp = file_last_changed; + SaveEventually(); + } + + // entry passed all the checks we cared about. + entry->basePath = getBasePath(base); + return entry; +} + +bool HttpMetaCache::updateEntry(MetaEntryPtr stale_entry) +{ + if (!m_entries.contains(stale_entry->baseId)) + { + qCritical() << "Cannot add entry with unknown base: " + << stale_entry->baseId.toLocal8Bit(); + return false; + } + if (stale_entry->stale) + { + qCritical() << "Cannot add stale entry: " << stale_entry->getFullPath().toLocal8Bit(); + return false; + } + m_entries[stale_entry->baseId].entry_list[stale_entry->relativePath] = stale_entry; + SaveEventually(); + return true; +} + +bool HttpMetaCache::evictEntry(MetaEntryPtr entry) +{ + if(entry) + { + entry->stale = true; + SaveEventually(); + return true; + } + return false; +} + +MetaEntryPtr HttpMetaCache::staleEntry(QString base, QString resource_path) +{ + auto foo = new MetaEntry(); + foo->baseId = base; + foo->basePath = getBasePath(base); + foo->relativePath = resource_path; + foo->stale = true; + return MetaEntryPtr(foo); +} + +void HttpMetaCache::addBase(QString base, QString base_root) +{ + // TODO: report error + if (m_entries.contains(base)) + return; + // TODO: check if the base path is valid + EntryMap foo; + foo.base_path = base_root; + m_entries[base] = foo; +} + +QString HttpMetaCache::getBasePath(QString base) +{ + if (m_entries.contains(base)) + { + return m_entries[base].base_path; + } + return QString(); +} + +void HttpMetaCache::Load() +{ + if(m_index_file.isNull()) + return; + + QFile index(m_index_file); + if (!index.open(QIODevice::ReadOnly)) + return; + + QJsonDocument json = QJsonDocument::fromJson(index.readAll()); + if (!json.isObject()) + return; + auto root = json.object(); + // check file version first + auto version_val = root.value("version"); + if (!version_val.isString()) + return; + if (version_val.toString() != "1") + return; + + // read the entry array + auto entries_val = root.value("entries"); + if (!entries_val.isArray()) + return; + QJsonArray array = entries_val.toArray(); + for (auto element : array) + { + if (!element.isObject()) + return; + auto element_obj = element.toObject(); + QString base = element_obj.value("base").toString(); + if (!m_entries.contains(base)) + continue; + auto &entrymap = m_entries[base]; + auto foo = new MetaEntry(); + foo->baseId = base; + QString path = foo->relativePath = element_obj.value("path").toString(); + foo->md5sum = element_obj.value("md5sum").toString(); + foo->etag = element_obj.value("etag").toString(); + foo->local_changed_timestamp = element_obj.value("last_changed_timestamp").toDouble(); + foo->remote_changed_timestamp = + element_obj.value("remote_changed_timestamp").toString(); + // presumed innocent until closer examination + foo->stale = false; + entrymap.entry_list[path] = MetaEntryPtr(foo); + } +} + +void HttpMetaCache::SaveEventually() +{ + // reset the save timer + saveBatchingTimer.stop(); + saveBatchingTimer.start(30000); +} + +void HttpMetaCache::SaveNow() +{ + if(m_index_file.isNull()) + return; + QJsonObject toplevel; + toplevel.insert("version", QJsonValue(QString("1"))); + QJsonArray entriesArr; + for (auto group : m_entries) + { + for (auto entry : group.entry_list) + { + // do not save stale entries. they are dead. + if(entry->stale) + { + continue; + } + QJsonObject entryObj; + entryObj.insert("base", QJsonValue(entry->baseId)); + entryObj.insert("path", QJsonValue(entry->relativePath)); + entryObj.insert("md5sum", QJsonValue(entry->md5sum)); + entryObj.insert("etag", QJsonValue(entry->etag)); + entryObj.insert("last_changed_timestamp", + QJsonValue(double(entry->local_changed_timestamp))); + if (!entry->remote_changed_timestamp.isEmpty()) + entryObj.insert("remote_changed_timestamp", + QJsonValue(entry->remote_changed_timestamp)); + entriesArr.append(entryObj); + } + } + toplevel.insert("entries", entriesArr); + + QJsonDocument doc(toplevel); + try + { + FS::write(m_index_file, doc.toJson()); + } + catch (Exception & e) + { + qWarning() << e.what(); + } +} diff --git a/libraries/logic/net/HttpMetaCache.h b/libraries/logic/net/HttpMetaCache.h new file mode 100644 index 00000000..7b626c70 --- /dev/null +++ b/libraries/logic/net/HttpMetaCache.h @@ -0,0 +1,125 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include + +#include "multimc_logic_export.h" + +class HttpMetaCache; + +class MULTIMC_LOGIC_EXPORT MetaEntry +{ +friend class HttpMetaCache; +protected: + MetaEntry() {} +public: + bool isStale() + { + return stale; + } + void setStale(bool stale) + { + this->stale = stale; + } + QString getFullPath(); + QString getRemoteChangedTimestamp() + { + return remote_changed_timestamp; + } + void setRemoteChangedTimestamp(QString remote_changed_timestamp) + { + this->remote_changed_timestamp = remote_changed_timestamp; + } + void setLocalChangedTimestamp(qint64 timestamp) + { + local_changed_timestamp = timestamp; + } + QString getETag() + { + return etag; + } + void setETag(QString etag) + { + this->etag = etag; + } + QString getMD5Sum() + { + return md5sum; + } + void setMD5Sum(QString md5sum) + { + this->md5sum = md5sum; + } +protected: + QString baseId; + QString basePath; + QString relativePath; + QString md5sum; + QString etag; + qint64 local_changed_timestamp = 0; + QString remote_changed_timestamp; // QString for now, RFC 2822 encoded time + bool stale = true; +}; + +typedef std::shared_ptr MetaEntryPtr; + +class MULTIMC_LOGIC_EXPORT HttpMetaCache : public QObject +{ + Q_OBJECT +public: + // supply path to the cache index file + HttpMetaCache(QString path = QString()); + ~HttpMetaCache(); + + // get the entry solely from the cache + // you probably don't want this, unless you have some specific caching needs. + MetaEntryPtr getEntry(QString base, QString resource_path); + + // get the entry from cache and verify that it isn't stale (within reason) + MetaEntryPtr resolveEntry(QString base, QString resource_path, + QString expected_etag = QString()); + + // add a previously resolved stale entry + bool updateEntry(MetaEntryPtr stale_entry); + + // evict selected entry from cache + bool evictEntry(MetaEntryPtr entry); + + void addBase(QString base, QString base_root); + + // (re)start a timer that calls SaveNow later. + void SaveEventually(); + void Load(); + QString getBasePath(QString base); +public +slots: + void SaveNow(); + +private: + // create a new stale entry, given the parameters + MetaEntryPtr staleEntry(QString base, QString resource_path); + struct EntryMap + { + QString base_path; + QMap entry_list; + }; + QMap m_entries; + QString m_index_file; + QTimer saveBatchingTimer; +}; diff --git a/libraries/logic/net/MD5EtagDownload.cpp b/libraries/logic/net/MD5EtagDownload.cpp new file mode 100644 index 00000000..3b4d5dcd --- /dev/null +++ b/libraries/logic/net/MD5EtagDownload.cpp @@ -0,0 +1,155 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Env.h" +#include "MD5EtagDownload.h" +#include +#include +#include + +MD5EtagDownload::MD5EtagDownload(QUrl url, QString target_path) : NetAction() +{ + m_url = url; + m_target_path = target_path; + m_status = Job_NotStarted; +} + +void MD5EtagDownload::start() +{ + QString filename = m_target_path; + m_output_file.setFileName(filename); + // if there already is a file and md5 checking is in effect and it can be opened + if (m_output_file.exists() && m_output_file.open(QIODevice::ReadOnly)) + { + // get the md5 of the local file. + m_local_md5 = + QCryptographicHash::hash(m_output_file.readAll(), QCryptographicHash::Md5) + .toHex() + .constData(); + m_output_file.close(); + // if we are expecting some md5sum, compare it with the local one + if (!m_expected_md5.isEmpty()) + { + // skip if they match + if(m_local_md5 == m_expected_md5) + { + qDebug() << "Skipping " << m_url.toString() << ": md5 match."; + emit succeeded(m_index_within_job); + return; + } + } + else + { + // no expected md5. we use the local md5sum as an ETag + } + } + if (!FS::ensureFilePathExists(filename)) + { + emit failed(m_index_within_job); + return; + } + + QNetworkRequest request(m_url); + + qDebug() << "Downloading " << m_url.toString() << " local MD5: " << m_local_md5; + + if(!m_local_md5.isEmpty()) + { + request.setRawHeader(QString("If-None-Match").toLatin1(), m_local_md5.toLatin1()); + } + if(!m_expected_md5.isEmpty()) + qDebug() << "Expecting " << m_expected_md5; + + request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)"); + + // Go ahead and try to open the file. + // If we don't do this, empty files won't be created, which breaks the updater. + // Plus, this way, we don't end up starting a download for a file we can't open. + if (!m_output_file.open(QIODevice::WriteOnly)) + { + emit failed(m_index_within_job); + return; + } + + auto worker = ENV.qnam(); + QNetworkReply *rep = worker->get(request); + + m_reply.reset(rep); + connect(rep, SIGNAL(downloadProgress(qint64, qint64)), + SLOT(downloadProgress(qint64, qint64))); + connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); + connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), + SLOT(downloadError(QNetworkReply::NetworkError))); + connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); +} + +void MD5EtagDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) +{ + m_total_progress = bytesTotal; + m_progress = bytesReceived; + emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal); +} + +void MD5EtagDownload::downloadError(QNetworkReply::NetworkError error) +{ + qCritical() << "Error" << error << ":" << m_reply->errorString() << "while downloading" + << m_reply->url(); + m_status = Job_Failed; +} + +void MD5EtagDownload::downloadFinished() +{ + // if the download succeeded + if (m_status != Job_Failed) + { + // nothing went wrong... + m_status = Job_Finished; + m_output_file.close(); + + // FIXME: compare with the real written data md5sum + // this is just an ETag + qDebug() << "Finished " << m_url.toString() << " got " << m_reply->rawHeader("ETag").constData(); + + m_reply.reset(); + emit succeeded(m_index_within_job); + return; + } + // else the download failed + else + { + m_output_file.close(); + m_output_file.remove(); + m_reply.reset(); + emit failed(m_index_within_job); + return; + } +} + +void MD5EtagDownload::downloadReadyRead() +{ + if (!m_output_file.isOpen()) + { + if (!m_output_file.open(QIODevice::WriteOnly)) + { + /* + * Can't open the file... the job failed + */ + m_reply->abort(); + emit failed(m_index_within_job); + return; + } + } + m_output_file.write(m_reply->readAll()); +} diff --git a/libraries/logic/net/MD5EtagDownload.h b/libraries/logic/net/MD5EtagDownload.h new file mode 100644 index 00000000..cd1cb550 --- /dev/null +++ b/libraries/logic/net/MD5EtagDownload.h @@ -0,0 +1,52 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "NetAction.h" +#include + +typedef std::shared_ptr Md5EtagDownloadPtr; +class MD5EtagDownload : public NetAction +{ + Q_OBJECT +public: + /// the expected md5 checksum. Only set from outside + QString m_expected_md5; + /// the md5 checksum of a file that already exists. + QString m_local_md5; + /// if saving to file, use the one specified in this string + QString m_target_path; + /// this is the output file, if any + QFile m_output_file; + +public: + explicit MD5EtagDownload(QUrl url, QString target_path); + static Md5EtagDownloadPtr make(QUrl url, QString target_path) + { + return Md5EtagDownloadPtr(new MD5EtagDownload(url, target_path)); + } + virtual ~MD5EtagDownload(){}; +protected +slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); + virtual void downloadError(QNetworkReply::NetworkError error); + virtual void downloadFinished(); + virtual void downloadReadyRead(); + +public +slots: + virtual void start(); +}; diff --git a/libraries/logic/net/NetAction.h b/libraries/logic/net/NetAction.h new file mode 100644 index 00000000..3c395605 --- /dev/null +++ b/libraries/logic/net/NetAction.h @@ -0,0 +1,96 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "multimc_logic_export.h" + +enum JobStatus +{ + Job_NotStarted, + Job_InProgress, + Job_Finished, + Job_Failed +}; + +typedef std::shared_ptr NetActionPtr; +class MULTIMC_LOGIC_EXPORT NetAction : public QObject +{ + Q_OBJECT +protected: + explicit NetAction() : QObject(0) {}; + +public: + virtual ~NetAction() {}; + +public: + virtual qint64 totalProgress() const + { + return m_total_progress; + } + virtual qint64 currentProgress() const + { + return m_progress; + } + virtual qint64 numberOfFailures() const + { + return m_failures; + } + +public: + /// the network reply + unique_qobject_ptr m_reply; + + /// the content of the content-type header + QString m_content_type; + + /// source URL + QUrl m_url; + + /// The file's status + JobStatus m_status = Job_NotStarted; + + /// index within the parent job + int m_index_within_job = 0; + + qint64 m_progress = 0; + qint64 m_total_progress = 1; + + /// number of failures up to this point + int m_failures = 0; + +signals: + void started(int index); + void netActionProgress(int index, qint64 current, qint64 total); + void succeeded(int index); + void failed(int index); + +protected +slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0; + virtual void downloadError(QNetworkReply::NetworkError error) = 0; + virtual void downloadFinished() = 0; + virtual void downloadReadyRead() = 0; + +public +slots: + virtual void start() = 0; +}; diff --git a/libraries/logic/net/NetJob.cpp b/libraries/logic/net/NetJob.cpp new file mode 100644 index 00000000..76c61c35 --- /dev/null +++ b/libraries/logic/net/NetJob.cpp @@ -0,0 +1,125 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "NetJob.h" +#include "MD5EtagDownload.h" +#include "ByteArrayDownload.h" +#include "CacheDownload.h" + +#include + +void NetJob::partSucceeded(int index) +{ + // do progress. all slots are 1 in size at least + auto &slot = parts_progress[index]; + partProgress(index, slot.total_progress, slot.total_progress); + + m_doing.remove(index); + m_done.insert(index); + downloads[index].get()->disconnect(this); + startMoreParts(); +} + +void NetJob::partFailed(int index) +{ + m_doing.remove(index); + auto &slot = parts_progress[index]; + if (slot.failures == 3) + { + m_failed.insert(index); + } + else + { + slot.failures++; + m_todo.enqueue(index); + } + downloads[index].get()->disconnect(this); + startMoreParts(); +} + +void NetJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal) +{ + auto &slot = parts_progress[index]; + + current_progress -= slot.current_progress; + slot.current_progress = bytesReceived; + current_progress += slot.current_progress; + + total_progress -= slot.total_progress; + slot.total_progress = bytesTotal; + total_progress += slot.total_progress; + setProgress(current_progress, total_progress); +} + +void NetJob::executeTask() +{ + qDebug() << m_job_name.toLocal8Bit() << " started."; + m_running = true; + for (int i = 0; i < downloads.size(); i++) + { + m_todo.enqueue(i); + } + // hack that delays early failures so they can be caught easier + QMetaObject::invokeMethod(this, "startMoreParts", Qt::QueuedConnection); +} + +void NetJob::startMoreParts() +{ + // check for final conditions if there's nothing in the queue + if(!m_todo.size()) + { + if(!m_doing.size()) + { + if(!m_failed.size()) + { + qDebug() << m_job_name << "succeeded."; + emitSucceeded(); + } + else + { + qCritical() << m_job_name << "failed."; + emitFailed(tr("Job '%1' failed to process:\n%2").arg(m_job_name).arg(getFailedFiles().join("\n"))); + } + } + return; + } + // otherwise try to start more parts + while (m_doing.size() < 6) + { + if(!m_todo.size()) + return; + int doThis = m_todo.dequeue(); + m_doing.insert(doThis); + auto part = downloads[doThis]; + // connect signals :D + connect(part.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int))); + connect(part.get(), SIGNAL(failed(int)), SLOT(partFailed(int))); + connect(part.get(), SIGNAL(netActionProgress(int, qint64, qint64)), + SLOT(partProgress(int, qint64, qint64))); + part->start(); + } +} + + +QStringList NetJob::getFailedFiles() +{ + QStringList failed; + for (auto index: m_failed) + { + failed.push_back(downloads[index]->m_url.toString()); + } + failed.sort(); + return failed; +} diff --git a/libraries/logic/net/NetJob.h b/libraries/logic/net/NetJob.h new file mode 100644 index 00000000..167fe176 --- /dev/null +++ b/libraries/logic/net/NetJob.h @@ -0,0 +1,117 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include "NetAction.h" +#include "ByteArrayDownload.h" +#include "MD5EtagDownload.h" +#include "CacheDownload.h" +#include "HttpMetaCache.h" +#include "tasks/Task.h" +#include "QObjectPtr.h" + +#include "multimc_logic_export.h" + +class NetJob; +typedef shared_qobject_ptr NetJobPtr; + +class MULTIMC_LOGIC_EXPORT NetJob : public Task +{ + Q_OBJECT +public: + explicit NetJob(QString job_name) : Task(), m_job_name(job_name) {} + virtual ~NetJob() {} + bool addNetAction(NetActionPtr action) + { + action->m_index_within_job = downloads.size(); + downloads.append(action); + part_info pi; + { + pi.current_progress = action->currentProgress(); + pi.total_progress = action->totalProgress(); + pi.failures = action->numberOfFailures(); + } + parts_progress.append(pi); + total_progress += pi.total_progress; + // if this is already running, the action needs to be started right away! + if (isRunning()) + { + setProgress(current_progress, total_progress); + connect(action.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int))); + connect(action.get(), SIGNAL(failed(int)), SLOT(partFailed(int))); + connect(action.get(), SIGNAL(netActionProgress(int, qint64, qint64)), + SLOT(partProgress(int, qint64, qint64))); + action->start(); + } + return true; + } + + NetActionPtr operator[](int index) + { + return downloads[index]; + } + const NetActionPtr at(const int index) + { + return downloads.at(index); + } + NetActionPtr first() + { + if (downloads.size()) + return downloads[0]; + return NetActionPtr(); + } + int size() const + { + return downloads.size(); + } + virtual bool isRunning() const + { + return m_running; + } + QStringList getFailedFiles(); + +private slots: + void startMoreParts(); + +public slots: + virtual void executeTask(); + // FIXME: implement + virtual bool abort() {return false;}; + +private slots: + void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal); + void partSucceeded(int index); + void partFailed(int index); + +private: + struct part_info + { + qint64 current_progress = 0; + qint64 total_progress = 1; + int failures = 0; + bool connected = false; + }; + QString m_job_name; + QList downloads; + QList parts_progress; + QQueue m_todo; + QSet m_doing; + QSet m_done; + QSet m_failed; + qint64 current_progress = 0; + qint64 total_progress = 0; + bool m_running = false; +}; diff --git a/libraries/logic/net/PasteUpload.cpp b/libraries/logic/net/PasteUpload.cpp new file mode 100644 index 00000000..4b671d6f --- /dev/null +++ b/libraries/logic/net/PasteUpload.cpp @@ -0,0 +1,99 @@ +#include "PasteUpload.h" +#include "Env.h" +#include +#include +#include + +PasteUpload::PasteUpload(QWidget *window, QString text, QString key) : m_window(window) +{ + m_key = key; + QByteArray temp; + temp = text.toUtf8(); + temp.replace('\n', "\r\n"); + m_textSize = temp.size(); + m_text = "key=" + m_key.toLatin1() + "&description=MultiMC5+Log+File&language=plain&format=json&expire=2592000&paste=" + temp.toPercentEncoding(); + buf = new QBuffer(&m_text); +} + +PasteUpload::~PasteUpload() +{ + if(buf) + { + delete buf; + } +} + +bool PasteUpload::validateText() +{ + return m_textSize <= maxSize(); +} + +void PasteUpload::executeTask() +{ + QNetworkRequest request(QUrl("http://paste.ee/api")); + request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)"); + + request.setRawHeader("Content-Type", "application/x-www-form-urlencoded"); + request.setRawHeader("Content-Length", QByteArray::number(m_text.size())); + + auto worker = ENV.qnam(); + QNetworkReply *rep = worker->post(request, buf); + + m_reply = std::shared_ptr(rep); + setStatus(tr("Uploading to paste.ee")); + connect(rep, &QNetworkReply::uploadProgress, this, &Task::setProgress); + connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), this, SLOT(downloadError(QNetworkReply::NetworkError))); + connect(rep, SIGNAL(finished()), this, SLOT(downloadFinished())); +} + +void PasteUpload::downloadError(QNetworkReply::NetworkError error) +{ + // error happened during download. + qCritical() << "Network error: " << error; + emitFailed(m_reply->errorString()); +} + +void PasteUpload::downloadFinished() +{ + // if the download succeeded + if (m_reply->error() == QNetworkReply::NetworkError::NoError) + { + QByteArray data = m_reply->readAll(); + m_reply.reset(); + QJsonParseError jsonError; + QJsonDocument doc = QJsonDocument::fromJson(data, &jsonError); + if (jsonError.error != QJsonParseError::NoError) + { + emitFailed(jsonError.errorString()); + return; + } + if (!parseResult(doc)) + { + emitFailed(tr("paste.ee returned an error. Please consult the logs for more information")); + return; + } + } + // else the download failed + else + { + emitFailed(QString("Network error: %1").arg(m_reply->errorString())); + m_reply.reset(); + return; + } + emitSucceeded(); +} + +bool PasteUpload::parseResult(QJsonDocument doc) +{ + auto object = doc.object(); + auto status = object.value("status").toString("error"); + if (status == "error") + { + qCritical() << "paste.ee reported error:" << QString(object.value("error").toString()); + return false; + } + m_pasteLink = object.value("paste").toObject().value("link").toString(); + m_pasteID = object.value("paste").toObject().value("id").toString(); + return true; +} + diff --git a/libraries/logic/net/PasteUpload.h b/libraries/logic/net/PasteUpload.h new file mode 100644 index 00000000..06e3f955 --- /dev/null +++ b/libraries/logic/net/PasteUpload.h @@ -0,0 +1,50 @@ +#pragma once +#include "tasks/Task.h" +#include +#include +#include + +#include "multimc_logic_export.h" + +class MULTIMC_LOGIC_EXPORT PasteUpload : public Task +{ + Q_OBJECT +public: + PasteUpload(QWidget *window, QString text, QString key = "public"); + virtual ~PasteUpload(); + QString pasteLink() + { + return m_pasteLink; + } + QString pasteID() + { + return m_pasteID; + } + uint32_t maxSize() + { + // 2MB for paste.ee - public + if(m_key == "public") + return 1024*1024*2; + // 12MB for paste.ee - with actual key + return 1024*1024*12; + } + bool validateText(); +protected: + virtual void executeTask(); + +private: + bool parseResult(QJsonDocument doc); + QByteArray m_text; + QString m_error; + QWidget *m_window; + QString m_pasteID; + QString m_pasteLink; + QString m_key; + int m_textSize = 0; + QBuffer * buf = nullptr; + std::shared_ptr m_reply; +public +slots: + void downloadError(QNetworkReply::NetworkError); + void downloadFinished(); +}; diff --git a/libraries/logic/net/URLConstants.cpp b/libraries/logic/net/URLConstants.cpp new file mode 100644 index 00000000..bd476b2c --- /dev/null +++ b/libraries/logic/net/URLConstants.cpp @@ -0,0 +1,16 @@ +#include "URLConstants.h" + +namespace URLConstants { + +QString getLegacyJarUrl(QString version) +{ + return "http://" + AWS_DOWNLOAD_VERSIONS + getJarPath(version); +} + +QString getJarPath(QString version) +{ + return version + "/" + version + ".jar"; +} + + +} diff --git a/libraries/logic/net/URLConstants.h b/libraries/logic/net/URLConstants.h new file mode 100644 index 00000000..8923ef54 --- /dev/null +++ b/libraries/logic/net/URLConstants.h @@ -0,0 +1,40 @@ +/* Copyright 2013-2015 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace URLConstants +{ +const QString AWS_DOWNLOAD_VERSIONS("s3.amazonaws.com/Minecraft.Download/versions/"); +const QString RESOURCE_BASE("resources.download.minecraft.net/"); +const QString LIBRARY_BASE("libraries.minecraft.net/"); +//const QString SKINS_BASE("skins.minecraft.net/MinecraftSkins/"); +const QString SKINS_BASE("crafatar.com/skins/"); +const QString AUTH_BASE("authserver.mojang.com/"); +const QString FORGE_LEGACY_URL("http://files.minecraftforge.net/minecraftforge/json"); +const QString FORGE_GRADLE_URL("http://files.minecraftforge.net/maven/net/minecraftforge/forge/json"); +const QString MOJANG_STATUS_URL("http://status.mojang.com/check"); +const QString MOJANG_STATUS_NEWS_URL("http://status.mojang.com/news"); +const QString LITELOADER_URL("http://dl.liteloader.com/versions/versions.json"); +const QString IMGUR_BASE_URL("https://api.imgur.com/3/"); +const QString FMLLIBS_OUR_BASE_URL("http://files.multimc.org/fmllibs/"); +const QString FMLLIBS_FORGE_BASE_URL("http://files.minecraftforge.net/fmllibs/"); +const QString TRANSLATIONS_BASE_URL("http://files.multimc.org/translations/"); + +QString getJarPath(QString version); +QString getLegacyJarUrl(QString version); +} -- cgit v1.2.3