From 210629e274a08c592d8f38f7b382c1c5a05a7ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sun, 17 Nov 2013 11:44:18 +0100 Subject: Use the forge mirrors for downloading forge libraries Let's hope we never, ever see a forge download error again. --- CMakeLists.txt | 2 + gui/MainWindow.cpp | 2 + logic/OneSixUpdate.cpp | 20 ++++++-- logic/net/CacheDownload.cpp | 6 +-- logic/net/CacheDownload.h | 2 - logic/net/FileDownload.cpp | 4 +- logic/net/FileDownload.h | 2 - logic/net/ForgeMirror.h | 10 ++++ logic/net/ForgeMirrors.cpp | 116 ++++++++++++++++++++++++++++++++++++++++++ logic/net/ForgeMirrors.h | 58 +++++++++++++++++++++ logic/net/ForgeXzDownload.cpp | 101 ++++++++++++++++++++++++++---------- logic/net/ForgeXzDownload.h | 19 +++++-- logic/net/NetJob.cpp | 1 + logic/net/NetJob.h | 10 ++++ 14 files changed, 306 insertions(+), 47 deletions(-) create mode 100644 logic/net/ForgeMirror.h create mode 100644 logic/net/ForgeMirrors.cpp create mode 100644 logic/net/ForgeMirrors.h diff --git a/CMakeLists.txt b/CMakeLists.txt index e150c459..aa78981b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -256,6 +256,8 @@ logic/net/ByteArrayDownload.h logic/net/ByteArrayDownload.cpp logic/net/CacheDownload.h logic/net/CacheDownload.cpp +logic/net/ForgeMirrors.h +logic/net/ForgeMirrors.cpp logic/net/ForgeXzDownload.h logic/net/ForgeXzDownload.cpp logic/net/NetJob.h diff --git a/gui/MainWindow.cpp b/gui/MainWindow.cpp index 1664a21d..4dadfada 100644 --- a/gui/MainWindow.cpp +++ b/gui/MainWindow.cpp @@ -180,6 +180,7 @@ MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent), ui(new Ui::MainWi { MMC->lwjgllist()->loadList(); } + /* assets_downloader = new OneSixAssets(); connect(assets_downloader, SIGNAL(indexStarted()), SLOT(assetsIndexStarted())); connect(assets_downloader, SIGNAL(filesStarted()), SLOT(assetsFilesStarted())); @@ -188,6 +189,7 @@ MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent), ui(new Ui::MainWi connect(assets_downloader, SIGNAL(failed()), SLOT(assetsFailed())); connect(assets_downloader, SIGNAL(finished()), SLOT(assetsFinished())); assets_downloader->start(); + */ } } diff --git a/logic/OneSixUpdate.cpp b/logic/OneSixUpdate.cpp index 2d8a167c..62d95ee6 100644 --- a/logic/OneSixUpdate.cpp +++ b/logic/OneSixUpdate.cpp @@ -28,6 +28,7 @@ #include "OneSixVersion.h" #include "OneSixLibrary.h" #include "OneSixInstance.h" +#include "net/ForgeMirrors.h" #include "pathutils.h" @@ -163,20 +164,33 @@ void OneSixUpdate::jarlibStart() libs.append(version->getActiveNormalLibs()); auto metacache = MMC->metacache(); + QList ForgeLibs; + bool already_forge_xz = false; for (auto lib : libs) { if (lib->hint() == "local") continue; - QString download_path = lib->downloadUrl(); auto entry = metacache->resolveEntry("libraries", lib->storagePath()); if (entry->stale) { if (lib->hint() == "forge-pack-xz") - jarlibDownloadJob->addNetAction(ForgeXzDownload::make(download_path, entry)); + { + ForgeLibs.append(ForgeXzDownload::make(lib->storagePath(), entry)); + } else - jarlibDownloadJob->addNetAction(CacheDownload::make(download_path, entry)); + { + jarlibDownloadJob->addNetAction(CacheDownload::make(lib->downloadUrl(), entry)); + } } } + // TODO: think about how to propagate this from the original json file... or IF AT ALL + QString forgeMirrorList = "http://files.minecraftforge.net/mirror-brand.list"; + if (!ForgeLibs.empty()) + { + jarlibDownloadJob->addNetAction( + ForgeMirrors::make(ForgeLibs, jarlibDownloadJob, forgeMirrorList)); + } + connect(jarlibDownloadJob.get(), SIGNAL(succeeded()), SLOT(jarlibFinished())); connect(jarlibDownloadJob.get(), SIGNAL(failed()), SLOT(jarlibFailed())); connect(jarlibDownloadJob.get(), SIGNAL(progress(qint64, qint64)), diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp index 4fe4e68e..873d3a2e 100644 --- a/logic/net/CacheDownload.cpp +++ b/logic/net/CacheDownload.cpp @@ -29,7 +29,6 @@ CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry) m_entry = entry; m_target_path = entry->getFullPath(); m_status = Job_NotStarted; - m_opened_for_saving = false; } void CacheDownload::start() @@ -87,7 +86,7 @@ void CacheDownload::downloadFinished() // nothing went wrong... m_status = Job_Finished; - if (m_opened_for_saving) + if (m_output_file.isOpen()) { // save the data to the downloadable if we aren't saving to file m_output_file.close(); @@ -133,7 +132,7 @@ void CacheDownload::downloadFinished() void CacheDownload::downloadReadyRead() { - if (!m_opened_for_saving) + if (!m_output_file.isOpen()) { if (!m_output_file.open(QIODevice::WriteOnly)) { @@ -144,7 +143,6 @@ void CacheDownload::downloadReadyRead() emit failed(index_within_job); return; } - m_opened_for_saving = true; } QByteArray ba = m_reply->readAll(); md5sum.addData(ba); diff --git a/logic/net/CacheDownload.h b/logic/net/CacheDownload.h index 2b9a5dd8..e25aecd2 100644 --- a/logic/net/CacheDownload.h +++ b/logic/net/CacheDownload.h @@ -26,8 +26,6 @@ class CacheDownload : public NetAction Q_OBJECT public: MetaEntryPtr m_entry; - /// is the saving file already open? - bool m_opened_for_saving; /// if saving to file, use the one specified in this string QString m_target_path; /// this is the output file, if any diff --git a/logic/net/FileDownload.cpp b/logic/net/FileDownload.cpp index 6b2aa66f..239af351 100644 --- a/logic/net/FileDownload.cpp +++ b/logic/net/FileDownload.cpp @@ -25,7 +25,6 @@ FileDownload::FileDownload(QUrl url, QString target_path) : NetAction() m_target_path = target_path; m_check_md5 = false; m_status = Job_NotStarted; - m_opened_for_saving = false; } void FileDownload::start() @@ -113,7 +112,7 @@ void FileDownload::downloadFinished() void FileDownload::downloadReadyRead() { - if (!m_opened_for_saving) + if (!m_output_file.isOpen()) { if (!m_output_file.open(QIODevice::WriteOnly)) { @@ -124,7 +123,6 @@ void FileDownload::downloadReadyRead() emit failed(index_within_job); return; } - m_opened_for_saving = true; } m_output_file.write(m_reply->readAll()); } diff --git a/logic/net/FileDownload.h b/logic/net/FileDownload.h index 31e0259c..58380e86 100644 --- a/logic/net/FileDownload.h +++ b/logic/net/FileDownload.h @@ -29,8 +29,6 @@ public: bool m_check_md5; /// the expected md5 checksum QString m_expected_md5; - /// is the saving file already open? - bool m_opened_for_saving; /// if saving to file, use the one specified in this string QString m_target_path; /// this is the output file, if any diff --git a/logic/net/ForgeMirror.h b/logic/net/ForgeMirror.h new file mode 100644 index 00000000..2518dffe --- /dev/null +++ b/logic/net/ForgeMirror.h @@ -0,0 +1,10 @@ +#pragma once +#include + +struct ForgeMirror +{ + QString name; + QString logo_url; + QString website_url; + QString mirror_url; +}; \ No newline at end of file diff --git a/logic/net/ForgeMirrors.cpp b/logic/net/ForgeMirrors.cpp new file mode 100644 index 00000000..fd7eccca --- /dev/null +++ b/logic/net/ForgeMirrors.cpp @@ -0,0 +1,116 @@ +#include "MultiMC.h" +#include "ForgeMirrors.h" +#include "logger/QsLog.h" +#include +#include + +ForgeMirrors::ForgeMirrors(QList &libs, NetJobPtr parent_job, + QString mirrorlist) +{ + m_libs = libs; + m_parent_job = parent_job; + m_url = QUrl(mirrorlist); + m_status = Job_NotStarted; +} + +void ForgeMirrors::start() +{ + QLOG_INFO() << "Downloading " << m_url.toString(); + QNetworkRequest request(m_url); + request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)"); + auto worker = MMC->qnam(); + QNetworkReply *rep = worker->get(request); + + m_reply = std::shared_ptr(rep); + connect(rep, SIGNAL(downloadProgress(qint64, qint64)), + SLOT(downloadProgress(qint64, qint64))); + connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); + connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), + SLOT(downloadError(QNetworkReply::NetworkError))); + connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); +} + +void ForgeMirrors::downloadError(QNetworkReply::NetworkError error) +{ + // error happened during download. + QLOG_ERROR() << "Error getting URL:" << m_url.toString().toLocal8Bit() + << "Network error: " << error; + m_status = Job_Failed; +} + +void ForgeMirrors::downloadFinished() +{ + // if the download succeeded + if (m_status != Job_Failed) + { + // nothing went wrong... ? + parseMirrorList(); + return; + } + // else the download failed, we use a fixed list + else + { + m_status = Job_Finished; + m_reply.reset(); + deferToFixedList(); + return; + } +} + +void ForgeMirrors::deferToFixedList() +{ + m_mirrors.clear(); + m_mirrors.append( + {"Minecraft Forge", "http://files.minecraftforge.net/forge_logo.png", + "http://files.minecraftforge.net/", "http://files.minecraftforge.net/maven/"}); + m_mirrors.append({"Creeper Host", + "http://files.minecraftforge.net/forge_logo.png", + "https://www.creeperhost.net/link.php?id=1", + "http://new.creeperrepo.net/forge/maven/"}); + injectDownloads(); + emit succeeded(index_within_job); +} + +void ForgeMirrors::parseMirrorList() +{ + m_status = Job_Finished; + auto data = m_reply->readAll(); + m_reply.reset(); + auto dataLines = data.split('\n'); + for(auto line: dataLines) + { + auto elements = line.split('!'); + if (elements.size() == 4) + { + m_mirrors.append({elements[0],elements[1],elements[2],elements[3]}); + } + } + if(!m_mirrors.size()) + deferToFixedList(); + injectDownloads(); + emit succeeded(index_within_job); +} + +void ForgeMirrors::injectDownloads() +{ + // shuffle the mirrors randomly + std::random_device rd; + std::mt19937 rng(rd()); + std::shuffle(m_mirrors.begin(), m_mirrors.end(), rng); + + // tell parent to download the libs + for(auto lib: m_libs) + { + lib->setMirrors(m_mirrors); + m_parent_job->addNetAction(lib); + } +} + +void ForgeMirrors::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) +{ + emit progress(index_within_job, bytesReceived, bytesTotal); +} + +void ForgeMirrors::downloadReadyRead() +{ +} diff --git a/logic/net/ForgeMirrors.h b/logic/net/ForgeMirrors.h new file mode 100644 index 00000000..990e49d6 --- /dev/null +++ b/logic/net/ForgeMirrors.h @@ -0,0 +1,58 @@ +/* Copyright 2013 MultiMC Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "NetAction.h" +#include "HttpMetaCache.h" +#include "ForgeXzDownload.h" +#include "NetJob.h" +#include +#include +typedef std::shared_ptr ForgeMirrorsPtr; + +class ForgeMirrors : public NetAction +{ + Q_OBJECT +public: + QList m_libs; + NetJobPtr m_parent_job; + QList m_mirrors; + +public: + explicit ForgeMirrors(QList &libs, NetJobPtr parent_job, + QString mirrorlist); + static ForgeMirrorsPtr make(QList &libs, NetJobPtr parent_job, + QString mirrorlist) + { + return ForgeMirrorsPtr(new ForgeMirrors(libs, parent_job, mirrorlist)); + } + +protected +slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); + virtual void downloadError(QNetworkReply::NetworkError error); + virtual void downloadFinished(); + virtual void downloadReadyRead(); + +private: + void parseMirrorList(); + void deferToFixedList(); + void injectDownloads(); + +public +slots: + virtual void start(); +}; diff --git a/logic/net/ForgeXzDownload.cpp b/logic/net/ForgeXzDownload.cpp index 384019d8..77161ab9 100644 --- a/logic/net/ForgeXzDownload.cpp +++ b/logic/net/ForgeXzDownload.cpp @@ -22,30 +22,44 @@ #include #include "logger/QsLog.h" -ForgeXzDownload::ForgeXzDownload(QUrl url, MetaEntryPtr entry) : NetAction() +ForgeXzDownload::ForgeXzDownload(QString relative_path, MetaEntryPtr entry) : NetAction() { - QString urlstr = url.toString(); - urlstr.append(".pack.xz"); - m_url = QUrl(urlstr); m_entry = entry; m_target_path = entry->getFullPath(); m_status = Job_NotStarted; - m_opened_for_saving = false; + m_url_path = relative_path; +} + +void ForgeXzDownload::setMirrors(QList &mirrors) +{ + m_mirror_index = 0; + m_mirrors = mirrors; + updateUrl(); } void ForgeXzDownload::start() { + m_status = Job_InProgress; if (!m_entry->stale) { + m_status = Job_Finished; emit succeeded(index_within_job); return; } // can we actually create the real, final file? if (!ensureFilePathExists(m_target_path)) { + m_status = Job_Failed; + emit failed(index_within_job); + return; + } + if (m_mirrors.empty()) + { + m_status = Job_Failed; emit failed(index_within_job); return; } + QLOG_INFO() << "Downloading " << m_url.toString(); QNetworkRequest request(m_url); request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); @@ -75,14 +89,53 @@ void ForgeXzDownload::downloadError(QNetworkReply::NetworkError error) m_status = Job_Failed; } +void ForgeXzDownload::failAndTryNextMirror() +{ + m_status = Job_Failed; + int next = m_mirror_index + 1; + if(m_mirrors.size() == next) + m_mirror_index = 0; + else + m_mirror_index = next; + + updateUrl(); + emit failed(index_within_job); +} + +void ForgeXzDownload::updateUrl() +{ + QLOG_INFO() << "Updating URL for " << m_url_path; + for (auto possible : m_mirrors) + { + QLOG_INFO() << "Possible: " << possible.name << " : " << possible.mirror_url; + } + QString aggregate = m_mirrors[m_mirror_index].mirror_url + m_url_path + ".pack.xz"; + m_url = QUrl(aggregate); +} + void ForgeXzDownload::downloadFinished() { + //TEST: defer to other possible mirrors (autofail the first one) + /* + QLOG_INFO() <<"dl " << index_within_job << " mirror " << m_mirror_index; + if( m_mirror_index == 0) + { + QLOG_INFO() <<"dl " << index_within_job << " AUTOFAIL"; + m_status = Job_Failed; + m_pack200_xz_file.close(); + m_pack200_xz_file.remove(); + m_reply.reset(); + failAndTryNextMirror(); + return; + } + */ + // if the download succeeded if (m_status != Job_Failed) { // nothing went wrong... m_status = Job_Finished; - if (m_opened_for_saving) + if (m_pack200_xz_file.isOpen()) { // we actually downloaded something! process and isntall it decompressAndInstall(); @@ -90,7 +143,7 @@ void ForgeXzDownload::downloadFinished() } else { - // something bad happened + // something bad happened -- on the local machine! m_status = Job_Failed; m_pack200_xz_file.remove(); m_reply.reset(); @@ -101,10 +154,11 @@ void ForgeXzDownload::downloadFinished() // else the download failed else { + m_status = Job_Failed; m_pack200_xz_file.close(); m_pack200_xz_file.remove(); m_reply.reset(); - emit failed(index_within_job); + failAndTryNextMirror(); return; } } @@ -112,7 +166,7 @@ void ForgeXzDownload::downloadFinished() void ForgeXzDownload::downloadReadyRead() { - if (!m_opened_for_saving) + if (!m_pack200_xz_file.isOpen()) { if (!m_pack200_xz_file.open()) { @@ -123,7 +177,6 @@ void ForgeXzDownload::downloadReadyRead() emit failed(index_within_job); return; } - m_opened_for_saving = true; } m_pack200_xz_file.write(m_reply->readAll()); } @@ -156,8 +209,7 @@ void ForgeXzDownload::decompressAndInstall() if (s == nullptr) { xz_dec_end(s); - m_status = Job_Failed; - emit failed(index_within_job); + failAndTryNextMirror(); return; } b.in = in; @@ -182,8 +234,7 @@ void ForgeXzDownload::decompressAndInstall() { // msg = "Write error\n"; xz_dec_end(s); - m_status = Job_Failed; - emit failed(index_within_job); + failAndTryNextMirror(); return; } @@ -217,44 +268,38 @@ void ForgeXzDownload::decompressAndInstall() case XZ_MEM_ERROR: QLOG_ERROR() << "Memory allocation failed\n"; xz_dec_end(s); - m_status = Job_Failed; - emit failed(index_within_job); + failAndTryNextMirror(); return; case XZ_MEMLIMIT_ERROR: QLOG_ERROR() << "Memory usage limit reached\n"; xz_dec_end(s); - m_status = Job_Failed; - emit failed(index_within_job); + failAndTryNextMirror(); return; case XZ_FORMAT_ERROR: QLOG_ERROR() << "Not a .xz file\n"; xz_dec_end(s); - m_status = Job_Failed; - emit failed(index_within_job); + failAndTryNextMirror(); return; case XZ_OPTIONS_ERROR: QLOG_ERROR() << "Unsupported options in the .xz headers\n"; xz_dec_end(s); - m_status = Job_Failed; - emit failed(index_within_job); + failAndTryNextMirror(); return; case XZ_DATA_ERROR: case XZ_BUF_ERROR: QLOG_ERROR() << "File is corrupt\n"; xz_dec_end(s); - m_status = Job_Failed; - emit failed(index_within_job); + failAndTryNextMirror(); return; default: QLOG_ERROR() << "Bug!\n"; xz_dec_end(s); - m_status = Job_Failed; - emit failed(index_within_job); + failAndTryNextMirror(); return; } } @@ -274,7 +319,7 @@ void ForgeXzDownload::decompressAndInstall() QFile f(m_target_path); if (f.exists()) f.remove(); - emit failed(index_within_job); + failAndTryNextMirror(); return; } @@ -283,7 +328,7 @@ void ForgeXzDownload::decompressAndInstall() if (!jar_file.open(QIODevice::ReadOnly)) { jar_file.remove(); - emit failed(index_within_job); + failAndTryNextMirror(); return; } m_entry->md5sum = QCryptographicHash::hash(jar_file.readAll(), QCryptographicHash::Md5) diff --git a/logic/net/ForgeXzDownload.h b/logic/net/ForgeXzDownload.h index 9f1bb029..990f91f0 100644 --- a/logic/net/ForgeXzDownload.h +++ b/logic/net/ForgeXzDownload.h @@ -19,6 +19,8 @@ #include "HttpMetaCache.h" #include #include +#include "ForgeMirror.h" + typedef std::shared_ptr ForgeXzDownloadPtr; class ForgeXzDownload : public NetAction @@ -26,19 +28,24 @@ class ForgeXzDownload : public NetAction Q_OBJECT public: MetaEntryPtr m_entry; - /// is the saving file already open? - bool m_opened_for_saving; /// if saving to file, use the one specified in this string QString m_target_path; /// this is the output file, if any QTemporaryFile m_pack200_xz_file; + /// mirror index (NOT OPTICS, I SWEAR) + int m_mirror_index = 0; + /// list of mirrors to use. Mirror has the url base + QList m_mirrors; + /// path relative to the mirror base + QString m_url_path; public: - explicit ForgeXzDownload(QUrl url, MetaEntryPtr entry); - static ForgeXzDownloadPtr make(QUrl url, MetaEntryPtr entry) + explicit ForgeXzDownload(QString relative_path, MetaEntryPtr entry); + static ForgeXzDownloadPtr make(QString relative_path, MetaEntryPtr entry) { - return ForgeXzDownloadPtr(new ForgeXzDownload(url, entry)); + return ForgeXzDownloadPtr(new ForgeXzDownload(relative_path, entry)); } + void setMirrors(QList & mirrors); protected slots: @@ -53,4 +60,6 @@ slots: private: void decompressAndInstall(); + void failAndTryNextMirror(); + void updateUrl(); }; diff --git a/logic/net/NetJob.cpp b/logic/net/NetJob.cpp index 21c6d3f7..333cdcbf 100644 --- a/logic/net/NetJob.cpp +++ b/logic/net/NetJob.cpp @@ -89,6 +89,7 @@ void NetJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal) void NetJob::start() { QLOG_INFO() << m_job_name.toLocal8Bit() << " started."; + m_running = true; for (auto iter : downloads) { connect(iter.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int))); diff --git a/logic/net/NetJob.h b/logic/net/NetJob.h index c5c0d00c..021a1550 100644 --- a/logic/net/NetJob.h +++ b/logic/net/NetJob.h @@ -40,6 +40,16 @@ public: downloads.append(action); parts_progress.append(part_info()); total_progress++; + // if this is already running, the action needs to be started right away! + if (isRunning()) + { + emit progress(current_progress, total_progress); + connect(base.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int))); + connect(base.get(), SIGNAL(failed(int)), SLOT(partFailed(int))); + connect(base.get(), SIGNAL(progress(int, qint64, qint64)), + SLOT(partProgress(int, qint64, qint64))); + base->start(); + } return true; } -- cgit v1.2.3