From 923347729557eed76e4f7e9f6f5f1a79216de0a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sat, 26 Oct 2013 19:55:48 +0200 Subject: S3 bucket listing support and network code refactors. * Adds support for listing all objects in an S3 bucket. * Renames a bunch of network related classes (Download->Action) * Net actions now have static constructors --- CMakeLists.txt | 9 ++- gui/LegacyModEditDialog.cpp | 4 +- gui/LegacyModEditDialog.h | 4 +- gui/OneSixModEditDialog.cpp | 4 +- gui/mainwindow.cpp | 43 ++++++----- logic/BaseUpdate.h | 2 +- logic/LegacyUpdate.cpp | 4 +- logic/LegacyUpdate.h | 4 +- logic/OneSixAssets.cpp | 99 ++++++++---------------- logic/OneSixAssets.h | 8 +- logic/OneSixUpdate.cpp | 14 ++-- logic/OneSixUpdate.h | 6 +- logic/lists/ForgeVersionList.cpp | 15 ++-- logic/lists/ForgeVersionList.h | 4 +- logic/net/ByteArrayDownload.cpp | 2 +- logic/net/ByteArrayDownload.h | 23 +++--- logic/net/CacheDownload.cpp | 2 +- logic/net/CacheDownload.h | 22 ++++-- logic/net/Download.h | 54 ------------- logic/net/DownloadJob.cpp | 138 --------------------------------- logic/net/DownloadJob.h | 86 --------------------- logic/net/FileDownload.cpp | 2 +- logic/net/FileDownload.h | 25 +++--- logic/net/ForgeXzDownload.cpp | 2 +- logic/net/ForgeXzDownload.h | 22 ++++-- logic/net/NetAction.h | 53 +++++++++++++ logic/net/NetJob.cpp | 98 ++++++++++++++++++++++++ logic/net/NetJob.h | 94 +++++++++++++++++++++++ logic/net/S3ListBucket.cpp | 161 +++++++++++++++++++++++++++++++++++++++ logic/net/S3ListBucket.h | 42 ++++++++++ 30 files changed, 604 insertions(+), 442 deletions(-) delete mode 100644 logic/net/Download.h delete mode 100644 logic/net/DownloadJob.cpp delete mode 100644 logic/net/DownloadJob.h create mode 100644 logic/net/NetAction.h create mode 100644 logic/net/NetJob.cpp create mode 100644 logic/net/NetJob.h create mode 100644 logic/net/S3ListBucket.cpp create mode 100644 logic/net/S3ListBucket.h diff --git a/CMakeLists.txt b/CMakeLists.txt index b7dd6ea3..16825be3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -241,7 +241,7 @@ logic/InstanceLauncher.h logic/InstanceLauncher.cpp # network stuffs -logic/net/Download.h +logic/net/NetAction.h logic/net/FileDownload.h logic/net/FileDownload.cpp logic/net/ByteArrayDownload.h @@ -250,12 +250,15 @@ logic/net/CacheDownload.h logic/net/CacheDownload.cpp logic/net/ForgeXzDownload.h logic/net/ForgeXzDownload.cpp -logic/net/DownloadJob.h -logic/net/DownloadJob.cpp +logic/net/NetJob.h +logic/net/NetJob.cpp logic/net/HttpMetaCache.h logic/net/HttpMetaCache.cpp logic/net/LoginTask.h logic/net/LoginTask.cpp +logic/net/S3ListBucket.h +logic/net/S3ListBucket.cpp + # legacy instances logic/LegacyInstance.h diff --git a/gui/LegacyModEditDialog.cpp b/gui/LegacyModEditDialog.cpp index b230193a..a7021bf9 100644 --- a/gui/LegacyModEditDialog.cpp +++ b/gui/LegacyModEditDialog.cpp @@ -218,8 +218,8 @@ void LegacyModEditDialog::on_addForgeBtn_clicked() auto entry = MMC->metacache()->resolveEntry("minecraftforge", forge->filename); if (entry->stale) { - DownloadJob *fjob = new DownloadJob("Forge download"); - fjob->addCacheDownload(forge->universal_url, entry); + NetJob *fjob = new NetJob("Forge download"); + fjob->addNetAction(CacheDownload::make(forge->universal_url, entry)); ProgressDialog dlg(this); dlg.exec(fjob); if (dlg.result() == QDialog::Accepted) diff --git a/gui/LegacyModEditDialog.h b/gui/LegacyModEditDialog.h index fc3ea1e6..d5582aef 100644 --- a/gui/LegacyModEditDialog.h +++ b/gui/LegacyModEditDialog.h @@ -17,7 +17,7 @@ #include #include "logic/LegacyInstance.h" -#include +#include namespace Ui { @@ -74,5 +74,5 @@ private: std::shared_ptr m_jarmods; std::shared_ptr m_texturepacks; LegacyInstance *m_inst; - DownloadJobPtr forgeJob; + NetJobPtr forgeJob; }; diff --git a/gui/OneSixModEditDialog.cpp b/gui/OneSixModEditDialog.cpp index 88738938..54f7289d 100644 --- a/gui/OneSixModEditDialog.cpp +++ b/gui/OneSixModEditDialog.cpp @@ -166,8 +166,8 @@ void OneSixModEditDialog::on_forgeBtn_clicked() auto entry = MMC->metacache()->resolveEntry("minecraftforge", forgeVersion->filename); if (entry->stale) { - DownloadJob *fjob = new DownloadJob("Forge download"); - fjob->addCacheDownload(forgeVersion->installer_url, entry); + NetJob *fjob = new NetJob("Forge download"); + fjob->addNetAction(CacheDownload::make(forgeVersion->installer_url, entry)); ProgressDialog dlg(this); dlg.exec(fjob); if (dlg.result() == QDialog::Accepted) diff --git a/gui/mainwindow.cpp b/gui/mainwindow.cpp index 46fd5bd7..5d3c52b5 100644 --- a/gui/mainwindow.cpp +++ b/gui/mainwindow.cpp @@ -176,7 +176,8 @@ MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent), ui(new Ui::MainWi assets_downloader = new OneSixAssets(); connect(assets_downloader, SIGNAL(indexStarted()), SLOT(assetsIndexStarted())); connect(assets_downloader, SIGNAL(filesStarted()), SLOT(assetsFilesStarted())); - connect(assets_downloader, SIGNAL(filesProgress(int, int, int)), SLOT(assetsFilesProgress(int, int, int))); + connect(assets_downloader, SIGNAL(filesProgress(int, int, int)), + SLOT(assetsFilesProgress(int, int, int))); connect(assets_downloader, SIGNAL(failed()), SLOT(assetsFailed())); connect(assets_downloader, SIGNAL(finished()), SLOT(assetsFinished())); assets_downloader->start(); @@ -465,8 +466,10 @@ void MainWindow::instanceActivated(QModelIndex index) (BaseInstance *)index.data(InstanceList::InstancePointerRole).value(); bool autoLogin = MMC->settings()->get("AutoLogin").toBool(); - if(autoLogin) doAutoLogin(); - else doLogin(); + if (autoLogin) + doAutoLogin(); + else + doLogin(); } void MainWindow::on_actionLaunchInstance_triggered() @@ -482,15 +485,15 @@ void MainWindow::doAutoLogin() if (!m_selectedInstance) return; - Keyring * k = Keyring::instance(); + Keyring *k = Keyring::instance(); QStringList accounts = k->getStoredAccounts("minecraft"); - if(!accounts.isEmpty()) + if (!accounts.isEmpty()) { QString username = accounts[0]; QString password = k->getPassword("minecraft", username); - if(!password.isEmpty()) + if (!password.isEmpty()) { QLOG_INFO() << "Automatically logging in with stored account: " << username; m_activeInst = m_selectedInstance; @@ -498,7 +501,8 @@ void MainWindow::doAutoLogin() } else { - QLOG_ERROR() << "Auto login set for account, but no password was found: " << username; + QLOG_ERROR() << "Auto login set for account, but no password was found: " + << username; doLogin(tr("Auto login attempted, but no password is stored.")); } } @@ -515,10 +519,8 @@ void MainWindow::doLogin(QString username, QString password) ProgressDialog *tDialog = new ProgressDialog(this); LoginTask *loginTask = new LoginTask(uInfo, tDialog); - connect(loginTask, SIGNAL(succeeded()), SLOT(onLoginComplete()), - Qt::QueuedConnection); - connect(loginTask, SIGNAL(failed(QString)), SLOT(doLogin(QString)), - Qt::QueuedConnection); + connect(loginTask, SIGNAL(succeeded()), SLOT(onLoginComplete()), Qt::QueuedConnection); + connect(loginTask, SIGNAL(failed(QString)), SLOT(doLogin(QString)), Qt::QueuedConnection); tDialog->exec(loginTask); } @@ -573,10 +575,13 @@ void MainWindow::onLoginComplete() delete updateTask; } - auto job = new DownloadJob("Player skin: " + m_activeLogin.player_name); + auto job = new NetJob("Player skin: " + m_activeLogin.player_name); auto meta = MMC->metacache()->resolveEntry("skins", m_activeLogin.player_name + ".png"); - job->addCacheDownload(QUrl("http://skins.minecraft.net/MinecraftSkins/" + m_activeLogin.player_name + ".png"), meta); + auto action = CacheDownload::make( + QUrl("http://skins.minecraft.net/MinecraftSkins/" + m_activeLogin.player_name + ".png"), + meta); + job->addNetAction(action); meta->stale = true; job->start(); @@ -586,7 +591,7 @@ void MainWindow::onLoginComplete() // Add skin mapping QByteArray data; { - if(!listFile.open(QIODevice::ReadWrite)) + if (!listFile.open(QIODevice::ReadWrite)) { QLOG_ERROR() << "Failed to open/make skins list JSON"; return; @@ -601,7 +606,7 @@ void MainWindow::onLoginComplete() QJsonObject mappings = root.value("mappings").toObject(); QJsonArray usernames = mappings.value(m_activeLogin.username).toArray(); - if(!usernames.contains(m_activeLogin.player_name)) + if (!usernames.contains(m_activeLogin.player_name)) { usernames.prepend(m_activeLogin.player_name); mappings[m_activeLogin.username] = usernames; @@ -642,12 +647,11 @@ void MainWindow::launchInstance(BaseInstance *instance, LoginResponse response) this->hide(); } - console = new ConsoleWindow(proc); connect(proc, SIGNAL(log(QString, MessageLevel::Enum)), console, SLOT(write(QString, MessageLevel::Enum))); - connect(proc, SIGNAL(ended(BaseInstance*)), this, SLOT(instanceEnded(BaseInstance*))); + connect(proc, SIGNAL(ended(BaseInstance *)), this, SLOT(instanceEnded(BaseInstance *))); if (instance->settings().get("ShowConsole").toBool()) { @@ -856,7 +860,7 @@ void MainWindow::checkSetDefaultJava() JavaUtils ju; java = ju.GetDefaultJava(); } - if(java) + if (java) MMC->settings()->set("JavaPath", java->path); else MMC->settings()->set("JavaPath", QString("java")); @@ -876,7 +880,8 @@ void MainWindow::assetsFilesStarted() void MainWindow::assetsFilesProgress(int succeeded, int failed, int total) { QString status = tr("Downloading assets: %1 / %2").arg(succeeded + failed).arg(total); - if(failed > 0) status += tr(" (%1 failed)").arg(failed); + if (failed > 0) + status += tr(" (%1 failed)").arg(failed); status += tr("..."); m_statusRight->setText(status); } diff --git a/logic/BaseUpdate.h b/logic/BaseUpdate.h index 6f1e26f3..9ada0770 100644 --- a/logic/BaseUpdate.h +++ b/logic/BaseUpdate.h @@ -19,7 +19,7 @@ #include #include -#include "net/DownloadJob.h" +#include "net/NetJob.h" #include "tasks/Task.h" diff --git a/logic/LegacyUpdate.cpp b/logic/LegacyUpdate.cpp index 66b4bf8a..5120b241 100644 --- a/logic/LegacyUpdate.cpp +++ b/logic/LegacyUpdate.cpp @@ -233,8 +233,8 @@ void LegacyUpdate::jarStart() QString intended_version_id = inst->intendedVersionId(); urlstr += intended_version_id + "/" + intended_version_id + ".jar"; - auto dljob = new DownloadJob("Minecraft.jar for version " + intended_version_id); - dljob->addFileDownload(QUrl(urlstr), inst->defaultBaseJar()); + auto dljob = new NetJob("Minecraft.jar for version " + intended_version_id); + dljob->addNetAction(FileDownload::make(QUrl(urlstr), inst->defaultBaseJar())); legacyDownloadJob.reset(dljob); connect(dljob, SIGNAL(succeeded()), SLOT(jarFinished())); connect(dljob, SIGNAL(failed()), SLOT(jarFailed())); diff --git a/logic/LegacyUpdate.h b/logic/LegacyUpdate.h index e84ec56a..69560f55 100644 --- a/logic/LegacyUpdate.h +++ b/logic/LegacyUpdate.h @@ -19,7 +19,7 @@ #include #include -#include "net/DownloadJob.h" +#include "net/NetJob.h" #include "tasks/Task.h" #include "BaseUpdate.h" @@ -66,7 +66,7 @@ private: QString lwjglTargetPath; QString lwjglNativesPath; private: - DownloadJobPtr legacyDownloadJob; + NetJobPtr legacyDownloadJob; }; diff --git a/logic/OneSixAssets.cpp b/logic/OneSixAssets.cpp index ff19a86b..375c87f1 100644 --- a/logic/OneSixAssets.cpp +++ b/logic/OneSixAssets.cpp @@ -2,19 +2,11 @@ #include #include #include "OneSixAssets.h" -#include "net/DownloadJob.h" +#include "net/NetJob.h" #include "net/HttpMetaCache.h" +#include "net/S3ListBucket.h" #include "MultiMC.h" -inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname) -{ - QDomNodeList elementList = parent.elementsByTagName(tagname); - if (elementList.count()) - return elementList.at(0).toElement(); - else - return QDomElement(); -} - class ThreadedDeleter : public QThread { Q_OBJECT @@ -22,18 +14,18 @@ public: void run() { QLOG_INFO() << "Cleaning up assets folder..."; - QDirIterator iter ( m_base, QDirIterator::Subdirectories ); + QDirIterator iter(m_base, QDirIterator::Subdirectories); int base_length = m_base.length(); - while ( iter.hasNext() ) + while (iter.hasNext()) { QString filename = iter.next(); - QFileInfo current ( filename ); + QFileInfo current(filename); // we keep the dirs... whatever - if ( current.isDir() ) + if (current.isDir()) continue; QString trimmedf = filename; - trimmedf.remove ( 0, base_length + 1 ); - if ( m_whitelist.contains ( trimmedf ) ) + trimmedf.remove(0, base_length + 1); + if (m_whitelist.contains(trimmedf)) { QLOG_TRACE() << trimmedf << " gets to live"; } @@ -41,7 +33,7 @@ public: { // DO NOT TOLERATE JUNK QLOG_TRACE() << trimmedf << " dies"; - QFile f ( filename ); + QFile f(filename); f.remove(); } } @@ -60,71 +52,41 @@ void OneSixAssets::downloadFinished() deleter->start(); } - -void OneSixAssets::fetchXMLFinished() +void OneSixAssets::S3BucketFinished() { - QString prefix ( "http://s3.amazonaws.com/Minecraft.Resources/" ); - QString fprefix ( "assets/" ); + QString prefix("http://s3.amazonaws.com/Minecraft.Resources/"); nuke_whitelist.clear(); emit filesStarted(); auto firstJob = index_job->first(); - QByteArray ba = std::dynamic_pointer_cast(firstJob)->m_data; + auto objectList = std::dynamic_pointer_cast(firstJob)->objects; - QString xmlErrorMsg; - QDomDocument doc; - if ( !doc.setContent ( ba, false, &xmlErrorMsg ) ) - { - QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:" << xmlErrorMsg << ba; - emit failed(); - return; - } - //QRegExp etag_match(".*([a-f0-9]{32}).*"); - QDomNodeList contents = doc.elementsByTagName ( "Contents" ); + NetJob *job = new NetJob("Assets"); - DownloadJob *job = new DownloadJob("Assets"); - connect ( job, SIGNAL(succeeded()), SLOT(downloadFinished()) ); - connect ( job, SIGNAL(failed()), SIGNAL(failed()) ); - connect ( job, SIGNAL(filesProgress(int, int, int)), SIGNAL(filesProgress(int, int, int)) ); + connect(job, SIGNAL(succeeded()), SLOT(downloadFinished())); + connect(job, SIGNAL(failed()), SIGNAL(failed())); + connect(job, SIGNAL(filesProgress(int, int, int)), SIGNAL(filesProgress(int, int, int))); auto metacache = MMC->metacache(); - - for ( int i = 0; i < contents.length(); i++ ) - { - QDomElement element = contents.at ( i ).toElement(); - - if ( element.isNull() ) - continue; - - QDomElement keyElement = getDomElementByTagName ( element, "Key" ); - QDomElement lastmodElement = getDomElementByTagName ( element, "LastModified" ); - QDomElement etagElement = getDomElementByTagName ( element, "ETag" ); - QDomElement sizeElement = getDomElementByTagName ( element, "Size" ); - if ( keyElement.isNull() || lastmodElement.isNull() || etagElement.isNull() || sizeElement.isNull() ) + for (auto object: objectList) + { + // Filter folder keys (zero size) + if (object.size == 0) continue; - QString keyStr = keyElement.text(); - QString lastModStr = lastmodElement.text(); - QString etagStr = etagElement.text(); - QString sizeStr = sizeElement.text(); - - //Filter folder keys - if ( sizeStr == "0" ) - continue; + nuke_whitelist.append(object.Key); - nuke_whitelist.append ( keyStr ); - - auto entry = metacache->resolveEntry("assets", keyStr, etagStr); - if(entry->stale) + auto entry = metacache->resolveEntry("assets", object.Key, object.ETag); + if (entry->stale) { - job->addCacheDownload(QUrl(prefix + keyStr), entry); + job->addNetAction(CacheDownload::make(QUrl(prefix + object.Key), entry)); } } - if(job->size()) + if (job->size()) { - files_job.reset ( job ); + files_job.reset(job); files_job->start(); } else @@ -136,11 +98,12 @@ void OneSixAssets::fetchXMLFinished() void OneSixAssets::start() { - auto job = new DownloadJob("Assets index"); - job->addByteArrayDownload(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" )); - connect ( job, SIGNAL(succeeded()), SLOT ( fetchXMLFinished() ) ); + auto job = new NetJob("Assets index"); + job->addNetAction( + S3ListBucket::make(QUrl("http://s3.amazonaws.com/Minecraft.Resources/"))); + connect(job, SIGNAL(succeeded()), SLOT(S3BucketFinished())); emit indexStarted(); - index_job.reset ( job ); + index_job.reset(job); job->start(); } diff --git a/logic/OneSixAssets.h b/logic/OneSixAssets.h index 6c6f2c36..72e8a843 100644 --- a/logic/OneSixAssets.h +++ b/logic/OneSixAssets.h @@ -1,5 +1,5 @@ #pragma once -#include "net/DownloadJob.h" +#include "net/NetJob.h" class Private; class ThreadedDeleter; @@ -15,13 +15,13 @@ signals: void filesProgress(int, int, int); public slots: - void fetchXMLFinished(); + void S3BucketFinished(); void downloadFinished(); public: void start(); private: ThreadedDeleter * deleter; QStringList nuke_whitelist; - DownloadJobPtr index_job; - DownloadJobPtr files_job; + NetJobPtr index_job; + NetJobPtr files_job; }; diff --git a/logic/OneSixUpdate.cpp b/logic/OneSixUpdate.cpp index b5f1d78b..5c421fbf 100644 --- a/logic/OneSixUpdate.cpp +++ b/logic/OneSixUpdate.cpp @@ -72,8 +72,8 @@ void OneSixUpdate::versionFileStart() QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/"); urlstr += targetVersion->descriptor() + "/" + targetVersion->descriptor() + ".json"; - auto job = new DownloadJob("Version index"); - job->addByteArrayDownload(QUrl(urlstr)); + auto job = new NetJob("Version index"); + job->addNetAction(ByteArrayDownload::make(QUrl(urlstr))); specificVersionDownloadJob.reset(job); connect(specificVersionDownloadJob.get(), SIGNAL(succeeded()), SLOT(versionFileFinished())); connect(specificVersionDownloadJob.get(), SIGNAL(failed()), SLOT(versionFileFailed())); @@ -84,7 +84,7 @@ void OneSixUpdate::versionFileStart() void OneSixUpdate::versionFileFinished() { - DownloadPtr DlJob = specificVersionDownloadJob->first(); + NetActionPtr DlJob = specificVersionDownloadJob->first(); OneSixInstance *inst = (OneSixInstance *)m_inst; QString version_id = targetVersion->descriptor(); @@ -154,8 +154,8 @@ void OneSixUpdate::jarlibStart() QString targetstr("versions/"); targetstr += version->id + "/" + version->id + ".jar"; - auto job = new DownloadJob("Libraries for instance " + inst->name()); - job->addFileDownload(QUrl(urlstr), targetstr); + auto job = new NetJob("Libraries for instance " + inst->name()); + job->addNetAction(FileDownload::make(QUrl(urlstr), targetstr)); jarlibDownloadJob.reset(job); auto libs = version->getActiveNativeLibs(); @@ -171,9 +171,9 @@ void OneSixUpdate::jarlibStart() if (entry->stale) { if (lib->hint() == "forge-pack-xz") - jarlibDownloadJob->addForgeXzDownload(download_path, entry); + jarlibDownloadJob->addNetAction(ForgeXzDownload::make(download_path, entry)); else - jarlibDownloadJob->addCacheDownload(download_path, entry); + jarlibDownloadJob->addNetAction(CacheDownload::make(download_path, entry)); } } connect(jarlibDownloadJob.get(), SIGNAL(succeeded()), SLOT(jarlibFinished())); diff --git a/logic/OneSixUpdate.h b/logic/OneSixUpdate.h index ed8dcbcd..7c6fce1b 100644 --- a/logic/OneSixUpdate.h +++ b/logic/OneSixUpdate.h @@ -18,7 +18,7 @@ #include #include #include -#include "net/DownloadJob.h" +#include "net/NetJob.h" #include "tasks/Task.h" #include "BaseUpdate.h" @@ -43,8 +43,8 @@ private slots: void jarlibFailed(); private: - DownloadJobPtr specificVersionDownloadJob; - DownloadJobPtr jarlibDownloadJob; + NetJobPtr specificVersionDownloadJob; + NetJobPtr jarlibDownloadJob; // target version, determined during this task std::shared_ptr targetVersion; diff --git a/logic/lists/ForgeVersionList.cpp b/logic/lists/ForgeVersionList.cpp index 491a43d7..27f567cd 100644 --- a/logic/lists/ForgeVersionList.cpp +++ b/logic/lists/ForgeVersionList.cpp @@ -14,7 +14,7 @@ */ #include "ForgeVersionList.h" -#include +#include #include "MultiMC.h" #include @@ -159,14 +159,14 @@ ForgeListLoadTask::ForgeListLoadTask(ForgeVersionList *vlist) : Task() void ForgeListLoadTask::executeTask() { - auto job = new DownloadJob("Version index"); + auto job = new NetJob("Version index"); // we do not care if the version is stale or not. auto forgeListEntry = MMC->metacache()->resolveEntry("minecraftforge", "list.json"); - + // verify by poking the server. forgeListEntry->stale = true; - - job->addCacheDownload(QUrl(JSON_URL), forgeListEntry); + + job->addNetAction(CacheDownload::make(QUrl(JSON_URL), forgeListEntry)); listJob.reset(job); connect(listJob.get(), SIGNAL(succeeded()), SLOT(list_downloaded())); connect(listJob.get(), SIGNAL(failed()), SLOT(list_failed())); @@ -178,7 +178,7 @@ void ForgeListLoadTask::list_failed() { auto DlJob = listJob->first(); auto reply = DlJob->m_reply; - if(reply) + if (reply) { QLOG_ERROR() << "Getting forge version list failed: " << reply->errorString(); } @@ -193,7 +193,7 @@ void ForgeListLoadTask::list_downloaded() auto DlJob = listJob->first(); auto filename = std::dynamic_pointer_cast(DlJob)->m_target_path; QFile listFile(filename); - if(!listFile.open(QIODevice::ReadOnly)) + if (!listFile.open(QIODevice::ReadOnly)) return; data = listFile.readAll(); DlJob.reset(); @@ -202,7 +202,6 @@ void ForgeListLoadTask::list_downloaded() QJsonParseError jsonError; QJsonDocument jsonDoc = QJsonDocument::fromJson(data, &jsonError); - if (jsonError.error != QJsonParseError::NoError) { emitFailed("Error parsing version list JSON:" + jsonError.errorString()); diff --git a/logic/lists/ForgeVersionList.h b/logic/lists/ForgeVersionList.h index 56a207c8..52593f94 100644 --- a/logic/lists/ForgeVersionList.h +++ b/logic/lists/ForgeVersionList.h @@ -23,7 +23,7 @@ #include #include "BaseVersionList.h" #include "logic/tasks/Task.h" -#include "logic/net/DownloadJob.h" +#include "logic/net/NetJob.h" class ForgeVersion; typedef std::shared_ptr ForgeVersionPtr; @@ -104,6 +104,6 @@ slots: void list_failed(); protected: - DownloadJobPtr listJob; + NetJobPtr listJob; ForgeVersionList *m_list; }; diff --git a/logic/net/ByteArrayDownload.cpp b/logic/net/ByteArrayDownload.cpp index ba771eef..25e6d51a 100644 --- a/logic/net/ByteArrayDownload.cpp +++ b/logic/net/ByteArrayDownload.cpp @@ -2,7 +2,7 @@ #include "MultiMC.h" #include -ByteArrayDownload::ByteArrayDownload(QUrl url) : Download() +ByteArrayDownload::ByteArrayDownload(QUrl url) : NetAction() { m_url = url; m_status = Job_NotStarted; diff --git a/logic/net/ByteArrayDownload.h b/logic/net/ByteArrayDownload.h index cfc6a8d0..fc32dc04 100644 --- a/logic/net/ByteArrayDownload.h +++ b/logic/net/ByteArrayDownload.h @@ -1,24 +1,29 @@ #pragma once -#include "Download.h" +#include "NetAction.h" -class ByteArrayDownload: public Download +typedef std::shared_ptr ByteArrayDownloadPtr; +class ByteArrayDownload : public NetAction { Q_OBJECT public: ByteArrayDownload(QUrl url); - + static ByteArrayDownloadPtr make(QUrl url) + { + return ByteArrayDownloadPtr(new ByteArrayDownload(url)); + } + public: /// if not saving to file, downloaded data is placed here QByteArray m_data; - -public slots: + +public +slots: virtual void start(); - -protected slots: + +protected +slots: void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); void downloadError(QNetworkReply::NetworkError error); void downloadFinished(); void downloadReadyRead(); }; - -typedef std::shared_ptr ByteArrayDownloadPtr; diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp index 309eb345..f8769576 100644 --- a/logic/net/CacheDownload.cpp +++ b/logic/net/CacheDownload.cpp @@ -8,7 +8,7 @@ #include CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry) - : Download(), md5sum(QCryptographicHash::Md5) + : NetAction(), md5sum(QCryptographicHash::Md5) { m_url = url; m_entry = entry; diff --git a/logic/net/CacheDownload.h b/logic/net/CacheDownload.h index 295391b1..1e70874c 100644 --- a/logic/net/CacheDownload.h +++ b/logic/net/CacheDownload.h @@ -1,11 +1,12 @@ #pragma once -#include "Download.h" +#include "NetAction.h" #include "HttpMetaCache.h" #include #include -class CacheDownload : public Download +typedef std::shared_ptr CacheDownloadPtr; +class CacheDownload : public NetAction { Q_OBJECT public: @@ -18,17 +19,22 @@ public: QFile m_output_file; /// the hash-as-you-download QCryptographicHash md5sum; + public: explicit CacheDownload(QUrl url, MetaEntryPtr entry); - -protected slots: + static CacheDownloadPtr make(QUrl url, MetaEntryPtr entry) + { + return CacheDownloadPtr(new CacheDownload(url, entry)); + } + +protected +slots: virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); virtual void downloadError(QNetworkReply::NetworkError error); virtual void downloadFinished(); virtual void downloadReadyRead(); - -public slots: + +public +slots: virtual void start(); }; - -typedef std::shared_ptr CacheDownloadPtr; diff --git a/logic/net/Download.h b/logic/net/Download.h deleted file mode 100644 index ca4bee9f..00000000 --- a/logic/net/Download.h +++ /dev/null @@ -1,54 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -enum JobStatus -{ - Job_NotStarted, - Job_InProgress, - Job_Finished, - Job_Failed -}; - -class Download : public QObject -{ - Q_OBJECT -protected: - explicit Download() : QObject(0) {}; - -public: - virtual ~Download() {}; - -public: - /// the network reply - std::shared_ptr m_reply; - - /// source URL - QUrl m_url; - - /// The file's status - JobStatus m_status; - - /// index within the parent job - int index_within_job = 0; - -signals: - void started(int index); - void progress(int index, qint64 current, qint64 total); - void succeeded(int index); - void failed(int index); - -protected slots: - virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0; - virtual void downloadError(QNetworkReply::NetworkError error) = 0; - virtual void downloadFinished() = 0; - virtual void downloadReadyRead() = 0; - -public slots: - virtual void start() = 0; -}; - -typedef std::shared_ptr DownloadPtr; diff --git a/logic/net/DownloadJob.cpp b/logic/net/DownloadJob.cpp deleted file mode 100644 index 38716a02..00000000 --- a/logic/net/DownloadJob.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include "DownloadJob.h" -#include "pathutils.h" -#include "MultiMC.h" -#include "FileDownload.h" -#include "ByteArrayDownload.h" -#include "CacheDownload.h" - -#include - -ByteArrayDownloadPtr DownloadJob::addByteArrayDownload(QUrl url) -{ - ByteArrayDownloadPtr ptr(new ByteArrayDownload(url)); - ptr->index_within_job = downloads.size(); - downloads.append(ptr); - parts_progress.append(part_info()); - total_progress++; - return ptr; -} - -FileDownloadPtr DownloadJob::addFileDownload(QUrl url, QString rel_target_path) -{ - FileDownloadPtr ptr(new FileDownload(url, rel_target_path)); - ptr->index_within_job = downloads.size(); - downloads.append(ptr); - parts_progress.append(part_info()); - total_progress++; - return ptr; -} - -CacheDownloadPtr DownloadJob::addCacheDownload(QUrl url, MetaEntryPtr entry) -{ - CacheDownloadPtr ptr(new CacheDownload(url, entry)); - ptr->index_within_job = downloads.size(); - downloads.append(ptr); - parts_progress.append(part_info()); - total_progress++; - return ptr; -} - -ForgeXzDownloadPtr DownloadJob::addForgeXzDownload(QUrl url, MetaEntryPtr entry) -{ - ForgeXzDownloadPtr ptr(new ForgeXzDownload(url, entry)); - ptr->index_within_job = downloads.size(); - downloads.append(ptr); - parts_progress.append(part_info()); - total_progress++; - return ptr; -} - -void DownloadJob::partSucceeded(int index) -{ - // do progress. all slots are 1 in size at least - auto &slot = parts_progress[index]; - partProgress(index, slot.total_progress, slot.total_progress); - - num_succeeded++; - QLOG_INFO() << m_job_name.toLocal8Bit() << "progress:" << num_succeeded << "/" - << downloads.size(); - emit filesProgress(num_succeeded, num_failed, downloads.size()); - - if (num_failed + num_succeeded == downloads.size()) - { - if (num_failed) - { - QLOG_ERROR() << m_job_name.toLocal8Bit() << "failed."; - emit failed(); - } - else - { - QLOG_INFO() << m_job_name.toLocal8Bit() << "succeeded."; - emit succeeded(); - } - } -} - -void DownloadJob::partFailed(int index) -{ - auto &slot = parts_progress[index]; - if (slot.failures == 3) - { - QLOG_ERROR() << "Part" << index << "failed 3 times (" << downloads[index]->m_url << ")"; - num_failed++; - emit filesProgress(num_succeeded, num_failed, downloads.size()); - if (num_failed + num_succeeded == downloads.size()) - { - QLOG_ERROR() << m_job_name.toLocal8Bit() << "failed."; - emit failed(); - } - } - else - { - QLOG_ERROR() << "Part" << index << "failed, restarting (" << downloads[index]->m_url - << ")"; - // restart the job - slot.failures++; - downloads[index]->start(); - } -} - -void DownloadJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal) -{ - auto &slot = parts_progress[index]; - - current_progress -= slot.current_progress; - slot.current_progress = bytesReceived; - current_progress += slot.current_progress; - - total_progress -= slot.total_progress; - slot.total_progress = bytesTotal; - total_progress += slot.total_progress; - emit progress(current_progress, total_progress); -} - -void DownloadJob::start() -{ - QLOG_INFO() << m_job_name.toLocal8Bit() << " started."; - for (auto iter : downloads) - { - connect(iter.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int))); - connect(iter.get(), SIGNAL(failed(int)), SLOT(partFailed(int))); - connect(iter.get(), SIGNAL(progress(int, qint64, qint64)), - SLOT(partProgress(int, qint64, qint64))); - iter->start(); - } -} - -QStringList DownloadJob::getFailedFiles() -{ - QStringList failed; - for (auto download : downloads) - { - if (download->m_status == Job_Failed) - { - failed.push_back(download->m_url.toString()); - } - } - return failed; -} diff --git a/logic/net/DownloadJob.h b/logic/net/DownloadJob.h deleted file mode 100644 index cc2a1d59..00000000 --- a/logic/net/DownloadJob.h +++ /dev/null @@ -1,86 +0,0 @@ -#pragma once -#include -#include -#include "Download.h" -#include "ByteArrayDownload.h" -#include "FileDownload.h" -#include "CacheDownload.h" -#include "HttpMetaCache.h" -#include "ForgeXzDownload.h" -#include "logic/tasks/ProgressProvider.h" - -class DownloadJob; -typedef std::shared_ptr DownloadJobPtr; - -/** - * A single file for the downloader/cache to process. - */ -class DownloadJob : public ProgressProvider -{ - Q_OBJECT -public: - explicit DownloadJob(QString job_name) - :ProgressProvider(), m_job_name(job_name){}; - - ByteArrayDownloadPtr addByteArrayDownload(QUrl url); - FileDownloadPtr addFileDownload(QUrl url, QString rel_target_path); - CacheDownloadPtr addCacheDownload(QUrl url, MetaEntryPtr entry); - ForgeXzDownloadPtr addForgeXzDownload(QUrl url, MetaEntryPtr entry); - - DownloadPtr operator[](int index) - { - return downloads[index]; - }; - DownloadPtr first() - { - if(downloads.size()) - return downloads[0]; - return DownloadPtr(); - } - int size() const - { - return downloads.size(); - } - virtual void getProgress(qint64& current, qint64& total) - { - current = current_progress; - total = total_progress; - }; - virtual QString getStatus() const - { - return m_job_name; - }; - virtual bool isRunning() const - { - return m_running; - }; - QStringList getFailedFiles(); -signals: - void started(); - void progress(qint64 current, qint64 total); - void filesProgress(int, int, int); - void succeeded(); - void failed(); -public slots: - virtual void start(); -private slots: - void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal); - void partSucceeded(int index); - void partFailed(int index); -private: - struct part_info - { - qint64 current_progress = 0; - qint64 total_progress = 1; - int failures = 0; - }; - QString m_job_name; - QList downloads; - QList parts_progress; - qint64 current_progress = 0; - qint64 total_progress = 0; - int num_succeeded = 0; - int num_failed = 0; - bool m_running = false; -}; - diff --git a/logic/net/FileDownload.cpp b/logic/net/FileDownload.cpp index 3f38b0fa..eefdd4da 100644 --- a/logic/net/FileDownload.cpp +++ b/logic/net/FileDownload.cpp @@ -6,7 +6,7 @@ FileDownload::FileDownload ( QUrl url, QString target_path ) - :Download() + :NetAction() { m_url = url; m_target_path = target_path; diff --git a/logic/net/FileDownload.h b/logic/net/FileDownload.h index 9abb590d..5f72587f 100644 --- a/logic/net/FileDownload.h +++ b/logic/net/FileDownload.h @@ -1,14 +1,16 @@ #pragma once -#include "Download.h" +#include "NetAction.h" #include -class FileDownload : public Download +typedef std::shared_ptr FileDownloadPtr; +class FileDownload : public NetAction { Q_OBJECT public: /// if true, check the md5sum against a provided md5sum - /// also, if a file exists, perform an md5sum first and don't download only if they don't match + /// also, if a file exists, perform an md5sum first and don't download only if they don't + /// match bool m_check_md5; /// the expected md5 checksum QString m_expected_md5; @@ -18,18 +20,21 @@ public: QString m_target_path; /// this is the output file, if any QFile m_output_file; - + public: explicit FileDownload(QUrl url, QString target_path); - -protected slots: + static FileDownloadPtr make(QUrl url, QString target_path) + { + return FileDownloadPtr(new FileDownload(url, target_path)); + } +protected +slots: virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); virtual void downloadError(QNetworkReply::NetworkError error); virtual void downloadFinished(); virtual void downloadReadyRead(); - -public slots: + +public +slots: virtual void start(); }; - -typedef std::shared_ptr FileDownloadPtr; diff --git a/logic/net/ForgeXzDownload.cpp b/logic/net/ForgeXzDownload.cpp index 0e5287d8..20279d99 100644 --- a/logic/net/ForgeXzDownload.cpp +++ b/logic/net/ForgeXzDownload.cpp @@ -8,7 +8,7 @@ #include ForgeXzDownload::ForgeXzDownload(QUrl url, MetaEntryPtr entry) - : Download() + : NetAction() { QString urlstr = url.toString(); urlstr.append(".pack.xz"); diff --git a/logic/net/ForgeXzDownload.h b/logic/net/ForgeXzDownload.h index 5d677947..0b73711e 100644 --- a/logic/net/ForgeXzDownload.h +++ b/logic/net/ForgeXzDownload.h @@ -1,11 +1,12 @@ #pragma once -#include "Download.h" +#include "NetAction.h" #include "HttpMetaCache.h" #include #include +typedef std::shared_ptr ForgeXzDownloadPtr; -class ForgeXzDownload : public Download +class ForgeXzDownload : public NetAction { Q_OBJECT public: @@ -19,17 +20,22 @@ public: public: explicit ForgeXzDownload(QUrl url, MetaEntryPtr entry); - -protected slots: + static ForgeXzDownloadPtr make(QUrl url, MetaEntryPtr entry) + { + return ForgeXzDownloadPtr(new ForgeXzDownload(url, entry)); + } + +protected +slots: virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); virtual void downloadError(QNetworkReply::NetworkError error); virtual void downloadFinished(); virtual void downloadReadyRead(); - -public slots: + +public +slots: virtual void start(); + private: void decompressAndInstall(); }; - -typedef std::shared_ptr ForgeXzDownloadPtr; diff --git a/logic/net/NetAction.h b/logic/net/NetAction.h new file mode 100644 index 00000000..b7c922f5 --- /dev/null +++ b/logic/net/NetAction.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include + +enum JobStatus +{ + Job_NotStarted, + Job_InProgress, + Job_Finished, + Job_Failed +}; + +typedef std::shared_ptr NetActionPtr; +class NetAction : public QObject +{ + Q_OBJECT +protected: + explicit NetAction() : QObject(0) {}; + +public: + virtual ~NetAction() {}; + +public: + /// the network reply + std::shared_ptr m_reply; + + /// source URL + QUrl m_url; + + /// The file's status + JobStatus m_status; + + /// index within the parent job + int index_within_job = 0; + +signals: + void started(int index); + void progress(int index, qint64 current, qint64 total); + void succeeded(int index); + void failed(int index); + +protected slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0; + virtual void downloadError(QNetworkReply::NetworkError error) = 0; + virtual void downloadFinished() = 0; + virtual void downloadReadyRead() = 0; + +public slots: + virtual void start() = 0; +}; diff --git a/logic/net/NetJob.cpp b/logic/net/NetJob.cpp new file mode 100644 index 00000000..c7ca4409 --- /dev/null +++ b/logic/net/NetJob.cpp @@ -0,0 +1,98 @@ +#include "NetJob.h" +#include "pathutils.h" +#include "MultiMC.h" +#include "FileDownload.h" +#include "ByteArrayDownload.h" +#include "CacheDownload.h" + +#include + +void NetJob::partSucceeded(int index) +{ + // do progress. all slots are 1 in size at least + auto &slot = parts_progress[index]; + partProgress(index, slot.total_progress, slot.total_progress); + + num_succeeded++; + QLOG_INFO() << m_job_name.toLocal8Bit() << "progress:" << num_succeeded << "/" + << downloads.size(); + emit filesProgress(num_succeeded, num_failed, downloads.size()); + + if (num_failed + num_succeeded == downloads.size()) + { + if (num_failed) + { + QLOG_ERROR() << m_job_name.toLocal8Bit() << "failed."; + emit failed(); + } + else + { + QLOG_INFO() << m_job_name.toLocal8Bit() << "succeeded."; + emit succeeded(); + } + } +} + +void NetJob::partFailed(int index) +{ + auto &slot = parts_progress[index]; + if (slot.failures == 3) + { + QLOG_ERROR() << "Part" << index << "failed 3 times (" << downloads[index]->m_url << ")"; + num_failed++; + emit filesProgress(num_succeeded, num_failed, downloads.size()); + if (num_failed + num_succeeded == downloads.size()) + { + QLOG_ERROR() << m_job_name.toLocal8Bit() << "failed."; + emit failed(); + } + } + else + { + QLOG_ERROR() << "Part" << index << "failed, restarting (" << downloads[index]->m_url + << ")"; + // restart the job + slot.failures++; + downloads[index]->start(); + } +} + +void NetJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal) +{ + auto &slot = parts_progress[index]; + + current_progress -= slot.current_progress; + slot.current_progress = bytesReceived; + current_progress += slot.current_progress; + + total_progress -= slot.total_progress; + slot.total_progress = bytesTotal; + total_progress += slot.total_progress; + emit progress(current_progress, total_progress); +} + +void NetJob::start() +{ + QLOG_INFO() << m_job_name.toLocal8Bit() << " started."; + for (auto iter : downloads) + { + connect(iter.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int))); + connect(iter.get(), SIGNAL(failed(int)), SLOT(partFailed(int))); + connect(iter.get(), SIGNAL(progress(int, qint64, qint64)), + SLOT(partProgress(int, qint64, qint64))); + iter->start(); + } +} + +QStringList NetJob::getFailedFiles() +{ + QStringList failed; + for (auto download : downloads) + { + if (download->m_status == Job_Failed) + { + failed.push_back(download->m_url.toString()); + } + } + return failed; +} diff --git a/logic/net/NetJob.h b/logic/net/NetJob.h new file mode 100644 index 00000000..01d12e60 --- /dev/null +++ b/logic/net/NetJob.h @@ -0,0 +1,94 @@ +#pragma once +#include +#include +#include "NetAction.h" +#include "ByteArrayDownload.h" +#include "FileDownload.h" +#include "CacheDownload.h" +#include "HttpMetaCache.h" +#include "ForgeXzDownload.h" +#include "logic/tasks/ProgressProvider.h" + +class NetJob; +typedef std::shared_ptr NetJobPtr; + +class NetJob : public ProgressProvider +{ + Q_OBJECT +public: + explicit NetJob(QString job_name) : ProgressProvider(), m_job_name(job_name) {}; + + template + bool addNetAction(T action) + { + NetActionPtr base = std::static_pointer_cast(action); + base->index_within_job = downloads.size(); + downloads.append(action); + parts_progress.append(part_info()); + total_progress++; + return true; + } + + NetActionPtr operator[](int index) + { + return downloads[index]; + } + ; + NetActionPtr first() + { + if (downloads.size()) + return downloads[0]; + return NetActionPtr(); + } + int size() const + { + return downloads.size(); + } + virtual void getProgress(qint64 ¤t, qint64 &total) + { + current = current_progress; + total = total_progress; + } + ; + virtual QString getStatus() const + { + return m_job_name; + } + ; + virtual bool isRunning() const + { + return m_running; + } + ; + QStringList getFailedFiles(); +signals: + void started(); + void progress(qint64 current, qint64 total); + void filesProgress(int, int, int); + void succeeded(); + void failed(); +public +slots: + virtual void start(); +private +slots: + void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal); + void partSucceeded(int index); + void partFailed(int index); + +private: + struct part_info + { + qint64 current_progress = 0; + qint64 total_progress = 1; + int failures = 0; + }; + QString m_job_name; + QList downloads; + QList parts_progress; + qint64 current_progress = 0; + qint64 total_progress = 0; + int num_succeeded = 0; + int num_failed = 0; + bool m_running = false; +}; diff --git a/logic/net/S3ListBucket.cpp b/logic/net/S3ListBucket.cpp new file mode 100644 index 00000000..643c3224 --- /dev/null +++ b/logic/net/S3ListBucket.cpp @@ -0,0 +1,161 @@ +#include "S3ListBucket.h" +#include "MultiMC.h" +#include +#include +#include +#include + +inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname) +{ + QDomNodeList elementList = parent.elementsByTagName(tagname); + if (elementList.count()) + return elementList.at(0).toElement(); + else + return QDomElement(); +} + +S3ListBucket::S3ListBucket(QUrl url) : NetAction() +{ + m_url = url; + m_status = Job_NotStarted; +} + +void S3ListBucket::start() +{ + QUrl finalUrl = m_url; + if (current_marker.size()) + { + QUrlQuery query; + query.addQueryItem("marker", current_marker); + finalUrl.setQuery(query); + } + QNetworkRequest request(finalUrl); + request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)"); + auto worker = MMC->qnam(); + QNetworkReply *rep = worker->get(request); + + m_reply = std::shared_ptr(rep); + connect(rep, SIGNAL(downloadProgress(qint64, qint64)), + SLOT(downloadProgress(qint64, qint64))); + connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); + connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), + SLOT(downloadError(QNetworkReply::NetworkError))); + connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); +} + +void S3ListBucket::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) +{ + emit progress(index_within_job, bytesSoFar + bytesReceived, bytesSoFar + bytesTotal); +} + +void S3ListBucket::downloadError(QNetworkReply::NetworkError error) +{ + // error happened during download. + QLOG_ERROR() << "Error getting URL:" << m_url.toString().toLocal8Bit() + << "Network error: " << error; + m_status = Job_Failed; +} + +void S3ListBucket::processValidReply() +{ + QLOG_TRACE() << "GOT: " << m_url.toString() << " marker:" << current_marker; + auto readContents = [&](QXmlStreamReader & xml) + { + QString Key, ETag, Size; + while (!(xml.tokenType() == QXmlStreamReader::EndElement && xml.name() == "Contents")) + { + if (xml.tokenType() == QXmlStreamReader::StartElement) + { + if (xml.name() == "Key") + { + Key = xml.readElementText(); + } + if (xml.name() == "ETag") + { + ETag = xml.readElementText(); + } + if (xml.name() == "Size") + { + Size = xml.readElementText(); + } + } + xml.readNext(); + } + if (xml.error() != QXmlStreamReader::NoError) + return; + objects.append({Key, ETag, Size.toLongLong()}); + }; + + // nothing went wrong... + QString prefix("http://s3.amazonaws.com/Minecraft.Resources/"); + QByteArray ba = m_reply->readAll(); + + QString xmlErrorMsg; + + bool is_truncated = false; + QXmlStreamReader xml(ba); + while (!xml.atEnd() && !xml.hasError()) + { + /* Read next element.*/ + QXmlStreamReader::TokenType token = xml.readNext(); + /* If token is just StartDocument, we'll go to next.*/ + if (token == QXmlStreamReader::StartDocument) + { + continue; + } + if (token == QXmlStreamReader::StartElement) + { + /* If it's named person, we'll dig the information from there.*/ + if (xml.name() == "Contents") + { + readContents(xml); + } + else if (xml.name() == "IsTruncated") + { + is_truncated = (xml.readElementText() == "true"); + } + } + } + if (xml.hasError()) + { + QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:" + << xml.errorString() << ba; + emit failed(index_within_job); + return; + } + if(is_truncated) + { + current_marker = objects.last().Key; + bytesSoFar += m_reply->size(); + m_reply.reset(); + start(); + } + else + { + m_status = Job_Finished; + m_reply.reset(); + emit succeeded(index_within_job); + } + return; +} + +void S3ListBucket::downloadFinished() +{ + // if the download succeeded + if (m_status != Job_Failed) + { + processValidReply(); + } + // else the download failed + else + { + m_reply.reset(); + emit failed(index_within_job); + return; + } +} + +void S3ListBucket::downloadReadyRead() +{ + // ~_~ +} diff --git a/logic/net/S3ListBucket.h b/logic/net/S3ListBucket.h new file mode 100644 index 00000000..f054532d --- /dev/null +++ b/logic/net/S3ListBucket.h @@ -0,0 +1,42 @@ +#pragma once +#include "NetAction.h" + +struct S3Object +{ + QString Key; + QString ETag; + qlonglong size; +}; + +typedef std::shared_ptr S3ListBucketPtr; +class S3ListBucket : public NetAction +{ + Q_OBJECT +public: + S3ListBucket(QUrl url); + static S3ListBucketPtr make(QUrl url) + { + return S3ListBucketPtr(new S3ListBucket(url)); + } + +public: + QList objects; + +public +slots: + virtual void start() override; + +protected +slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) override; + virtual void downloadError(QNetworkReply::NetworkError error) override; + virtual void downloadFinished() override; + virtual void downloadReadyRead() override; + +private: + void processValidReply(); + +private: + qint64 bytesSoFar = 0; + QString current_marker; +}; -- cgit v1.2.3