From 923347729557eed76e4f7e9f6f5f1a79216de0a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sat, 26 Oct 2013 19:55:48 +0200 Subject: S3 bucket listing support and network code refactors. * Adds support for listing all objects in an S3 bucket. * Renames a bunch of network related classes (Download->Action) * Net actions now have static constructors --- logic/net/ByteArrayDownload.cpp | 2 +- logic/net/ByteArrayDownload.h | 23 +++--- logic/net/CacheDownload.cpp | 2 +- logic/net/CacheDownload.h | 22 ++++-- logic/net/Download.h | 54 -------------- logic/net/DownloadJob.cpp | 138 ---------------------------------- logic/net/DownloadJob.h | 86 --------------------- logic/net/FileDownload.cpp | 2 +- logic/net/FileDownload.h | 25 ++++--- logic/net/ForgeXzDownload.cpp | 2 +- logic/net/ForgeXzDownload.h | 22 ++++-- logic/net/NetAction.h | 53 +++++++++++++ logic/net/NetJob.cpp | 98 ++++++++++++++++++++++++ logic/net/NetJob.h | 94 +++++++++++++++++++++++ logic/net/S3ListBucket.cpp | 161 ++++++++++++++++++++++++++++++++++++++++ logic/net/S3ListBucket.h | 42 +++++++++++ 16 files changed, 509 insertions(+), 317 deletions(-) delete mode 100644 logic/net/Download.h delete mode 100644 logic/net/DownloadJob.cpp delete mode 100644 logic/net/DownloadJob.h create mode 100644 logic/net/NetAction.h create mode 100644 logic/net/NetJob.cpp create mode 100644 logic/net/NetJob.h create mode 100644 logic/net/S3ListBucket.cpp create mode 100644 logic/net/S3ListBucket.h (limited to 'logic/net') diff --git a/logic/net/ByteArrayDownload.cpp b/logic/net/ByteArrayDownload.cpp index ba771eef..25e6d51a 100644 --- a/logic/net/ByteArrayDownload.cpp +++ b/logic/net/ByteArrayDownload.cpp @@ -2,7 +2,7 @@ #include "MultiMC.h" #include -ByteArrayDownload::ByteArrayDownload(QUrl url) : Download() +ByteArrayDownload::ByteArrayDownload(QUrl url) : NetAction() { m_url = url; m_status = Job_NotStarted; diff --git a/logic/net/ByteArrayDownload.h b/logic/net/ByteArrayDownload.h index cfc6a8d0..fc32dc04 100644 --- a/logic/net/ByteArrayDownload.h +++ b/logic/net/ByteArrayDownload.h @@ -1,24 +1,29 @@ #pragma once -#include "Download.h" +#include "NetAction.h" -class ByteArrayDownload: public Download +typedef std::shared_ptr ByteArrayDownloadPtr; +class ByteArrayDownload : public NetAction { Q_OBJECT public: ByteArrayDownload(QUrl url); - + static ByteArrayDownloadPtr make(QUrl url) + { + return ByteArrayDownloadPtr(new ByteArrayDownload(url)); + } + public: /// if not saving to file, downloaded data is placed here QByteArray m_data; - -public slots: + +public +slots: virtual void start(); - -protected slots: + +protected +slots: void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); void downloadError(QNetworkReply::NetworkError error); void downloadFinished(); void downloadReadyRead(); }; - -typedef std::shared_ptr ByteArrayDownloadPtr; diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp index 309eb345..f8769576 100644 --- a/logic/net/CacheDownload.cpp +++ b/logic/net/CacheDownload.cpp @@ -8,7 +8,7 @@ #include CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry) - : Download(), md5sum(QCryptographicHash::Md5) + : NetAction(), md5sum(QCryptographicHash::Md5) { m_url = url; m_entry = entry; diff --git a/logic/net/CacheDownload.h b/logic/net/CacheDownload.h index 295391b1..1e70874c 100644 --- a/logic/net/CacheDownload.h +++ b/logic/net/CacheDownload.h @@ -1,11 +1,12 @@ #pragma once -#include "Download.h" +#include "NetAction.h" #include "HttpMetaCache.h" #include #include -class CacheDownload : public Download +typedef std::shared_ptr CacheDownloadPtr; +class CacheDownload : public NetAction { Q_OBJECT public: @@ -18,17 +19,22 @@ public: QFile m_output_file; /// the hash-as-you-download QCryptographicHash md5sum; + public: explicit CacheDownload(QUrl url, MetaEntryPtr entry); - -protected slots: + static CacheDownloadPtr make(QUrl url, MetaEntryPtr entry) + { + return CacheDownloadPtr(new CacheDownload(url, entry)); + } + +protected +slots: virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); virtual void downloadError(QNetworkReply::NetworkError error); virtual void downloadFinished(); virtual void downloadReadyRead(); - -public slots: + +public +slots: virtual void start(); }; - -typedef std::shared_ptr CacheDownloadPtr; diff --git a/logic/net/Download.h b/logic/net/Download.h deleted file mode 100644 index ca4bee9f..00000000 --- a/logic/net/Download.h +++ /dev/null @@ -1,54 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -enum JobStatus -{ - Job_NotStarted, - Job_InProgress, - Job_Finished, - Job_Failed -}; - -class Download : public QObject -{ - Q_OBJECT -protected: - explicit Download() : QObject(0) {}; - -public: - virtual ~Download() {}; - -public: - /// the network reply - std::shared_ptr m_reply; - - /// source URL - QUrl m_url; - - /// The file's status - JobStatus m_status; - - /// index within the parent job - int index_within_job = 0; - -signals: - void started(int index); - void progress(int index, qint64 current, qint64 total); - void succeeded(int index); - void failed(int index); - -protected slots: - virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0; - virtual void downloadError(QNetworkReply::NetworkError error) = 0; - virtual void downloadFinished() = 0; - virtual void downloadReadyRead() = 0; - -public slots: - virtual void start() = 0; -}; - -typedef std::shared_ptr DownloadPtr; diff --git a/logic/net/DownloadJob.cpp b/logic/net/DownloadJob.cpp deleted file mode 100644 index 38716a02..00000000 --- a/logic/net/DownloadJob.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include "DownloadJob.h" -#include "pathutils.h" -#include "MultiMC.h" -#include "FileDownload.h" -#include "ByteArrayDownload.h" -#include "CacheDownload.h" - -#include - -ByteArrayDownloadPtr DownloadJob::addByteArrayDownload(QUrl url) -{ - ByteArrayDownloadPtr ptr(new ByteArrayDownload(url)); - ptr->index_within_job = downloads.size(); - downloads.append(ptr); - parts_progress.append(part_info()); - total_progress++; - return ptr; -} - -FileDownloadPtr DownloadJob::addFileDownload(QUrl url, QString rel_target_path) -{ - FileDownloadPtr ptr(new FileDownload(url, rel_target_path)); - ptr->index_within_job = downloads.size(); - downloads.append(ptr); - parts_progress.append(part_info()); - total_progress++; - return ptr; -} - -CacheDownloadPtr DownloadJob::addCacheDownload(QUrl url, MetaEntryPtr entry) -{ - CacheDownloadPtr ptr(new CacheDownload(url, entry)); - ptr->index_within_job = downloads.size(); - downloads.append(ptr); - parts_progress.append(part_info()); - total_progress++; - return ptr; -} - -ForgeXzDownloadPtr DownloadJob::addForgeXzDownload(QUrl url, MetaEntryPtr entry) -{ - ForgeXzDownloadPtr ptr(new ForgeXzDownload(url, entry)); - ptr->index_within_job = downloads.size(); - downloads.append(ptr); - parts_progress.append(part_info()); - total_progress++; - return ptr; -} - -void DownloadJob::partSucceeded(int index) -{ - // do progress. all slots are 1 in size at least - auto &slot = parts_progress[index]; - partProgress(index, slot.total_progress, slot.total_progress); - - num_succeeded++; - QLOG_INFO() << m_job_name.toLocal8Bit() << "progress:" << num_succeeded << "/" - << downloads.size(); - emit filesProgress(num_succeeded, num_failed, downloads.size()); - - if (num_failed + num_succeeded == downloads.size()) - { - if (num_failed) - { - QLOG_ERROR() << m_job_name.toLocal8Bit() << "failed."; - emit failed(); - } - else - { - QLOG_INFO() << m_job_name.toLocal8Bit() << "succeeded."; - emit succeeded(); - } - } -} - -void DownloadJob::partFailed(int index) -{ - auto &slot = parts_progress[index]; - if (slot.failures == 3) - { - QLOG_ERROR() << "Part" << index << "failed 3 times (" << downloads[index]->m_url << ")"; - num_failed++; - emit filesProgress(num_succeeded, num_failed, downloads.size()); - if (num_failed + num_succeeded == downloads.size()) - { - QLOG_ERROR() << m_job_name.toLocal8Bit() << "failed."; - emit failed(); - } - } - else - { - QLOG_ERROR() << "Part" << index << "failed, restarting (" << downloads[index]->m_url - << ")"; - // restart the job - slot.failures++; - downloads[index]->start(); - } -} - -void DownloadJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal) -{ - auto &slot = parts_progress[index]; - - current_progress -= slot.current_progress; - slot.current_progress = bytesReceived; - current_progress += slot.current_progress; - - total_progress -= slot.total_progress; - slot.total_progress = bytesTotal; - total_progress += slot.total_progress; - emit progress(current_progress, total_progress); -} - -void DownloadJob::start() -{ - QLOG_INFO() << m_job_name.toLocal8Bit() << " started."; - for (auto iter : downloads) - { - connect(iter.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int))); - connect(iter.get(), SIGNAL(failed(int)), SLOT(partFailed(int))); - connect(iter.get(), SIGNAL(progress(int, qint64, qint64)), - SLOT(partProgress(int, qint64, qint64))); - iter->start(); - } -} - -QStringList DownloadJob::getFailedFiles() -{ - QStringList failed; - for (auto download : downloads) - { - if (download->m_status == Job_Failed) - { - failed.push_back(download->m_url.toString()); - } - } - return failed; -} diff --git a/logic/net/DownloadJob.h b/logic/net/DownloadJob.h deleted file mode 100644 index cc2a1d59..00000000 --- a/logic/net/DownloadJob.h +++ /dev/null @@ -1,86 +0,0 @@ -#pragma once -#include -#include -#include "Download.h" -#include "ByteArrayDownload.h" -#include "FileDownload.h" -#include "CacheDownload.h" -#include "HttpMetaCache.h" -#include "ForgeXzDownload.h" -#include "logic/tasks/ProgressProvider.h" - -class DownloadJob; -typedef std::shared_ptr DownloadJobPtr; - -/** - * A single file for the downloader/cache to process. - */ -class DownloadJob : public ProgressProvider -{ - Q_OBJECT -public: - explicit DownloadJob(QString job_name) - :ProgressProvider(), m_job_name(job_name){}; - - ByteArrayDownloadPtr addByteArrayDownload(QUrl url); - FileDownloadPtr addFileDownload(QUrl url, QString rel_target_path); - CacheDownloadPtr addCacheDownload(QUrl url, MetaEntryPtr entry); - ForgeXzDownloadPtr addForgeXzDownload(QUrl url, MetaEntryPtr entry); - - DownloadPtr operator[](int index) - { - return downloads[index]; - }; - DownloadPtr first() - { - if(downloads.size()) - return downloads[0]; - return DownloadPtr(); - } - int size() const - { - return downloads.size(); - } - virtual void getProgress(qint64& current, qint64& total) - { - current = current_progress; - total = total_progress; - }; - virtual QString getStatus() const - { - return m_job_name; - }; - virtual bool isRunning() const - { - return m_running; - }; - QStringList getFailedFiles(); -signals: - void started(); - void progress(qint64 current, qint64 total); - void filesProgress(int, int, int); - void succeeded(); - void failed(); -public slots: - virtual void start(); -private slots: - void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal); - void partSucceeded(int index); - void partFailed(int index); -private: - struct part_info - { - qint64 current_progress = 0; - qint64 total_progress = 1; - int failures = 0; - }; - QString m_job_name; - QList downloads; - QList parts_progress; - qint64 current_progress = 0; - qint64 total_progress = 0; - int num_succeeded = 0; - int num_failed = 0; - bool m_running = false; -}; - diff --git a/logic/net/FileDownload.cpp b/logic/net/FileDownload.cpp index 3f38b0fa..eefdd4da 100644 --- a/logic/net/FileDownload.cpp +++ b/logic/net/FileDownload.cpp @@ -6,7 +6,7 @@ FileDownload::FileDownload ( QUrl url, QString target_path ) - :Download() + :NetAction() { m_url = url; m_target_path = target_path; diff --git a/logic/net/FileDownload.h b/logic/net/FileDownload.h index 9abb590d..5f72587f 100644 --- a/logic/net/FileDownload.h +++ b/logic/net/FileDownload.h @@ -1,14 +1,16 @@ #pragma once -#include "Download.h" +#include "NetAction.h" #include -class FileDownload : public Download +typedef std::shared_ptr FileDownloadPtr; +class FileDownload : public NetAction { Q_OBJECT public: /// if true, check the md5sum against a provided md5sum - /// also, if a file exists, perform an md5sum first and don't download only if they don't match + /// also, if a file exists, perform an md5sum first and don't download only if they don't + /// match bool m_check_md5; /// the expected md5 checksum QString m_expected_md5; @@ -18,18 +20,21 @@ public: QString m_target_path; /// this is the output file, if any QFile m_output_file; - + public: explicit FileDownload(QUrl url, QString target_path); - -protected slots: + static FileDownloadPtr make(QUrl url, QString target_path) + { + return FileDownloadPtr(new FileDownload(url, target_path)); + } +protected +slots: virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); virtual void downloadError(QNetworkReply::NetworkError error); virtual void downloadFinished(); virtual void downloadReadyRead(); - -public slots: + +public +slots: virtual void start(); }; - -typedef std::shared_ptr FileDownloadPtr; diff --git a/logic/net/ForgeXzDownload.cpp b/logic/net/ForgeXzDownload.cpp index 0e5287d8..20279d99 100644 --- a/logic/net/ForgeXzDownload.cpp +++ b/logic/net/ForgeXzDownload.cpp @@ -8,7 +8,7 @@ #include ForgeXzDownload::ForgeXzDownload(QUrl url, MetaEntryPtr entry) - : Download() + : NetAction() { QString urlstr = url.toString(); urlstr.append(".pack.xz"); diff --git a/logic/net/ForgeXzDownload.h b/logic/net/ForgeXzDownload.h index 5d677947..0b73711e 100644 --- a/logic/net/ForgeXzDownload.h +++ b/logic/net/ForgeXzDownload.h @@ -1,11 +1,12 @@ #pragma once -#include "Download.h" +#include "NetAction.h" #include "HttpMetaCache.h" #include #include +typedef std::shared_ptr ForgeXzDownloadPtr; -class ForgeXzDownload : public Download +class ForgeXzDownload : public NetAction { Q_OBJECT public: @@ -19,17 +20,22 @@ public: public: explicit ForgeXzDownload(QUrl url, MetaEntryPtr entry); - -protected slots: + static ForgeXzDownloadPtr make(QUrl url, MetaEntryPtr entry) + { + return ForgeXzDownloadPtr(new ForgeXzDownload(url, entry)); + } + +protected +slots: virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); virtual void downloadError(QNetworkReply::NetworkError error); virtual void downloadFinished(); virtual void downloadReadyRead(); - -public slots: + +public +slots: virtual void start(); + private: void decompressAndInstall(); }; - -typedef std::shared_ptr ForgeXzDownloadPtr; diff --git a/logic/net/NetAction.h b/logic/net/NetAction.h new file mode 100644 index 00000000..b7c922f5 --- /dev/null +++ b/logic/net/NetAction.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include + +enum JobStatus +{ + Job_NotStarted, + Job_InProgress, + Job_Finished, + Job_Failed +}; + +typedef std::shared_ptr NetActionPtr; +class NetAction : public QObject +{ + Q_OBJECT +protected: + explicit NetAction() : QObject(0) {}; + +public: + virtual ~NetAction() {}; + +public: + /// the network reply + std::shared_ptr m_reply; + + /// source URL + QUrl m_url; + + /// The file's status + JobStatus m_status; + + /// index within the parent job + int index_within_job = 0; + +signals: + void started(int index); + void progress(int index, qint64 current, qint64 total); + void succeeded(int index); + void failed(int index); + +protected slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0; + virtual void downloadError(QNetworkReply::NetworkError error) = 0; + virtual void downloadFinished() = 0; + virtual void downloadReadyRead() = 0; + +public slots: + virtual void start() = 0; +}; diff --git a/logic/net/NetJob.cpp b/logic/net/NetJob.cpp new file mode 100644 index 00000000..c7ca4409 --- /dev/null +++ b/logic/net/NetJob.cpp @@ -0,0 +1,98 @@ +#include "NetJob.h" +#include "pathutils.h" +#include "MultiMC.h" +#include "FileDownload.h" +#include "ByteArrayDownload.h" +#include "CacheDownload.h" + +#include + +void NetJob::partSucceeded(int index) +{ + // do progress. all slots are 1 in size at least + auto &slot = parts_progress[index]; + partProgress(index, slot.total_progress, slot.total_progress); + + num_succeeded++; + QLOG_INFO() << m_job_name.toLocal8Bit() << "progress:" << num_succeeded << "/" + << downloads.size(); + emit filesProgress(num_succeeded, num_failed, downloads.size()); + + if (num_failed + num_succeeded == downloads.size()) + { + if (num_failed) + { + QLOG_ERROR() << m_job_name.toLocal8Bit() << "failed."; + emit failed(); + } + else + { + QLOG_INFO() << m_job_name.toLocal8Bit() << "succeeded."; + emit succeeded(); + } + } +} + +void NetJob::partFailed(int index) +{ + auto &slot = parts_progress[index]; + if (slot.failures == 3) + { + QLOG_ERROR() << "Part" << index << "failed 3 times (" << downloads[index]->m_url << ")"; + num_failed++; + emit filesProgress(num_succeeded, num_failed, downloads.size()); + if (num_failed + num_succeeded == downloads.size()) + { + QLOG_ERROR() << m_job_name.toLocal8Bit() << "failed."; + emit failed(); + } + } + else + { + QLOG_ERROR() << "Part" << index << "failed, restarting (" << downloads[index]->m_url + << ")"; + // restart the job + slot.failures++; + downloads[index]->start(); + } +} + +void NetJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal) +{ + auto &slot = parts_progress[index]; + + current_progress -= slot.current_progress; + slot.current_progress = bytesReceived; + current_progress += slot.current_progress; + + total_progress -= slot.total_progress; + slot.total_progress = bytesTotal; + total_progress += slot.total_progress; + emit progress(current_progress, total_progress); +} + +void NetJob::start() +{ + QLOG_INFO() << m_job_name.toLocal8Bit() << " started."; + for (auto iter : downloads) + { + connect(iter.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int))); + connect(iter.get(), SIGNAL(failed(int)), SLOT(partFailed(int))); + connect(iter.get(), SIGNAL(progress(int, qint64, qint64)), + SLOT(partProgress(int, qint64, qint64))); + iter->start(); + } +} + +QStringList NetJob::getFailedFiles() +{ + QStringList failed; + for (auto download : downloads) + { + if (download->m_status == Job_Failed) + { + failed.push_back(download->m_url.toString()); + } + } + return failed; +} diff --git a/logic/net/NetJob.h b/logic/net/NetJob.h new file mode 100644 index 00000000..01d12e60 --- /dev/null +++ b/logic/net/NetJob.h @@ -0,0 +1,94 @@ +#pragma once +#include +#include +#include "NetAction.h" +#include "ByteArrayDownload.h" +#include "FileDownload.h" +#include "CacheDownload.h" +#include "HttpMetaCache.h" +#include "ForgeXzDownload.h" +#include "logic/tasks/ProgressProvider.h" + +class NetJob; +typedef std::shared_ptr NetJobPtr; + +class NetJob : public ProgressProvider +{ + Q_OBJECT +public: + explicit NetJob(QString job_name) : ProgressProvider(), m_job_name(job_name) {}; + + template + bool addNetAction(T action) + { + NetActionPtr base = std::static_pointer_cast(action); + base->index_within_job = downloads.size(); + downloads.append(action); + parts_progress.append(part_info()); + total_progress++; + return true; + } + + NetActionPtr operator[](int index) + { + return downloads[index]; + } + ; + NetActionPtr first() + { + if (downloads.size()) + return downloads[0]; + return NetActionPtr(); + } + int size() const + { + return downloads.size(); + } + virtual void getProgress(qint64 ¤t, qint64 &total) + { + current = current_progress; + total = total_progress; + } + ; + virtual QString getStatus() const + { + return m_job_name; + } + ; + virtual bool isRunning() const + { + return m_running; + } + ; + QStringList getFailedFiles(); +signals: + void started(); + void progress(qint64 current, qint64 total); + void filesProgress(int, int, int); + void succeeded(); + void failed(); +public +slots: + virtual void start(); +private +slots: + void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal); + void partSucceeded(int index); + void partFailed(int index); + +private: + struct part_info + { + qint64 current_progress = 0; + qint64 total_progress = 1; + int failures = 0; + }; + QString m_job_name; + QList downloads; + QList parts_progress; + qint64 current_progress = 0; + qint64 total_progress = 0; + int num_succeeded = 0; + int num_failed = 0; + bool m_running = false; +}; diff --git a/logic/net/S3ListBucket.cpp b/logic/net/S3ListBucket.cpp new file mode 100644 index 00000000..643c3224 --- /dev/null +++ b/logic/net/S3ListBucket.cpp @@ -0,0 +1,161 @@ +#include "S3ListBucket.h" +#include "MultiMC.h" +#include +#include +#include +#include + +inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname) +{ + QDomNodeList elementList = parent.elementsByTagName(tagname); + if (elementList.count()) + return elementList.at(0).toElement(); + else + return QDomElement(); +} + +S3ListBucket::S3ListBucket(QUrl url) : NetAction() +{ + m_url = url; + m_status = Job_NotStarted; +} + +void S3ListBucket::start() +{ + QUrl finalUrl = m_url; + if (current_marker.size()) + { + QUrlQuery query; + query.addQueryItem("marker", current_marker); + finalUrl.setQuery(query); + } + QNetworkRequest request(finalUrl); + request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)"); + auto worker = MMC->qnam(); + QNetworkReply *rep = worker->get(request); + + m_reply = std::shared_ptr(rep); + connect(rep, SIGNAL(downloadProgress(qint64, qint64)), + SLOT(downloadProgress(qint64, qint64))); + connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); + connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), + SLOT(downloadError(QNetworkReply::NetworkError))); + connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); +} + +void S3ListBucket::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) +{ + emit progress(index_within_job, bytesSoFar + bytesReceived, bytesSoFar + bytesTotal); +} + +void S3ListBucket::downloadError(QNetworkReply::NetworkError error) +{ + // error happened during download. + QLOG_ERROR() << "Error getting URL:" << m_url.toString().toLocal8Bit() + << "Network error: " << error; + m_status = Job_Failed; +} + +void S3ListBucket::processValidReply() +{ + QLOG_TRACE() << "GOT: " << m_url.toString() << " marker:" << current_marker; + auto readContents = [&](QXmlStreamReader & xml) + { + QString Key, ETag, Size; + while (!(xml.tokenType() == QXmlStreamReader::EndElement && xml.name() == "Contents")) + { + if (xml.tokenType() == QXmlStreamReader::StartElement) + { + if (xml.name() == "Key") + { + Key = xml.readElementText(); + } + if (xml.name() == "ETag") + { + ETag = xml.readElementText(); + } + if (xml.name() == "Size") + { + Size = xml.readElementText(); + } + } + xml.readNext(); + } + if (xml.error() != QXmlStreamReader::NoError) + return; + objects.append({Key, ETag, Size.toLongLong()}); + }; + + // nothing went wrong... + QString prefix("http://s3.amazonaws.com/Minecraft.Resources/"); + QByteArray ba = m_reply->readAll(); + + QString xmlErrorMsg; + + bool is_truncated = false; + QXmlStreamReader xml(ba); + while (!xml.atEnd() && !xml.hasError()) + { + /* Read next element.*/ + QXmlStreamReader::TokenType token = xml.readNext(); + /* If token is just StartDocument, we'll go to next.*/ + if (token == QXmlStreamReader::StartDocument) + { + continue; + } + if (token == QXmlStreamReader::StartElement) + { + /* If it's named person, we'll dig the information from there.*/ + if (xml.name() == "Contents") + { + readContents(xml); + } + else if (xml.name() == "IsTruncated") + { + is_truncated = (xml.readElementText() == "true"); + } + } + } + if (xml.hasError()) + { + QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:" + << xml.errorString() << ba; + emit failed(index_within_job); + return; + } + if(is_truncated) + { + current_marker = objects.last().Key; + bytesSoFar += m_reply->size(); + m_reply.reset(); + start(); + } + else + { + m_status = Job_Finished; + m_reply.reset(); + emit succeeded(index_within_job); + } + return; +} + +void S3ListBucket::downloadFinished() +{ + // if the download succeeded + if (m_status != Job_Failed) + { + processValidReply(); + } + // else the download failed + else + { + m_reply.reset(); + emit failed(index_within_job); + return; + } +} + +void S3ListBucket::downloadReadyRead() +{ + // ~_~ +} diff --git a/logic/net/S3ListBucket.h b/logic/net/S3ListBucket.h new file mode 100644 index 00000000..f054532d --- /dev/null +++ b/logic/net/S3ListBucket.h @@ -0,0 +1,42 @@ +#pragma once +#include "NetAction.h" + +struct S3Object +{ + QString Key; + QString ETag; + qlonglong size; +}; + +typedef std::shared_ptr S3ListBucketPtr; +class S3ListBucket : public NetAction +{ + Q_OBJECT +public: + S3ListBucket(QUrl url); + static S3ListBucketPtr make(QUrl url) + { + return S3ListBucketPtr(new S3ListBucket(url)); + } + +public: + QList objects; + +public +slots: + virtual void start() override; + +protected +slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) override; + virtual void downloadError(QNetworkReply::NetworkError error) override; + virtual void downloadFinished() override; + virtual void downloadReadyRead() override; + +private: + void processValidReply(); + +private: + qint64 bytesSoFar = 0; + QString current_marker; +}; -- cgit v1.2.3