S3 bucket listing support and network code refactors.

* Adds support for listing all objects in an S3 bucket.
* Renames a bunch of network related classes (Download->Action)
* Net actions now have static constructors

22 files changed, 369 insertions, 215 deletions

diff --git a/logic/BaseUpdate.h b/logic/BaseUpdate.h
index 6f1e26f3..9ada0770 100644
--- a/logic/BaseUpdate.h
+++ b/logic/BaseUpdate.h
@@ -19,7 +19,7 @@
 #include <QList>
 #include <QUrl>
 
-#include "net/DownloadJob.h"
+#include "net/NetJob.h"
 
 #include "tasks/Task.h"
 
diff --git a/logic/LegacyUpdate.cpp b/logic/LegacyUpdate.cpp
index 66b4bf8a..5120b241 100644
--- a/logic/LegacyUpdate.cpp
+++ b/logic/LegacyUpdate.cpp
@@ -233,8 +233,8 @@ void LegacyUpdate::jarStart()
 	QString intended_version_id = inst->intendedVersionId();
 	urlstr += intended_version_id + "/" + intended_version_id + ".jar";
 
-	auto dljob = new DownloadJob("Minecraft.jar for version " + intended_version_id);
-	dljob->addFileDownload(QUrl(urlstr), inst->defaultBaseJar());
+	auto dljob = new NetJob("Minecraft.jar for version " + intended_version_id);
+	dljob->addNetAction(FileDownload::make(QUrl(urlstr), inst->defaultBaseJar()));
 	legacyDownloadJob.reset(dljob);
 	connect(dljob, SIGNAL(succeeded()), SLOT(jarFinished()));
 	connect(dljob, SIGNAL(failed()), SLOT(jarFailed()));
diff --git a/logic/LegacyUpdate.h b/logic/LegacyUpdate.h
index e84ec56a..69560f55 100644
--- a/logic/LegacyUpdate.h
+++ b/logic/LegacyUpdate.h
@@ -19,7 +19,7 @@
 #include <QList>
 #include <QUrl>
 
-#include "net/DownloadJob.h"
+#include "net/NetJob.h"
 #include "tasks/Task.h"
 #include "BaseUpdate.h"
 
@@ -66,7 +66,7 @@ private:
 	QString lwjglTargetPath;
 	QString lwjglNativesPath;
 private:
-	DownloadJobPtr legacyDownloadJob;
+	NetJobPtr legacyDownloadJob;
 };
 
 
diff --git a/logic/OneSixAssets.cpp b/logic/OneSixAssets.cpp
index ff19a86b..375c87f1 100644
--- a/logic/OneSixAssets.cpp
+++ b/logic/OneSixAssets.cpp
@@ -2,19 +2,11 @@
 #include <logger/QsLog.h>
 #include <QtXml/QtXml>
 #include "OneSixAssets.h"
-#include "net/DownloadJob.h"
+#include "net/NetJob.h"
 #include "net/HttpMetaCache.h"
+#include "net/S3ListBucket.h"
 #include "MultiMC.h"
 
-inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
-{
-	QDomNodeList elementList = parent.elementsByTagName(tagname);
-	if (elementList.count())
-		return elementList.at(0).toElement();
-	else
-		return QDomElement();
-}
-
 class ThreadedDeleter : public QThread
 {
 	Q_OBJECT
@@ -22,18 +14,18 @@ public:
 	void run()
 	{
 		QLOG_INFO() << "Cleaning up assets folder...";
-		QDirIterator iter ( m_base, QDirIterator::Subdirectories );
+		QDirIterator iter(m_base, QDirIterator::Subdirectories);
 		int base_length = m_base.length();
-		while ( iter.hasNext() )
+		while (iter.hasNext())
 		{
 			QString filename = iter.next();
-			QFileInfo current ( filename );
+			QFileInfo current(filename);
 			// we keep the dirs... whatever
-			if ( current.isDir() )
+			if (current.isDir())
 				continue;
 			QString trimmedf = filename;
-			trimmedf.remove ( 0, base_length + 1 );
-			if ( m_whitelist.contains ( trimmedf ) )
+			trimmedf.remove(0, base_length + 1);
+			if (m_whitelist.contains(trimmedf))
 			{
 				QLOG_TRACE() << trimmedf << " gets to live";
 			}
@@ -41,7 +33,7 @@ public:
 			{
 				// DO NOT TOLERATE JUNK
 				QLOG_TRACE() << trimmedf << " dies";
-				QFile f ( filename );
+				QFile f(filename);
 				f.remove();
 			}
 		}
@@ -60,71 +52,41 @@ void OneSixAssets::downloadFinished()
 	deleter->start();
 }
 
-
-void OneSixAssets::fetchXMLFinished()
+void OneSixAssets::S3BucketFinished()
 {
-	QString prefix ( "http://s3.amazonaws.com/Minecraft.Resources/" );
-	QString fprefix ( "assets/" );
+	QString prefix("http://s3.amazonaws.com/Minecraft.Resources/");
 	nuke_whitelist.clear();
 
 	emit filesStarted();
 
 	auto firstJob = index_job->first();
-	QByteArray ba  = std::dynamic_pointer_cast<ByteArrayDownload>(firstJob)->m_data;
+	auto objectList = std::dynamic_pointer_cast<S3ListBucket>(firstJob)->objects;
 
-	QString xmlErrorMsg;
-	QDomDocument doc;
-	if ( !doc.setContent ( ba, false, &xmlErrorMsg ) )
-	{
-		QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:" << xmlErrorMsg << ba;
-		emit failed();
-		return;
-	}
-	//QRegExp etag_match(".*([a-f0-9]{32}).*");
-	QDomNodeList contents = doc.elementsByTagName ( "Contents" );
+	NetJob *job = new NetJob("Assets");
 
-	DownloadJob *job = new DownloadJob("Assets");
-	connect ( job, SIGNAL(succeeded()), SLOT(downloadFinished()) );
-	connect ( job, SIGNAL(failed()), SIGNAL(failed()) );
-	connect ( job, SIGNAL(filesProgress(int, int, int)), SIGNAL(filesProgress(int, int, int)) );
+	connect(job, SIGNAL(succeeded()), SLOT(downloadFinished()));
+	connect(job, SIGNAL(failed()), SIGNAL(failed()));
+	connect(job, SIGNAL(filesProgress(int, int, int)), SIGNAL(filesProgress(int, int, int)));
 
 	auto metacache = MMC->metacache();
-	
-	for ( int i = 0; i < contents.length(); i++ )
-	{
-		QDomElement element = contents.at ( i ).toElement();
-
-		if ( element.isNull() )
-			continue;
-
-		QDomElement keyElement = getDomElementByTagName ( element, "Key" );
-		QDomElement lastmodElement = getDomElementByTagName ( element, "LastModified" );
-		QDomElement etagElement = getDomElementByTagName ( element, "ETag" );
-		QDomElement sizeElement = getDomElementByTagName ( element, "Size" );
 
-		if ( keyElement.isNull() || lastmodElement.isNull() || etagElement.isNull() || sizeElement.isNull() )
+	for (auto object: objectList)
+	{
+		// Filter folder keys (zero size)
+		if (object.size == 0)
 			continue;
 
-		QString keyStr = keyElement.text();
-		QString lastModStr = lastmodElement.text();
-		QString etagStr = etagElement.text();
-		QString sizeStr = sizeElement.text();
-
-		//Filter folder keys
-		if ( sizeStr == "0" )
-			continue;
+		nuke_whitelist.append(object.Key);
 
-		nuke_whitelist.append ( keyStr );
-		
-		auto entry = metacache->resolveEntry("assets", keyStr, etagStr);
-		if(entry->stale)
+		auto entry = metacache->resolveEntry("assets", object.Key, object.ETag);
+		if (entry->stale)
 		{
-			job->addCacheDownload(QUrl(prefix + keyStr), entry);
+			job->addNetAction(CacheDownload::make(QUrl(prefix + object.Key), entry));
 		}
 	}
-	if(job->size())
+	if (job->size())
 	{
-		files_job.reset ( job );
+		files_job.reset(job);
 		files_job->start();
 	}
 	else
@@ -136,11 +98,12 @@ void OneSixAssets::fetchXMLFinished()
 
 void OneSixAssets::start()
 {
-	auto job = new DownloadJob("Assets index");
-	job->addByteArrayDownload(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" ));
-	connect ( job, SIGNAL(succeeded()), SLOT ( fetchXMLFinished() ) );
+	auto job = new NetJob("Assets index");
+	job->addNetAction(
+		S3ListBucket::make(QUrl("http://s3.amazonaws.com/Minecraft.Resources/")));
+	connect(job, SIGNAL(succeeded()), SLOT(S3BucketFinished()));
 	emit indexStarted();
-	index_job.reset ( job );
+	index_job.reset(job);
 	job->start();
 }
 
diff --git a/logic/OneSixAssets.h b/logic/OneSixAssets.h
index 6c6f2c36..72e8a843 100644
--- a/logic/OneSixAssets.h
+++ b/logic/OneSixAssets.h
@@ -1,5 +1,5 @@
 #pragma once
-#include "net/DownloadJob.h"
+#include "net/NetJob.h"
 
 class Private;
 class ThreadedDeleter;
@@ -15,13 +15,13 @@ signals:
 	void filesProgress(int, int, int);
 
 public slots:
-	void fetchXMLFinished();
+	void S3BucketFinished();
 	void downloadFinished();
 public:
 	void start();
 private:
 	ThreadedDeleter * deleter;
 	QStringList nuke_whitelist;
-	DownloadJobPtr index_job;
-	DownloadJobPtr files_job;
+	NetJobPtr index_job;
+	NetJobPtr files_job;
 };
diff --git a/logic/OneSixUpdate.cpp b/logic/OneSixUpdate.cpp
index b5f1d78b..5c421fbf 100644
--- a/logic/OneSixUpdate.cpp
+++ b/logic/OneSixUpdate.cpp
@@ -72,8 +72,8 @@ void OneSixUpdate::versionFileStart()
 
 	QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/");
 	urlstr += targetVersion->descriptor() + "/" + targetVersion->descriptor() + ".json";
-	auto job = new DownloadJob("Version index");
-	job->addByteArrayDownload(QUrl(urlstr));
+	auto job = new NetJob("Version index");
+	job->addNetAction(ByteArrayDownload::make(QUrl(urlstr)));
 	specificVersionDownloadJob.reset(job);
 	connect(specificVersionDownloadJob.get(), SIGNAL(succeeded()), SLOT(versionFileFinished()));
 	connect(specificVersionDownloadJob.get(), SIGNAL(failed()), SLOT(versionFileFailed()));
@@ -84,7 +84,7 @@ void OneSixUpdate::versionFileStart()
 
 void OneSixUpdate::versionFileFinished()
 {
-	DownloadPtr DlJob = specificVersionDownloadJob->first();
+	NetActionPtr DlJob = specificVersionDownloadJob->first();
 	OneSixInstance *inst = (OneSixInstance *)m_inst;
 
 	QString version_id = targetVersion->descriptor();
@@ -154,8 +154,8 @@ void OneSixUpdate::jarlibStart()
 	QString targetstr("versions/");
 	targetstr += version->id + "/" + version->id + ".jar";
 
-	auto job = new DownloadJob("Libraries for instance " + inst->name());
-	job->addFileDownload(QUrl(urlstr), targetstr);
+	auto job = new NetJob("Libraries for instance " + inst->name());
+	job->addNetAction(FileDownload::make(QUrl(urlstr), targetstr));
 	jarlibDownloadJob.reset(job);
 
 	auto libs = version->getActiveNativeLibs();
@@ -171,9 +171,9 @@ void OneSixUpdate::jarlibStart()
 		if (entry->stale)
 		{
 			if (lib->hint() == "forge-pack-xz")
-				jarlibDownloadJob->addForgeXzDownload(download_path, entry);
+				jarlibDownloadJob->addNetAction(ForgeXzDownload::make(download_path, entry));
 			else
-				jarlibDownloadJob->addCacheDownload(download_path, entry);
+				jarlibDownloadJob->addNetAction(CacheDownload::make(download_path, entry));
 		}
 	}
 	connect(jarlibDownloadJob.get(), SIGNAL(succeeded()), SLOT(jarlibFinished()));
diff --git a/logic/OneSixUpdate.h b/logic/OneSixUpdate.h
index ed8dcbcd..7c6fce1b 100644
--- a/logic/OneSixUpdate.h
+++ b/logic/OneSixUpdate.h
@@ -18,7 +18,7 @@
 #include <QObject>
 #include <QList>
 #include <QUrl>
-#include "net/DownloadJob.h"
+#include "net/NetJob.h"
 
 #include "tasks/Task.h"
 #include "BaseUpdate.h"
@@ -43,8 +43,8 @@ private slots:
 	void jarlibFailed();
 	
 private:
-	DownloadJobPtr specificVersionDownloadJob;
-	DownloadJobPtr jarlibDownloadJob;
+	NetJobPtr specificVersionDownloadJob;
+	NetJobPtr jarlibDownloadJob;
 	
 	// target version, determined during this task
 	std::shared_ptr<MinecraftVersion> targetVersion;
diff --git a/logic/lists/ForgeVersionList.cpp b/logic/lists/ForgeVersionList.cpp
index 491a43d7..27f567cd 100644
--- a/logic/lists/ForgeVersionList.cpp
+++ b/logic/lists/ForgeVersionList.cpp
@@ -14,7 +14,7 @@
  */
 
 #include "ForgeVersionList.h"
-#include <logic/net/DownloadJob.h>
+#include <logic/net/NetJob.h>
 #include "MultiMC.h"
 
 #include <QtNetwork>
@@ -159,14 +159,14 @@ ForgeListLoadTask::ForgeListLoadTask(ForgeVersionList *vlist) : Task()
 
 void ForgeListLoadTask::executeTask()
 {
-	auto job = new DownloadJob("Version index");
+	auto job = new NetJob("Version index");
 	// we do not care if the version is stale or not.
 	auto forgeListEntry = MMC->metacache()->resolveEntry("minecraftforge", "list.json");
-	
+
 	// verify by poking the server.
 	forgeListEntry->stale = true;
-	
-	job->addCacheDownload(QUrl(JSON_URL), forgeListEntry);
+
+	job->addNetAction(CacheDownload::make(QUrl(JSON_URL), forgeListEntry));
 	listJob.reset(job);
 	connect(listJob.get(), SIGNAL(succeeded()), SLOT(list_downloaded()));
 	connect(listJob.get(), SIGNAL(failed()), SLOT(list_failed()));
@@ -178,7 +178,7 @@ void ForgeListLoadTask::list_failed()
 {
 	auto DlJob = listJob->first();
 	auto reply = DlJob->m_reply;
-	if(reply)
+	if (reply)
 	{
 		QLOG_ERROR() << "Getting forge version list failed: " << reply->errorString();
 	}
@@ -193,7 +193,7 @@ void ForgeListLoadTask::list_downloaded()
 		auto DlJob = listJob->first();
 		auto filename = std::dynamic_pointer_cast<CacheDownload>(DlJob)->m_target_path;
 		QFile listFile(filename);
-		if(!listFile.open(QIODevice::ReadOnly))
+		if (!listFile.open(QIODevice::ReadOnly))
 			return;
 		data = listFile.readAll();
 		DlJob.reset();
@@ -202,7 +202,6 @@ void ForgeListLoadTask::list_downloaded()
 	QJsonParseError jsonError;
 	QJsonDocument jsonDoc = QJsonDocument::fromJson(data, &jsonError);
 
-
 	if (jsonError.error != QJsonParseError::NoError)
 	{
 		emitFailed("Error parsing version list JSON:" + jsonError.errorString());
diff --git a/logic/lists/ForgeVersionList.h b/logic/lists/ForgeVersionList.h
index 56a207c8..52593f94 100644
--- a/logic/lists/ForgeVersionList.h
+++ b/logic/lists/ForgeVersionList.h
@@ -23,7 +23,7 @@
 #include <QNetworkReply>
 #include "BaseVersionList.h"
 #include "logic/tasks/Task.h"
-#include "logic/net/DownloadJob.h"
+#include "logic/net/NetJob.h"
 
 class ForgeVersion;
 typedef std::shared_ptr<ForgeVersion> ForgeVersionPtr;
@@ -104,6 +104,6 @@ slots:
 	void list_failed();
 
 protected:
-	DownloadJobPtr listJob;
+	NetJobPtr listJob;
 	ForgeVersionList *m_list;
 };
diff --git a/logic/net/ByteArrayDownload.cpp b/logic/net/ByteArrayDownload.cpp
index ba771eef..25e6d51a 100644
--- a/logic/net/ByteArrayDownload.cpp
+++ b/logic/net/ByteArrayDownload.cpp
@@ -2,7 +2,7 @@
 #include "MultiMC.h"
 #include <logger/QsLog.h>
 
-ByteArrayDownload::ByteArrayDownload(QUrl url) : Download()
+ByteArrayDownload::ByteArrayDownload(QUrl url) : NetAction()
 {
 	m_url = url;
 	m_status = Job_NotStarted;
diff --git a/logic/net/ByteArrayDownload.h b/logic/net/ByteArrayDownload.h
index cfc6a8d0..fc32dc04 100644
--- a/logic/net/ByteArrayDownload.h
+++ b/logic/net/ByteArrayDownload.h
@@ -1,24 +1,29 @@
 #pragma once
-#include "Download.h"
+#include "NetAction.h"
 
-class ByteArrayDownload: public Download
+typedef std::shared_ptr<class ByteArrayDownload> ByteArrayDownloadPtr;
+class ByteArrayDownload : public NetAction
 {
 	Q_OBJECT
 public:
 	ByteArrayDownload(QUrl url);
-	
+	static ByteArrayDownloadPtr make(QUrl url)
+	{
+		return ByteArrayDownloadPtr(new ByteArrayDownload(url));
+	}
+
 public:
 	/// if not saving to file, downloaded data is placed here
 	QByteArray m_data;
-	
-public slots:
+
+public
+slots:
 	virtual void start();
-	
-protected slots:
+
+protected
+slots:
 	void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
 	void downloadError(QNetworkReply::NetworkError error);
 	void downloadFinished();
 	void downloadReadyRead();
 };
-
-typedef std::shared_ptr<ByteArrayDownload> ByteArrayDownloadPtr;
diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp
index 309eb345..f8769576 100644
--- a/logic/net/CacheDownload.cpp
+++ b/logic/net/CacheDownload.cpp
@@ -8,7 +8,7 @@
 #include <logger/QsLog.h>
 
 CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry)
-	: Download(), md5sum(QCryptographicHash::Md5)
+	: NetAction(), md5sum(QCryptographicHash::Md5)
 {
 	m_url = url;
 	m_entry = entry;
diff --git a/logic/net/CacheDownload.h b/logic/net/CacheDownload.h
index 295391b1..1e70874c 100644
--- a/logic/net/CacheDownload.h
+++ b/logic/net/CacheDownload.h
@@ -1,11 +1,12 @@
 #pragma once
 
-#include "Download.h"
+#include "NetAction.h"
 #include "HttpMetaCache.h"
 #include <QFile>
 #include <qcryptographichash.h>
 
-class CacheDownload : public Download
+typedef std::shared_ptr<class CacheDownload> CacheDownloadPtr;
+class CacheDownload : public NetAction
 {
 	Q_OBJECT
 public:
@@ -18,17 +19,22 @@ public:
 	QFile m_output_file;
 	/// the hash-as-you-download
 	QCryptographicHash md5sum;
+
 public:
 	explicit CacheDownload(QUrl url, MetaEntryPtr entry);
-	
-protected slots:
+	static CacheDownloadPtr make(QUrl url, MetaEntryPtr entry)
+	{
+		return CacheDownloadPtr(new CacheDownload(url, entry));
+	}
+
+protected
+slots:
 	virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
 	virtual void downloadError(QNetworkReply::NetworkError error);
 	virtual void downloadFinished();
 	virtual void downloadReadyRead();
-	
-public slots:
+
+public
+slots:
 	virtual void start();
 };
-
-typedef std::shared_ptr<CacheDownload> CacheDownloadPtr;
diff --git a/logic/net/FileDownload.cpp b/logic/net/FileDownload.cpp
index 3f38b0fa..eefdd4da 100644
--- a/logic/net/FileDownload.cpp
+++ b/logic/net/FileDownload.cpp
@@ -6,7 +6,7 @@
 
 
 FileDownload::FileDownload ( QUrl url, QString target_path )
-	:Download()
+	:NetAction()
 {
 	m_url = url;
 	m_target_path = target_path;
diff --git a/logic/net/FileDownload.h b/logic/net/FileDownload.h
index 9abb590d..5f72587f 100644
--- a/logic/net/FileDownload.h
+++ b/logic/net/FileDownload.h
@@ -1,14 +1,16 @@
 #pragma once
 
-#include "Download.h"
+#include "NetAction.h"
 #include <QFile>
 
-class FileDownload : public Download
+typedef std::shared_ptr<class FileDownload> FileDownloadPtr;
+class FileDownload : public NetAction
 {
 	Q_OBJECT
 public:
 	/// if true, check the md5sum against a provided md5sum
-	/// also, if a file exists, perform an md5sum first and don't download only if they don't match
+	/// also, if a file exists, perform an md5sum first and don't download only if they don't
+	/// match
 	bool m_check_md5;
 	/// the expected md5 checksum
 	QString m_expected_md5;
@@ -18,18 +20,21 @@ public:
 	QString m_target_path;
 	/// this is the output file, if any
 	QFile m_output_file;
-	
+
 public:
 	explicit FileDownload(QUrl url, QString target_path);
-	
-protected slots:
+	static FileDownloadPtr make(QUrl url, QString target_path)
+	{
+		return FileDownloadPtr(new FileDownload(url, target_path));
+	}
+protected
+slots:
 	virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
 	virtual void downloadError(QNetworkReply::NetworkError error);
 	virtual void downloadFinished();
 	virtual void downloadReadyRead();
-	
-public slots:
+
+public
+slots:
 	virtual void start();
 };
-
-typedef std::shared_ptr<FileDownload> FileDownloadPtr;
diff --git a/logic/net/ForgeXzDownload.cpp b/logic/net/ForgeXzDownload.cpp
index 0e5287d8..20279d99 100644
--- a/logic/net/ForgeXzDownload.cpp
+++ b/logic/net/ForgeXzDownload.cpp
@@ -8,7 +8,7 @@
 #include <logger/QsLog.h>
 
 ForgeXzDownload::ForgeXzDownload(QUrl url, MetaEntryPtr entry)
-	: Download()
+	: NetAction()
 {
 	QString urlstr = url.toString();
 	urlstr.append(".pack.xz");
diff --git a/logic/net/ForgeXzDownload.h b/logic/net/ForgeXzDownload.h
index 5d677947..0b73711e 100644
--- a/logic/net/ForgeXzDownload.h
+++ b/logic/net/ForgeXzDownload.h
@@ -1,11 +1,12 @@
 #pragma once
 
-#include "Download.h"
+#include "NetAction.h"
 #include "HttpMetaCache.h"
 #include <QFile>
 #include <QTemporaryFile>
+typedef std::shared_ptr<class ForgeXzDownload> ForgeXzDownloadPtr;
 
-class ForgeXzDownload : public Download
+class ForgeXzDownload : public NetAction
 {
 	Q_OBJECT
 public:
@@ -19,17 +20,22 @@ public:
 
 public:
 	explicit ForgeXzDownload(QUrl url, MetaEntryPtr entry);
-	
-protected slots:
+	static ForgeXzDownloadPtr make(QUrl url, MetaEntryPtr entry)
+	{
+		return ForgeXzDownloadPtr(new ForgeXzDownload(url, entry));
+	}
+
+protected
+slots:
 	virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
 	virtual void downloadError(QNetworkReply::NetworkError error);
 	virtual void downloadFinished();
 	virtual void downloadReadyRead();
-	
-public slots:
+
+public
+slots:
 	virtual void start();
+
 private:
 	void decompressAndInstall();
 };
-
-typedef std::shared_ptr<ForgeXzDownload> ForgeXzDownloadPtr;
diff --git a/logic/net/Download.h b/logic/net/NetAction.h
index ca4bee9f..b7c922f5 100644
--- a/logic/net/Download.h
+++ b/logic/net/NetAction.h
@@ -13,14 +13,15 @@ enum JobStatus
 	Job_Failed
 };
 
-class Download : public QObject
+typedef std::shared_ptr<class NetAction> NetActionPtr;
+class NetAction : public QObject
 {
 	Q_OBJECT
 protected:
-	explicit Download() : QObject(0) {};
+	explicit NetAction() : QObject(0) {};
 
 public:
-	virtual ~Download() {};
+	virtual ~NetAction() {};
 
 public:
 	/// the network reply
@@ -50,5 +51,3 @@ protected slots:
 public slots:
 	virtual void start() = 0;
 };
-
-typedef std::shared_ptr<Download> DownloadPtr;
diff --git a/logic/net/DownloadJob.cpp b/logic/net/NetJob.cpp
index 38716a02..c7ca4409 100644
--- a/logic/net/DownloadJob.cpp
+++ b/logic/net/NetJob.cpp
@@ -1,4 +1,4 @@
-#include "DownloadJob.h"
+#include "NetJob.h"
 #include "pathutils.h"
 #include "MultiMC.h"
 #include "FileDownload.h"
@@ -7,47 +7,7 @@
 
 #include <logger/QsLog.h>
 
-ByteArrayDownloadPtr DownloadJob::addByteArrayDownload(QUrl url)
-{
-	ByteArrayDownloadPtr ptr(new ByteArrayDownload(url));
-	ptr->index_within_job = downloads.size();
-	downloads.append(ptr);
-	parts_progress.append(part_info());
-	total_progress++;
-	return ptr;
-}
-
-FileDownloadPtr DownloadJob::addFileDownload(QUrl url, QString rel_target_path)
-{
-	FileDownloadPtr ptr(new FileDownload(url, rel_target_path));
-	ptr->index_within_job = downloads.size();
-	downloads.append(ptr);
-	parts_progress.append(part_info());
-	total_progress++;
-	return ptr;
-}
-
-CacheDownloadPtr DownloadJob::addCacheDownload(QUrl url, MetaEntryPtr entry)
-{
-	CacheDownloadPtr ptr(new CacheDownload(url, entry));
-	ptr->index_within_job = downloads.size();
-	downloads.append(ptr);
-	parts_progress.append(part_info());
-	total_progress++;
-	return ptr;
-}
-
-ForgeXzDownloadPtr DownloadJob::addForgeXzDownload(QUrl url, MetaEntryPtr entry)
-{
-	ForgeXzDownloadPtr ptr(new ForgeXzDownload(url, entry));
-	ptr->index_within_job = downloads.size();
-	downloads.append(ptr);
-	parts_progress.append(part_info());
-	total_progress++;
-	return ptr;
-}
-
-void DownloadJob::partSucceeded(int index)
+void NetJob::partSucceeded(int index)
 {
 	// do progress. all slots are 1 in size at least
 	auto &slot = parts_progress[index];
@@ -73,7 +33,7 @@ void DownloadJob::partSucceeded(int index)
 	}
 }
 
-void DownloadJob::partFailed(int index)
+void NetJob::partFailed(int index)
 {
 	auto &slot = parts_progress[index];
 	if (slot.failures == 3)
@@ -97,7 +57,7 @@ void DownloadJob::partFailed(int index)
 	}
 }
 
-void DownloadJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal)
+void NetJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal)
 {
 	auto &slot = parts_progress[index];
 
@@ -111,7 +71,7 @@ void DownloadJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTota
 	emit progress(current_progress, total_progress);
 }
 
-void DownloadJob::start()
+void NetJob::start()
 {
 	QLOG_INFO() << m_job_name.toLocal8Bit() << " started.";
 	for (auto iter : downloads)
@@ -124,7 +84,7 @@ void DownloadJob::start()
 	}
 }
 
-QStringList DownloadJob::getFailedFiles()
+QStringList NetJob::getFailedFiles()
 {
 	QStringList failed;
 	for (auto download : downloads)
diff --git a/logic/net/DownloadJob.h b/logic/net/NetJob.h
index cc2a1d59..01d12e60 100644
--- a/logic/net/DownloadJob.h
+++ b/logic/net/NetJob.h
@@ -1,7 +1,7 @@
 #pragma once
 #include <QtNetwork>
 #include <QLabel>
-#include "Download.h"
+#include "NetAction.h"
 #include "ByteArrayDownload.h"
 #include "FileDownload.h"
 #include "CacheDownload.h"
@@ -9,51 +9,57 @@
 #include "ForgeXzDownload.h"
 #include "logic/tasks/ProgressProvider.h"
 
-class DownloadJob;
-typedef std::shared_ptr<DownloadJob> DownloadJobPtr;
+class NetJob;
+typedef std::shared_ptr<NetJob> NetJobPtr;
 
-/**
- * A single file for the downloader/cache to process.
- */
-class DownloadJob : public ProgressProvider
+class NetJob : public ProgressProvider
 {
 	Q_OBJECT
 public:
-	explicit DownloadJob(QString job_name)
-		:ProgressProvider(), m_job_name(job_name){};
-	
-	ByteArrayDownloadPtr addByteArrayDownload(QUrl url);
-	FileDownloadPtr      addFileDownload(QUrl url, QString rel_target_path);
-	CacheDownloadPtr     addCacheDownload(QUrl url, MetaEntryPtr entry);
-	ForgeXzDownloadPtr   addForgeXzDownload(QUrl url, MetaEntryPtr entry);
-	
-	DownloadPtr operator[](int index)
+	explicit NetJob(QString job_name) : ProgressProvider(), m_job_name(job_name) {};
+
+	template <typename T>
+	bool addNetAction(T action)
+	{
+		NetActionPtr base = std::static_pointer_cast<NetAction>(action); 
+		base->index_within_job = downloads.size();
+		downloads.append(action);
+		parts_progress.append(part_info());
+		total_progress++;
+		return true;
+	}
+
+	NetActionPtr operator[](int index)
 	{
 		return downloads[index];
-	};
-	DownloadPtr first()
+	}
+	;
+	NetActionPtr first()
 	{
-		if(downloads.size())
+		if (downloads.size())
 			return downloads[0];
-		return DownloadPtr();
+		return NetActionPtr();
 	}
 	int size() const
 	{
 		return downloads.size();
 	}
-	virtual void getProgress(qint64& current, qint64& total)
+	virtual void getProgress(qint64 &current, qint64 &total)
 	{
 		current = current_progress;
 		total = total_progress;
-	};
+	}
+	;
 	virtual QString getStatus() const
 	{
 		return m_job_name;
-	};
+	}
+	;
 	virtual bool isRunning() const
 	{
 		return m_running;
-	};
+	}
+	;
 	QStringList getFailedFiles();
 signals:
 	void started();
@@ -61,12 +67,15 @@ signals:
 	void filesProgress(int, int, int);
 	void succeeded();
 	void failed();
-public slots:
+public
+slots:
 	virtual void start();
-private slots:
+private
+slots:
 	void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal);
 	void partSucceeded(int index);
 	void partFailed(int index);
+
 private:
 	struct part_info
 	{
@@ -75,7 +84,7 @@ private:
 		int failures = 0;
 	};
 	QString m_job_name;
-	QList<DownloadPtr> downloads;
+	QList<NetActionPtr> downloads;
 	QList<part_info> parts_progress;
 	qint64 current_progress = 0;
 	qint64 total_progress = 0;
@@ -83,4 +92,3 @@ private:
 	int num_failed = 0;
 	bool m_running = false;
 };
-
diff --git a/logic/net/S3ListBucket.cpp b/logic/net/S3ListBucket.cpp
new file mode 100644
index 00000000..643c3224
--- /dev/null
+++ b/logic/net/S3ListBucket.cpp
@@ -0,0 +1,161 @@
+#include "S3ListBucket.h"
+#include "MultiMC.h"
+#include <logger/QsLog.h>
+#include <QUrlQuery>
+#include <qxmlstream.h>
+#include <QDomDocument>
+
+inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
+{
+	QDomNodeList elementList = parent.elementsByTagName(tagname);
+	if (elementList.count())
+		return elementList.at(0).toElement();
+	else
+		return QDomElement();
+}
+
+S3ListBucket::S3ListBucket(QUrl url) : NetAction()
+{
+	m_url = url;
+	m_status = Job_NotStarted;
+}
+
+void S3ListBucket::start()
+{
+	QUrl finalUrl = m_url;
+	if (current_marker.size())
+	{
+		QUrlQuery query;
+		query.addQueryItem("marker", current_marker);
+		finalUrl.setQuery(query);
+	}
+	QNetworkRequest request(finalUrl);
+	request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)");
+	auto worker = MMC->qnam();
+	QNetworkReply *rep = worker->get(request);
+
+	m_reply = std::shared_ptr<QNetworkReply>(rep);
+	connect(rep, SIGNAL(downloadProgress(qint64, qint64)),
+			SLOT(downloadProgress(qint64, qint64)));
+	connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
+	connect(rep, SIGNAL(error(QNetworkReply::NetworkError)),
+			SLOT(downloadError(QNetworkReply::NetworkError)));
+	connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead()));
+}
+
+void S3ListBucket::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
+{
+	emit progress(index_within_job, bytesSoFar + bytesReceived, bytesSoFar + bytesTotal);
+}
+
+void S3ListBucket::downloadError(QNetworkReply::NetworkError error)
+{
+	// error happened during download.
+	QLOG_ERROR() << "Error getting URL:" << m_url.toString().toLocal8Bit()
+				 << "Network error: " << error;
+	m_status = Job_Failed;
+}
+
+void S3ListBucket::processValidReply()
+{
+	QLOG_TRACE() << "GOT: " << m_url.toString() << " marker:" << current_marker;
+	auto readContents = [&](QXmlStreamReader & xml)
+	{
+		QString Key, ETag, Size;
+		while (!(xml.tokenType() == QXmlStreamReader::EndElement && xml.name() == "Contents"))
+		{
+			if (xml.tokenType() == QXmlStreamReader::StartElement)
+			{
+				if (xml.name() == "Key")
+				{
+					Key = xml.readElementText();
+				}
+				if (xml.name() == "ETag")
+				{
+					ETag = xml.readElementText();
+				}
+				if (xml.name() == "Size")
+				{
+					Size = xml.readElementText();
+				}
+			}
+			xml.readNext();
+		}
+		if (xml.error() != QXmlStreamReader::NoError)
+			return;
+		objects.append({Key, ETag, Size.toLongLong()});
+	};
+
+	// nothing went wrong...
+	QString prefix("http://s3.amazonaws.com/Minecraft.Resources/");
+	QByteArray ba = m_reply->readAll();
+
+	QString xmlErrorMsg;
+
+	bool is_truncated = false;
+	QXmlStreamReader xml(ba);
+	while (!xml.atEnd() && !xml.hasError())
+	{
+		/* Read next element.*/
+		QXmlStreamReader::TokenType token = xml.readNext();
+		/* If token is just StartDocument, we'll go to next.*/
+		if (token == QXmlStreamReader::StartDocument)
+		{
+			continue;
+		}
+		if (token == QXmlStreamReader::StartElement)
+		{
+			/* If it's named person, we'll dig the information from there.*/
+			if (xml.name() == "Contents")
+			{
+				readContents(xml);
+			}
+			else if (xml.name() == "IsTruncated")
+			{
+				is_truncated = (xml.readElementText() == "true");
+			}
+		}
+	}
+	if (xml.hasError())
+	{
+		QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:"
+					 << xml.errorString() << ba;
+		emit failed(index_within_job);
+		return;
+	}
+	if(is_truncated)
+	{
+		current_marker = objects.last().Key;
+		bytesSoFar += m_reply->size();
+		m_reply.reset();
+		start();
+	}
+	else
+	{
+		m_status = Job_Finished;
+		m_reply.reset();
+		emit succeeded(index_within_job);
+	}
+	return;
+}
+
+void S3ListBucket::downloadFinished()
+{
+	// if the download succeeded
+	if (m_status != Job_Failed)
+	{
+		processValidReply();
+	}
+	// else the download failed
+	else
+	{
+		m_reply.reset();
+		emit failed(index_within_job);
+		return;
+	}
+}
+
+void S3ListBucket::downloadReadyRead()
+{
+	// ~_~
+}
diff --git a/logic/net/S3ListBucket.h b/logic/net/S3ListBucket.h
new file mode 100644
index 00000000..f054532d
--- /dev/null
+++ b/logic/net/S3ListBucket.h
@@ -0,0 +1,42 @@
+#pragma once
+#include "NetAction.h"
+
+struct S3Object
+{
+	QString Key;
+	QString ETag;
+	qlonglong size;
+};
+
+typedef std::shared_ptr<class S3ListBucket> S3ListBucketPtr;
+class S3ListBucket : public NetAction
+{
+	Q_OBJECT
+public:
+	S3ListBucket(QUrl url);
+	static S3ListBucketPtr make(QUrl url)
+	{
+		return S3ListBucketPtr(new S3ListBucket(url));
+	}
+
+public:
+	QList<S3Object> objects;
+
+public
+slots:
+	virtual void start() override;
+
+protected
+slots:
+	virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) override;
+	virtual void downloadError(QNetworkReply::NetworkError error) override;
+	virtual void downloadFinished() override;
+	virtual void downloadReadyRead() override;
+
+private:
+	void processValidReply();
+
+private:
+	qint64 bytesSoFar = 0;
+	QString current_marker;
+};