Use HttpMetaCache to minimize network use.

17 files changed, 693 insertions, 269 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 27f8eaba..cbc3d842 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -184,6 +184,10 @@ logic/ModList.h
 logic/InstanceLauncher.h
 
 # network stuffs
+logic/net/Download.h
+logic/net/FileDownload.h
+logic/net/ByteArrayDownload.h
+logic/net/CacheDownload.h
 logic/net/DownloadJob.h
 logic/net/HttpMetaCache.h
 
@@ -252,6 +256,9 @@ logic/ModList.cpp
 logic/InstanceLauncher.cpp
 
 # network stuffs - to be moved into a depend lib ~_~
+logic/net/FileDownload.cpp
+logic/net/ByteArrayDownload.cpp
+logic/net/CacheDownload.cpp
 logic/net/DownloadJob.cpp
 logic/net/HttpMetaCache.cpp
 
diff --git a/MultiMC.cpp b/MultiMC.cpp
index 88eb3e62..f212d830 100644
--- a/MultiMC.cpp
+++ b/MultiMC.cpp
@@ -8,6 +8,7 @@
 #include "logic/lists/InstanceList.h"
 #include "logic/lists/IconList.h"
 #include "logic/InstanceLauncher.h"
+#include "logic/net/HttpMetaCache.h"
 
 
 #include "pathutils.h"
@@ -112,11 +113,16 @@ MultiMC::MultiMC ( int& argc, char** argv )
 	
 	// load settings
 	initGlobalSettings();
+	
 	// and instances
 	m_instances = new InstanceList(m_settings->get("InstanceDir").toString(),this);
 	std::cout << "Loading Instances..." << std::endl;
 	m_instances->loadList();
-	// network manager
+	
+	// init the http meta cache
+	initHttpMetaCache();
+	
+	// create the global network manager
 	m_qnam = new QNetworkAccessManager(this);
 	
 	// Register meta types.
@@ -137,11 +143,12 @@ MultiMC::MultiMC ( int& argc, char** argv )
 MultiMC::~MultiMC()
 {
 	delete m_settings;
+	delete m_metacache;
 }
 
 void MultiMC::initGlobalSettings()
 {
-	m_settings = new INISettingsObject(applicationDirPath() + "/multimc.cfg", this);
+	m_settings = new INISettingsObject("multimc.cfg", this);
 		// Updates
 	m_settings->registerSetting(new Setting("UseDevBuilds", false));
 	m_settings->registerSetting(new Setting("AutoUpdate", true));
@@ -189,6 +196,16 @@ void MultiMC::initGlobalSettings()
 	m_settings->registerSetting(new Setting("TheCat", false));
 }
 
+void MultiMC::initHttpMetaCache()
+{
+	m_metacache = new HttpMetaCache("metacache");
+	m_metacache->addBase("assets", QDir("assets").absolutePath());
+	m_metacache->addBase("versions", QDir("versions").absolutePath());
+	m_metacache->addBase("libraries", QDir("libraries").absolutePath());
+	m_metacache->Load();
+}
+
+
 IconList* MultiMC::icons()
 {
 	if ( !m_icons )
diff --git a/MultiMC.h b/MultiMC.h
index 99d90b99..0c8b673b 100644
--- a/MultiMC.h
+++ b/MultiMC.h
@@ -4,6 +4,7 @@
 #include "MultiMCVersion.h"
 #include "config.h"
 
+class HttpMetaCache;
 class SettingsObject;
 class InstanceList;
 class IconList;
@@ -55,14 +56,21 @@ public:
 	{
 		return m_qnam;
 	}
+	
+	HttpMetaCache * metacache()
+	{
+		return m_metacache;
+	}
 private:
 	void initGlobalSettings();
 	
+	void initHttpMetaCache();
 private:
 	SettingsObject * m_settings = nullptr;
 	InstanceList * m_instances = nullptr;
 	IconList * m_icons = nullptr;
 	QNetworkAccessManager * m_qnam = nullptr;
+	HttpMetaCache * m_metacache = nullptr;
 	Status m_status = MultiMC::Failed;
 	MultiMCVersion m_version = {VERSION_MAJOR, VERSION_MINOR, VERSION_REVISION, VERSION_BUILD};
 };
\ No newline at end of file
diff --git a/logic/LegacyUpdate.cpp b/logic/LegacyUpdate.cpp
index 626ad1e0..c75a0011 100644
--- a/logic/LegacyUpdate.cpp
+++ b/logic/LegacyUpdate.cpp
@@ -227,7 +227,9 @@ void LegacyUpdate::jarStart()
 	QString intended_version_id = inst->intendedVersionId();
 	urlstr += intended_version_id + "/" + intended_version_id + ".jar";
 	
-	legacyDownloadJob.reset(new DownloadJob(QUrl(urlstr), inst->defaultBaseJar()));
+	auto dljob = new DownloadJob("Minecraft.jar for version " + intended_version_id);
+	dljob->add(QUrl(urlstr), inst->defaultBaseJar());
+	legacyDownloadJob.reset();
 	connect(legacyDownloadJob.data(), SIGNAL(finished()), SLOT(jarFinished()));
 	connect(legacyDownloadJob.data(), SIGNAL(failed()), SLOT(jarFailed()));
 	connect(legacyDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64)));
diff --git a/logic/OneSixAssets.cpp b/logic/OneSixAssets.cpp
index c65ee607..5bdd29d7 100644
--- a/logic/OneSixAssets.cpp
+++ b/logic/OneSixAssets.cpp
@@ -3,6 +3,8 @@
 #include <QtXml/QtXml>
 #include "OneSixAssets.h"
 #include "net/DownloadJob.h"
+#include "net/HttpMetaCache.h"
+#include "MultiMC.h"
 
 inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
 {
@@ -65,7 +67,7 @@ void OneSixAssets::fetchXMLFinished()
 	nuke_whitelist.clear();
 
 	auto firstJob = index_job->first();
-	QByteArray ba = firstJob->m_data;
+	QByteArray ba  = firstJob.dynamicCast<ByteArrayDownload>()->m_data;
 
 	QString xmlErrorMsg;
 	QDomDocument doc;
@@ -76,10 +78,12 @@ void OneSixAssets::fetchXMLFinished()
 	//QRegExp etag_match(".*([a-f0-9]{32}).*");
 	QDomNodeList contents = doc.elementsByTagName ( "Contents" );
 
-	DownloadJob *job = new DownloadJob();
+	DownloadJob *job = new DownloadJob("Assets");
 	connect ( job, SIGNAL(succeeded()), SLOT(downloadFinished()) );
 	connect ( job, SIGNAL(failed()), SIGNAL(failed()) );
 
+	auto metacache = MMC->metacache();
+	
 	for ( int i = 0; i < contents.length(); i++ )
 	{
 		QDomElement element = contents.at ( i ).toElement();
@@ -104,22 +108,12 @@ void OneSixAssets::fetchXMLFinished()
 		if ( sizeStr == "0" )
 			continue;
 
-		QString filename = fprefix + keyStr;
-		QFile check_file ( filename );
-		QString client_etag = "nonsense";
-		// if there already is a file and md5 checking is in effect and it can be opened
-		if ( check_file.exists() && check_file.open ( QIODevice::ReadOnly ) )
-		{
-			// check the md5 against the expected one
-			client_etag = QCryptographicHash::hash ( check_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
-			check_file.close();
-		}
-		
-		QString trimmedEtag = etagStr.remove ( '"' );
 		nuke_whitelist.append ( keyStr );
-		if(trimmedEtag != client_etag)
+		
+		auto entry = metacache->resolveEntry("assets", keyStr, etagStr);
+		if(entry->stale)
 		{
-			job->add ( QUrl ( prefix + keyStr ), filename );
+			job->add(QUrl(prefix + keyStr), entry);
 		}
 	}
 	if(job->size())
@@ -135,7 +129,8 @@ void OneSixAssets::fetchXMLFinished()
 }
 void OneSixAssets::start()
 {
-	DownloadJob * job = new DownloadJob(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" ));
+	auto job = new DownloadJob("Assets index");
+	job->add(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" ));
 	connect ( job, SIGNAL(succeeded()), SLOT ( fetchXMLFinished() ) );
 	index_job.reset ( job );
 	job->start();
diff --git a/logic/OneSixUpdate.cpp b/logic/OneSixUpdate.cpp
index 428d6ef7..ce71bde0 100644
--- a/logic/OneSixUpdate.cpp
+++ b/logic/OneSixUpdate.cpp
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
+#include "MultiMC.h"
 #include "OneSixUpdate.h"
 
 #include <QtNetwork>
@@ -72,7 +72,9 @@ void OneSixUpdate::versionFileStart()
 	
 	QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/");
 	urlstr += targetVersion->descriptor + "/" + targetVersion->descriptor + ".json";
-	specificVersionDownloadJob.reset(new DownloadJob(QUrl(urlstr)));
+	auto job = new DownloadJob("Version index");
+	job->add(QUrl(urlstr));
+	specificVersionDownloadJob.reset(job);
 	connect(specificVersionDownloadJob.data(), SIGNAL(succeeded()), SLOT(versionFileFinished()));
 	connect(specificVersionDownloadJob.data(), SIGNAL(failed()), SLOT(versionFileFailed()));
 	connect(specificVersionDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64)));
@@ -92,7 +94,7 @@ void OneSixUpdate::versionFileFinished()
 		// FIXME: detect errors here, download to a temp file, swap
 		QFile  vfile1 (version1);
 		vfile1.open(QIODevice::Truncate | QIODevice::WriteOnly );
-		vfile1.write(DlJob->m_data);
+		vfile1.write(DlJob.dynamicCast<ByteArrayDownload>()->m_data);
 		vfile1.close();
 	}
 	
@@ -134,16 +136,22 @@ void OneSixUpdate::jarlibStart()
 	QString targetstr ("versions/");
 	targetstr += version->id + "/" + version->id + ".jar";
 	
-	jarlibDownloadJob.reset(new DownloadJob(QUrl(urlstr), targetstr));
+	auto job = new DownloadJob("Libraries for instance " + inst->name());
+	job->add(QUrl(urlstr), targetstr);
+	jarlibDownloadJob.reset(job);
 	
 	auto libs = version->getActiveNativeLibs();
 	libs.append(version->getActiveNormalLibs());
 	
+	auto metacache = MMC->metacache();
 	for(auto lib: libs)
 	{
 		QString download_path = lib->downloadPath();
-		QString storage_path = "libraries/" + lib->storagePath();
-		jarlibDownloadJob->add(download_path, storage_path);
+		auto entry = metacache->resolveEntry("libraries", lib->storagePath());
+		if(entry->stale)
+		{
+			jarlibDownloadJob->add(download_path, entry);
+		}
 	}
 	connect(jarlibDownloadJob.data(), SIGNAL(succeeded()), SLOT(jarlibFinished()));
 	connect(jarlibDownloadJob.data(), SIGNAL(failed()), SLOT(jarlibFailed()));
diff --git a/logic/net/ByteArrayDownload.cpp b/logic/net/ByteArrayDownload.cpp
new file mode 100644
index 00000000..6ae3f121
--- /dev/null
+++ b/logic/net/ByteArrayDownload.cpp
@@ -0,0 +1,62 @@
+#include "ByteArrayDownload.h"
+#include "MultiMC.h"
+#include <QDebug>
+
+ByteArrayDownload::ByteArrayDownload ( QUrl url )
+	: Download()
+{
+	m_url = url;
+	m_status = Job_NotStarted;
+}
+
+void ByteArrayDownload::start()
+{
+	qDebug() << "Downloading " << m_url.toString();
+	QNetworkRequest request ( m_url );
+	auto worker = MMC->qnam();
+	QNetworkReply * rep = worker->get ( request );
+	
+	m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
+	connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
+	connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
+	connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
+	connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
+}
+
+void ByteArrayDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
+{
+	emit progress (index_within_job, bytesReceived, bytesTotal );
+}
+
+void ByteArrayDownload::downloadError ( QNetworkReply::NetworkError error )
+{
+	// error happened during download.
+	// TODO: log the reason why
+	m_status = Job_Failed;
+}
+
+void ByteArrayDownload::downloadFinished()
+{
+	// if the download succeeded
+	if ( m_status != Job_Failed )
+	{
+		// nothing went wrong...
+		m_status = Job_Finished;
+		m_data = m_reply->readAll();
+		m_reply.clear();
+		emit succeeded(index_within_job);
+		return;
+	}
+	// else the download failed
+	else
+	{
+		m_reply.clear();
+		emit failed(index_within_job);
+		return;
+	}
+}
+
+void ByteArrayDownload::downloadReadyRead()
+{
+	// ~_~
+}
diff --git a/logic/net/ByteArrayDownload.h b/logic/net/ByteArrayDownload.h
new file mode 100644
index 00000000..428b21db
--- /dev/null
+++ b/logic/net/ByteArrayDownload.h
@@ -0,0 +1,24 @@
+#pragma once
+#include "Download.h"
+
+class ByteArrayDownload: public Download
+{
+	Q_OBJECT
+public:
+	ByteArrayDownload(QUrl url);
+	
+public:
+	/// if not saving to file, downloaded data is placed here
+	QByteArray m_data;
+	
+public slots:
+	virtual void start();
+	
+protected slots:
+	void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
+	void downloadError(QNetworkReply::NetworkError error);
+	void downloadFinished();
+	void downloadReadyRead();
+};
+
+typedef QSharedPointer<ByteArrayDownload> ByteArrayDownloadPtr;
diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp
new file mode 100644
index 00000000..c0074574
--- /dev/null
+++ b/logic/net/CacheDownload.cpp
@@ -0,0 +1,122 @@
+#include "MultiMC.h"
+#include "CacheDownload.h"
+#include <pathutils.h>
+
+#include <QCryptographicHash>
+#include <QFileInfo>
+#include <QDateTime>
+#include <QDebug>
+
+CacheDownload::CacheDownload (QUrl url, MetaEntryPtr entry )
+	:Download(), md5sum(QCryptographicHash::Md5)
+{
+	m_url = url;
+	m_entry = entry;
+	m_target_path = entry->getFullPath();
+	m_status = Job_NotStarted;
+	m_opened_for_saving = false;
+}
+
+void CacheDownload::start()
+{
+	if(!m_entry->stale)
+	{
+		emit succeeded(index_within_job);
+		return;
+	}
+	m_output_file.setFileName(m_target_path);
+	// if there already is a file and md5 checking is in effect and it can be opened
+	if(!ensureFilePathExists(m_target_path))
+	{
+		emit failed(index_within_job);
+		return;
+	}
+	qDebug() << "Downloading " << m_url.toString();
+	QNetworkRequest request ( m_url );
+	request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); 
+	
+	auto worker = MMC->qnam();
+	QNetworkReply * rep = worker->get ( request );
+	
+	m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
+	connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
+	connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
+	connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
+	connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
+}
+
+void CacheDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
+{
+	emit progress (index_within_job, bytesReceived, bytesTotal );
+}
+
+void CacheDownload::downloadError ( QNetworkReply::NetworkError error )
+{
+	// error happened during download.
+	// TODO: log the reason why
+	m_status = Job_Failed;
+}
+void CacheDownload::downloadFinished()
+{
+	// if the download succeeded
+	if ( m_status != Job_Failed )
+	{
+		
+		// nothing went wrong...
+		m_status = Job_Finished;
+		if(m_opened_for_saving)
+		{
+			// save the data to the downloadable if we aren't saving to file
+			m_output_file.close();
+			m_entry->md5sum = md5sum.result().toHex().constData();
+		}
+		else
+		{
+			if ( m_output_file.open ( QIODevice::ReadOnly ) )
+			{
+				m_entry->md5sum = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
+				m_output_file.close();
+			}
+		}
+		QFileInfo output_file_info(m_target_path);
+		
+		
+		m_entry->etag = m_reply->rawHeader("ETag").constData();
+		m_entry->last_changed_timestamp = output_file_info.lastModified().toUTC().toMSecsSinceEpoch();
+		m_entry->stale = false;
+		MMC->metacache()->updateEntry(m_entry);
+		
+		m_reply.clear();
+		emit succeeded(index_within_job);
+		return;
+	}
+	// else the download failed
+	else
+	{
+		m_output_file.close();
+		m_output_file.remove();
+		m_reply.clear();
+		emit failed(index_within_job);
+		return;
+	}
+}
+
+void CacheDownload::downloadReadyRead()
+{
+	if(!m_opened_for_saving)
+	{
+		if ( !m_output_file.open ( QIODevice::WriteOnly ) )
+		{
+			/*
+			* Can't open the file... the job failed
+			*/
+			m_reply->abort();
+			emit failed(index_within_job);
+			return;
+		}
+		m_opened_for_saving = true;
+	}
+	QByteArray ba = m_reply->readAll();
+	md5sum.addData(ba);
+	m_output_file.write ( ba );
+}
diff --git a/logic/net/CacheDownload.h b/logic/net/CacheDownload.h
new file mode 100644
index 00000000..f95dabd5
--- /dev/null
+++ b/logic/net/CacheDownload.h
@@ -0,0 +1,34 @@
+#pragma once
+
+#include "Download.h"
+#include "HttpMetaCache.h"
+#include <QFile>
+#include <qcryptographichash.h>
+
+class CacheDownload : public Download
+{
+	Q_OBJECT
+public:
+	MetaEntryPtr m_entry;
+	/// is the saving file already open?
+	bool m_opened_for_saving;
+	/// if saving to file, use the one specified in this string
+	QString m_target_path;
+	/// this is the output file, if any
+	QFile m_output_file;
+	/// the hash-as-you-download
+	QCryptographicHash md5sum;
+public:
+	explicit CacheDownload(QUrl url, MetaEntryPtr entry);
+	
+protected slots:
+	virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
+	virtual void downloadError(QNetworkReply::NetworkError error);
+	virtual void downloadFinished();
+	virtual void downloadReadyRead();
+	
+public slots:
+	virtual void start();
+};
+
+typedef QSharedPointer<CacheDownload> CacheDownloadPtr;
diff --git a/logic/net/Download.h b/logic/net/Download.h
new file mode 100644
index 00000000..91f09dec
--- /dev/null
+++ b/logic/net/Download.h
@@ -0,0 +1,54 @@
+#pragma once
+
+#include <QObject>
+#include <QUrl>
+#include <QSharedPointer>
+#include <QNetworkReply>
+
+
+enum JobStatus
+{
+	Job_NotStarted,
+	Job_InProgress,
+	Job_Finished,
+	Job_Failed
+};
+
+class Download : public QObject
+{
+	Q_OBJECT
+protected:
+	explicit Download(): QObject(0){};
+public:
+	virtual ~Download() {};
+
+public:
+	/// the network reply
+	QSharedPointer<QNetworkReply> m_reply;
+	
+	/// source URL
+	QUrl m_url;
+	
+	/// The file's status
+	JobStatus m_status;
+	
+	/// index within the parent job
+	int index_within_job = 0;
+
+signals:
+	void started(int index);
+	void progress(int index, qint64 current, qint64 total);
+	void succeeded(int index);
+	void failed(int index);
+
+protected slots:
+	virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0;
+	virtual void downloadError(QNetworkReply::NetworkError error) = 0;
+	virtual void downloadFinished() = 0;
+	virtual void downloadReadyRead() = 0;
+	
+public slots:
+	virtual void start() = 0;
+};
+
+typedef QSharedPointer<Download> DownloadPtr;
diff --git a/logic/net/DownloadJob.cpp b/logic/net/DownloadJob.cpp
index cad9ae72..cbc2c5fe 100644
--- a/logic/net/DownloadJob.cpp
+++ b/logic/net/DownloadJob.cpp
@@ -1,136 +1,29 @@
 #include "DownloadJob.h"
 #include "pathutils.h"
 #include "MultiMC.h"
+#include "FileDownload.h"
+#include "ByteArrayDownload.h"
+#include "CacheDownload.h"
 
-Download::Download (QUrl url, QString target_path, QString expected_md5 )
-	:Job()
+ByteArrayDownloadPtr DownloadJob::add ( QUrl url )
 {
-	m_url = url;
-	m_target_path = target_path;
-	m_expected_md5 = expected_md5;
-
-	m_check_md5 = m_expected_md5.size();
-	m_save_to_file = m_target_path.size();
-	m_status = Job_NotStarted;
-	m_opened_for_saving = false;
-}
-
-void Download::start()
-{
-	if ( m_save_to_file )
-	{
-		QString filename = m_target_path;
-		m_output_file.setFileName ( filename );
-		// if there already is a file and md5 checking is in effect and it can be opened
-		if ( m_output_file.exists() && m_output_file.open ( QIODevice::ReadOnly ) )
-		{
-			// check the md5 against the expected one
-			QString hash = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
-			m_output_file.close();
-			// skip this file if they match
-			if ( m_check_md5 && hash == m_expected_md5 )
-			{
-				qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
-				emit succeeded(index_within_job);
-				return;
-			}
-			else
-			{
-				m_expected_md5 = hash;
-			}
-		}
-		if(!ensureFilePathExists(filename))
-		{
-			emit failed(index_within_job);
-			return;
-		}
-	}
-	qDebug() << "Downloading " << m_url.toString();
-	QNetworkRequest request ( m_url );
-	request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1()); 
-	
-	auto worker = MMC->qnam();
-	QNetworkReply * rep = worker->get ( request );
-	
-	m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
-	connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
-	connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
-	connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
-	connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
-}
-
-void Download::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
-{
-	emit progress (index_within_job, bytesReceived, bytesTotal );
-}
-
-void Download::downloadError ( QNetworkReply::NetworkError error )
-{
-	// error happened during download.
-	// TODO: log the reason why
-	m_status = Job_Failed;
-}
-
-void Download::downloadFinished()
-{
-	// if the download succeeded
-	if ( m_status != Job_Failed )
-	{
-		// nothing went wrong...
-		m_status = Job_Finished;
-		// save the data to the downloadable if we aren't saving to file
-		if ( !m_save_to_file )
-		{
-			m_data = m_reply->readAll();
-		}
-		else
-		{
-			m_output_file.close();
-		}
-
-		//TODO: check md5 here!
-		m_reply.clear();
-		emit succeeded(index_within_job);
-		return;
-	}
-	// else the download failed
-	else
-	{
-		if ( m_save_to_file )
-		{
-			m_output_file.close();
-			m_output_file.remove();
-		}
-		m_reply.clear();
-		emit failed(index_within_job);
-		return;
-	}
+	ByteArrayDownloadPtr ptr (new ByteArrayDownload(url));
+	ptr->index_within_job = downloads.size();
+	downloads.append(ptr);
+	return ptr;
 }
 
-void Download::downloadReadyRead()
+FileDownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path)
 {
-	if( m_save_to_file )
-	{
-		if(!m_opened_for_saving)
-		{
-			if ( !m_output_file.open ( QIODevice::WriteOnly ) )
-			{
-				/*
-				* Can't open the file... the job failed
-				*/
-				m_reply->abort();
-				emit failed(index_within_job);
-				return;
-			}
-			m_opened_for_saving = true;
-		}
-		m_output_file.write ( m_reply->readAll() );
-	}
+	FileDownloadPtr ptr (new FileDownload(url, rel_target_path));
+	ptr->index_within_job = downloads.size();
+	downloads.append(ptr);
+	return ptr;
 }
 
-DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expected_md5 )
+CacheDownloadPtr DownloadJob::add ( QUrl url, MetaEntryPtr entry)
 {
-	DownloadPtr ptr (new Download(url, rel_target_path, expected_md5));
+	CacheDownloadPtr ptr (new CacheDownload(url, entry));
 	ptr->index_within_job = downloads.size();
 	downloads.append(ptr);
 	return ptr;
@@ -139,16 +32,17 @@ DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expect
 void DownloadJob::partSucceeded ( int index )
 {
 	num_succeeded++;
+	qDebug() << m_job_name.toLocal8Bit() << " progress: " << num_succeeded << "/" << downloads.size();
 	if(num_failed + num_succeeded == downloads.size())
 	{
 		if(num_failed)
 		{
-			qDebug() << "Download JOB failed: " << this;
+			qDebug() << m_job_name.toLocal8Bit() << " failed.";
 			emit failed();
 		}
 		else
 		{
-			qDebug() << "Download JOB succeeded: " << this;
+			qDebug() << m_job_name.toLocal8Bit() << " succeeded.";
 			emit succeeded();
 		}
 	}
@@ -159,7 +53,7 @@ void DownloadJob::partFailed ( int index )
 	num_failed++;
 	if(num_failed + num_succeeded == downloads.size())
 	{
-		qDebug() << "Download JOB failed: " << this;
+		qDebug() << m_job_name.toLocal8Bit() << " failed.";
 		emit failed();
 	}
 }
@@ -172,7 +66,7 @@ void DownloadJob::partProgress ( int index, qint64 bytesReceived, qint64 bytesTo
 
 void DownloadJob::start()
 {
-	qDebug() << "Download JOB started: " << this;
+	qDebug() << m_job_name.toLocal8Bit() << " started.";
 	for(auto iter: downloads)
 	{
 		connect(iter.data(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));
diff --git a/logic/net/DownloadJob.h b/logic/net/DownloadJob.h
index adeef646..71466282 100644
--- a/logic/net/DownloadJob.h
+++ b/logic/net/DownloadJob.h
@@ -1,98 +1,28 @@
 #pragma once
 #include <QtNetwork>
+#include "Download.h"
+#include "ByteArrayDownload.h"
+#include "FileDownload.h"
+#include "CacheDownload.h"
+#include "HttpMetaCache.h"
 
 class DownloadJob;
-class Download;
 typedef QSharedPointer<DownloadJob> DownloadJobPtr;
-typedef QSharedPointer<Download> DownloadPtr;
-
-enum JobStatus
-{
-	Job_NotStarted,
-	Job_InProgress,
-	Job_Finished,
-	Job_Failed
-};
-
-class Job : public QObject
-{
-	Q_OBJECT
-protected:
-	explicit Job(): QObject(0){};
-public:
-	virtual ~Job() {};
-
-public slots:
-	virtual void start() = 0;
-};
-
-class Download: public Job
-{
-	friend class DownloadJob;
-	Q_OBJECT
-protected:
-	Download(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
-public:
-	/// the network reply
-	QSharedPointer<QNetworkReply> m_reply;
-	/// source URL
-	QUrl m_url;
-	
-	/// if true, check the md5sum against a provided md5sum
-	/// also, if a file exists, perform an md5sum first and don't download only if they don't match
-	bool m_check_md5;
-	/// the expected md5 checksum
-	QString m_expected_md5;
-	
-	/// save to file?
-	bool m_save_to_file;
-	/// is the saving file already open?
-	bool m_opened_for_saving;
-	/// if saving to file, use the one specified in this string
-	QString m_target_path;
-	/// this is the output file, if any
-	QFile m_output_file;
-	/// if not saving to file, downloaded data is placed here
-	QByteArray m_data;
-	
-	int currentProgress = 0;
-	int totalProgress = 0;
-	
-	/// The file's status
-	JobStatus m_status;
-	
-	int index_within_job = 0;
-signals:
-	void started(int index);
-	void progress(int index, qint64 current, qint64 total);
-	void succeeded(int index);
-	void failed(int index);
-public slots:
-	virtual void start();
-	
-private slots:
-	void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);;
-	void downloadError(QNetworkReply::NetworkError error);
-	void downloadFinished();
-	void downloadReadyRead();
-};
 
 /**
  * A single file for the downloader/cache to process.
  */
-class DownloadJob : public Job
+class DownloadJob : public QObject
 {
 	Q_OBJECT
 public:
-	explicit DownloadJob()
-		:Job(){};
-	explicit DownloadJob(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString())
-		:Job()
-	{
-		add(url, rel_target_path, expected_md5);
-	};
+	explicit DownloadJob(QString job_name)
+		:QObject(), m_job_name(job_name){};
+	
+	ByteArrayDownloadPtr add(QUrl url);
+	FileDownloadPtr      add(QUrl url, QString rel_target_path);
+	CacheDownloadPtr     add(QUrl url, MetaEntryPtr entry);
 	
-	DownloadPtr add(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
 	DownloadPtr operator[](int index)
 	{
 		return downloads[index];
@@ -119,6 +49,7 @@ private slots:
 	void partSucceeded(int index);
 	void partFailed(int index);
 private:
+	QString m_job_name;
 	QList<DownloadPtr> downloads;
 	int num_succeeded = 0;
 	int num_failed = 0;
diff --git a/logic/net/FileDownload.cpp b/logic/net/FileDownload.cpp
new file mode 100644
index 00000000..aad4a991
--- /dev/null
+++ b/logic/net/FileDownload.cpp
@@ -0,0 +1,111 @@
+#include "MultiMC.h"
+#include "FileDownload.h"
+#include <pathutils.h>
+#include <QCryptographicHash>
+#include <QDebug>
+
+
+FileDownload::FileDownload ( QUrl url, QString target_path )
+	:Download()
+{
+	m_url = url;
+	m_target_path = target_path;
+	m_check_md5 = false;
+	m_status = Job_NotStarted;
+	m_opened_for_saving = false;
+}
+
+void FileDownload::start()
+{
+	QString filename = m_target_path;
+	m_output_file.setFileName ( filename );
+	// if there already is a file and md5 checking is in effect and it can be opened
+	if ( m_output_file.exists() && m_output_file.open ( QIODevice::ReadOnly ) )
+	{
+		// check the md5 against the expected one
+		QString hash = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
+		m_output_file.close();
+		// skip this file if they match
+		if ( m_check_md5 && hash == m_expected_md5 )
+		{
+			qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
+			emit succeeded(index_within_job);
+			return;
+		}
+		else
+		{
+			m_expected_md5 = hash;
+		}
+	}
+	if(!ensureFilePathExists(filename))
+	{
+		emit failed(index_within_job);
+		return;
+	}
+	
+	qDebug() << "Downloading " << m_url.toString();
+	QNetworkRequest request ( m_url );
+	request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1()); 
+	
+	auto worker = MMC->qnam();
+	QNetworkReply * rep = worker->get ( request );
+	
+	m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
+	connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
+	connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
+	connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
+	connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
+}
+
+void FileDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
+{
+	emit progress (index_within_job, bytesReceived, bytesTotal );
+}
+
+void FileDownload::downloadError ( QNetworkReply::NetworkError error )
+{
+	// error happened during download.
+	// TODO: log the reason why
+	m_status = Job_Failed;
+}
+
+void FileDownload::downloadFinished()
+{
+	// if the download succeeded
+	if ( m_status != Job_Failed )
+	{
+		// nothing went wrong...
+		m_status = Job_Finished;
+		m_output_file.close();
+
+		m_reply.clear();
+		emit succeeded(index_within_job);
+		return;
+	}
+	// else the download failed
+	else
+	{
+		m_output_file.close();
+		m_reply.clear();
+		emit failed(index_within_job);
+		return;
+	}
+}
+
+void FileDownload::downloadReadyRead()
+{
+	if(!m_opened_for_saving)
+	{
+		if ( !m_output_file.open ( QIODevice::WriteOnly ) )
+		{
+			/*
+			* Can't open the file... the job failed
+			*/
+			m_reply->abort();
+			emit failed(index_within_job);
+			return;
+		}
+		m_opened_for_saving = true;
+	}
+	m_output_file.write ( m_reply->readAll() );
+}
diff --git a/logic/net/FileDownload.h b/logic/net/FileDownload.h
new file mode 100644
index 00000000..190bee58
--- /dev/null
+++ b/logic/net/FileDownload.h
@@ -0,0 +1,35 @@
+#pragma once
+
+#include "Download.h"
+#include <QFile>
+
+class FileDownload : public Download
+{
+	Q_OBJECT
+public:
+	/// if true, check the md5sum against a provided md5sum
+	/// also, if a file exists, perform an md5sum first and don't download only if they don't match
+	bool m_check_md5;
+	/// the expected md5 checksum
+	QString m_expected_md5;
+	/// is the saving file already open?
+	bool m_opened_for_saving;
+	/// if saving to file, use the one specified in this string
+	QString m_target_path;
+	/// this is the output file, if any
+	QFile m_output_file;
+	
+public:
+	explicit FileDownload(QUrl url, QString target_path);
+	
+protected slots:
+	virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
+	virtual void downloadError(QNetworkReply::NetworkError error);
+	virtual void downloadFinished();
+	virtual void downloadReadyRead();
+	
+public slots:
+	virtual void start();
+};
+
+typedef QSharedPointer<FileDownload> FileDownloadPtr;
diff --git a/logic/net/HttpMetaCache.cpp b/logic/net/HttpMetaCache.cpp
index 87741dc9..50a1136e 100644
--- a/logic/net/HttpMetaCache.cpp
+++ b/logic/net/HttpMetaCache.cpp
@@ -1,38 +1,136 @@
+#include "MultiMC.h"
 #include "HttpMetaCache.h"
 #include <pathutils.h>
+
+#include <QFileInfo>
 #include <QFile>
-#include <qjsondocument.h>
-#include <qjsonarray.h>
-#include <qjsonobject.h>
-#include <qfileinfo.h>
-#include <qtemporaryfile.h>
-#include <qsavefile.h>
+#include <QTemporaryFile>
+#include <QSaveFile>
+#include <QDateTime>
+#include <QCryptographicHash>
+
+#include <QDebug>
+
+#include <QJsonDocument>
+#include <QJsonArray>
+#include <QJsonObject>
+
+QString MetaEntry::getFullPath()
+{
+	return PathCombine(MMC->metacache()->getBasePath(base), path);
+}
+
 
 HttpMetaCache::HttpMetaCache(QString path)
+	:QObject()
 {
 	m_index_file = path;
+	saveBatchingTimer.setSingleShot(true);
+	saveBatchingTimer.setTimerType(Qt::VeryCoarseTimer);
+	connect(&saveBatchingTimer,SIGNAL(timeout()),SLOT(SaveNow()));
 }
+
 HttpMetaCache::~HttpMetaCache()
 {
-	Save();
+	saveBatchingTimer.stop();
+	SaveNow();
 }
 
-void HttpMetaCache::addEntry ( QString base, QString resource_path, QString etag )
+void HttpMetaCache::SaveEventually()
+{
+	saveBatchingTimer.stop();
+	saveBatchingTimer.start(30000);
+}
+
+MetaEntryPtr HttpMetaCache::getEntry ( QString base, QString resource_path )
 {
 	// no base. no base path. can't store
 	if(!m_entries.contains(base))
-		return;
-	QString real_path = PathCombine(m_entries[base].base_path, resource_path);
+	{
+		// TODO: log problem
+		return MetaEntryPtr();
+	}
+	EntryMap & map = m_entries[base];
+	if(map.entry_list.contains(resource_path))
+	{
+		return map.entry_list[resource_path];
+	}
+	return MetaEntryPtr();
+}
+
+MetaEntryPtr HttpMetaCache::resolveEntry ( QString base, QString resource_path, QString expected_etag )
+{
+	auto entry = getEntry(base, resource_path);
+	// it's not present? generate a default stale entry
+	if(!entry)
+	{
+		return staleEntry(base, resource_path);
+	}
+	
+	auto & selected_base = m_entries[base];
+	QString real_path = PathCombine(selected_base.base_path, resource_path);
 	QFileInfo finfo(real_path);
 	
-	// just ignore it, it's garbage if it's not a proper file
+	// is the file really there? if not -> stale
 	if(!finfo.isFile() || !finfo.isReadable())
 	{
-		// TODO: log problem
-		return;
+		// if the file doesn't exist, we disown the entry
+		selected_base.entry_list.remove(resource_path);
+		return staleEntry(base, resource_path);
+	}
+	
+	if(!expected_etag.isEmpty() && expected_etag != entry->etag)
+	{
+		// if the etag doesn't match expected, we disown the entry
+		selected_base.entry_list.remove(resource_path);
+		return staleEntry(base, resource_path);
 	}
 	
-	Save();
+	// if the file changed, check md5sum
+	qint64 file_last_changed = finfo.lastModified().toUTC().toMSecsSinceEpoch();
+	if(file_last_changed != entry->last_changed_timestamp)
+	{
+		QFile input(real_path);
+		input.open(QIODevice::ReadOnly);
+		QString md5sum = QCryptographicHash::hash(input.readAll(), QCryptographicHash::Md5).toHex().constData();
+		if(entry->md5sum != md5sum)
+		{
+			selected_base.entry_list.remove(resource_path);
+			return staleEntry(base, resource_path);
+		}
+		// md5sums matched... keep entry and save the new state to file
+		entry->last_changed_timestamp = file_last_changed;
+		SaveEventually();
+	}
+
+	// entry passed all the checks we cared about.
+	return entry;
+}
+
+bool HttpMetaCache::updateEntry ( MetaEntryPtr stale_entry )
+{
+	if(!m_entries.contains(stale_entry->base))
+	{
+		qDebug() << "Cannot add entry with unknown base: " << stale_entry->base.toLocal8Bit();
+		return false;
+	}
+	if(stale_entry->stale)
+	{
+		qDebug() << "Cannot add stale entry: " << stale_entry->getFullPath().toLocal8Bit();
+		return false;
+	}
+	m_entries[stale_entry->base].entry_list[stale_entry->path] = stale_entry;
+	SaveEventually();
+	return true;
+}
+
+MetaEntryPtr HttpMetaCache::staleEntry(QString base, QString resource_path)
+{
+	auto foo = new MetaEntry;
+	foo->base = base;
+	foo->path = resource_path;
+	foo->stale = true;
+	return MetaEntryPtr(foo);
 }
 
 void HttpMetaCache::addBase ( QString base, QString base_root )
@@ -46,6 +144,16 @@ void HttpMetaCache::addBase ( QString base, QString base_root )
 	m_entries[base] = foo;
 }
 
+QString HttpMetaCache::getBasePath ( QString base )
+{
+	if(m_entries.contains(base))
+	{
+		return m_entries[base].base_path;
+	}
+	return QString();
+}
+
+
 void HttpMetaCache::Load()
 {
 	QFile index(m_index_file);
@@ -65,12 +173,12 @@ void HttpMetaCache::Load()
 	
 	// read the entry array
 	auto entries_val =root.value("entries");
-	if(!version_val.isArray())
+	if(!entries_val.isArray())
 		return;
-	QJsonArray array = json.array();
+	QJsonArray array = entries_val.toArray();
 	for(auto element: array)
 	{
-		if(!element.isObject());
+		if(!element.isObject())
 			return;
 		auto element_obj = element.toObject();
 		QString base = element_obj.value("base").toString();
@@ -83,11 +191,13 @@ void HttpMetaCache::Load()
 		foo->md5sum = element_obj.value("md5sum").toString();
 		foo->etag = element_obj.value("etag").toString();
 		foo->last_changed_timestamp = element_obj.value("last_changed_timestamp").toDouble();
+		// presumed innocent until closer examination
+		foo->stale = false;
 		entrymap.entry_list[path] = MetaEntryPtr( foo );
 	}
 }
 
-void HttpMetaCache::Save()
+void HttpMetaCache::SaveNow()
 {
 	QSaveFile tfile(m_index_file);
 	if(!tfile.open(QIODevice::WriteOnly | QIODevice::Truncate))
@@ -118,14 +228,3 @@ void HttpMetaCache::Save()
 		return;
 	tfile.commit();
 }
-
-
-MetaEntryPtr HttpMetaCache::getEntryForResource ( QString base, QString resource_path )
-{
-	if(!m_entries.contains(base))
-		return MetaEntryPtr();
-	auto & entrymap = m_entries[base];
-	if(!entrymap.entry_list.contains(resource_path))
-		return MetaEntryPtr();
-	return entrymap.entry_list[resource_path];
-}
diff --git a/logic/net/HttpMetaCache.h b/logic/net/HttpMetaCache.h
index 161483ad..fac6bec3 100644
--- a/logic/net/HttpMetaCache.h
+++ b/logic/net/HttpMetaCache.h
@@ -2,6 +2,7 @@
 #include <QString>
 #include <QSharedPointer>
 #include <QMap>
+#include <qtimer.h>
 
 struct MetaEntry
 {
@@ -9,23 +10,42 @@ struct MetaEntry
 	QString path;
 	QString md5sum;
 	QString etag;
-	quint64 last_changed_timestamp = 0;
+	qint64 last_changed_timestamp = 0;
+	bool stale = true;
+	QString getFullPath();
 };
 
 typedef QSharedPointer<MetaEntry> MetaEntryPtr;
 
-class HttpMetaCache
+class HttpMetaCache : public QObject
 {
+	Q_OBJECT
 public:
 	// supply path to the cache index file
 	HttpMetaCache(QString path);
 	~HttpMetaCache();
-	MetaEntryPtr getEntryForResource(QString base, QString resource_path);
-	void addEntry(QString base, QString resource_path, QString etag);
+	
+	// get the entry solely from the cache
+	// you probably don't want this, unless you have some specific caching needs.
+	MetaEntryPtr getEntry(QString base, QString resource_path);
+	
+	// get the entry from cache and verify that it isn't stale (within reason)
+	MetaEntryPtr resolveEntry(QString base, QString resource_path, QString expected_etag = QString());
+	
+	// add a previously resolved stale entry
+	bool updateEntry(MetaEntryPtr stale_entry);
+	
 	void addBase(QString base, QString base_root);
-private:
-	void Save();
+	
+	// (re)start a timer that calls SaveNow later.
+	void SaveEventually();
 	void Load();
+	QString getBasePath ( QString base );
+public slots:
+	void SaveNow();
+private:
+	// create a new stale entry, given the parameters
+	MetaEntryPtr staleEntry(QString base, QString resource_path);
 	struct EntryMap
 	{
 		QString base_path;
@@ -33,4 +53,5 @@ private:
 	};
 	QMap<QString, EntryMap> m_entries;
 	QString m_index_file;
+	QTimer saveBatchingTimer;
 };
\ No newline at end of file