From 6bea4ec988b7caeac63353fb9d2a354f2fd47dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sun, 8 Sep 2013 02:15:20 +0200 Subject: Use HttpMetaCache to minimize network use. --- logic/net/HttpMetaCache.cpp | 157 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 128 insertions(+), 29 deletions(-) (limited to 'logic/net/HttpMetaCache.cpp') diff --git a/logic/net/HttpMetaCache.cpp b/logic/net/HttpMetaCache.cpp index 87741dc9..50a1136e 100644 --- a/logic/net/HttpMetaCache.cpp +++ b/logic/net/HttpMetaCache.cpp @@ -1,38 +1,136 @@ +#include "MultiMC.h" #include "HttpMetaCache.h" #include + +#include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +QString MetaEntry::getFullPath() +{ + return PathCombine(MMC->metacache()->getBasePath(base), path); +} + HttpMetaCache::HttpMetaCache(QString path) + :QObject() { m_index_file = path; + saveBatchingTimer.setSingleShot(true); + saveBatchingTimer.setTimerType(Qt::VeryCoarseTimer); + connect(&saveBatchingTimer,SIGNAL(timeout()),SLOT(SaveNow())); } + HttpMetaCache::~HttpMetaCache() { - Save(); + saveBatchingTimer.stop(); + SaveNow(); } -void HttpMetaCache::addEntry ( QString base, QString resource_path, QString etag ) +void HttpMetaCache::SaveEventually() +{ + saveBatchingTimer.stop(); + saveBatchingTimer.start(30000); +} + +MetaEntryPtr HttpMetaCache::getEntry ( QString base, QString resource_path ) { // no base. no base path. can't store if(!m_entries.contains(base)) - return; - QString real_path = PathCombine(m_entries[base].base_path, resource_path); + { + // TODO: log problem + return MetaEntryPtr(); + } + EntryMap & map = m_entries[base]; + if(map.entry_list.contains(resource_path)) + { + return map.entry_list[resource_path]; + } + return MetaEntryPtr(); +} + +MetaEntryPtr HttpMetaCache::resolveEntry ( QString base, QString resource_path, QString expected_etag ) +{ + auto entry = getEntry(base, resource_path); + // it's not present? generate a default stale entry + if(!entry) + { + return staleEntry(base, resource_path); + } + + auto & selected_base = m_entries[base]; + QString real_path = PathCombine(selected_base.base_path, resource_path); QFileInfo finfo(real_path); - // just ignore it, it's garbage if it's not a proper file + // is the file really there? if not -> stale if(!finfo.isFile() || !finfo.isReadable()) { - // TODO: log problem - return; + // if the file doesn't exist, we disown the entry + selected_base.entry_list.remove(resource_path); + return staleEntry(base, resource_path); + } + + if(!expected_etag.isEmpty() && expected_etag != entry->etag) + { + // if the etag doesn't match expected, we disown the entry + selected_base.entry_list.remove(resource_path); + return staleEntry(base, resource_path); } - Save(); + // if the file changed, check md5sum + qint64 file_last_changed = finfo.lastModified().toUTC().toMSecsSinceEpoch(); + if(file_last_changed != entry->last_changed_timestamp) + { + QFile input(real_path); + input.open(QIODevice::ReadOnly); + QString md5sum = QCryptographicHash::hash(input.readAll(), QCryptographicHash::Md5).toHex().constData(); + if(entry->md5sum != md5sum) + { + selected_base.entry_list.remove(resource_path); + return staleEntry(base, resource_path); + } + // md5sums matched... keep entry and save the new state to file + entry->last_changed_timestamp = file_last_changed; + SaveEventually(); + } + + // entry passed all the checks we cared about. + return entry; +} + +bool HttpMetaCache::updateEntry ( MetaEntryPtr stale_entry ) +{ + if(!m_entries.contains(stale_entry->base)) + { + qDebug() << "Cannot add entry with unknown base: " << stale_entry->base.toLocal8Bit(); + return false; + } + if(stale_entry->stale) + { + qDebug() << "Cannot add stale entry: " << stale_entry->getFullPath().toLocal8Bit(); + return false; + } + m_entries[stale_entry->base].entry_list[stale_entry->path] = stale_entry; + SaveEventually(); + return true; +} + +MetaEntryPtr HttpMetaCache::staleEntry(QString base, QString resource_path) +{ + auto foo = new MetaEntry; + foo->base = base; + foo->path = resource_path; + foo->stale = true; + return MetaEntryPtr(foo); } void HttpMetaCache::addBase ( QString base, QString base_root ) @@ -46,6 +144,16 @@ void HttpMetaCache::addBase ( QString base, QString base_root ) m_entries[base] = foo; } +QString HttpMetaCache::getBasePath ( QString base ) +{ + if(m_entries.contains(base)) + { + return m_entries[base].base_path; + } + return QString(); +} + + void HttpMetaCache::Load() { QFile index(m_index_file); @@ -65,12 +173,12 @@ void HttpMetaCache::Load() // read the entry array auto entries_val =root.value("entries"); - if(!version_val.isArray()) + if(!entries_val.isArray()) return; - QJsonArray array = json.array(); + QJsonArray array = entries_val.toArray(); for(auto element: array) { - if(!element.isObject()); + if(!element.isObject()) return; auto element_obj = element.toObject(); QString base = element_obj.value("base").toString(); @@ -83,11 +191,13 @@ void HttpMetaCache::Load() foo->md5sum = element_obj.value("md5sum").toString(); foo->etag = element_obj.value("etag").toString(); foo->last_changed_timestamp = element_obj.value("last_changed_timestamp").toDouble(); + // presumed innocent until closer examination + foo->stale = false; entrymap.entry_list[path] = MetaEntryPtr( foo ); } } -void HttpMetaCache::Save() +void HttpMetaCache::SaveNow() { QSaveFile tfile(m_index_file); if(!tfile.open(QIODevice::WriteOnly | QIODevice::Truncate)) @@ -118,14 +228,3 @@ void HttpMetaCache::Save() return; tfile.commit(); } - - -MetaEntryPtr HttpMetaCache::getEntryForResource ( QString base, QString resource_path ) -{ - if(!m_entries.contains(base)) - return MetaEntryPtr(); - auto & entrymap = m_entries[base]; - if(!entrymap.entry_list.contains(resource_path)) - return MetaEntryPtr(); - return entrymap.entry_list[resource_path]; -} -- cgit v1.2.3