From e49b81869832a83ae02580909608348c43895e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sun, 30 Jun 2013 22:39:57 +0200 Subject: Add queued downloader, some super-minor UI tweaks in stuff that's not even visible yet. --- asset_test.cpp | 354 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 354 insertions(+) create mode 100644 asset_test.cpp (limited to 'asset_test.cpp') diff --git a/asset_test.cpp b/asset_test.cpp new file mode 100644 index 00000000..411e270d --- /dev/null +++ b/asset_test.cpp @@ -0,0 +1,354 @@ +#include +#include +#include +#include + +enum DlStatus +{ + Dl_NotStarted, + Dl_InProgress, + Dl_Finished, + Dl_Failed +}; + +/** + * A single file for the downloader/cache to process. + */ +struct Downloadable +{ + Downloadable(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()) + :m_url(url), m_rel_target_path(rel_target_path), m_expected_md5(expected_md5) + { + m_check_md5 = m_expected_md5.size(); + m_save_to_file = m_rel_target_path.size(); + status = Dl_NotStarted; + }; + /// source URL + QUrl m_url; + + /// if true, check the md5sum against a provided md5sum + /// also, if a file exists, perform an md5sum first and don't download only if they don't match + bool m_check_md5; + /// the expected md5 checksum + QString m_expected_md5; + + /// save to file? + bool m_save_to_file; + /// if saving to file, use the one specified in this string + QString m_rel_target_path; + /// if not saving to file, downloaded data is placed here + QByteArray data; + /// The file's status + DlStatus status; +}; +typedef QSharedPointer DownloadablePtr; + +class Downloader; + +/** + * A downloader job, which can have multiple files + * the user of the downloader is responsible for creating it + * and connecting its signals to his own slots. + */ +class DLJob : public QObject +{ + friend class Downloader; + Q_OBJECT +public: + + DLJob ( QUrl what , QObject* parent = 0 ) : QObject(parent) + { + m_status = Dl_NotStarted; + m_downloads.append(DownloadablePtr(new Downloadable(what))); + } + DlStatus getStatus() + { + return m_status; + } + QByteArray getFirstFileData() + { + if(!m_downloads.size()) + return QByteArray(); + else return m_downloads[0]->data; + } +private: + void emitStart() + { + m_status = Dl_InProgress; + emit started(); + } + void emitFail() + { + m_status = Dl_Failed; + emit failed(); + } + void emitFinish() + { + m_status = Dl_Finished; + emit finished(); + } +private: + QVector m_downloads; + /// The job's status + DlStatus m_status; +signals: + void started(); + void finished(); + void failed(); +}; +typedef QSharedPointer DLJobPtr; + +/** + * The downloader itself. Make one, use it. + * User is responsible for keeping it around until all the jobs either finish or fail. + */ +class Downloader : public QObject +{ + Q_OBJECT +public: + Downloader(QObject *p = 0): + QObject(p), + nam(new QNetworkAccessManager(this)), + currentReply(0), + currentIndex(0){} + + void enqueue(DLJobPtr job) + { + if(jobs.empty()) + QTimer::singleShot(0, this, SLOT(startNextJob())); + jobs.enqueue(job); + } + + private slots: + void startNextJob() + { + if (jobs.isEmpty()) + { + currentJob.clear(); + currentIndex = 0; + emit finishedAllJobs(); + return; + } + + currentJob = jobs.dequeue(); + currentIndex = 0; + currentJob->emitStart(); + + startNextDownload(); + } + + void startNextDownload() + { + // NO-OP... makes no sense, should be detected as error likely + if(!currentJob) + return; + + // we finished the current job. Good job. + if(currentIndex >= currentJob->m_downloads.size()) + { + currentJob->emitFinish(); + QTimer::singleShot(0, this, SLOT(startNextJob())); + return; + } + + DownloadablePtr dlable = currentJob->m_downloads[currentIndex]; + if(dlable->m_save_to_file) + { + QString filename = dlable->m_rel_target_path; + currentOutput.setFileName(filename); + // if there already is a file and md5 checking is in effect + if(currentOutput.exists() && dlable->m_check_md5) + { + // and it can be opened + if(currentOutput.open(QIODevice::ReadOnly)) + { + // check the md5 against the expected one + QString hash = QCryptographicHash::hash(currentOutput.readAll(), QCryptographicHash::Md5).toHex().constData(); + currentOutput.close(); + // skip this file if they match + if(hash == dlable->m_expected_md5) + { + currentIndex++; + QTimer::singleShot(0, this, SLOT(startNextDownload())); + return; + } + } + } + if (!currentOutput.open(QIODevice::WriteOnly)) + { + /* + * TODO: Can't open the file... the job failed + */ + currentJob->emitFail(); + currentJob.clear(); + currentIndex = 0; + QTimer::singleShot(0, this, SLOT(startNextJob())); + + return; + } + } + + QNetworkRequest request(dlable->m_url); + QNetworkReply * rep = nam->get(request); + currentReply = QSharedPointer(rep, &QObject::deleteLater); + connect(rep, SIGNAL(downloadProgress(qint64,qint64)), SLOT(downloadProgress(qint64,qint64))); + connect(rep, SIGNAL(finished()), SLOT(downloadFinished())); + connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(downloadError(QNetworkReply::NetworkError))); + connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead())); + } + void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) + { + // eventually, progress bars. yeah. + }; + + void downloadError(QNetworkReply::NetworkError error) + { + // error happened during download. :< + DownloadablePtr dlable = currentJob->m_downloads[currentIndex]; + //TODO: log the reason why + dlable->status = Dl_Failed; + } + + void downloadFinished() + { + DownloadablePtr dlable = currentJob->m_downloads[currentIndex]; + // if the download succeeded + if(dlable->status != Dl_Failed) + { + // nothing went wrong... + dlable->status = Dl_Finished; + // save the data to the downloadable if we aren't saving to file + if(!dlable->m_save_to_file) + { + dlable->data = currentReply->readAll(); + } + else + { + currentOutput.close(); + } + + //TODO: check md5 here! + + // continue with the next download, if any + currentIndex ++; + QTimer::singleShot(0, this, SLOT(startNextDownload())); + return; + } + // else the download failed + else + { + if(dlable->m_save_to_file) + { + currentOutput.close(); + currentOutput.remove(); + } + } + } + void downloadReadyRead() + { + DownloadablePtr dlable = currentJob->m_downloads[currentIndex]; + if(dlable->m_save_to_file) + { + currentOutput.write(currentReply->readAll()); + } + }; + +signals: + void finishedAllJobs(); + +public slots: + +private: + QSharedPointer nam; + DLJobPtr currentJob; + QSharedPointer currentReply; + QQueue jobs; + QFile currentOutput; + unsigned currentIndex; +}; + +inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname) +{ + QDomNodeList elementList = parent.elementsByTagName(tagname); + if (elementList.count()) + return elementList.at(0).toElement(); + else + return QDomElement(); +} + +class DlMachine : public QObject +{ + Q_OBJECT +public slots: + void fetchFinished() + { + QByteArray ba = jptr->getFirstFileData(); + + QString xmlErrorMsg; + QDomDocument doc; + if (!doc.setContent(ba, false, &xmlErrorMsg)) + { + qDebug() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:" << + xmlErrorMsg << ba; + } + QRegExp etag_match(".*([a-f0-9]{32}).*"); + QDomNodeList contents = doc.elementsByTagName("Contents"); + + for (int i = 0; i < contents.length(); i++) + { + QDomElement element = contents.at(i).toElement(); + + if (element.isNull()) + continue; + + QDomElement keyElement = getDomElementByTagName(element, "Key"); + QDomElement lastmodElement = getDomElementByTagName(element, "LastModified"); + QDomElement etagElement = getDomElementByTagName(element, "ETag"); + QDomElement sizeElement = getDomElementByTagName(element, "Size"); + + if (keyElement.isNull() || lastmodElement.isNull() || etagElement.isNull() || sizeElement.isNull()) + continue; + + QString keyStr = keyElement.text(); + QString lastModStr = lastmodElement.text(); + QString etagStr = etagElement.text(); + QString sizeStr = sizeElement.text(); + + //Filter folder keys + if (sizeStr == "0") + continue; + + //TODO:Need to get ETag keys to be valid strings not "" "" + + qDebug() << keyStr << " " << lastModStr << " " << etagStr << sizeStr; + } + + qApp->quit(); + } + void fetchStarted() + { + qDebug() << " Started downloading!"; + } +public: + void start() + { + DLJob *job = new DLJob(QUrl("http://s3.amazonaws.com/Minecraft.Resources/")); + connect(job, SIGNAL(finished()), SLOT(fetchFinished())); + connect(job, SIGNAL(started()), SLOT(fetchStarted())); + jptr.reset(job); + dl.enqueue(jptr); + } + Downloader dl; + DLJobPtr jptr; +}; + +int main(int argc, char *argv[]) +{ + QCoreApplication app(argc, argv); + + DlMachine dl; + dl.start(); + + return app.exec(); +} +#include "asset_test.moc" \ No newline at end of file -- cgit v1.2.3