From 55f311a9084aba9f0601388183ff10abb0baac8c Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 24 Sep 2016 11:35:45 +0900 Subject: [PATCH] Add --keep-unfinished-download-result option This option keeps unfinished download results even if doing so exceeds --max-download-result. This is useful if all unfinished downloads must be saved in session file (see --save-session option). Please keep in mind that there is no upper bound to the number of unfinished download result to keep. User should use this option only when they know the total number of downloads in advance. --- doc/manual-src/en/aria2c.rst | 11 +++++++++++ doc/xmlrpc/aria2rpc | 3 +++ src/OptionHandlerFactory.cc | 9 +++++++++ src/RequestGroupMan.cc | 12 ++++++++++-- src/RequestGroupMan.h | 10 ++++++++++ src/SessionSerializer.cc | 38 +++++++++++++++++++++++++++++------- src/prefs.cc | 3 +++ src/prefs.h | 2 ++ src/usage_text.h | 11 ++++++++++- 9 files changed, 89 insertions(+), 10 deletions(-) diff --git a/doc/manual-src/en/aria2c.rst b/doc/manual-src/en/aria2c.rst index 0cb21fac..ef709e20 100644 --- a/doc/manual-src/en/aria2c.rst +++ b/doc/manual-src/en/aria2c.rst @@ -1391,6 +1391,16 @@ Advanced Options system doesn't have :manpage:`getifaddrs(3)`, this option doesn't accept interface name. +.. option:: --keep-unfinished-download-result[=true|false] + + Keep unfinished download results even if doing so exceeds + :option:`--max-download-result`. This is useful if all unfinished + downloads must be saved in session file (see + :option:`--save-session` option). Please keep in mind that there is + no upper bound to the number of unfinished download result to keep. + User should use this option only when they know the total number of + downloads in advance. Default: ``false`` + .. option:: --max-download-result= Set maximum number of download result kept in memory. The download @@ -3240,6 +3250,7 @@ For information on the *secret* parameter, see :ref:`rpc_auth`. * :option:`bt-max-open-files <--bt-max-open-files>` * :option:`download-result <--download-result>` + * :option:`keep-unfinished-download-result <--keep-unfinished-download-result>` * :option:`log <-l>` * :option:`log-level <--log-level>` * :option:`max-concurrent-downloads <-j>` diff --git a/doc/xmlrpc/aria2rpc b/doc/xmlrpc/aria2rpc index 934fbdbc..f52b7f6a 100755 --- a/doc/xmlrpc/aria2rpc +++ b/doc/xmlrpc/aria2rpc @@ -228,6 +228,9 @@ OptionParser.new do |opt| opt.on("-l","--log FILE"){|val| options["log"]=val} opt.on("--max-download-result NUM"){|val| options["max-download-result"]=val} opt.on("--download-result OPT"){|val| options["download-result"]=val} + opt.on("--keep-unfinished-download-result [BOOL]",["true","false"]){|val| + options["keep-unfinished-download-result"]=val||"true" + } opt.on("--save-session FILE"){|val| options["save-session"]=val} opt.on("--server-stat-of FILE"){|val| options["server-stat-of"]=val} opt.on("--save-cookies FILE"){|val| options["save-cookies"]=val} diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index 842ae56f..e718382b 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -403,6 +403,15 @@ std::vector OptionHandlerFactory::createOptionHandlers() op->addTag(TAG_ADVANCED); handlers.push_back(op); } + { + OptionHandler* op( + new BooleanOptionHandler(PREF_KEEP_UNFINISHED_DOWNLOAD_RESULT, + TEXT_KEEP_UNFINISHED_DOWNLOAD_RESULT, + A2_V_FALSE, OptionHandler::OPT_ARG)); + op->addTag(TAG_ADVANCED); + op->setChangeGlobalOption(true); + handlers.push_back(op); + } { OptionHandler* op(new DefaultOptionHandler( PREF_LOG, TEXT_LOG, NO_DEFAULT_VALUE, PATH_TO_FILE_STDOUT, diff --git a/src/RequestGroupMan.cc b/src/RequestGroupMan.cc index 19ee63ad..169a6041 100644 --- a/src/RequestGroupMan.cc +++ b/src/RequestGroupMan.cc @@ -920,13 +920,21 @@ void RequestGroupMan::addDownloadResult( bool rv = downloadResults_.push_back(dr->gid->getNumericId(), dr); assert(rv); while (downloadResults_.size() > maxDownloadResult_) { - DownloadResultList::iterator i = downloadResults_.begin(); // Save last encountered error code so that we can report it // later. - const std::shared_ptr& dr = *i; + const auto& dr = downloadResults_[0]; if (dr->belongsTo == 0 && dr->result != error_code::FINISHED) { removedLastErrorResult_ = dr->result; ++removedErrorResult_; + + // Keep unfinished download result, so that we can save them by + // SessionSerializer. + if (option_->getAsBool(PREF_KEEP_UNFINISHED_DOWNLOAD_RESULT)) { + if (dr->result != error_code::REMOVED || + dr->option->getAsBool(PREF_FORCE_SAVE)) { + unfinishedDownloadResults_.push_back(dr); + } + } } downloadResults_.pop_front(); } diff --git a/src/RequestGroupMan.h b/src/RequestGroupMan.h index 566b77d3..b8aa8d4f 100644 --- a/src/RequestGroupMan.h +++ b/src/RequestGroupMan.h @@ -71,6 +71,10 @@ private: RequestGroupList requestGroups_; RequestGroupList reservedGroups_; DownloadResultList downloadResults_; + // This includes download result which did not finish, and deleted + // from downloadResults_. This is used to save them in + // SessionSerializer. + std::vector> unfinishedDownloadResults_; int maxConcurrentDownloads_; @@ -261,6 +265,12 @@ public: void addDownloadResult(const std::shared_ptr& downloadResult); + const std::vector>& + getUnfinishedDownloadResult() const + { + return unfinishedDownloadResults_; + } + std::shared_ptr findServerStat(const std::string& hostname, const std::string& protocol) const; diff --git a/src/SessionSerializer.cc b/src/SessionSerializer.cc index 0d260a3f..8dd9008a 100644 --- a/src/SessionSerializer.cc +++ b/src/SessionSerializer.cc @@ -272,11 +272,14 @@ bool writeDownloadResult(IOFile& fp, std::set& metainfoCache, } } // namespace -bool SessionSerializer::save(IOFile& fp) const +namespace { +template +bool saveDownloadResult(IOFile& fp, std::set& metainfoCache, + InputIt first, InputIt last, bool saveInProgress, + bool saveError) { - std::set metainfoCache; - const DownloadResultList& results = rgman_->getDownloadResults(); - for (const auto& dr : results) { + for (; first != last; ++first) { + const auto& dr = *first; auto save = false; switch (dr->result) { case error_code::FINISHED: @@ -284,20 +287,41 @@ bool SessionSerializer::save(IOFile& fp) const save = dr->option->getAsBool(PREF_FORCE_SAVE); break; case error_code::IN_PROGRESS: - save = saveInProgress_; + save = saveInProgress; break; case error_code::RESOURCE_NOT_FOUND: case error_code::MAX_FILE_NOT_FOUND: - save = saveError_ && dr->option->getAsBool(PREF_SAVE_NOT_FOUND); + save = saveError && dr->option->getAsBool(PREF_SAVE_NOT_FOUND); break; default: - save = saveError_; + save = saveError; break; } if (save && !writeDownloadResult(fp, metainfoCache, dr, false)) { return false; } } + return true; +} +} // namespace + +bool SessionSerializer::save(IOFile& fp) const +{ + std::set metainfoCache; + + const auto& unfinishedResults = rgman_->getUnfinishedDownloadResult(); + if (!saveDownloadResult(fp, metainfoCache, std::begin(unfinishedResults), + std::end(unfinishedResults), saveInProgress_, + saveError_)) { + return false; + } + + const auto& results = rgman_->getDownloadResults(); + if (!saveDownloadResult(fp, metainfoCache, std::begin(results), + std::end(results), saveInProgress_, saveError_)) { + return false; + } + { // Save active downloads. const RequestGroupList& groups = rgman_->getRequestGroups(); diff --git a/src/prefs.cc b/src/prefs.cc index 8ec80569..c0e1af6d 100644 --- a/src/prefs.cc +++ b/src/prefs.cc @@ -374,6 +374,9 @@ PrefPtr PREF_SOCKET_RECV_BUFFER_SIZE = makePref("socket-recv-buffer-size"); PrefPtr PREF_MAX_MMAP_LIMIT = makePref("max-mmap-limit"); // value: true | false PrefPtr PREF_STDERR = makePref("stderr"); +// value: true | false +PrefPtr PREF_KEEP_UNFINISHED_DOWNLOAD_RESULT = + makePref("keep-unfinished-download-result"); /** * FTP related preferences diff --git a/src/prefs.h b/src/prefs.h index cc759f28..fb064727 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -328,6 +328,8 @@ extern PrefPtr PREF_SOCKET_RECV_BUFFER_SIZE; extern PrefPtr PREF_MAX_MMAP_LIMIT; // value: true | false extern PrefPtr PREF_STDERR; +// value: true | false +extern PrefPtr PREF_KEEP_UNFINISHED_DOWNLOAD_RESULT; /** * FTP related preferences diff --git a/src/usage_text.h b/src/usage_text.h index 4a4bde97..a9240732 100644 --- a/src/usage_text.h +++ b/src/usage_text.h @@ -1099,5 +1099,14 @@ #define TEXT_STDERR \ _(" --stderr[=true|false] Redirect all console output that would be\n" \ " otherwise printed in stdout to stderr.") - +#define TEXT_KEEP_UNFINISHED_DOWNLOAD_RESULT \ + _(" --keep-unfinished-download-result[=true|false]\n" \ + " Keep unfinished download results even if doing\n" \ + " so exceeds --max-download-result. This is useful\n" \ + " if all unfinished downloads must be saved in\n" \ + " session file (see --save-session option). Please\n" \ + " keep in mind that there is no upper bound to the\n" \ + " number of unfinished download result to keep.\n" \ + " User should use this option only when they know\n" \ + " the total number of downloads in advance.") // clang-format on