diff -p -up ./src/AbstractCommand.cc.adaptive ./src/AbstractCommand.cc --- ./src/AbstractCommand.cc.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/AbstractCommand.cc 2008-10-13 11:43:37.000000000 +0200 @@ -188,7 +188,7 @@ bool AbstractCommand::execute() { void AbstractCommand::tryReserved() { _requestGroup->removeServerHost(cuid); Commands commands; - _requestGroup->createNextCommand(commands, e, 1); + _requestGroup->createNextCommand(commands, e, 1, true); e->setNoWait(true); e->addCommand(commands); } diff -p -up ./src/AdaptiveURISelector.cc.adaptive ./src/AdaptiveURISelector.cc --- ./src/AdaptiveURISelector.cc.adaptive 2008-10-13 11:43:37.000000000 +0200 +++ ./src/AdaptiveURISelector.cc 2008-10-14 14:37:40.000000000 +0200 @@ -0,0 +1,272 @@ +/* <!-- copyright */ +/* + * aria2 - The high speed download utility + * + * Copyright (C) 2006 Tatsuhiro Tsujikawa + * Copyright (C) 2008 Aurelien Lefebvre, Mandriva + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * In addition, as a special exception, the copyright holders give + * permission to link the code of portions of this program with the + * OpenSSL library under certain conditions as described in each + * individual source file, and distribute linked combinations + * including the two. + * You must obey the GNU General Public License in all respects + * for all of the code used other than OpenSSL. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you + * do not wish to do so, delete this exception statement from your + * version. If you delete this exception statement from all source + * files in the program, then also delete it here. + */ +/* copyright --> */ +#include "AdaptiveURISelector.h" +#include "ServerStatURISelector.h" +#include "URISelector.h" +#include "Util.h" +#include "DownloadCommand.h" +#include "ServerStatMan.h" +#include "ServerStat.h" +#include "RequestGroup.h" +#include "Request.h" +#include "LogFactory.h" +#include "A2STR.h" +#include "prefs.h" +#include "Option.h" +#include "SimpleRandomizer.h" +#include <cstdlib> +#include <algorithm> +#include <cmath> + +namespace aria2 { + +/* In that URI Selector, select method returns one of the bests + * mirrors for first and reserved connections. For supplementary + * ones, it returns mirrors which has not been tested yet, and + * if each of them already tested, returns mirrors which has to + * be tested again. Otherwise, it doesn't return anymore mirrors. + */ + +AdaptiveURISelector::AdaptiveURISelector +(const SharedHandle<ServerStatMan>& serverStatMan, + const SharedHandle<RequestGroup>& requestGroup): + _serverStatMan(serverStatMan), + _requestGroup(requestGroup), + _nbConnections(0), + _logger(LogFactory::getInstance()) + { + const Option* op = _requestGroup->getOption(); + _nbServerToEvaluate = op->getAsInt(PREF_METALINK_SERVERS) - 1; + } + +AdaptiveURISelector::~AdaptiveURISelector() {} + +std::string AdaptiveURISelector::select(std::deque<std::string>& uris, bool reserved) +{ + std::string selected = selectOne(uris, reserved); + + if(selected != A2STR::NIL) + uris.erase(std::find(uris.begin(), uris.end(), selected)); + + return selected; +} + +std::string AdaptiveURISelector::selectOne(std::deque<std::string>& uris, bool reserved) +{ + + if(uris.empty()) { + return A2STR::NIL; + } else { + + _nbConnections++; + /* At least, 3 mirrors must be tested */ + if(getNbTestedServers(uris) < 3) { + std::string notTested = getFirstNotTestedUri(uris); + if(notTested != A2STR::NIL) { + _logger->debug("AdaptiveURISelector: choosing the first non tested mirror: %s", notTested.c_str()); + --_nbServerToEvaluate; + return notTested; + } + } + + if(!reserved && _nbConnections > 1 && _nbServerToEvaluate > 0) { + _nbServerToEvaluate--; + std::string notTested = getFirstNotTestedUri(uris); + if(notTested != A2STR::NIL) { + /* Here we return the first untested mirror */ + _logger->debug("AdaptiveURISelector: choosing non tested mirror %s for connection #%d", notTested.c_str(), _nbConnections); + return notTested; + } else { + /* Here we return a mirror which need to be tested again */ + std::string toReTest = getFirstToTestUri(uris); + _logger->debug("AdaptiveURISelector: choosing mirror %s which has not been tested recently for connection #%d", toReTest.c_str(), _nbConnections); + return toReTest; + } + } + else { + /* Here we return one of the bests mirrors */ + unsigned int max = getMaxDownloadSpeed(uris); + unsigned int min = max-(int)(max*0.25); + std::deque<std::string> bests = getUrisBySpeed(uris, min); + + if (bests.size() < 2) { + std::string uri = getMaxDownloadSpeedUri(uris); + _logger->debug("AdaptiveURISelector: choosing the best mirror : %.2fKB/s %s (other mirrors are at least 25%% slower)", (float) max/1024, uri.c_str()); + return uri; + } else { + std::string uri = selectRandomUri(bests); + _logger->debug("AdaptiveURISelector: choosing randomly one of the best mirrors (range [%.2fKB/s, %.2fKB/s]): %s", (float) min/1024, (float) max/1024, uri.c_str()); + return uri; + } + } + } +} + +void AdaptiveURISelector::resetCounters() { + const Option* op = _requestGroup->getOption(); + _nbConnections = 0; + _nbServerToEvaluate = op->getAsInt(PREF_METALINK_SERVERS) - 1; +} + +void AdaptiveURISelector::tuneDownloadCommand(std::deque<std::string>& uris, DownloadCommand *command) { + adjustLowestSpeedLimit(uris, command); +} + +void AdaptiveURISelector::adjustLowestSpeedLimit(std::deque<std::string>& uris, DownloadCommand *command) { + const Option* op = _requestGroup->getOption(); + unsigned int lowest = op->getAsInt(PREF_LOWEST_SPEED_LIMIT); + if (lowest > 0) { + unsigned int low_lowest = 4 * 1024; + unsigned int max = getMaxDownloadSpeed(uris); + if (max > 0 && lowest > max / 4) { + _logger->notice("Lowering lowest-speed-limit since known max speed is too near (new:%d was:%d max:%d)", max / 4, lowest, max); + command->setLowestDownloadSpeedLimit(max / 4); + } else if (max == 0 && lowest > low_lowest) { + _logger->notice("Lowering lowest-speed-limit since we have no clue about available speed (now:%d was:%d)", low_lowest, lowest); + command->setLowestDownloadSpeedLimit(low_lowest); + } + } +} + +unsigned int _getUriMaxSpeed(SharedHandle<ServerStat> ss) +{ + return std::max(ss->getSingleConnectionAvgSpeed(), ss->getMultiConnectionAvgSpeed()); +} + +unsigned int AdaptiveURISelector::getMaxDownloadSpeed(std::deque<std::string>& uris) +{ + std::string uri = getMaxDownloadSpeedUri(uris); + if(uri == A2STR::NIL) + return 0; + return _getUriMaxSpeed(getServerStats(uri)); +} + +std::string AdaptiveURISelector::getMaxDownloadSpeedUri(std::deque<std::string>& uris) +{ + int max = -1; + std::string uri = A2STR::NIL; + for(std::deque<std::string>::iterator i = uris.begin(); + i != uris.end(); ++i) { + SharedHandle<ServerStat> ss = getServerStats(*i); + if(ss.isNull()) + continue; + + if((int)ss->getSingleConnectionAvgSpeed() > max) { + max = ss->getSingleConnectionAvgSpeed(); + uri = (*i); + } + if((int)ss->getMultiConnectionAvgSpeed() > max) { + max = ss->getMultiConnectionAvgSpeed(); + uri = (*i); + } + } + return uri; +} + +std::deque<std::string> AdaptiveURISelector::getUrisBySpeed(std::deque<std::string>& uris, unsigned int min) +{ + std::deque<std::string> bests; + for(std::deque<std::string>::iterator i = uris.begin(); + i != uris.end(); ++i) { + SharedHandle<ServerStat> ss = getServerStats(*i); + if(ss.isNull()) + continue; + if(ss->getSingleConnectionAvgSpeed() > min || + ss->getMultiConnectionAvgSpeed() > min) { + bests.push_back(*i); + } + } + return bests; +} + +std::string AdaptiveURISelector::selectRandomUri(std::deque<std::string>& uris) +{ + int pos = SimpleRandomizer::getInstance()->getRandomNumber(uris.size()); + std::deque<std::string>::iterator i = uris.begin(); + i = i+pos; + return *i; +} + +std::string AdaptiveURISelector::getFirstNotTestedUri(std::deque<std::string>& uris) +{ + for(std::deque<std::string>::iterator i = uris.begin(); + i != uris.end(); ++i) { + SharedHandle<ServerStat> ss = getServerStats(*i); + if(ss.isNull()) + return *i; + } + return A2STR::NIL; +} + +std::string AdaptiveURISelector::getFirstToTestUri(std::deque<std::string>& uris) { + unsigned int counter; + int power; + for(std::deque<std::string>::iterator i = uris.begin(); + i != uris.end(); ++i) { + SharedHandle<ServerStat> ss = getServerStats(*i); + if(ss.isNull()) + continue; + counter = ss->getCounter(); + if(counter > 8) + continue; + power = (int)pow(2.0, (float)counter); + /* We test the mirror another time if it has not been + * tested since 2^counter days */ + if(ss->getLastUpdated().difference() > power*24*60*60) { + return *i; + } + } + return A2STR::NIL; +} + +SharedHandle<ServerStat> AdaptiveURISelector::getServerStats(std::string uri) { + Request r; + r.setUrl(uri); + return _serverStatMan->find(r.getHost(), r.getProtocol()); +} + +unsigned int AdaptiveURISelector::getNbTestedServers(std::deque<std::string>& uris) { + unsigned int counter = 0; + for(std::deque<std::string>::iterator i = uris.begin(); + i != uris.end(); ++i) { + SharedHandle<ServerStat> ss = getServerStats(*i); + if(ss.isNull()) + counter++; + } + return uris.size() - counter; +} + +} // namespace aria2 diff -p -up ./src/AdaptiveURISelector.h.adaptive ./src/AdaptiveURISelector.h --- ./src/AdaptiveURISelector.h.adaptive 2008-10-13 11:43:37.000000000 +0200 +++ ./src/AdaptiveURISelector.h 2008-10-13 16:40:41.000000000 +0200 @@ -0,0 +1,78 @@ +/* <!-- copyright */ +/* + * aria2 - The high speed download utility + * + * Copyright (C) 2006 Tatsuhiro Tsujikawa + * Copyright (C) 2008 Aurelien Lefebvre, Mandriva + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * In addition, as a special exception, the copyright holders give + * permission to link the code of portions of this program with the + * OpenSSL library under certain conditions as described in each + * individual source file, and distribute linked combinations + * including the two. + * You must obey the GNU General Public License in all respects + * for all of the code used other than OpenSSL. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you + * do not wish to do so, delete this exception statement from your + * version. If you delete this exception statement from all source + * files in the program, then also delete it here. + */ +/* copyright --> */ +#ifndef _D_ADAPTIVE_URI_SELECTOR_H_ +#define _D_ADAPTIVE_URI_SELECTOR_H_ +#include "URISelector.h" +#include "SharedHandle.h" +#include "ServerStatMan.h" +#include "RequestGroup.h" +#include "ServerStat.h" + +namespace aria2 { + +class AdaptiveURISelector:public URISelector { +private: + SharedHandle<ServerStatMan> _serverStatMan; + SharedHandle<RequestGroup> _requestGroup; + unsigned int _nbServerToEvaluate; + unsigned int _nbConnections; + + Logger* _logger; + + std::string selectOne(std::deque<std::string>& uris, bool reserved); + void adjustLowestSpeedLimit(std::deque<std::string>& uris, DownloadCommand *command); + unsigned int getMaxDownloadSpeed(std::deque<std::string>& uris); + std::string getMaxDownloadSpeedUri(std::deque<std::string>& uris); + std::deque<std::string> getUrisBySpeed(std::deque<std::string>& uris, unsigned int min); + std::string selectRandomUri(std::deque<std::string>& uris); + std::string getFirstNotTestedUri(std::deque<std::string>& uris); + std::string getFirstToTestUri(std::deque<std::string>& uris); + SharedHandle<ServerStat> getServerStats(std::string uri); + unsigned int getNbTestedServers(std::deque<std::string>& uris); +public: + AdaptiveURISelector(const SharedHandle<ServerStatMan>& serverStatMan, + const SharedHandle<RequestGroup>& requestGroup); + + virtual ~AdaptiveURISelector(); + + virtual std::string select(std::deque<std::string>& uris, bool reserved); + virtual void tuneDownloadCommand(std::deque<std::string>& uris, DownloadCommand *command); + + virtual void resetCounters(); +}; + +} // namespace aria2 +#endif // _D_ADAPTIVE_URI_SELECTOR_H_ diff -p -up ./src/FtpNegotiationCommand.cc.adaptive ./src/FtpNegotiationCommand.cc --- ./src/FtpNegotiationCommand.cc.adaptive 2008-09-18 14:48:03.000000000 +0200 +++ ./src/FtpNegotiationCommand.cc 2008-10-13 16:30:13.000000000 +0200 @@ -97,6 +97,7 @@ bool FtpNegotiationCommand::executeInter _requestGroup->removeURIWhoseHostnameIs(sv->getHostname()); } } + _requestGroup->tuneDownloadCommand(command); e->commands.push_back(command); return true; } else if(sequence == SEQ_HEAD_OK || sequence == SEQ_DOWNLOAD_ALREADY_COMPLETED) { diff -p -up ./src/HttpResponseCommand.cc.adaptive ./src/HttpResponseCommand.cc --- ./src/HttpResponseCommand.cc.adaptive 2008-09-18 14:48:03.000000000 +0200 +++ ./src/HttpResponseCommand.cc 2008-10-13 16:30:44.000000000 +0200 @@ -311,6 +311,8 @@ HttpDownloadCommand* HttpResponseCommand _requestGroup->setFileAllocationEnabled(false); } + _requestGroup->tuneDownloadCommand(command); + return command; } diff -p -up ./src/InOrderURISelector.cc.adaptive ./src/InOrderURISelector.cc --- ./src/InOrderURISelector.cc.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/InOrderURISelector.cc 2008-10-13 11:43:37.000000000 +0200 @@ -41,7 +41,7 @@ InOrderURISelector::InOrderURISelector() InOrderURISelector::~InOrderURISelector() {} -std::string InOrderURISelector::select(std::deque<std::string>& uris) +std::string InOrderURISelector::select(std::deque<std::string>& uris, bool reserved) { if(uris.empty()) { return A2STR::NIL; diff -p -up ./src/InOrderURISelector.h.adaptive ./src/InOrderURISelector.h --- ./src/InOrderURISelector.h.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/InOrderURISelector.h 2008-10-13 15:45:17.000000000 +0200 @@ -44,7 +44,7 @@ public: virtual ~InOrderURISelector(); - virtual std::string select(std::deque<std::string>& uris); + virtual std::string select(std::deque<std::string>& uris, bool reserved); }; } // namespace aria2 diff -p -up ./src/Makefile.am.adaptive ./src/Makefile.am --- ./src/Makefile.am.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/Makefile.am 2008-10-13 11:43:37.000000000 +0200 @@ -191,6 +191,7 @@ SRCS = Socket.h\ ServerStat.cc ServerStat.h\ ServerStatMan.cc ServerStatMan.h\ URISelector.h\ + AdaptiveURISelector.cc AdaptiveURISelector.h\ InOrderURISelector.cc InOrderURISelector.h\ ServerStatURISelector.cc ServerStatURISelector.h\ NsCookieParser.cc NsCookieParser.h\ @@ -486,4 +487,4 @@ AM_CPPFLAGS = -Wall\ @LIBGNUTLS_CFLAGS@ @LIBGCRYPT_CFLAGS@ @OPENSSL_CFLAGS@ @XML_CPPFLAGS@\ @LIBARES_CPPFLAGS@ @LIBCARES_CPPFLAGS@ @LIBEXPAT_CPPFLAGS@\ @LIBZ_CPPFLAGS@ @SQLITE3_CPPFLAGS@\ - -DLOCALEDIR=\"$(localedir)\" @DEFS@ #-pg \ No newline at end of file + -DLOCALEDIR=\"$(localedir)\" @DEFS@ #-pg diff -p -up ./src/Makefile.in.adaptive ./src/Makefile.in --- ./src/Makefile.in.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/Makefile.in 2008-10-13 11:43:37.000000000 +0200 @@ -411,6 +411,7 @@ am__libaria2c_a_SOURCES_DIST = Socket.h Decoder.h ChunkedDecoder.cc ChunkedDecoder.h Signature.cc \ Signature.h ServerStat.cc ServerStat.h ServerStatMan.cc \ ServerStatMan.h URISelector.h InOrderURISelector.cc \ + AdaptiveURISelector.h AdaptiveURISelector.cc \ InOrderURISelector.h ServerStatURISelector.cc \ ServerStatURISelector.h NsCookieParser.cc NsCookieParser.h \ CookieStorage.cc CookieStorage.h SocketBuffer.cc \ @@ -807,6 +808,7 @@ am__objects_18 = SocketCore.$(OBJEXT) Co RarestPieceSelector.$(OBJEXT) ChunkedDecoder.$(OBJEXT) \ Signature.$(OBJEXT) ServerStat.$(OBJEXT) \ ServerStatMan.$(OBJEXT) InOrderURISelector.$(OBJEXT) \ + AdaptiveURISelector.$(OBJEXT) \ ServerStatURISelector.$(OBJEXT) NsCookieParser.$(OBJEXT) \ CookieStorage.$(OBJEXT) SocketBuffer.$(OBJEXT) \ $(am__objects_1) $(am__objects_2) $(am__objects_3) \ @@ -1135,6 +1137,7 @@ SRCS = Socket.h SocketCore.cc SocketCore Decoder.h ChunkedDecoder.cc ChunkedDecoder.h Signature.cc \ Signature.h ServerStat.cc ServerStat.h ServerStatMan.cc \ ServerStatMan.h URISelector.h InOrderURISelector.cc \ + AdaptiveURISelector.h AdaptiveURISelector.cc \ InOrderURISelector.h ServerStatURISelector.cc \ ServerStatURISelector.h NsCookieParser.cc NsCookieParser.h \ CookieStorage.cc CookieStorage.h SocketBuffer.cc \ @@ -1242,6 +1245,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AbstractProxyResponseCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AbstractSingleDiskAdaptor.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ActivePeerConnectionCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AdaptiveURISelector.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AnnounceList.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AsyncNameResolver.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AuthConfig.Po@am__quote@ diff -p -up ./src/OptionHandlerFactory.cc.adaptive ./src/OptionHandlerFactory.cc --- ./src/OptionHandlerFactory.cc.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/OptionHandlerFactory.cc 2008-10-13 11:43:37.000000000 +0200 @@ -144,7 +144,7 @@ OptionHandlers OptionHandlerFactory::cre } handlers.push_back(SH(new BooleanOptionHandler(PREF_BT_SEED_UNVERIFIED))); { - const std::string params[] = { V_INORDER, V_FEEDBACK }; + const std::string params[] = { V_INORDER, V_FEEDBACK, V_ADAPTIVE }; handlers.push_back(SH(new ParameterOptionHandler (PREF_URI_SELECTOR, std::deque<std::string> diff -p -up ./src/RequestGroup.cc.adaptive ./src/RequestGroup.cc --- ./src/RequestGroup.cc.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/RequestGroup.cc 2008-10-13 16:42:25.000000000 +0200 @@ -293,7 +293,7 @@ void RequestGroup::createInitialCommand( // TODO I assume here when totallength is set to DownloadContext and it is // not 0, then filepath is also set DownloadContext correctly.... if(_downloadContext->getTotalLength() == 0) { - createNextCommand(commands, e, 1); + createNextCommand(commands, e, 1, false); }else { if(e->_requestGroupMan->isSameFileBeingDownloaded(this)) { throw DownloadFailureException @@ -498,18 +498,21 @@ void RequestGroup::createNextCommandWith numCommand += numAdj; } if(numCommand > 0) { - createNextCommand(commands, e, numCommand); + createNextCommand(commands, e, numCommand, false); } } void RequestGroup::createNextCommand(std::deque<Command*>& commands, DownloadEngine* e, unsigned int numCommand, + bool reserved, const std::string& method) { std::deque<std::string> pendingURIs; for(; !_uris.empty() && numCommand--; ) { - std::string uri = _uriSelector->select(_uris); + std::string uri = _uriSelector->select(_uris, reserved); + if(uri.size() == 0) + continue; RequestHandle req(new Request()); if(req->setUrl(uri)) { ServerHostHandle sv; @@ -971,6 +974,7 @@ void RequestGroup::reportDownloadFinishe { _logger->notice(MSG_FILE_DOWNLOAD_COMPLETED, getFilePath().c_str()); + _uriSelector->resetCounters(); #ifdef ENABLE_BITTORRENT TransferStat stat = calculateStat(); SharedHandle<BtContext> ctx = dynamic_pointer_cast<BtContext>(_downloadContext); @@ -1057,4 +1061,9 @@ void RequestGroup::increaseAndValidateFi } } +void RequestGroup::tuneDownloadCommand(DownloadCommand *command) +{ + _uriSelector->tuneDownloadCommand(_uris, command); +} + } // namespace aria2 diff -p -up ./src/RequestGroup.h.adaptive ./src/RequestGroup.h --- ./src/RequestGroup.h.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/RequestGroup.h 2008-10-13 16:27:31.000000000 +0200 @@ -48,6 +48,7 @@ class DownloadEngine; class SegmentMan; class SegmentManFactory; class Command; +class DownloadCommand; class DownloadContext; class PieceStorage; class BtProgressInfoFile; @@ -156,7 +157,7 @@ public: void createNextCommand(std::deque<Command*>& commands, DownloadEngine* e, unsigned int numCommand, - const std::string& method = "GET"); + bool reserved, const std::string& method = "GET"); void addURI(const std::string& uri) { @@ -372,6 +373,8 @@ public: void updateLastModifiedTime(const Time& time); void increaseAndValidateFileNotFoundCount(); + + void tuneDownloadCommand(DownloadCommand *command); }; typedef SharedHandle<RequestGroup> RequestGroupHandle; diff -p -up ./src/RequestGroupMan.cc.adaptive ./src/RequestGroupMan.cc --- ./src/RequestGroupMan.cc.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/RequestGroupMan.cc 2008-10-14 10:38:28.000000000 +0200 @@ -49,6 +49,7 @@ #include "SegmentMan.h" #include "ServerStatURISelector.h" #include "InOrderURISelector.h" +#include "AdaptiveURISelector.h" #include "Option.h" #include "prefs.h" #include "File.h" @@ -197,15 +198,26 @@ public: if(!group->getSegmentMan().isNull()) { const std::deque<SharedHandle<PeerStat> >& peerStats = group->getSegmentMan()->getPeerStats(); + for(std::deque<SharedHandle<PeerStat> >::const_iterator i = peerStats.begin(); i != peerStats.end(); ++i) { if((*i)->getHostname().empty() || (*i)->getProtocol().empty()) { continue; } + int speed = (*i)->getAvgDownloadSpeed(); + if (speed == 0) continue; + SharedHandle<ServerStat> ss = _requestGroupMan->getOrCreateServerStat((*i)->getHostname(), (*i)->getProtocol()); - ss->updateDownloadSpeed((*i)->getAvgDownloadSpeed()); + ss->increaseCounter(); + ss->updateDownloadSpeed(speed); + if(peerStats.size() == 1) { + ss->updateSingleConnectionAvgSpeed(speed); + } + else { + ss->updateMultiConnectionAvgSpeed(speed); + } } } } @@ -255,6 +267,9 @@ void RequestGroupMan::configureRequestGr } else if(uriSelectorValue == V_INORDER) { requestGroup->setURISelector (SharedHandle<URISelector>(new InOrderURISelector())); + } else if(uriSelectorValue == V_ADAPTIVE) { + requestGroup->setURISelector + (SharedHandle<URISelector>(new AdaptiveURISelector(_serverStatMan, requestGroup))); } } diff -p -up ./src/ServerStat.cc.adaptive ./src/ServerStat.cc --- ./src/ServerStat.cc.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/ServerStat.cc 2008-10-14 15:10:48.000000000 +0200 @@ -34,6 +34,7 @@ /* copyright --> */ #include "ServerStat.h" #include "array_fun.h" +#include "LogFactory.h" #include <ostream> #include <algorithm> @@ -49,7 +50,12 @@ ServerStat::ServerStat(const std::string _hostname(hostname), _protocol(protocol), _downloadSpeed(0), - _status(OK) {} + _singleConnectionAvgSpeed(0), + _multiConnectionAvgSpeed(0), + _counter(0), + _status(OK), + _logger(LogFactory::getInstance()) + {} ServerStat::~ServerStat() {} @@ -92,6 +98,79 @@ void ServerStat::updateDownloadSpeed(uns _lastUpdated.reset(); } +unsigned int ServerStat::getSingleConnectionAvgSpeed() const +{ + return _singleConnectionAvgSpeed; +} + +void ServerStat::setSingleConnectionAvgSpeed(unsigned int singleConnectionAvgSpeed) +{ + _singleConnectionAvgSpeed = singleConnectionAvgSpeed; +} + +void ServerStat::updateSingleConnectionAvgSpeed(unsigned int downloadSpeed) +{ + float avgDownloadSpeed; + if(_counter == 0) + return; + if(_counter < 5) { + avgDownloadSpeed = ((((float)_counter-1)/(float)_counter)*(float)_singleConnectionAvgSpeed) + + ((1.0/(float)_counter)*(float)downloadSpeed); + } + else { + avgDownloadSpeed = ((4.0/5.0)*(float)_singleConnectionAvgSpeed) + + ((1.0/5.0)*(float)downloadSpeed); + } + if(avgDownloadSpeed < (int)(0.80*_singleConnectionAvgSpeed)) { + _logger->debug("ServerStat:%s: resetting counter since single connection speed dropped", getHostname().c_str()); + _counter = 0; + } + _logger->debug("ServerStat:%s: _singleConnectionAvgSpeed old:%.2fKB/s new:%.2fKB/s last:%.2fKB/s", getHostname().c_str(), (float) _singleConnectionAvgSpeed/1024, (float) avgDownloadSpeed/1024, (float) downloadSpeed / 1024); + _singleConnectionAvgSpeed = (int)avgDownloadSpeed; +} + +unsigned int ServerStat::getMultiConnectionAvgSpeed() const +{ + return _multiConnectionAvgSpeed; +} + +void ServerStat::setMultiConnectionAvgSpeed(unsigned int multiConnectionAvgSpeed) +{ + _multiConnectionAvgSpeed = multiConnectionAvgSpeed; +} + +void ServerStat::updateMultiConnectionAvgSpeed(unsigned int downloadSpeed) +{ + float avgDownloadSpeed; + if(_counter == 0) + return; + if(_counter < 5) { + avgDownloadSpeed = ((((float)_counter-1)/(float)_counter)*(float)_multiConnectionAvgSpeed) + + ((1.0/(float)_counter)*(float)downloadSpeed); + } + else { + avgDownloadSpeed = ((4.0/5.0)*(float)_multiConnectionAvgSpeed) + + ((1.0/5.0)*(float)downloadSpeed); + } + _logger->debug("ServerStat:%s: _multiConnectionAvgSpeed old:%.2fKB/s new:%.2fKB/s last:%.2fKB/s", getHostname().c_str(), (float) _multiConnectionAvgSpeed/1024, (float) avgDownloadSpeed/1024, (float) downloadSpeed / 1024); + _multiConnectionAvgSpeed = (int)avgDownloadSpeed; +} + +unsigned int ServerStat::getCounter() const +{ + return _counter; +} + +void ServerStat::increaseCounter() +{ + _counter++; +} + +void ServerStat::setCounter(unsigned int value) +{ + _counter = value; +} + void ServerStat::setStatus(STATUS status) { _status = status; @@ -160,7 +239,10 @@ std::ostream& operator<<(std::ostream& o o << "host=" << serverStat.getHostname() << ", " << "protocol=" << serverStat.getProtocol() << ", " << "dl_speed=" << serverStat.getDownloadSpeed() << ", " + << "sc_avg_speed=" << serverStat.getSingleConnectionAvgSpeed() << ", " + << "mc_avg_speed=" << serverStat.getMultiConnectionAvgSpeed() << ", " << "last_updated=" << serverStat.getLastUpdated().getTime() << ", " + << "counter=" << serverStat.getCounter() << ", " << "status=" << ServerStat::STATUS_STRING[serverStat.getStatus()]; return o; } diff -p -up ./src/ServerStat.h.adaptive ./src/ServerStat.h --- ./src/ServerStat.h.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/ServerStat.h 2008-10-14 10:00:32.000000000 +0200 @@ -41,6 +41,8 @@ namespace aria2 { +class Logger; + // ServerStatMan: has many ServerStat // URISelector: interface // ServerStatURISelector: Has a reference of ServerStatMan @@ -75,6 +77,18 @@ public: // set download speed. This method doesn't update _lastUpdate. void setDownloadSpeed(unsigned int downloadSpeed); + unsigned int getSingleConnectionAvgSpeed() const; + void updateSingleConnectionAvgSpeed(unsigned int downloadSpeed); + void setSingleConnectionAvgSpeed(unsigned int singleConnectionAvgSpeed); + + unsigned int getMultiConnectionAvgSpeed() const; + void updateMultiConnectionAvgSpeed(unsigned int downloadSpeed); + void setMultiConnectionAvgSpeed(unsigned int singleConnectionAvgSpeed); + + unsigned int getCounter() const; + void increaseCounter(); + void setCounter(unsigned int value); + // This method doesn't update _lastUpdate. void setStatus(STATUS status); @@ -104,6 +118,14 @@ private: std::string _protocol; unsigned int _downloadSpeed; + + unsigned int _singleConnectionAvgSpeed; + + unsigned int _multiConnectionAvgSpeed; + + unsigned int _counter; + + Logger* _logger; STATUS _status; diff -p -up ./src/ServerStatMan.cc.adaptive ./src/ServerStatMan.cc --- ./src/ServerStatMan.cc.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/ServerStatMan.cc 2008-10-13 15:36:18.000000000 +0200 @@ -86,7 +86,10 @@ bool ServerStatMan::load(std::istream& i static const std::string S_HOST = "host"; static const std::string S_PROTOCOL = "protocol"; static const std::string S_DL_SPEED = "dl_speed"; + static const std::string S_SC_AVG_SPEED = "sc_avg_speed"; + static const std::string S_MC_AVG_SPEED = "mc_avg_speed"; static const std::string S_LAST_UPDATED = "last_updated"; + static const std::string S_COUNTER = "counter"; static const std::string S_STATUS = "status"; std::string line; @@ -111,7 +114,10 @@ bool ServerStatMan::load(std::istream& i SharedHandle<ServerStat> sstat(new ServerStat(m[S_HOST], m[S_PROTOCOL])); try { sstat->setDownloadSpeed(Util::parseUInt(m[S_DL_SPEED])); + sstat->setSingleConnectionAvgSpeed(Util::parseUInt(m[S_SC_AVG_SPEED])); + sstat->setMultiConnectionAvgSpeed(Util::parseUInt(m[S_MC_AVG_SPEED])); sstat->setLastUpdated(Time(Util::parseInt(m[S_LAST_UPDATED]))); + sstat->setCounter(Util::parseUInt(m[S_COUNTER])); sstat->setStatus(m[S_STATUS]); add(sstat); } catch(RecoverableException* e) { diff -p -up ./src/ServerStatURISelector.cc.adaptive ./src/ServerStatURISelector.cc --- ./src/ServerStatURISelector.cc.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/ServerStatURISelector.cc 2008-10-13 11:43:37.000000000 +0200 @@ -57,7 +57,7 @@ public: } }; -std::string ServerStatURISelector::select(std::deque<std::string>& uris) +std::string ServerStatURISelector::select(std::deque<std::string>& uris, bool reserved) { if(uris.empty()) { return A2STR::NIL; diff -p -up ./src/ServerStatURISelector.h.adaptive ./src/ServerStatURISelector.h --- ./src/ServerStatURISelector.h.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/ServerStatURISelector.h 2008-10-13 11:43:37.000000000 +0200 @@ -50,7 +50,7 @@ public: virtual ~ServerStatURISelector(); - virtual std::string select(std::deque<std::string>& uris); + virtual std::string select(std::deque<std::string>& uris, bool reserved); }; } // namespace aria2 diff -p -up ./src/SpeedCalc.cc.adaptive ./src/SpeedCalc.cc --- ./src/SpeedCalc.cc.adaptive 2008-09-18 14:48:03.000000000 +0200 +++ ./src/SpeedCalc.cc 2008-10-14 17:17:45.000000000 +0200 @@ -115,7 +115,9 @@ void SpeedCalc::changeSw() { unsigned int SpeedCalc::calculateAvgSpeed() const { uint64_t milliElapsed = start.differenceInMillis(); - if(milliElapsed) { + + // if milliElapsed is too small, the average speed is rubish, better return 0 + if(milliElapsed && milliElapsed > 4) { unsigned int speed = accumulatedLength*1000/milliElapsed; return speed; } else { diff -p -up ./src/URISelector.h.adaptive ./src/URISelector.h --- ./src/URISelector.h.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/URISelector.h 2008-10-13 16:28:54.000000000 +0200 @@ -40,11 +40,17 @@ namespace aria2 { +class DownloadCommand; + class URISelector { public: virtual ~URISelector() {} - virtual std::string select(std::deque<std::string>& uris) = 0; + virtual std::string select(std::deque<std::string>& uris, bool reserved) = 0; + + virtual void tuneDownloadCommand(std::deque<std::string>& uris, DownloadCommand *command) {}; + + virtual void resetCounters() { return; }; }; } // namespace aria2 diff -p -up ./src/prefs.cc.adaptive ./src/prefs.cc --- ./src/prefs.cc.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/prefs.cc 2008-10-13 11:43:37.000000000 +0200 @@ -137,10 +137,11 @@ const std::string V_INFO("info"); const std::string V_NOTICE("notice"); const std::string V_WARN("warn"); const std::string V_ERROR("error"); -// value: inorder | feedback +// value: inorder | feedback | adaptive const std::string PREF_URI_SELECTOR("uri-selector"); const std::string V_INORDER("inorder"); const std::string V_FEEDBACK("feedback"); +const std::string V_ADAPTIVE("adaptive"); // value: 1*digit const std::string PREF_SERVER_STAT_TIMEOUT("server-stat-timeout"); // value: string that your file system recognizes as a file name. diff -p -up ./src/prefs.h.adaptive ./src/prefs.h --- ./src/prefs.h.adaptive 2008-09-18 14:56:02.000000000 +0200 +++ ./src/prefs.h 2008-10-13 11:43:37.000000000 +0200 @@ -141,10 +141,11 @@ extern const std::string V_INFO; extern const std::string V_NOTICE; extern const std::string V_WARN; extern const std::string V_ERROR; -// value: inorder | feedback +// value: inorder | feedback | adaptive extern const std::string PREF_URI_SELECTOR; extern const std::string V_INORDER; extern const std::string V_FEEDBACK; +extern const std::string V_ADAPTIVE; // value: 1*digit extern const std::string PREF_SERVER_STAT_TIMEOUT; // value: string that your file system recognizes as a file name.