Repository: ytdl-org/youtube-dl

Issue:
rate limit exceeds
youtube-dl -o '%(autonumber)s-%(title)s.%(ext)s' --extract-audio -a youtube.txt  --no-playlist -r "50K"
I ran this command and I saw following

[download]  74.5% of 5.07MiB at 94.41KiB/s ETA 00:14S

when I've impose a data rate limit of 50K and it should have obeyed that.
But still it's downloading at almost double rate.

Error When downloading 
Command

youtube-dl -i -w -c -r 100k -f mp4 -v -o "%%(uploader)s\%%(playlist)s\%%(playlist_index)s-%%(title)s-%%(id)s.%%(ext)s" http://www.youtube.com/playlist?list=UUQJ6ESCWQotBwtJm0Ff_gyQ

Log

[download] Downloading video #82 of 885
[youtube] 7EwyLLlLhWU: Downloading video webpage
ERROR: Unable to download video webpage: [Errno 10054] An existing connection wa
s forcibly closed by the remote host; please report this issue on https://yt-dl.
org/bug . Be sure to call youtube-dl with the --verbose flag and include its com
plete output. Make sure you are using the latest version; type  youtube-dl -U  t
o update.
Traceback (most recent call last):
  File "youtube_dl\extractor\youtube.pyo", line 1265, in _real_extract
  File "urllib2.pyo", line 127, in urlopen
  File "urllib2.pyo", line 404, in open
  File "urllib2.pyo", line 422, in _open
  File "urllib2.pyo", line 382, in _call_chain
  File "youtube_dl\utils.pyo", line 561, in https_open
  File "urllib2.pyo", line 1187, in do_open
  File "httplib.pyo", line 1045, in getresponse
  File "httplib.pyo", line 409, in begin
  File "httplib.pyo", line 365, in _read_status
  File "socket.pyo", line 476, in readline
  File "ssl.pyo", line 241, in recv
  File "ssl.pyo", line 160, in read
error: [Errno 10054] An existing connection was forcibly closed by the remote ho
st
Traceback (most recent call last):
  File "youtube_dl\YoutubeDL.pyo", line 429, in extract_info
  File "youtube_dl\extractor\common.pyo", line 131, in extract
  File "youtube_dl\extractor\youtube.pyo", line 1267, in _real_extract
ExtractorError: Unable to download video webpage: [Errno 10054] An existing conn
ection was forcibly closed by the remote host; please report this issue on https
://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and includ
e its complete output. Make sure you are using the latest version; type  youtube
-dl -U  to update.

[download] Downloading video #83 of 885



Discussion:
See #595, for long downloads it should finally reach the target speed.
Thanks for the report



Resolving patch:
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -77,8 +77,10 @@ def format_percent(percent):
     def calc_eta(start, now, total, current):
         if total is None:
             return None
+        if now is None:
+            now = time.time()
         dif = now - start
-        if current == 0 or dif < 0.001: # One millisecond
+        if current == 0 or dif < 0.001:  # One millisecond
             return None
         rate = float(current) / dif
         return int((float(total) - float(current)) / rate)
@@ -92,7 +94,7 @@ def format_eta(eta):
     @staticmethod
     def calc_speed(start, now, bytes):
         dif = now - start
-        if bytes == 0 or dif < 0.001: # One millisecond
+        if bytes == 0 or dif < 0.001:  # One millisecond
             return None
         return float(bytes) / dif
 
@@ -105,7 +107,7 @@ def format_speed(speed):
     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)
-        new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
+        new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
         if elapsed_time < 0.001:
             return int(new_max)
         rate = bytes / elapsed_time
@@ -143,18 +145,19 @@ def report_warning(self, *args, **kargs):
     def report_error(self, *args, **kargs):
         self.ydl.report_error(*args, **kargs)
 
-    def slow_down(self, start_time, byte_counter):
+    def slow_down(self, start_time, now, byte_counter):
         """Sleep if the download speed is over the rate limit."""
         rate_limit = self.params.get('ratelimit', None)
         if rate_limit is None or byte_counter == 0:
             return
-        now = time.time()
+        if now is None:
+            now = time.time()
         elapsed = now - start_time
         if elapsed <= 0.0:
             return
         speed = float(byte_counter) / elapsed
         if speed > rate_limit:
-            time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
+            time.sleep(max((byte_counter / rate_limit) - elapsed, 0))
 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -128,16 +128,21 @@ def real_download(self, filename, info_dict):
         byte_counter = 0 + resume_len
         block_size = self.params.get('buffersize', 1024)
         start = time.time()
+
+        # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
+        now = None  # needed for slow_down() in the first loop run
+        before = start  # start measuring
         while True:
+
             # Download and write
-            before = time.time()
             data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
-            after = time.time()
+            byte_counter += len(data_block)
+
+            # exit loop when download is finished
             if len(data_block) == 0:
                 break
-            byte_counter += len(data_block)
 
-            # Open file just in time
+            # Open destination file just in time
             if stream is None:
                 try:
                     (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
@@ -153,11 +158,22 @@ def real_download(self, filename, info_dict):
                 self.to_stderr(u"\n")
                 self.report_error(u'unable to write data: %s' % str(err))
                 return False
+
+            # Apply rate limit
+            self.slow_down(start, now, byte_counter - resume_len)
+
+            # end measuring of one loop run
+            now = time.time()
+            after = now
+
+            # Adjust block size
             if not self.params.get('noresizebuffer', False):
                 block_size = self.best_block_size(after - before, len(data_block))
 
+            before = after
+
             # Progress message
-            speed = self.calc_speed(start, time.time(), byte_counter - resume_len)
+            speed = self.calc_speed(start, now, byte_counter - resume_len)
             if data_len is None:
                 eta = percent = None
             else:
@@ -178,9 +194,6 @@ def real_download(self, filename, info_dict):
             if is_test and byte_counter == data_len:
                 break
 
-            # Apply rate limit
-            self.slow_down(start, byte_counter - resume_len)
-
         if stream is None:
             self.to_stderr(u"\n")
             self.report_error(u'Did not get any data blocks')


Test patch: