Repository: ytdl-org/youtube-dl

Issue:
[fc2] 403 error
```
youtube-dl http://video.fc2.com/en/content/20130113eqtNRAv5 -v
[debug] System config: []
[debug] User config: ['--age-limit', '17']
[debug] Command-line args: ['http://video.fc2.com/en/content/20130113eqtNRAv5', '-v']
[debug] Encodings: locale UTF-8, fs UTF-8, out UTF-8, pref UTF-8
[debug] youtube-dl version 2014.05.13
[debug] Git HEAD: 1800eee
[debug] Python version 2.7.6 - Linux-3.13-1-amd64-x86_64-with-debian-jessie-sid
[debug] Proxy map: {}
[fc2] 20130113eqtNRAv5: Downloading webpage
[fc2] 20130113eqtNRAv5: Downloading info page
ERROR: Error code: 403; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type  youtube-dl -U  to update.
Traceback (most recent call last):
  File "youtube_dl/YoutubeDL.py", line 516, in extract_info
    ie_result = ie.extract(url)
  File "youtube_dl/extractor/common.py", line 161, in extract
    return self._real_extract(url)
  File "youtube_dl/extractor/fc2.py", line 50, in _real_extract
    raise ExtractorError('Error code: %s' % info['err_code'][0])
ExtractorError: Error code: 403; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type  youtube-dl -U  to update.
```

It looks like the problem is the missing PHPSESSID cookie. We should fix cookie handling in fc2.



Discussion:
Getting same error on some FC2 videos, on Windows with youtube-dl version 2014.06.09.

<pre><code>youtube-dl -u EMAIL -p PASSWORD http://video.fc2.com/a/content/20140519VeQg1FwP -v
[debug] System config: []
[debug] User config: []
[debug] Command-line args: ['-u', '<PRIVATE>', '-p', '<PRIVATE>', 'http://video.fc2.com/a/content/20140519VeQg1FwP', '-v']
[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
[debug] youtube-dl version 2014.06.09
[debug] Python version 2.7.5 - Windows-8-6.2.9200
[debug] Proxy map: {}
[fc2] 20140519VeQg1FwP: Downloading webpage
[fc2] 20140519VeQg1FwP: Downloading info page
ERROR: Error code: 403; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type  youtube-dl -U  to update.
Traceback (most recent call last):
  File "youtube_dl\YoutubeDL.pyo", line 516, in extract_info
  File "youtube_dl\extractor\common.pyo", line 167, in extract
  File "youtube_dl\extractor\fc2.pyo", line 50, in _real_extract
ExtractorError: Error code: 403; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type  youtube-dl -U  to update.</code></pre> 


Resolving patch:
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -7,14 +7,16 @@
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
 )
 
 
 class FC2IE(InfoExtractor):
-    _VALID_URL = r'^http://video\.fc2\.com/((?P<lang>[^/]+)/)?content/(?P<id>[^/]+)'
+    _VALID_URL = r'^http://video\.fc2\.com/((?P<lang>[^/]+)/)?(a/)?content/(?P<id>[^/]+)'
     IE_NAME = 'fc2'
+    _NETRC_MACHINE = 'fc2'
     _TEST = {
         'url': 'http://video.fc2.com/en/content/20121103kUan1KHs',
         'md5': 'a6ebe8ebe0396518689d963774a54eb7',
@@ -25,17 +27,53 @@ class FC2IE(InfoExtractor):
         },
     }
 
+    #def _real_initialize(self):
+    #    self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if (username is None) or (password is None):
+           self.to_screen('unable to log in: will be downloading in non authorized mode') # report_warning
+           return False
+
+        # Log in
+        login_form_strs = {
+            'email':    username,
+            'password': password,
+            'done':     'video',
+            'Submit':   ' Login ',
+        }
+
+        # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
+        # chokes on unicode
+        login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
+        login_data = compat_urllib_parse.urlencode(login_form).encode('utf-8')
+        request    = compat_urllib_request.Request(
+            'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
+
+        login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
+        if 'mode=redirect&login=done' not in login_results:
+            self.to_screen('unable to log in: bad username or password') # report_warning
+            return False
+        
+        # this is also needed
+        login_redir = compat_urllib_request.Request('http://id.fc2.com/?mode=redirect&login=done')
+        redir_res   = self._download_webpage(login_redir, None, note='Login redirect', errnote='Something is not right')
+
+        return True
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
         webpage = self._download_webpage(url, video_id)
         self._downloader.cookiejar.clear_session_cookies()  # must clear
+        self._login()
 
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
-        refer = url.replace('/content/', '/a/content/')
 
+        refer = (url if '/a/content/' in url else url.replace('/content/', '/a/content/'));
         mimi = hashlib.md5((video_id + '_gGddgPfeaf_gzyr').encode('utf-8')).hexdigest()
 
         info_url = (
@@ -47,7 +85,12 @@ def _real_extract(self, url):
         info = compat_urlparse.parse_qs(info_webpage)
 
         if 'err_code' in info:
-            raise ExtractorError('Error code: %s' % info['err_code'][0])
+            #raise ExtractorError('Error code: %s' % info['err_code'][0])
+            # most of the time we can still download wideo even if err_code is 403 or 602
+            self.to_screen('Error code was: %s... but still trying' % info['err_code'][0]) # report_warning
+
+        if 'filepath' not in info:
+            raise ExtractorError('Cannot download file. Are you logged?')
 
         video_url = info['filepath'][0] + '?mid=' + info['mid'][0]
         title_info = info.get('title')


Test patch: