Repository: kmyk/online-judge-tools
Pull request #209 (2018-12-17 12:56:33)

RELEASE: 0.1.46


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index edfedf3..39249e0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,13 @@
 # Change Log
 
-## 0.1.45
+## 0.1.46 / 2018-12-17
+
+-   [#208](https://github.com/kmyk/online-judge-tools/issues/208) features for HackerRank are removed
+-   [#206](https://github.com/kmyk/online-judge-tools/issues/206) support the new `atcoder.jp` domain
+-   [#202](https://github.com/kmyk/online-judge-tools/issues/202) write document
+-   [#201](https://github.com/kmyk/online-judge-tools/issues/201) add `--json` option to `download` command
+
+## 0.1.45 / 2018-11-26
 
 -   [#193](https://github.com/kmyk/online-judge-tools/issues/193) make languages selectable with their IDs with
 -   [#198](https://github.com/kmyk/online-judge-tools/issues/198) suppress leading spaces of samples at Codeforces
diff --git a/docs/onlinejudge.rst b/docs/onlinejudge.rst
index 9e17a4e..c1e4386 100644
--- a/docs/onlinejudge.rst
+++ b/docs/onlinejudge.rst
@@ -59,14 +59,6 @@ onlinejudge\.dispatch module
     :undoc-members:
     :show-inheritance:
 
-onlinejudge\.hackerrank module
-------------------------------
-
-.. automodule:: onlinejudge.hackerrank
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 onlinejudge\.problem module
 ---------------------------
 
diff --git a/onlinejudge/__init__.py b/onlinejudge/__init__.py
index e62d157..d7fbaa8 100644
--- a/onlinejudge/__init__.py
+++ b/onlinejudge/__init__.py
@@ -4,7 +4,6 @@ import onlinejudge.atcoder
 import onlinejudge.yukicoder
 import onlinejudge.anarchygolf
 import onlinejudge.codeforces
-import onlinejudge.hackerrank
 import onlinejudge.aoj
 import onlinejudge.csacademy
 import onlinejudge.topcoder
diff --git a/onlinejudge/atcoder.py b/onlinejudge/atcoder.py
index ad6703b..913b08c 100644
--- a/onlinejudge/atcoder.py
+++ b/onlinejudge/atcoder.py
@@ -162,7 +162,7 @@ class AtCoderProblem(onlinejudge.problem.Problem):
         # example: https://beta.atcoder.jp/contests/abc073/tasks/abc073_a
         m = re.match(r'^/contests/([\w\-_]+)/tasks/([\w\-_]+)$', utils.normpath(result.path))
         if result.scheme in ('', 'http', 'https') \
-                and result.netloc == 'beta.atcoder.jp' \
+                and result.netloc in ('atcoder.jp', 'beta.atcoder.jp') \
                 and m:
             contest_id = m.group(1)
             problem_id = m.group(2)
@@ -307,7 +307,7 @@ class AtCoderSubmission(onlinejudge.submission.Submission):
         # example: https://beta.atcoder.jp/contests/abc073/submissions/1592381
         m = re.match(r'^/contests/([\w\-_]+)/submissions/(\d+)$', utils.normpath(result.path))
         if result.scheme in ('', 'http', 'https') \
-                and result.netloc == 'beta.atcoder.jp' \
+                and result.netloc == ('atcoder.jp', 'beta.atcoder.jp') \
                 and m:
             contest_id = m.group(1)
             try:
diff --git a/onlinejudge/hackerrank.py b/onlinejudge/hackerrank.py
deleted file mode 100644
index 95f653a..0000000
--- a/onlinejudge/hackerrank.py
+++ /dev/null
@@ -1,208 +0,0 @@
-# Python Version: 3.x
-import onlinejudge.service
-import onlinejudge.problem
-from onlinejudge.problem import LabeledString, TestCase
-import onlinejudge.submission
-import onlinejudge.dispatch
-import onlinejudge.implementation.utils as utils
-import onlinejudge.implementation.logging as log
-import re
-import bs4
-import requests
-import urllib.parse
-import posixpath
-import json
-import datetime
-import time
-from typing import *
-
-
-@utils.singleton
-class HackerRankService(onlinejudge.service.Service):
-
-    def login(self, get_credentials: onlinejudge.service.CredentialsProvider, session: Optional[requests.Session] = None) -> bool:
-        session = session or utils.new_default_session()
-        url = 'https://www.hackerrank.com/login'
-        # get
-        resp = utils.request('GET', url, session=session)
-        if resp.url != url:
-            log.info('You have already signed in.')
-            return True
-        # parse
-        soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser)
-        form = soup.find('form', id='legacy-login')
-        if not form:
-            log.error('form not found')
-            return False
-        csrftoken = soup.find('meta', attrs={ 'name': 'csrf-token' }).attrs['content']
-        # post
-        username, password = get_credentials()
-        form = utils.FormSender(form, url=resp.url)
-        form.set('login', username)
-        form.set('password', password)
-        form.set('remember_me', 'true')
-        form.set('fallback', 'true')
-        resp = form.request(session, action='/auth/login', headers={ 'X-CSRF-Token': csrftoken })
-        resp.raise_for_status()
-        # result
-        if resp.url != url:
-            log.success('You signed in.')
-            return True
-        else:
-            log.failure('You failed to sign in. Wrong user ID or password.')
-            return False
-
-    def get_url(self) -> str:
-        return 'https://www.hackerrank.com/'
-
-    def get_name(self) -> str:
-        return 'hackerrank'
-
-    @classmethod
-    def from_url(cls, s: str) -> Optional['HackerRankService']:
-        # example: https://www.hackerrank.com/dashboard
-        result = urllib.parse.urlparse(s)
-        if result.scheme in ('', 'http', 'https') \
-                and result.netloc in ('hackerrank.com', 'www.hackerrank.com'):
-            return cls()
-        return None
-
-
-class HackerRankProblem(onlinejudge.problem.Problem):
-    def __init__(self, contest_slug: str, challenge_slug: str):
-        self.contest_slug = contest_slug
-        self.challenge_slug = challenge_slug
-
-    def download(self, session: Optional[requests.Session] = None, method: str = 'run_code') -> List[TestCase]:
-        if method == 'run_code':
-            return self.download_with_running_code(session=session)
-        elif method == 'parse_html':
-            return self.download_with_parsing_html(session=session)
-        else:
-            raise ValueError
-
-    def download_with_running_code(self, session: Optional[requests.Session] = None) -> List[TestCase]:
-        session = session or utils.new_default_session()
-        # get
-        resp = utils.request('GET', self.get_url(), session=session)
-        # parse
-        soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser)
-        csrftoken = soup.find('meta', attrs={ 'name': 'csrf-token' }).attrs['content']
-        # post
-        url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}/compile_tests'.format(self.contest_slug, self.challenge_slug)
-        payload = { 'code': ':', 'language': 'bash', 'customtestcase': False }
-        log.debug('payload: %s', payload)
-        resp = utils.request('POST', url, session=session, json=payload, headers={ 'X-CSRF-Token': csrftoken })
-        # parse
-        it = json.loads(resp.content.decode())
-        log.debug('json: %s', it)
-        if not it['status']:
-            log.error('Run Code: failed')
-            return []
-        model_id = it['model']['id']
-        now = datetime.datetime.now()
-        unixtime = int(datetime.datetime.now().timestamp() * 10**3)
-        url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}/compile_tests/{}?_={}'.format(self.contest_slug, self.challenge_slug, it['model']['id'], unixtime)
-        # sleep
-        log.status('sleep(3)')
-        time.sleep(3)
-        # get
-        resp = utils.request('GET', url, session=session, headers={ 'X-CSRF-Token': csrftoken })
-        # parse
-        it = json.loads(resp.content.decode())
-        log.debug('json: %s', it)
-        if not it['status']:
-            log.error('Run Code: failed')
-            return []
-        samples: List[TestCase] = []
-        for i, (inf, outf) in enumerate(zip(it['model']['stdin'], it['model']['expected_output'])):
-            inname  = 'Testcase {} Input'.format(i)
-            outname = 'Testcase {} Expected Output'.format(i)
-            samples += [ TestCase(
-                LabeledString( inname, utils.textfile(inf)),
-                LabeledString(outname, utils.textfile(outf)),
-                ) ]
-        return samples
-
-
-    def download_with_parsing_html(self, session: Optional[requests.Session] = None) -> List[TestCase]:
-        session = session or utils.new_default_session()
-        url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}'.format(self.contest_slug, self.challenge_slug)
-        raise NotImplementedError
-
-    def get_url(self) -> str:
-        if self.contest_slug == 'master':
-            return 'https://www.hackerrank.com/challenges/{}'.format(self.challenge_slug)
-        else:
-            return 'https://www.hackerrank.com/contests/{}/challenges/{}'.format(self.contest_slug, self.challenge_slug)
-
-    def get_service(self) -> HackerRankService:
-        return HackerRankService()
-
-    @classmethod
-    def from_url(cls, s: str) -> Optional['HackerRankProblem']:
-        # example: https://www.hackerrank.com/contests/university-codesprint-2/challenges/the-story-of-a-tree
-        # example: https://www.hackerrank.com/challenges/fp-hello-world
-        result = urllib.parse.urlparse(s)
-        if result.scheme in ('', 'http', 'https') \
-                and result.netloc in ('hackerrank.com', 'www.hackerrank.com'):
-            m = re.match(r'^/contests/([0-9A-Za-z-]+)/challenges/([0-9A-Za-z-]+)$', utils.normpath(result.path))
-            if m:
-                return cls(m.group(1), m.group(2))
-            m = re.match(r'^/challenges/([0-9A-Za-z-]+)$', utils.normpath(result.path))
-            if m:
-                return cls('master', m.group(1))
-        return None
-
-    def _get_model(self, session: Optional[requests.Session] = None) -> Dict[str, Any]:
-        session = session or utils.new_default_session()
-        # get
-        url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}'.format(self.contest_slug, self.challenge_slug)
-        resp = utils.request('GET', url, session=session)
-        # parse
-        it = json.loads(resp.content.decode())
-        log.debug('json: %s', it)
-        if not it['status']:
-            log.error('get model: failed')
-            raise onlinejudge.problem.SubmissionError
-        return it['model']
-
-    def get_language_dict(self, session: Optional[requests.Session] = None) -> Dict[str, Dict[str, str]]:
-        session = session or utils.new_default_session()
-        info = self._get_model(session=session)
-        # lang_display_mapping from https://hrcdn.net/hackerrank/assets/codeshell/dist/codeshell-449bb296b091277fedc42b23f7c9c447.js, Sun Feb 19 02:25:36 JST 2017
-        lang_display_mapping = { 'c': 'C', 'cpp': 'C++', 'java': 'Java 7', 'csharp': 'C#', 'haskell': 'Haskell', 'php': 'PHP', 'python': 'Python 2', 'pypy': 'Pypy 2', 'pypy3': 'Pypy 3', 'ruby': 'Ruby', 'perl': 'Perl', 'bash': 'BASH', 'oracle': 'Oracle', 'mysql': 'MySQL', 'sql': 'SQL', 'clojure': 'Clojure', 'scala': 'Scala', 'code': 'Generic', 'text': 'Plain Text', 'brainfuck': 'Brainfuck', 'javascript': 'Javascript', 'typescript': 'Typescript', 'lua': 'Lua', 'sbcl': 'Common Lisp (SBCL)', 'erlang': 'Erlang', 'go': 'Go', 'd': 'D', 'ocaml': 'OCaml', 'pascal': 'Pascal', 'python3': 'Python 3', 'groovy': 'Groovy', 'objectivec': 'Objective-C', 'text_pseudo': 'Plain Text', 'fsharp': 'F#', 'visualbasic': 'VB. NET', 'cobol': 'COBOL', 'tsql': 'MS SQL Server', 'lolcode': 'LOLCODE', 'smalltalk': 'Smalltalk', 'tcl': 'Tcl', 'whitespace': 'Whitespace', 'css': 'CSS', 'html': 'HTML', 'java8': 'Java 8', 'db2': 'DB2', 'octave': 'Octave', 'r': 'R', 'xquery': 'XQuery', 'racket': 'Racket', 'xml': 'XML', 'rust': 'Rust', 'swift': 'Swift', 'elixir': 'Elixir', 'fortran': 'Fortran', 'ada': 'Ada', 'nim': 'Nim', 'julia': 'Julia', 'cpp14': 'C++14', 'coffeescript': 'Coffeescript' }
-        result = {}
-        for lang in info['languages']:
-            descr = lang_display_mapping.get(lang)
-            if descr is None:
-                log.warning('display mapping for language `%s\' not found', lang)
-                descr = lang
-            result[lang] = { 'description': descr }
-        return result
-
-    def submit(self, code: str, language: str, session: Optional[requests.Session] = None) -> onlinejudge.submission.Submission:
-        session = session or utils.new_default_session()
-        # get
-        resp = utils.request('GET', self.get_url(), session=session)
-        # parse
-        soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser)
-        csrftoken = soup.find('meta', attrs={ 'name': 'csrf-token' }).attrs['content']
-        # post
-        url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}/submissions'.format(self.contest_slug, self.challenge_slug)
-        payload = { 'code': code, 'language': language }
-        log.debug('payload: %s', payload)
-        resp = utils.request('POST', url, session=session, json=payload, headers={ 'X-CSRF-Token': csrftoken })
-        # parse
-        it = json.loads(resp.content.decode())
-        log.debug('json: %s', it)
-        if not it['status']:
-            log.failure('Submit Code: failed')
-            raise onlinejudge.problem.SubmissionError
-        model_id = it['model']['id']
-        url = self.get_url().rstrip('/') + '/submissions/code/{}'.format(model_id)
-        log.success('success: result: %s', url)
-        return onlinejudge.submission.CompatibilitySubmission(url, problem=self)
-
-onlinejudge.dispatch.services += [ HackerRankService ]
-onlinejudge.dispatch.problems += [ HackerRankProblem ]
diff --git a/onlinejudge/implementation/command/download.py b/onlinejudge/implementation/command/download.py
index 60202c0..713236f 100644
--- a/onlinejudge/implementation/command/download.py
+++ b/onlinejudge/implementation/command/download.py
@@ -1,14 +1,24 @@
 # Python Version: 3.x
 import onlinejudge
+import onlinejudge.problem
 import onlinejudge.implementation.utils as utils
 import onlinejudge.implementation.logging as log
 import os
 import colorama
 import sys
+import json
 from typing import *
 if TYPE_CHECKING:
     import argparse
 
+def convert_sample_to_dict(sample: onlinejudge.problem.TestCase) -> dict:
+    data = {}
+    data["input"] = sample.input.data
+    data["output"] = sample.output.data
+    if sample.input.name == sample.output.name:
+        data["name"] = sample.input.name
+    return data
+
 def download(args: 'argparse.Namespace') -> None:
     # prepare values
     problem = onlinejudge.dispatch.problem_from_url(args.url)
@@ -62,3 +72,7 @@ def download(args: 'argparse.Namespace') -> None:
             with open(path, 'w') as fh:
                 fh.write(data)
             log.success('saved to: %s', path)
+
+    # print json
+    if args.json:
+        print(json.dumps(list(map(convert_sample_to_dict, samples))))
diff --git a/onlinejudge/implementation/main.py b/onlinejudge/implementation/main.py
index 4e39425..ae2a4d6 100644
--- a/onlinejudge/implementation/main.py
+++ b/onlinejudge/implementation/main.py
@@ -44,10 +44,11 @@ supported services:
   Aizu Online Judge
   AtCoder
   Codeforces
-  HackerRank
   Yukicoder
   CS Academy
 
+  (HackerRank has been removed)
+
 supported services with --system:
   Aizu Online Judge
   Yukicoder
@@ -66,6 +67,7 @@ format string for --format:
     subparser.add_argument('--overwrite', action='store_true')
     subparser.add_argument('-n', '--dry-run', action='store_true', help='don\'t write to files')
     subparser.add_argument('-a', '--system', action='store_true', help='download system testcases')
+    subparser.add_argument('--json', action='store_true')
 
     # login
     subparser = subparsers.add_parser('login',
@@ -76,10 +78,11 @@ format string for --format:
 supported services:
   AtCoder
   Codeforces
-  HackerRank
   Yukicoder
   TopCoder
 
+  (HackerRank has been removed)
+
 strings for --method:
   github                for yukicoder, login via github (default)
   twitter               for yukicoder, login via twitter (not implementated yet)
@@ -157,9 +160,9 @@ tips:
             epilog='''\
 supported services:
   AtCoder
-  HackerRank
 
   (Yukicoder has been removed)
+  (HackerRank has been removed)
 
 example:
   http://agc001.contest.atcoder.jp/tasks/agc001_a
diff --git a/onlinejudge/implementation/version.py b/onlinejudge/implementation/version.py
index cce4d25..0a82179 100644
--- a/onlinejudge/implementation/version.py
+++ b/onlinejudge/implementation/version.py
@@ -4,5 +4,5 @@ __author__  = 'Kimiyuki Onaka'
 __email__ = 'kimiyuki95@gmail.com'
 __license__ = 'MIT License'
 __url__ = 'https://github.com/kmyk/online-judge-tools'
-__version_info__ = ( 0, 1, 45, 'final', 0 )
+__version_info__ = ( 0, 1, 46, 'final', 0 )
 __version__ = '.'.join(map(str, __version_info__[: 3]))
diff --git a/onlinejudge/yukicoder.py b/onlinejudge/yukicoder.py
index ff3fb33..2dd75f0 100644
--- a/onlinejudge/yukicoder.py
+++ b/onlinejudge/yukicoder.py
@@ -275,7 +275,6 @@ class YukicoderProblem(onlinejudge.problem.Problem):
                 name = basename
                 if os.path.splitext(name)[1] == '.in':  # ".in" extension is confusing
                     name = os.path.splitext(name)[0]
-                print(filename, name)  # TODO: what is this?
                 basenames[basename][kind] = LabeledString(name, content)
         samples: List[TestCase] = []
         for basename in sorted(basenames.keys()):
diff --git a/readme.md b/readme.md
index bb6584a..720954e 100644
--- a/readme.md
+++ b/readme.md
@@ -20,7 +20,7 @@ Tools for online judge services. Downloading sample cases, Testing/Submitting yo
     -   Yukicoder
     -   Anarchy Golf
     -   Codeforces
-    -   HackerRank (using `Run Code`)
+    -   ~~HackerRank~~ (removed)
     -   Aizu Online Judge
     -   CS Academy
 -   Download system test cases
@@ -30,12 +30,12 @@ Tools for online judge services. Downloading sample cases, Testing/Submitting yo
     -   AtCoder
     -   Yukicoder (via github.com)
     -   Codeforces
-    -   HackerRank
+    -   ~~HackerRank~~ (removed)
     -   TopCoder
 -   Submit your solution
     -   AtCoder
     -   ~~Yukicoder~~ (removed)
-    -   HackerRank
+    -   ~~HackerRank~~ (removed)
     -   TopCoder (Marathon Match)
 -   Generate scanner for input  (experimental)
     -   AtCoder
@@ -126,17 +126,6 @@ $ oj download http://agc001.contest.atcoder.jp/tasks/agc001_a
 [+] saved to: test/sample-4.out
 ```
 
-## FAQ
-
--   I cannot install this tool on my Windows machine. How should I do?
-    -   Use Windows Subsystem for Linux (WSL). If your Windows is too and WSL is not supported, you can use virtual machines, MinGW or Cygwin, but you should buy a new machine.
--   Are there features to manage templates or snippets?
-    -   No. They are not the responsibility of this tool. You should use plugins of your editor, like [thinca/vim-template](https://github.com/thinca/vim-template) or [Shougo/deoplete.nvim](https://github.com/Shougo/deoplete.nvim).
--   I usually make one directory per one contest (or, site). Is there a support for this style?
-    -   Yes. You can use `--directory` (`-d`) option. However, I recommend to make one directory per one problem.
--   Can I specify problems by their IDs or names, instead of URLs?
-    -   No. I have tried it once, but it is actually not so convenient and only increases the maintenance cost.
-
 ## Related Tools
 
 conflicted:


Test patch:
diff --git a/tests/download.py b/tests/download.py
index b8997c8..324aca6 100644
--- a/tests/download.py
+++ b/tests/download.py
@@ -8,7 +8,22 @@ import subprocess
 import sys
 import tempfile
 
-def snippet_call_download(self, url, files, is_system=False):
+def get_files_from_json(samples):
+    files = {}
+    for i, sample in enumerate(samples):
+        for ext in ('in', 'out'):
+            if 'name' in sample:
+                name = sample['name'] + '.' + ext
+            else:
+                name = 'sample-{}.{}'.format(i + 1, ext)
+            files[name] = hashlib.md5(sample[ext + 'put'].encode()).hexdigest()
+    return files
+
+def snippet_call_download(self, url, files, is_system=False, type='files'):
+    assert type in 'files' or 'json'
+    if type == 'json':
+        files = get_files_from_json(files)
+
     cwd = os.getcwd()
     ojtools = os.path.join( cwd, 'oj' )
     try:
@@ -35,18 +50,6 @@ class DownloadOthersTest(unittest.TestCase):
     def snippet_call_download(self, *args, **kwargs):
         snippet_call_download(self, *args, **kwargs)
 
-    def test_call_download_hackerrank_beautiful_array(self):
-        if 'CI' in os.environ:
-            print('NOTE: this test is skipped since login is required')
-            return
-        self.snippet_call_download(
-            'https://www.hackerrank.com/contests/hourrank-1/challenges/beautiful-array', {
-                'sample-1.in':  'fb3f7e56dac548ce73f9d8e485e5336b',
-                'sample-2.out': '897316929176464ebc9ad085f31e7284',
-                'sample-2.in':  '6047a07c8defde4d696513d26e871b20',
-                'sample-1.out': '6d7fce9fee471194aa8b5b6e47267f03',
-            })
-
     def test_call_download_anarchygolf_the_b_programming_language(self):
         self.snippet_call_download(
             'http://golf.shinh.org/p.rb?The+B+Programming+Language', {
diff --git a/tests/download_atcoder.py b/tests/download_atcoder.py
index 9a57339..e4f87bb 100644
--- a/tests/download_atcoder.py
+++ b/tests/download_atcoder.py
@@ -80,3 +80,17 @@ class DownloadAtCoderTest(unittest.TestCase):
                 'sample-3.in':  'ed5d34c74e59d16bd6d5b3683db655c3',
                 'sample-3.out': 'e14b420b7266f69a2b2b457f3bbec804',
             })
+    def test_call_download_atcoder_agc029_a(self):
+        self.snippet_call_download(
+            'https://atcoder.jp/contests/agc029/tasks/agc029_a', {
+                'sample-1.in':  '13152b3514c161917e9a771d51b77699',
+                'sample-1.out': '26ab0db90d72e28ad0ba1e22ee510510',
+                'sample-2.in':  '5bdcc020d84b69530288143e38b643b9',
+                'sample-2.out': '9ae0ea9e3c9c6e1b9b6252c8395efdc1',
+            })
+    def test_call_download_atcoder_abc114_c(self):
+        self.snippet_call_download('https://atcoder.jp/contests/abc114/tasks/abc114_c', [
+                {"input": "575\n", "output": "4\n"},
+                {"input": "3600\n", "output": "13\n"},
+                {"input": "999999999\n", "output": "26484\n"},
+            ], type='json')