def fetch(self, category=CATEGORY_QUESTION, offset=DEFAULT_OFFSET):

if not offset:<EOL><INDENT>offset = DEFAULT_OFFSET<EOL><DEDENT>kwargs = {"<STR_LIT>": offset}<EOL>items = super().fetch(category, **kwargs)<EOL>return items<EOL>

Fetch questions from the Kitsune url.

        :param category: the category of items to fetch
        :offset: obtain questions after offset
        :returns: a generator of questions


def fetch_items(self, category, **kwargs):

offset = kwargs['<STR_LIT>']<EOL>logger.info("<STR_LIT>",<EOL>self.url, str(offset))<EOL>nquestions = <NUM_LIT:0>  <EOL>tquestions = <NUM_LIT:0>  <EOL>equestions = <NUM_LIT:0>  <EOL>page = int(offset / KitsuneClient.ITEMS_PER_PAGE)<EOL>page_offset = page * KitsuneClient.ITEMS_PER_PAGE<EOL>drop_questions = offset - page_offset<EOL>current_offset = offset<EOL>questions_page = self.client.get_questions(offset)<EOL>while True:<EOL><INDENT>try:<EOL><INDENT>raw_questions = next(questions_page)<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>except requests.exceptions.HTTPError as e:<EOL><INDENT>if e.response.status_code == <NUM_LIT>:<EOL><INDENT>logger.exception(e)<EOL>logger.error("<STR_LIT>"<EOL>"<STR_LIT>",<EOL>KitsuneClient.ITEMS_PER_PAGE)<EOL>equestions += KitsuneClient.ITEMS_PER_PAGE<EOL>current_offset += KitsuneClient.ITEMS_PER_PAGE<EOL>questions_page = self.client.get_questions(current_offset)<EOL>continue<EOL><DEDENT>else:<EOL><INDENT>raise e<EOL><DEDENT><DEDENT>try:<EOL><INDENT>questions_data = json.loads(raw_questions)<EOL>tquestions = questions_data['<STR_LIT:count>']<EOL>questions = questions_data['<STR_LIT>']<EOL><DEDENT>except (ValueError, KeyError) as ex:<EOL><INDENT>logger.error(ex)<EOL>cause = ("<STR_LIT>" % (raw_questions))<EOL>raise ParseError(cause=cause)<EOL><DEDENT>for question in questions:<EOL><INDENT>if drop_questions > <NUM_LIT:0>:<EOL><INDENT>drop_questions -= <NUM_LIT:1><EOL>continue<EOL><DEDENT>question['<STR_LIT>'] = current_offset<EOL>current_offset += <NUM_LIT:1><EOL>question['<STR_LIT>'] = []<EOL>for raw_answers in self.client.get_question_answers(question['<STR_LIT:id>']):<EOL><INDENT>answers = json.loads(raw_answers)['<STR_LIT>']<EOL>question['<STR_LIT>'] += answers<EOL><DEDENT>yield question<EOL>nquestions += <NUM_LIT:1><EOL><DEDENT>logger.debug("<STR_LIT>", nquestions + offset, tquestions)<EOL><DEDENT>logger.info("<STR_LIT>", nquestions, tquestions)<EOL>logger.info("<STR_LIT>", equestions)<EOL>

Fetch questions from the Kitsune url

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items


def metadata(self, item, filter_classified=False):

item = super().metadata(item, filter_classified=filter_classified)<EOL>item['<STR_LIT>'] = item['<STR_LIT:data>'].pop('<STR_LIT>')<EOL>return item<EOL>

Kitsune metadata.

        This method takes items overrides `metadata` method to add extra
        information related to Kitsune (offset of the question).

        :param item: an item fetched by a backend
        :param filter_classified: sets if classified fields were filtered


return True<EOL>

Returns whether it supports archiving items on the fetch process.

        :returns: this backend supports items archive


return True<EOL>

Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming


return str(item['<STR_LIT:id>'])<EOL>

Extracts the identifier from a Kitsune item.


return float(str_to_datetime(item['<STR_LIT>']).timestamp())<EOL>

Extracts the update time from a Kitsune item.

        The timestamp is extracted from 'updated' field.
        This date is a UNIX timestamp but needs to be converted to
        a float value.

        :param item: item generated by the backend

        :returns: a UNIX timestamp


return CATEGORY_QUESTION<EOL>

Extracts the category from a Kitsune item.

        This backend only generates one type of item which is
        'question'.


return KitsuneClient(self.url, self.archive, from_archive)<EOL>


page = KitsuneClient.FIRST_PAGE<EOL>if offset:<EOL><INDENT>page += int(offset / KitsuneClient.ITEMS_PER_PAGE)<EOL><DEDENT>while True:<EOL><INDENT>api_questions_url = urijoin(self.base_url, '<STR_LIT>') + '<STR_LIT:/>'<EOL>params = {<EOL>"<STR_LIT>": page,<EOL>"<STR_LIT>": "<STR_LIT>"<EOL>}<EOL>questions = self.fetch(api_questions_url, params)<EOL>yield questions<EOL>questions_json = json.loads(questions)<EOL>next_uri = questions_json['<STR_LIT>']<EOL>if not next_uri:<EOL><INDENT>break<EOL><DEDENT>page += <NUM_LIT:1><EOL><DEDENT>

Retrieve questions from older to newer updated starting offset


page = KitsuneClient.FIRST_PAGE<EOL>while True:<EOL><INDENT>api_answers_url = urijoin(self.base_url, '<STR_LIT>') + '<STR_LIT:/>'<EOL>params = {<EOL>"<STR_LIT>": page,<EOL>"<STR_LIT>": question_id,<EOL>"<STR_LIT>": "<STR_LIT>"<EOL>}<EOL>answers_raw = self.fetch(api_answers_url, params)<EOL>yield answers_raw<EOL>answers = json.loads(answers_raw)<EOL>if not answers['<STR_LIT>']:<EOL><INDENT>break<EOL><DEDENT>page += <NUM_LIT:1><EOL><DEDENT>

Retrieve all answers for a question from older to newer (updated)


logger.debug("<STR_LIT>",<EOL>url, str(params))<EOL>response = super().fetch(url, payload=params)<EOL>return response.text<EOL>

Return the textual content associated to the Response object


parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,<EOL>offset=True,<EOL>archive=True)<EOL>parser.parser.add_argument('<STR_LIT:url>', nargs='<STR_LIT:?>',<EOL>default="<STR_LIT>",<EOL>help="<STR_LIT>")<EOL>return parser<EOL>


logger = logging.getLogger(__name__)<EOL>logger.exception(exc)<EOL>

Handle an internal exception that was caught and suppressed.

        :param exc: Exception to process.
        :type exc: Exception


def _make_response(self, body='<STR_LIT>', headers=None, status_code=<NUM_LIT:200>):

res = Response()<EOL>res.status_code = status_code<EOL>if headers is not None:<EOL><INDENT>res.headers.update(headers)<EOL><DEDENT>res.raw = StringIO(body)<EOL>return res<EOL>

Return a response object from the given parameters.

        :param body: Buffer/string containing the response body.
        :type body: str
        :param headers: Dict of headers to include in the requests.
        :type headers: dict
        :param status_code: HTTP status code.
        :type status_code: int
        :rtype: requests.Response


def _make_redirect_error_response(self, redirect_uri, err):

params = {<EOL>'<STR_LIT:error>': err,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None<EOL>}<EOL>redirect = utils.build_url(redirect_uri, params)<EOL>return self._make_response(headers={'<STR_LIT>': redirect},<EOL>status_code=<NUM_LIT>)<EOL>

Return a HTTP 302 redirect response object containing the error.

        :param redirect_uri: Client redirect URI.
        :type redirect_uri: str
        :param err: OAuth error message.
        :type err: str
        :rtype: requests.Response


def _make_json_response(self, data, headers=None, status_code=<NUM_LIT:200>):

response_headers = {}<EOL>if headers is not None:<EOL><INDENT>response_headers.update(headers)<EOL><DEDENT>response_headers['<STR_LIT:Content-Type>'] = '<STR_LIT>'<EOL>response_headers['<STR_LIT>'] = '<STR_LIT>'<EOL>response_headers['<STR_LIT>'] = '<STR_LIT>'<EOL>return self._make_response(json.dumps(data),<EOL>response_headers,<EOL>status_code)<EOL>

Return a response object from the given JSON data.

        :param data: Data to JSON-encode.
        :type data: mixed
        :param headers: Dict of headers to include in the requests.
        :type headers: dict
        :param status_code: HTTP status code.
        :type status_code: int
        :rtype: requests.Response


return self._make_json_response({'<STR_LIT:error>': err}, status_code=<NUM_LIT>)<EOL>

Return a JSON-encoded response object representing the error.

        :param err: OAuth error message.
        :type err: str
        :rtype: requests.Response


return self._make_json_error_response('<STR_LIT>')<EOL>

What to return when the redirect_uri parameter is missing.

        :rtype: requests.Response


return <NUM_LIT><EOL>

Property method to get the length used to generate tokens.

        :rtype: int


return '<STR_LIT>'<EOL>

Property method to get the access token type.

        :rtype: str


return <NUM_LIT><EOL>

Property method to get the token expiration time in seconds.

        :rtype: int


return utils.random_ascii_string(self.token_length)<EOL>

Generate a random authorization code.

        :rtype: str


Generate a random access token.

        :rtype: str


Generate a random refresh token.

        :rtype: str


<EOL>if response_type != '<STR_LIT:code>':<EOL><INDENT>err = '<STR_LIT>'<EOL>return self._make_redirect_error_response(redirect_uri, err)<EOL><DEDENT>is_valid_redirect_uri = self.validate_redirect_uri(client_id,<EOL>redirect_uri)<EOL>if not is_valid_redirect_uri:<EOL><INDENT>return self._invalid_redirect_uri_response()<EOL><DEDENT>is_valid_client_id = self.validate_client_id(client_id)<EOL>is_valid_access = self.validate_access()<EOL>scope = params.get('<STR_LIT>', '<STR_LIT>')<EOL>is_valid_scope = self.validate_scope(client_id, scope)<EOL>if not is_valid_client_id:<EOL><INDENT>err = '<STR_LIT>'<EOL>return self._make_redirect_error_response(redirect_uri, err)<EOL><DEDENT>if not is_valid_access:<EOL><INDENT>err = '<STR_LIT>'<EOL>return self._make_redirect_error_response(redirect_uri, err)<EOL><DEDENT>if not is_valid_scope:<EOL><INDENT>err = '<STR_LIT>'<EOL>return self._make_redirect_error_response(redirect_uri, err)<EOL><DEDENT>code = self.generate_authorization_code()<EOL>self.persist_authorization_code(client_id=client_id,<EOL>code=code,<EOL>scope=scope)<EOL>params.update({<EOL>'<STR_LIT:code>': code,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None<EOL>})<EOL>redirect = utils.build_url(redirect_uri, params)<EOL>return self._make_response(headers={'<STR_LIT>': redirect},<EOL>status_code=<NUM_LIT>)<EOL>

Generate authorization code HTTP response.

        :param response_type: Desired response type. Must be exactly "code".
        :type response_type: str
        :param client_id: Client ID.
        :type client_id: str
        :param redirect_uri: Client redirect URI.
        :type redirect_uri: str
        :rtype: requests.Response


<EOL>if grant_type != '<STR_LIT>':<EOL><INDENT>return self._make_json_error_response('<STR_LIT>')<EOL><DEDENT>is_valid_client_id = self.validate_client_id(client_id)<EOL>is_valid_client_secret = self.validate_client_secret(client_id,<EOL>client_secret)<EOL>scope = params.get('<STR_LIT>', '<STR_LIT>')<EOL>is_valid_scope = self.validate_scope(client_id, scope)<EOL>data = self.from_refresh_token(client_id, refresh_token, scope)<EOL>is_valid_refresh_token = data is not None<EOL>if not (is_valid_client_id and is_valid_client_secret):<EOL><INDENT>return self._make_json_error_response('<STR_LIT>')<EOL><DEDENT>if not is_valid_scope:<EOL><INDENT>return self._make_json_error_response('<STR_LIT>')<EOL><DEDENT>if not is_valid_refresh_token:<EOL><INDENT>return self._make_json_error_response('<STR_LIT>')<EOL><DEDENT>self.discard_refresh_token(client_id, refresh_token)<EOL>access_token = self.generate_access_token()<EOL>token_type = self.token_type<EOL>expires_in = self.token_expires_in<EOL>refresh_token = self.generate_refresh_token()<EOL>self.persist_token_information(client_id=client_id,<EOL>scope=scope,<EOL>access_token=access_token,<EOL>token_type=token_type,<EOL>expires_in=expires_in,<EOL>refresh_token=refresh_token,<EOL>data=data)<EOL>return self._make_json_response({<EOL>'<STR_LIT>': access_token,<EOL>'<STR_LIT>': token_type,<EOL>'<STR_LIT>': expires_in,<EOL>'<STR_LIT>': refresh_token<EOL>})<EOL>

Generate access token HTTP response from a refresh token.

        :param grant_type: Desired grant type. Must be "refresh_token".
        :type grant_type: str
        :param client_id: Client ID.
        :type client_id: str
        :param client_secret: Client secret.
        :type client_secret: str
        :param refresh_token: Refresh token.
        :type refresh_token: str
        :rtype: requests.Response


<EOL>if grant_type != '<STR_LIT>':<EOL><INDENT>return self._make_json_error_response('<STR_LIT>')<EOL><DEDENT>is_valid_client_id = self.validate_client_id(client_id)<EOL>is_valid_client_secret = self.validate_client_secret(client_id,<EOL>client_secret)<EOL>is_valid_redirect_uri = self.validate_redirect_uri(client_id,<EOL>redirect_uri)<EOL>scope = params.get('<STR_LIT>', '<STR_LIT>')<EOL>is_valid_scope = self.validate_scope(client_id, scope)<EOL>data = self.from_authorization_code(client_id, code, scope)<EOL>is_valid_grant = data is not None<EOL>if not (is_valid_client_id and is_valid_client_secret):<EOL><INDENT>return self._make_json_error_response('<STR_LIT>')<EOL><DEDENT>if not is_valid_grant or not is_valid_redirect_uri:<EOL><INDENT>return self._make_json_error_response('<STR_LIT>')<EOL><DEDENT>if not is_valid_scope:<EOL><INDENT>return self._make_json_error_response('<STR_LIT>')<EOL><DEDENT>self.discard_authorization_code(client_id, code)<EOL>access_token = self.generate_access_token()<EOL>token_type = self.token_type<EOL>expires_in = self.token_expires_in<EOL>refresh_token = self.generate_refresh_token()<EOL>self.persist_token_information(client_id=client_id,<EOL>scope=scope,<EOL>access_token=access_token,<EOL>token_type=token_type,<EOL>expires_in=expires_in,<EOL>refresh_token=refresh_token,<EOL>data=data)<EOL>return self._make_json_response({<EOL>'<STR_LIT>': access_token,<EOL>'<STR_LIT>': token_type,<EOL>'<STR_LIT>': expires_in,<EOL>'<STR_LIT>': refresh_token<EOL>})<EOL>

Generate access token HTTP response.

        :param grant_type: Desired grant type. Must be "authorization_code".
        :type grant_type: str
        :param client_id: Client ID.
        :type client_id: str
        :param client_secret: Client secret.
        :type client_secret: str
        :param redirect_uri: Client redirect URI.
        :type redirect_uri: str
        :param code: Authorization code.
        :type code: str
        :rtype: requests.Response


params = utils.url_query_params(uri)<EOL>try:<EOL><INDENT>if '<STR_LIT>' not in params:<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>if '<STR_LIT>' not in params:<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>if '<STR_LIT>' not in params:<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>return self.get_authorization_code(**params)<EOL><DEDENT>except TypeError as exc:<EOL><INDENT>self._handle_exception(exc)<EOL>err = '<STR_LIT>'<EOL>if '<STR_LIT>' in params:<EOL><INDENT>u = params['<STR_LIT>']<EOL>return self._make_redirect_error_response(u, err)<EOL><DEDENT>else:<EOL><INDENT>return self._invalid_redirect_uri_response()<EOL><DEDENT><DEDENT>except StandardError as exc:<EOL><INDENT>self._handle_exception(exc)<EOL>err = '<STR_LIT>'<EOL>u = params['<STR_LIT>']<EOL>return self._make_redirect_error_response(u, err)<EOL><DEDENT>

Get authorization code response from a URI. This method will
        ignore the domain and path of the request, instead
        automatically parsing the query string parameters.

        :param uri: URI to parse for authorization information.
        :type uri: str
        :rtype: requests.Response


try:<EOL><INDENT>for x in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if not data.get(x):<EOL><INDENT>raise TypeError("<STR_LIT>".format(x))<EOL><DEDENT><DEDENT>if '<STR_LIT>' in data:<EOL><INDENT>return self.refresh_token(**data)<EOL><DEDENT>for x in ['<STR_LIT>', '<STR_LIT:code>']:<EOL><INDENT>if not data.get(x):<EOL><INDENT>raise TypeError("<STR_LIT>".format(x))            <EOL><DEDENT><DEDENT>return self.get_token(**data)<EOL><DEDENT>except TypeError as exc:<EOL><INDENT>self._handle_exception(exc)<EOL>return self._make_json_error_response('<STR_LIT>')<EOL><DEDENT>except StandardError as exc:<EOL><INDENT>self._handle_exception(exc)<EOL>return self._make_json_error_response('<STR_LIT>')<EOL><DEDENT>

Get a token response from POST data.

        :param data: POST data containing authorization information.
        :type data: dict
        :rtype: requests.Response


auth = self.authorization_class()<EOL>header = self.get_authorization_header()<EOL>if not header or not header.split:<EOL><INDENT>return auth<EOL><DEDENT>header = header.split()<EOL>if len(header) > <NUM_LIT:1> and header[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>auth.is_oauth = True<EOL>access_token = header[<NUM_LIT:1>]<EOL>self.validate_access_token(access_token, auth)<EOL>if not auth.is_valid:<EOL><INDENT>auth.error = '<STR_LIT>'<EOL><DEDENT><DEDENT>return auth<EOL>

Get authorization object representing status of authentication.


def __init__(self, client_id, client_secret, redirect_uri,authorization_uri, token_uri):

self.client_id = client_id<EOL>self.client_secret = client_secret<EOL>self.redirect_uri = redirect_uri<EOL>self.authorization_uri = authorization_uri<EOL>self.token_uri = token_uri<EOL>

Constructor for OAuth 2.0 Client.

        :param client_id: Client ID.
        :type client_id: str
        :param client_secret: Client secret.
        :type client_secret: str
        :param redirect_uri: Client redirect URI: handle provider response.
        :type redirect_uri: str
        :param authorization_uri: Provider authorization URI.
        :type authorization_uri: str
        :param token_uri: Provider token URI.
        :type token_uri: str


if not url.startswith('<STR_LIT>'):<EOL><INDENT>raise ValueError('<STR_LIT>' % url)<EOL><DEDENT>return requests.post(url, data, verify=True)<EOL>

POST to URL and get result as a response object.

        :param url: URL to POST.
        :type url: str
        :param data: Data to send in the form body.
        :type data: str
        :rtype: requests.Response


if '<STR_LIT>' not in params:<EOL><INDENT>params['<STR_LIT>'] = self.default_response_type<EOL><DEDENT>params.update({'<STR_LIT>': self.client_id,<EOL>'<STR_LIT>': self.redirect_uri})<EOL>return utils.build_url(self.authorization_uri, params)<EOL>

Construct a full URL that can be used to obtain an authorization
        code from the provider authorization_uri. Use this URI in a client
        frame to cause the provider to generate an authorization code.

        :rtype: str


params['<STR_LIT:code>'] = code<EOL>if '<STR_LIT>' not in params:<EOL><INDENT>params['<STR_LIT>'] = self.default_grant_type<EOL><DEDENT>params.update({'<STR_LIT>': self.client_id,<EOL>'<STR_LIT>': self.client_secret,<EOL>'<STR_LIT>': self.redirect_uri})<EOL>response = self.http_post(self.token_uri, params)<EOL>try:<EOL><INDENT>return response.json()<EOL><DEDENT>except TypeError:<EOL><INDENT>return response.json<EOL><DEDENT>

Get an access token from the provider token URI.

        :param code: Authorization code.
        :type code: str
        :return: Dict containing access token, refresh token, etc.
        :rtype: dict


return dict(urlparse.parse_qsl(urlparse.urlparse(url).query, True))<EOL>

Return query parameters as a dict from the specified URL.

    :param url: URL.
    :type url: str
    :rtype: dict


url = urlparse.urlparse(url)<EOL>return urlparse.urlunparse((url.scheme,<EOL>url.netloc,<EOL>url.path,<EOL>url.params,<EOL>'<STR_LIT>',<EOL>url.fragment))<EOL>

Return a URL with the query component removed.

    :param url: URL to dequery.
    :type url: str
    :rtype: str


url = urlparse.urlparse(base)<EOL>query_params = {}<EOL>query_params.update(urlparse.parse_qsl(url.query, True))<EOL>if additional_params is not None:<EOL><INDENT>query_params.update(additional_params)<EOL>for k, v in additional_params.iteritems():<EOL><INDENT>if v is None:<EOL><INDENT>query_params.pop(k)<EOL><DEDENT><DEDENT><DEDENT>return urlparse.urlunparse((url.scheme,<EOL>url.netloc,<EOL>url.path,<EOL>url.params,<EOL>urllib.urlencode(query_params),<EOL>url.fragment))<EOL>

Construct a URL based off of base containing all parameters in
    the query portion of base plus any additional parameters.

    :param base: Base URL
    :type base: str
    ::param additional_params: Additional query parameters to include.
    :type additional_params: dict
    :rtype: str


with io.open(fname, "<STR_LIT:r>", encoding='<STR_LIT:utf-8>') as fp:<EOL><INDENT>return json.loads(fp.read())<EOL><DEDENT>


self.parser = ctllib.PARSER<EOL>self.base = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>

Initialize the parser, required arguments


self.places = ctllib.Places(config='<STR_LIT>', messages='<STR_LIT>')<EOL>def _cleanup():<EOL><INDENT>for d in self.places:<EOL><INDENT>if os.path.exists(d):<EOL><INDENT>shutil.rmtree(d)<EOL><DEDENT><DEDENT><DEDENT>_cleanup()<EOL>self.addCleanup(_cleanup)<EOL>for d in self.places:<EOL><INDENT>os.mkdir(d)<EOL><DEDENT>

Set up configuration and build/cleanup directories


self.events = []<EOL>


self.events.append(('<STR_LIT>', name, contents))<EOL>


self.events.append(('<STR_LIT>', name))<EOL>


self.events.append(('<STR_LIT>', contents))<EOL>


pass<EOL>


self.testDirectory = os.path.join(os.getcwd(), '<STR_LIT>')<EOL>def _cleanup():<EOL><INDENT>if os.path.exists(self.testDirectory):<EOL><INDENT>shutil.rmtree(self.testDirectory)<EOL><DEDENT><DEDENT>self.addCleanup(_cleanup)<EOL>_cleanup()<EOL>os.makedirs(self.testDirectory)<EOL>


name = os.path.join(self.testDirectory, name)<EOL>with open(name, '<STR_LIT:wb>') as fp:<EOL><INDENT>fp.write(content)<EOL><DEDENT>


name = os.path.join(self.testDirectory, name)<EOL>os.remove(name)<EOL>


DirectoryBasedTest.setUp(self)<EOL>self.receiver = EventRecorder()<EOL>self.message = directory_monitor.messages(self.testDirectory,<EOL>self.receiver)<EOL>self.assertFalse(self.receiver.events)<EOL>


DirectoryBasedTest.setUp(self)<EOL>self.receiver = EventRecorder()<EOL>self.monitor = directory_monitor.checker(self.testDirectory,<EOL>self.receiver)<EOL>self.assertFalse(self.receiver.events)<EOL>


if childName in self.children:<EOL><INDENT>return self.children[childName]<EOL><DEDENT>ret = DummyFile(self.name + "<STR_LIT:/>" + childName)<EOL>self.children[childName] = ret<EOL>return ret<EOL>


self.content = content<EOL>


self.removed = True<EOL>


def _cleanup(testDir):<EOL><INDENT>if os.path.exists(testDir):<EOL><INDENT>shutil.rmtree(testDir)<EOL><DEDENT><DEDENT>self.testDirs = {}<EOL>for subd in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>testDir = self.testDirs[subd] = os.path.join(os.getcwd(), subd)<EOL>self.addCleanup(_cleanup, testDir)<EOL>_cleanup(testDir)<EOL>os.makedirs(testDir)<EOL><DEDENT>self.my_reactor = test_procmon.DummyProcessReactor()<EOL>self.service = service.get(self.testDirs['<STR_LIT>'],<EOL>self.testDirs['<STR_LIT>'],<EOL><NUM_LIT:5>, reactor=self.my_reactor)<EOL>self._finishSetUp()<EOL>


self.opt = service.Options()<EOL>self.basic = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>


self.events = []<EOL>

Initialize to record which events we got


def addProcess(self, name, args, uid=None, gid=None, env=None):

if env is None:<EOL><INDENT>env = {}<EOL><DEDENT>self.events.append(('<STR_LIT>', name, args, uid, gid, env))<EOL>

Add a process

        TODO: document arguments


Remove a process

        TODO: document arguments


Stop (really, restart) a process.

        TODO: document arguments


self.monitor = DummyProcessMonitor()<EOL>self.receiver = process_events.Receiver(self.monitor)<EOL>self.assertFalse(self.monitor.events)<EOL>self.logMessages = []<EOL>def _observer(msg):<EOL><INDENT>self.logMessages.append('<STR_LIT>'.join(msg['<STR_LIT:message>']))<EOL><DEDENT>self.addCleanup(log.removeObserver, _observer)<EOL>log.addObserver(_observer)<EOL>self.assertFalse(self.logMessages)<EOL>


return json.dumps(something).encode('<STR_LIT:utf-8>')<EOL>


def request(self, method, url, headers, body):

d = defer.Deferred()<EOL>self.calls.append((method, url, headers, body))<EOL>self.pending[url].append(d)<EOL>return d<EOL>


self.opt = beatcheck.Options()<EOL>self.basic = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>


return '<STR_LIT:U+0020>'.join('<STR_LIT:U+0020>'.join('<STR_LIT>' % (key, vpart)<EOL>for vpart in value.split())<EOL>for key, value in six.iteritems(self.args)).split()<EOL>


path = filepath.FilePath(location)<EOL>files = set()<EOL>filesContents = {}<EOL>def _check(path):<EOL><INDENT>currentFiles = set(fname for fname in os.listdir(location)<EOL>if not fname.endswith('<STR_LIT>'))<EOL>removed = files - currentFiles<EOL>added = currentFiles - files<EOL>for fname in added:<EOL><INDENT>contents = path.child(fname).getContent()<EOL>filesContents[fname] = contents<EOL>receiver.add(fname, contents)<EOL><DEDENT>for fname in removed:<EOL><INDENT>receiver.remove(fname)<EOL><DEDENT>same = currentFiles & files<EOL>for fname in same:<EOL><INDENT>newContents = path.child(fname).getContent()<EOL>oldContents = filesContents[fname]<EOL>if newContents == oldContents:<EOL><INDENT>continue<EOL><DEDENT>receiver.remove(fname)<EOL>filesContents[fname] = newContents<EOL>receiver.add(fname, newContents)<EOL><DEDENT>files.clear()<EOL>files.update(currentFiles)<EOL><DEDENT>return functools.partial(_check, path)<EOL>

Construct a function that checks a directory for process configuration

    The function checks for additions or removals
    of JSON process configuration files and calls the appropriate receiver
    methods.

    :param location: string, the directory to monitor
    :param receiver: IEventReceiver
    :returns: a function with no parameters


path = filepath.FilePath(location)<EOL>def _check(path):<EOL><INDENT>messageFiles = path.globChildren('<STR_LIT:*>')<EOL>for message in messageFiles:<EOL><INDENT>if message.basename().endswith('<STR_LIT>'):<EOL><INDENT>continue<EOL><DEDENT>receiver.message(message.getContent())<EOL>message.remove()<EOL><DEDENT><DEDENT>return functools.partial(_check, path)<EOL>

Construct a function that checks a directory for messages

    The function checks for new messages and
    calls the appropriate method on the receiver. Sent messages are
    deleted.

    :param location: string, the directory to monitor
    :param receiver: IEventReceiver
    :returns: a function with no parameters


children = {child.basename(): child for child in location.children()}<EOL>last = set(states)<EOL>current = set(children)<EOL>gone = last - current<EOL>added = current - last<EOL>for name in gone:<EOL><INDENT>states[name].close()<EOL>del states[name]<EOL><DEDENT>for name in added:<EOL><INDENT>states[name] = State(location=children[name], settings=settings)<EOL><DEDENT>return [name for name, state in six.iteritems(states) if state.check()]<EOL>


for bad in checker():<EOL><INDENT>restarter(bad)<EOL><DEDENT>

Run restarter on the checker's output

    :params restarter: something to run on the output of the checker
    :params checker: a function expected to get one argument (current time)
                     and return a list of stale names
    :params timer: a function of zero arguments, intended to return current
                   time
    :returns: None


restarter, path = beatcheck.parseConfig(opt)<EOL>pool = client.HTTPConnectionPool(reactor)<EOL>agent = client.Agent(reactor=reactor, pool=pool)<EOL>settings = Settings(reactor=reactor, agent=agent)<EOL>states = {}<EOL>checker = functools.partial(check, settings, states, path)<EOL>httpcheck = tainternet.TimerService(opt['<STR_LIT>'], run, restarter, checker)<EOL>httpcheck.setName('<STR_LIT>')<EOL>return heart.wrapHeart(httpcheck)<EOL>

Make a service

    :params opt: dictionary-like object with 'freq', 'config' and 'messages'
    :returns: twisted.application.internet.TimerService that at opt['freq']
              checks for stale processes in opt['config'], and sends
              restart messages through opt['messages']


return self.bad > self.maxBad<EOL>


self.bad += <NUM_LIT:1><EOL>


self.bad = <NUM_LIT:0><EOL>


if self.closed:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if self.call is not None:<EOL><INDENT>self.call.cancel()<EOL><DEDENT>self.closed = True<EOL>

Discard data and cancel all calls.

        Instance cannot be reused after closing.


if self.closed:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>self._maybeReset()<EOL>if self.url is None:<EOL><INDENT>return False<EOL><DEDENT>return self._maybeCheck()<EOL>


pass<EOL>


def runProcess(args, timeout, grace, reactor):

deferred = defer.Deferred()<EOL>protocol = ProcessProtocol(deferred)<EOL>process = reactor.spawnProcess(protocol, args[<NUM_LIT:0>], args, env=os.environ)<EOL>def _logEnded(err):<EOL><INDENT>err.trap(tierror.ProcessDone, tierror.ProcessTerminated)<EOL>print(err.value)<EOL><DEDENT>deferred.addErrback(_logEnded)<EOL>def _cancelTermination(dummy):<EOL><INDENT>for termination in terminations:<EOL><INDENT>if termination.active():<EOL><INDENT>termination.cancel()<EOL><DEDENT><DEDENT><DEDENT>deferred.addCallback(_cancelTermination)<EOL>terminations = []<EOL>terminations.append(reactor.callLater(timeout, process.signalProcess,<EOL>"<STR_LIT>"))<EOL>terminations.append(reactor.callLater(timeout+grace,<EOL>process.signalProcess, "<STR_LIT>"))<EOL>return deferred<EOL>

Run a process, return a deferred that fires when it is done

    :params args: Process arguments
    :params timeout: Time before terminating process
    :params grace: Time before killing process after terminating it
    :params reactor: IReactorProcess and IReactorTime
    :returns: deferred that fires with success when the process ends,
              or fails if there was a problem spawning/terminating
              the process


ser = tainternet.TimerService(opts['<STR_LIT>'], runProcess, opts['<STR_LIT:args>'],<EOL>opts['<STR_LIT>'], opts['<STR_LIT>'], tireactor)<EOL>ret = service.MultiService()<EOL>ser.setName('<STR_LIT>')<EOL>ser.setServiceParent(ret)<EOL>heart.maybeAddHeart(ret)<EOL>return ret<EOL>

Make scheduler service

    :params opts: dict-like object.
       keys: frequency, args, timeout, grace


for line in data.splitlines():<EOL><INDENT>print('<STR_LIT>' % fd, line)<EOL><DEDENT>

Log data from process

        :params fd: File descriptor data is coming from
        :params data: The bytes the process returned


self.deferred.errback(reason)<EOL>

Report process end to deferred

        :params reason: a Failure


if myEnv is None:<EOL><INDENT>myEnv = buildEnv()<EOL><DEDENT>oldEnviron = os.environ<EOL>def _cleanup():<EOL><INDENT>os.environ = oldEnviron<EOL><DEDENT>case.addCleanup(_cleanup)<EOL>os.environ = myEnv<EOL>

Replace environment temporarily, restoring it at end of test

    :params myEnv: a dict-like object


def checkHeartService(case, service, statusName='<STR_LIT>'):

case.assertIsInstance(service, tainternet.TimerService)<EOL>case.assertEquals(service.step, <NUM_LIT:10>/<NUM_LIT:3>)<EOL>func, args, kwargs = service.call<EOL>case.assertFalse(args)<EOL>case.assertFalse(kwargs)<EOL>myHeart = _getSelf(func)<EOL>case.assertIs(_getFunc(func), _getFunc(heart.Heart.beat))<EOL>case.assertIsInstance(myHeart, heart.Heart)<EOL>fp = myHeart.getFile()<EOL>case.assertIsInstance(fp, filepath.FilePath)<EOL>case.assertEquals(fp.basename(), statusName)<EOL>

Check that a heart service is correct

    :params case: a unittest.TestCase
    :params service: a heart timer service


if params is None:<EOL><INDENT>params = dict(status='<STR_LIT>', period=<NUM_LIT:10>, grace=<NUM_LIT:3>)<EOL><DEDENT>config = {'<STR_LIT>': params}<EOL>configJSON = json.dumps(config)<EOL>myEnv = dict(os.environ)<EOL>myEnv['<STR_LIT>'] = configJSON<EOL>return myEnv<EOL>

Build an environment with NCOLONY_CONFIG

    :returns: copy of the environment dict and NCOLONY_CONFIG


self.touched += <NUM_LIT:1><EOL>

Note how many times this method was called


configJSON = os.environ.get('<STR_LIT>')<EOL>if configJSON is None:<EOL><INDENT>return None<EOL><DEDENT>config = json.loads(configJSON)<EOL>params = config.get('<STR_LIT>')<EOL>if params is None:<EOL><INDENT>return None<EOL><DEDENT>myFilePath = filepath.FilePath(params['<STR_LIT:status>'])<EOL>if myFilePath.isdir():<EOL><INDENT>name = os.environ['<STR_LIT>']<EOL>myFilePath = myFilePath.child(name)<EOL><DEDENT>heart = Heart(myFilePath)<EOL>ret = tainternet.TimerService(params['<STR_LIT>']/<NUM_LIT:3>, heart.beat)<EOL>return ret<EOL>

Make a service

    :returns: an IService


heartSer = makeService()<EOL>if heartSer is None:<EOL><INDENT>return<EOL><DEDENT>heartSer.setName('<STR_LIT>')<EOL>heartSer.setServiceParent(master)<EOL>

Add a heart to a service collection

    Add a heart to a service.IServiceCollector if
    the heart is not None.

    :params master: a service.IServiceCollector


master = taservice.MultiService()<EOL>service.setServiceParent(master)<EOL>maybeAddHeart(master)<EOL>return master<EOL>

Wrap a service in a MultiService with a heart


return self.path<EOL>


self.path.touch()<EOL>


def __init__(self, monitor, environ=None):

if environ is None:<EOL><INDENT>environ = os.environ<EOL><DEDENT>self.environ = environ<EOL>self.monitor = monitor<EOL>self._groupToProcess = collections.defaultdict(set)<EOL>self._processToGroups = {}<EOL>


parsedContents = json.loads(contents.decode('<STR_LIT:utf-8>'))<EOL>parsedContents = {key: value<EOL>for key, value in six.iteritems(parsedContents)<EOL>if key in VALID_KEYS}<EOL>parsedContents['<STR_LIT:name>'] = name<EOL>parsedContents['<STR_LIT>'] = parsedContents.get('<STR_LIT>', {})<EOL>for key in parsedContents.pop('<STR_LIT>', []):<EOL><INDENT>parsedContents['<STR_LIT>'][key] = self.environ.get(key, '<STR_LIT>')<EOL><DEDENT>groups = parsedContents.pop('<STR_LIT>', [])<EOL>for key in groups:<EOL><INDENT>self._groupToProcess[key].add(name)<EOL><DEDENT>self._processToGroups[name] = groups<EOL>parsedContents['<STR_LIT>']['<STR_LIT>'] = contents<EOL>parsedContents['<STR_LIT>']['<STR_LIT>'] = name<EOL>self.monitor.addProcess(**parsedContents)<EOL>log.msg("<STR_LIT>", name)<EOL>

Add a process

        :params name: string, name of process
        :params contents: string, contents
           parsed as JSON for process params
        :returns: None


self.monitor.removeProcess(name)<EOL>log.msg("<STR_LIT>", name)<EOL>for group in self._processToGroups.pop(name):<EOL><INDENT>self._groupToProcess[group].remove(name)<EOL><DEDENT>

Remove a process

        :params name: string, name of process


contents = json.loads(contents.decode('<STR_LIT:utf-8>'))<EOL>tp = contents['<STR_LIT:type>']<EOL>if tp == '<STR_LIT>':<EOL><INDENT>self.monitor.stopProcess(contents['<STR_LIT:name>'])<EOL>log.msg("<STR_LIT>", contents['<STR_LIT:name>'])<EOL><DEDENT>elif tp == '<STR_LIT>':<EOL><INDENT>self.monitor.restartAll()<EOL>log.msg("<STR_LIT>")<EOL><DEDENT>elif tp == '<STR_LIT>':<EOL><INDENT>log.msg("<STR_LIT>", contents['<STR_LIT>'])<EOL>for name in self._groupToProcess[contents['<STR_LIT>']]:<EOL><INDENT>log.msg("<STR_LIT>", name)<EOL>self.monitor.stopProcess(name)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>', contents)<EOL><DEDENT>

Respond to a restart or a restart-all message

        :params contents: string, contents of message
           parsed as JSON, and assumed to have a 'type'
           key, with value either 'restart' or 'restart-all'.
           If the value is 'restart', another key
           ('value') should exist with a logical process
           name.


def add(places, name, cmd, args, env=None, uid=None, gid=None, extras=None,<EOL>env_inherit=None):

args = [cmd]+args<EOL>config = filepath.FilePath(places.config)<EOL>fle = config.child(name)<EOL>details = dict(args=args)<EOL>if env is not None:<EOL><INDENT>newEnv = {}<EOL>for thing in env:<EOL><INDENT>name, value = thing.split('<STR_LIT:=>', <NUM_LIT:1>)<EOL>newEnv[name] = value<EOL><DEDENT>details['<STR_LIT>'] = newEnv<EOL><DEDENT>if uid is not None:<EOL><INDENT>details['<STR_LIT>'] = uid<EOL><DEDENT>if gid is not None:<EOL><INDENT>details['<STR_LIT>'] = gid<EOL><DEDENT>if env_inherit is not None:<EOL><INDENT>details['<STR_LIT>'] = env_inherit<EOL><DEDENT>if extras is not None:<EOL><INDENT>details.update(extras)<EOL><DEDENT>content = _dumps(details)<EOL>fle.setContent(content)<EOL>

Add a process.

    :param places: a Places instance
    :param name: string, the logical name of the process
    :param cmd: string, executable
    :param args: list of strings, command-line arguments
    :param env: dictionary mapping strings to strings
         (will be environment in subprocess)
    :param uid: integer, uid to run the new process as
    :param gid: integer, gid to run the new process as
    :param extras: a dictionary with additional parameters
    :param env_inherit: a list of environment variables to inherit
    :returns: None


config = filepath.FilePath(places.config)<EOL>fle = config.child(name)<EOL>fle.remove()<EOL>

Remove a process

    :params places: a Places instance
    :params name: string, the logical name of the process
    :returns: None


content = _dumps(dict(type='<STR_LIT>', name=name))<EOL>_addMessage(places, content)<EOL>

Restart a process

    :params places: a Places instance
    :params name: string, the logical name of the process
    :returns: None


content = _dumps(dict(type='<STR_LIT>'))<EOL>_addMessage(places, content)<EOL>

Restart all processes

    :params places: a Places instance
    :returns: None


results = vars(results)<EOL>places = Places(config=results.pop('<STR_LIT>'),<EOL>messages=results.pop('<STR_LIT>'))<EOL>func = results.pop('<STR_LIT>')<EOL>func(places, **results)<EOL>

Call results.func on the attributes of results

    :params result: dictionary-like object
    :returns: None


ns = PARSER.parse_args(argv[<NUM_LIT:1>:])<EOL>call(ns)<EOL>

command-line entry point

        --messages: messages directory

        --config: configuration directory

    subcommands:
        add:
            name (positional)

            --cmd (required) -- executable

            --arg -- add an argument

            --env -- add an environment variable (VAR=value)

            --uid -- set uid

            --gid -- set gid

            --extras -- a JSON file with more parameters

            --env-inherit -- add an environment variable to inherit

        remove:
            name (positional)
        restart:
            name (positional)
        restart-all:
            no arguments


return [child.basename() for child in path.children()<EOL>if _isbad(child, start, now)]<EOL>

check which processes need to be restarted

    :params path: a twisted.python.filepath.FilePath with configurations
    :params start: when the checker started running
    :params now: current time
    :returns: list of strings


for bad in checker(timer()):<EOL><INDENT>restarter(bad)<EOL><DEDENT>

Run restarter on the checker's output

    :params restarter: something to run on the output of the checker
    :params checker: a function expected to get one argument (current time)
                     and return a list of stale names
    :params timer: a function of zero arguments, intended to return current
                   time
    :returns: None


places = ctllib.Places(config=opt['<STR_LIT>'], messages=opt['<STR_LIT>'])<EOL>restarter = functools.partial(ctllib.restart, places)<EOL>path = filepath.FilePath(opt['<STR_LIT>'])<EOL>return restarter, path<EOL>

Parse configuration

    :params opt: dict-like object with config and messages keys
    :returns: restarter, path


restarter, path = parseConfig(opt)<EOL>now = time.time()<EOL>checker = functools.partial(check, path, now)<EOL>beatcheck = tainternet.TimerService(opt['<STR_LIT>'], run, restarter,<EOL>checker, time.time)<EOL>beatcheck.setName('<STR_LIT>')<EOL>return heart.wrapHeart(beatcheck)<EOL>

Make a service

    :params opt: dictionary-like object with 'freq', 'config' and 'messages'
    :returns: twisted.application.internet.TimerService that at opt['freq']
              checks for stale processes in opt['config'], and sends
              restart messages through opt['messages']


for param in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>if self[param] is None:<EOL><INDENT>raise usage.UsageError("<STR_LIT>", param)<EOL><DEDENT><DEDENT>

Checks that required messages/config directories are present


def get(config, messages, freq, pidDir=None, reactor=None):

ret = taservice.MultiService()<EOL>args = ()<EOL>if reactor is not None:<EOL><INDENT>args = reactor,<EOL><DEDENT>procmon = procmonlib.ProcessMonitor(*args)<EOL>if pidDir is not None:<EOL><INDENT>protocols = TransportDirectoryDict(pidDir)<EOL>procmon.protocols = protocols<EOL><DEDENT>procmon.setName('<STR_LIT>')<EOL>receiver = process_events.Receiver(procmon)<EOL>confcheck = directory_monitor.checker(config, receiver)<EOL>confserv = internet.TimerService(freq, confcheck)<EOL>confserv.setServiceParent(ret)<EOL>messagecheck = directory_monitor.messages(messages, receiver)<EOL>messageserv = internet.TimerService(freq, messagecheck)<EOL>messageserv.setServiceParent(ret)<EOL>procmon.setServiceParent(ret)<EOL>return ret<EOL>

Return a service which monitors processes based on directory contents

    Construct and return a service that, when started, will run processes
    based on the contents of the 'config' directory, restarting them
    if file contents change and stopping them if the file is removed.

    It also listens for restart and restart-all messages on the 'messages'
    directory.

    :param config: string, location of configuration directory
    :param messages: string, location of messages directory
    :param freq: number, frequency to check for new messages and configuration
                 updates
    :param pidDir: {twisted.python.filepath.FilePath} or None,
                   location to keep pid files
    :param reactor: something implementing the interfaces
                       {twisted.internet.interfaces.IReactorTime} and
                       {twisted.internet.interfaces.IReactorProcess} and
    :returns: service, {twisted.application.interfaces.IService}


ret = get(config=opt['<STR_LIT>'], messages=opt['<STR_LIT>'],<EOL>pidDir=opt['<STR_LIT>'], freq=opt['<STR_LIT>'])<EOL>pm = ret.getServiceNamed("<STR_LIT>")<EOL>pm.threshold = opt["<STR_LIT>"]<EOL>pm.killTime = opt["<STR_LIT>"]<EOL>pm.minRestartDelay = opt["<STR_LIT>"]<EOL>pm.maxRestartDelay = opt["<STR_LIT>"]<EOL>return ret<EOL>

Return a service based on parsed command-line options

    :param opt: dict-like object. Relevant keys are config, messages,
                pid, frequency, threshold, killtime, minrestartdelay
                and maxrestartdelay
    :returns: service, {twisted.application.interfaces.IService}


super(TransportDirectoryDict, self).__init__()<EOL>self.output = output<EOL>

Initialize

        :param output: a {twisted.python.filepath.FilePath} object


def hash_eth2(data: Union[bytes, bytearray]) -> Hash32:

return Hash32(keccak(data))<EOL>

Return Keccak-256 hashed result.
Note: it's a placeholder and we aim to migrate to a S[T/N]ARK-friendly hash function in
a future Ethereum 2.0 deployment phase.


def modular_squareroot_in_FQ2(value: FQ2) -> FQ2:

candidate_squareroot = value ** ((FQ2_order + <NUM_LIT:8>) // <NUM_LIT:16>)<EOL>check = candidate_squareroot ** <NUM_LIT:2> / value<EOL>if check in eighth_roots_of_unity[::<NUM_LIT:2>]:<EOL><INDENT>x1 = candidate_squareroot / eighth_roots_of_unity[eighth_roots_of_unity.index(check) // <NUM_LIT:2>]<EOL>x2 = -x1<EOL>x1_re, x1_im = x1.coeffs<EOL>x2_re, x2_im = x2.coeffs<EOL>return x1 if (x1_im > x2_im or (x1_im == x2_im and x1_re > x2_re)) else x2<EOL><DEDENT>return None<EOL>

``modular_squareroot_in_FQ2(x)`` returns the value ``y`` such that ``y**2 % q == x``,
and None if this is not possible. In cases where there are two solutions,
the value with higher imaginary component is favored;
if both solutions have equal imaginary component the value with higher real
component is favored.


def compress_G1(pt: G1Uncompressed) -> G1Compressed:

if is_inf(pt):<EOL><INDENT>return G1Compressed(POW_2_383 + POW_2_382)<EOL><DEDENT>else:<EOL><INDENT>x, y = normalize(pt)<EOL>a_flag = (y.n * <NUM_LIT:2>) // q<EOL>return G1Compressed(x.n + a_flag * POW_2_381 + POW_2_383)<EOL><DEDENT>

A compressed point is a 384-bit integer with the bit order (c_flag, b_flag, a_flag, x),
where the c_flag bit is always set to 1,
the b_flag bit indicates infinity when set to 1,
the a_flag bit helps determine the y-coordinate when decompressing,
and the 381-bit integer x is the x-coordinate of the point.


def decompress_G1(z: G1Compressed) -> G1Uncompressed:

<EOL>b_flag = (z % POW_2_383) // POW_2_382<EOL>if b_flag == <NUM_LIT:1>:<EOL><INDENT>return Z1<EOL><DEDENT>x = z % POW_2_381<EOL>y = pow((x**<NUM_LIT:3> + b.n) % q, (q + <NUM_LIT:1>) // <NUM_LIT:4>, q)<EOL>if pow(y, <NUM_LIT:2>, q) != (x**<NUM_LIT:3> + b.n) % q:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>a_flag = (z % POW_2_382) // POW_2_381<EOL>if (y * <NUM_LIT:2>) // q != a_flag:<EOL><INDENT>y = q - y<EOL><DEDENT>return (FQ(x), FQ(y), FQ(<NUM_LIT:1>))<EOL>

Recovers x and y coordinates from the compressed point.


def compress_G2(pt: G2Uncompressed) -> G2Compressed:

if not is_on_curve(pt, b2):<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>if is_inf(pt):<EOL><INDENT>return G2Compressed((POW_2_383 + POW_2_382, <NUM_LIT:0>))<EOL><DEDENT>x, y = normalize(pt)<EOL>x_re, x_im = x.coeffs<EOL>y_re, y_im = y.coeffs<EOL>a_flag1 = (y_im * <NUM_LIT:2>) // q if y_im > <NUM_LIT:0> else (y_re * <NUM_LIT:2>) // q<EOL>z1 = x_im + a_flag1 * POW_2_381 + POW_2_383<EOL>z2 = x_re<EOL>return G2Compressed((z1, z2))<EOL>

The compressed point (z1, z2) has the bit order:
z1: (c_flag1, b_flag1, a_flag1, x1)
z2: (c_flag2, b_flag2, a_flag2, x2)
where
- c_flag1 is always set to 1
- b_flag1 indicates infinity when set to 1
- a_flag1 helps determine the y-coordinate when decompressing,
- a_flag2, b_flag2, and c_flag2 are always set to 0


def decompress_G2(p: G2Compressed) -> G2Uncompressed:

z1, z2 = p<EOL>b_flag1 = (z1 % POW_2_383) // POW_2_382<EOL>if b_flag1 == <NUM_LIT:1>:<EOL><INDENT>return Z2<EOL><DEDENT>x1 = z1 % POW_2_381<EOL>x2 = z2<EOL>x = FQ2([x2, x1])<EOL>y = modular_squareroot_in_FQ2(x**<NUM_LIT:3> + b2)<EOL>if y is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>a_flag1 = (z1 % POW_2_382) // POW_2_381<EOL>y_re, y_im = y.coeffs<EOL>if (y_im > <NUM_LIT:0> and (y_im * <NUM_LIT:2>) // q != a_flag1) or (y_im == <NUM_LIT:0> and (y_re * <NUM_LIT:2>) // q != a_flag1):<EOL><INDENT>y = FQ2((y * -<NUM_LIT:1>).coeffs)<EOL><DEDENT>if not is_on_curve((x, y, FQ2([<NUM_LIT:1>, <NUM_LIT:0>])), b2):<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>return (x, y, FQ2([<NUM_LIT:1>, <NUM_LIT:0>]))<EOL>

Recovers x and y coordinates from the compressed point (z1, z2).


def prime_field_inv(a: int, n: int) -> int:

if a == <NUM_LIT:0>:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>lm, hm = <NUM_LIT:1>, <NUM_LIT:0><EOL>low, high = a % n, n<EOL>while low > <NUM_LIT:1>:<EOL><INDENT>r = high // low<EOL>nm, new = hm - lm * r, high - low * r<EOL>lm, low, hm, high = nm, new, lm, low<EOL><DEDENT>return lm % n<EOL>

Extended euclidean algorithm to find modular inverses for integers


def create_access_request(pid_value, users, confirmed):

datastore = current_app.extensions['<STR_LIT>'].datastore<EOL>receiver = datastore.get_user(users['<STR_LIT>']['<STR_LIT:id>'])<EOL>sender = datastore.get_user(users['<STR_LIT>']['<STR_LIT:id>'])<EOL>return AccessRequest.create(<EOL>recid=pid_value,<EOL>receiver=receiver,<EOL>sender_full_name="<STR_LIT>",<EOL>sender_email="<STR_LIT>",<EOL>sender=sender if confirmed else None,<EOL>justification="<STR_LIT>",<EOL>)<EOL>


return val<EOL>


def create_token(self, obj_id, extra_data):

return self.dumps(<EOL>dict(<EOL>id=obj_id,<EOL>data=extra_data,<EOL>rnd=binascii.hexlify(os.urandom(<NUM_LIT:4>)).decode('<STR_LIT:utf-8>')<EOL>)<EOL>)<EOL>

Create a token referencing the object id with extra data.

        Note random data is added to ensure that no two tokens are identical.


def validate_token(self, token, expected_data=None):

try:<EOL><INDENT>data = self.load_token(token)<EOL>if expected_data:<EOL><INDENT>for k in expected_data:<EOL><INDENT>if expected_data[k] != data["<STR_LIT:data>"].get(k):<EOL><INDENT>return None<EOL><DEDENT><DEDENT><DEDENT>return data<EOL><DEDENT>except BadData:<EOL><INDENT>return None<EOL><DEDENT>

Validate secret link token.

        :param token: Token value.
        :param expected_data: A dictionary of key/values that must be present
            in the data part of the token (i.e. included via ``extra_data`` in
            ``create_token``).


def load_token(self, token, force=False):

try:<EOL><INDENT>data = self.loads(token)<EOL><DEDENT>except SignatureExpired as e:<EOL><INDENT>if not force:<EOL><INDENT>raise<EOL><DEDENT>data = e.payload<EOL><DEDENT>del data["<STR_LIT>"]<EOL>return data<EOL>

Load data in a token.

        :param token: Token to load.
        :param force: Load token data even if signature expired.
                      Default: False.


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>from cryptography.fernet import Fernet<EOL>from cryptography.hazmat.backends import default_backend<EOL>from cryptography.hazmat.primitives import hashes<EOL>digest = hashes.Hash(hashes.SHA256(), backend=default_backend())<EOL>digest.update(current_app.config['<STR_LIT>'].encode('<STR_LIT:utf8>'))<EOL>fernet_key = urlsafe_b64encode(digest.finalize())<EOL>self._engine = Fernet(fernet_key)<EOL><DEDENT>return self._engine<EOL>


return self.engine.encrypt(<EOL>super(EncryptedTokenMixIn, self).create_token(obj_id, extra_data)<EOL>)<EOL>

Create a token referencing the object id with extra data.


return super(EncryptedTokenMixIn, self).load_token(<EOL>self.engine.decrypt(token), force=force<EOL>)<EOL>


def __init__(self, expires_in=None, **kwargs):

dt = expires_in orcurrent_app.config['<STR_LIT>']<EOL>super(EmailConfirmationSerializer, self).__init__(<EOL>current_app.config['<STR_LIT>'],<EOL>expires_in=dt,<EOL>salt='<STR_LIT>',<EOL>**kwargs<EOL>)<EOL>


@classmethod<EOL><INDENT>def compat_validate_token(cls, *args, **kwargs):<DEDENT>

data = None<EOL>for algorithm in SUPPORTED_DIGEST_ALGORITHMS:<EOL><INDENT>data = cls(algorithm_name=algorithm).validate_token(<EOL>*args, **kwargs)<EOL>if not data:  <EOL><INDENT>continue<EOL><DEDENT><DEDENT>return data<EOL>

Multiple algorithm-compatible token validation.


super(SecretLinkSerializer, self).__init__(<EOL>current_app.config['<STR_LIT>'],<EOL>salt='<STR_LIT>',<EOL>**kwargs<EOL>)<EOL>


def __init__(self, expires_at=None, **kwargs):

assert isinstance(expires_at, datetime) or expires_at is None<EOL>dt = expires_at - datetime.now() if expires_at else None<EOL>super(TimedSecretLinkSerializer, self).__init__(<EOL>current_app.config['<STR_LIT>'],<EOL>expires_in=int(dt.total_seconds()) if dt else None,<EOL>salt='<STR_LIT>',<EOL>**kwargs<EOL>)<EOL>


@classmethod<EOL><INDENT>def create_token(cls, obj_id, data, expires_at=None):<DEDENT>

if expires_at:<EOL><INDENT>s = TimedSecretLinkSerializer(expires_at=expires_at)<EOL><DEDENT>else:<EOL><INDENT>s = SecretLinkSerializer()<EOL><DEDENT>return s.create_token(obj_id, data)<EOL>


@classmethod<EOL><INDENT>def validate_token(cls, token, expected_data=None):<DEDENT>

for algorithm in SUPPORTED_DIGEST_ALGORITHMS:<EOL><INDENT>s = SecretLinkSerializer(algorithm_name=algorithm)<EOL>st = TimedSecretLinkSerializer(algorithm_name=algorithm)<EOL>try:<EOL><INDENT>for serializer in (s, st):<EOL><INDENT>data = serializer.validate_token(<EOL>token, expected_data=expected_data)<EOL>if data:<EOL><INDENT>return data<EOL><DEDENT><DEDENT><DEDENT>except SignatureExpired:  <EOL><INDENT>raise<EOL><DEDENT>except BadData:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>

Validate a secret link token (non-expiring + expiring).


@classmethod<EOL><INDENT>def load_token(cls, token, force=False):<DEDENT>

for algorithm in SUPPORTED_DIGEST_ALGORITHMS:<EOL><INDENT>s = SecretLinkSerializer(algorithm_name=algorithm)<EOL>st = TimedSecretLinkSerializer(algorithm_name=algorithm)<EOL>for serializer in (s, st):<EOL><INDENT>try:<EOL><INDENT>data = serializer.load_token(token, force=force)<EOL>if data:<EOL><INDENT>return data<EOL><DEDENT><DEDENT>except SignatureExpired:<EOL><INDENT>raise  <EOL><DEDENT>except BadData:<EOL><INDENT>continue<EOL><DEDENT><DEDENT><DEDENT>


try:<EOL><INDENT>from .models import SecretLink<EOL>token = request.args['<STR_LIT>']<EOL>if token and SecretLink.validate_token(token, {}):<EOL><INDENT>session['<STR_LIT>'] = token<EOL><DEDENT><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>

Verify token and save in session if it's valid.


from .receivers import connect_receivers<EOL>self.app = app<EOL>connect_receivers()<EOL>


if app:<EOL><INDENT>self.init_app(app)<EOL><DEDENT>


app.before_request(verify_token)<EOL>self.init_config(app)<EOL>state = _AppState(app=app)<EOL>app.extensions['<STR_LIT>'] = state<EOL>


app.config.setdefault(<EOL>"<STR_LIT>",<EOL>app.config.get("<STR_LIT>",<EOL>"<STR_LIT>"))<EOL>app.config.setdefault(<EOL>"<STR_LIT>",<EOL>app.config.get("<STR_LIT>",<EOL>"<STR_LIT>"))<EOL>for k in dir(config):<EOL><INDENT>if k.startswith('<STR_LIT>'):<EOL><INDENT>app.config.setdefault(k, getattr(config, k))<EOL><DEDENT><DEDENT>


if form.accept.data:<EOL><INDENT>if not field.data or datetime.utcnow().date() >= field.data:<EOL><INDENT>raise validators.StopValidation(_(<EOL>"<STR_LIT>"<EOL>))<EOL><DEDENT>if not field.data ordatetime.utcnow().date() + timedelta(days=<NUM_LIT>) < field.data:<EOL><INDENT>raise validators.StopValidation(_(<EOL>"<STR_LIT>"<EOL>))<EOL><DEDENT><DEDENT>


if field.data and form.reject.data:<EOL><INDENT>raise validators.ValidationError(<EOL>_("<STR_LIT>")<EOL>)<EOL><DEDENT>


if field.data and form.accept.data:<EOL><INDENT>raise validators.ValidationError(<EOL>_("<STR_LIT>")<EOL>)<EOL><DEDENT>


if form.reject.data and not field.data.strip():<EOL><INDENT>raise validators.ValidationError(<EOL>_("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>)<EOL><DEDENT>


self.options = options<EOL>if selected in options:<EOL><INDENT>self._selected = selected<EOL>self.asc = True<EOL><DEDENT>elif selected and selected[<NUM_LIT:0>] == '<STR_LIT:->' and selected[<NUM_LIT:1>:] in options:<EOL><INDENT>self._selected = selected[<NUM_LIT:1>:]<EOL>self.asc = False<EOL><DEDENT>else:<EOL><INDENT>self._selected = None<EOL>self.asc = None<EOL><DEDENT>

Initialize ordering with possible options the selected option.

        :param options: List of column options.
        :param selected: Selected column. Prefix name with ``-`` to denote
            descending ordering.


if col in self.options:<EOL><INDENT>if self.is_selected(col):<EOL><INDENT>return col if not self.asc else '<STR_LIT>'.format(col)<EOL><DEDENT>else:<EOL><INDENT>return col<EOL><DEDENT><DEDENT>return None<EOL>


def dir(self, col, asc='<STR_LIT>', desc='<STR_LIT>'):

if col == self._selected and self.asc is not None:<EOL><INDENT>return asc if self.asc else desc<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

Get direction (ascending/descending) of ordering.


return col == self._selected<EOL>


if self._selected:<EOL><INDENT>return self._selected if self.asc else"<STR_LIT>".format(self._selected)<EOL><DEDENT>return None<EOL>


def __init__(self, query, options, selected):

super(QueryOrdering, self).__init__(options, selected)<EOL>self.query = query<EOL>


if self.asc is not None:<EOL><INDENT>if self._selected and self.asc:<EOL><INDENT>return self.query.order_by(self._selected)<EOL><DEDENT>elif self._selected and not self.asc:<EOL><INDENT>return self.query.order_by(desc(self._selected))<EOL><DEDENT><DEDENT>return self.query<EOL>


def create_secret_link(request, message=None, expires_at=None):

pid, record = get_record(request.recid)<EOL>if not record:<EOL><INDENT>raise RecordNotFound(request.recid)<EOL><DEDENT>description = render_template(<EOL>"<STR_LIT>",<EOL>request=request,<EOL>record=record,<EOL>pid=pid,<EOL>expires_at=expires_at,<EOL>message=message,<EOL>)<EOL>request.create_secret_link(<EOL>record["<STR_LIT:title>"],<EOL>description=description,<EOL>expires_at=expires_at<EOL>)<EOL>


def send_accept_notification(request, message=None, expires_at=None):

Receiver for request-accepted signal to send email notification.


pid, record = get_record(request.recid)<EOL>if record is None:<EOL><INDENT>current_app.logger.error("<STR_LIT>"<EOL>% request.recid)<EOL>return<EOL><DEDENT>title = _("<STR_LIT>", record=record["<STR_LIT:title>"])<EOL>_send_notification(<EOL>request.receiver.email,<EOL>title,<EOL>"<STR_LIT>",<EOL>request=request,<EOL>record=record,<EOL>pid=pid,<EOL>)<EOL>_send_notification(<EOL>request.sender_email,<EOL>title,<EOL>"<STR_LIT>",<EOL>request=request,<EOL>record=record,<EOL>pid=pid,<EOL>)<EOL>

Receiver for request-confirmed signal to send email notification.


Receiver for request-created signal to send email notification.


Receiver for request-rejected signal to send email notification.


def _send_notification(to, subject, template, **ctx):

msg = Message(<EOL>subject,<EOL>sender=current_app.config.get('<STR_LIT>'),<EOL>recipients=[to]<EOL>)<EOL>msg.body = render_template(template, **ctx)<EOL>send_email.delay(msg.__dict__)<EOL>


self._icon = icon<EOL>


kwargs.setdefault('<STR_LIT:id>', field.id)<EOL>kwargs.setdefault('<STR_LIT:type>', self.input_type)<EOL>kwargs.setdefault('<STR_LIT:value>', "<STR_LIT:y>")<EOL>return HTMLString('<STR_LIT>' % (<EOL>html_params(name=field.name, **kwargs),<EOL>"<STR_LIT>" % self._icon if self._icon else "<STR_LIT>",<EOL>text_type(field.label.text)<EOL>))<EOL>


return resolver.resolve(str(recid))<EOL>


return current_app.config['<STR_LIT>'].encode('<STR_LIT:utf-8>')<EOL>


@classmethod<EOL><INDENT>def create(cls, title, owner, extra_data, description="<STR_LIT>", expires_at=None):<DEDENT>

if isinstance(expires_at, date):<EOL><INDENT>expires_at = datetime.combine(expires_at, datetime.min.time())<EOL><DEDENT>with db.session.begin_nested():<EOL><INDENT>obj = cls(<EOL>owner=owner,<EOL>title=title,<EOL>description=description,<EOL>expires_at=expires_at,<EOL>token='<STR_LIT>',<EOL>)<EOL>db.session.add(obj)<EOL><DEDENT>with db.session.begin_nested():<EOL><INDENT>obj.token = SecretLinkFactory.create_token(<EOL>obj.id, extra_data, expires_at=expires_at<EOL>).decode('<STR_LIT:utf8>')<EOL><DEDENT>link_created.send(obj)<EOL>return obj<EOL>


@classmethod<EOL><INDENT>def validate_token(cls, token, expected_data):<DEDENT>

data = SecretLinkFactory.validate_token(<EOL>token, expected_data=expected_data<EOL>)<EOL>if data:<EOL><INDENT>link = cls.query.get(data['<STR_LIT:id>'])<EOL>if link and link.is_valid():<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Validate a secret link token.

        Only queries the database if token is valid to determine that the token
        has not been revoked.


return cls.query.filter_by(<EOL>owner_user_id=user.id<EOL>)<EOL>


if self.token:<EOL><INDENT>return SecretLinkFactory.load_token(self.token, force=True)["<STR_LIT:data>"]<EOL><DEDENT>return None<EOL>

Load token data stored in token (ignores expiry date of tokens).


copy = deepcopy(self.extra_data)<EOL>if '<STR_LIT>' in copy:<EOL><INDENT>copy['<STR_LIT>'] = copy.pop('<STR_LIT>')<EOL><DEDENT>return url_for(<EOL>endpoint, token=self.token,<EOL>_external=True, **(copy or {})<EOL>)<EOL>

Get absolute for secret link (using https scheme).

        The endpoint is passed to ``url_for`` with ``token`` and ``extra_data``
        as keyword arguments. E.g.::

            >>> link.extra_data
            dict(recid=1)
            >>> link.get_absolute_url('record.metadata')

        translates into::

            >>> url_for('record.metadata', token="...", recid=1, )


if self.revoked_at is None:<EOL><INDENT>with db.session.begin_nested():<EOL><INDENT>self.revoked_at = datetime.utcnow()<EOL><DEDENT>link_revoked.send(self)<EOL>return True<EOL><DEDENT>return False<EOL>


if self.expires_at:<EOL><INDENT>return datetime.utcnow() > self.expires_at<EOL><DEDENT>return False<EOL>


return self.revoked_at is not None<EOL>


return not(self.is_expired() or self.is_revoked())<EOL>


@classmethod<EOL><INDENT>def create(cls, recid=None, receiver=None, sender_full_name=None,<EOL>sender_email=None, justification=None, sender=None):<DEDENT>

sender_user_id = None if sender is None else sender.id<EOL>assert recid<EOL>assert receiver<EOL>assert sender_full_name<EOL>assert sender_email<EOL>assert justification<EOL>status = RequestStatus.EMAIL_VALIDATION<EOL>if sender and sender.confirmed_at:<EOL><INDENT>status = RequestStatus.PENDING<EOL><DEDENT>with db.session.begin_nested():<EOL><INDENT>obj = cls(<EOL>status=status,<EOL>recid=recid,<EOL>receiver_user_id=receiver.id,<EOL>sender_user_id=sender_user_id,<EOL>sender_full_name=sender_full_name,<EOL>sender_email=sender_email,<EOL>justification=justification<EOL>)<EOL>db.session.add(obj)<EOL><DEDENT>if obj.status == RequestStatus.EMAIL_VALIDATION:<EOL><INDENT>request_created.send(obj)<EOL><DEDENT>else:<EOL><INDENT>request_confirmed.send(obj)<EOL><DEDENT>return obj<EOL>

Create a new access request.

        :param recid: Record id (required).
        :param receiver: User object of receiver (required).
        :param sender_full_name: Full name of sender (required).
        :param sender_email: Email address of sender (required).
        :param justification: Justification message (required).
        :param sender: User object of sender (optional).


return cls.query.filter_by(<EOL>receiver_user_id=user.id<EOL>)<EOL>

Get access requests for a specific receiver.


@classmethod<EOL><INDENT>def get_by_receiver(cls, request_id, user):<DEDENT>

return cls.query.filter_by(<EOL>id=request_id,<EOL>receiver_user_id=user.id<EOL>).first()<EOL>

Get access request for a specific receiver.


with db.session.begin_nested():<EOL><INDENT>if self.status != RequestStatus.EMAIL_VALIDATION:<EOL><INDENT>raise InvalidRequestStateError(RequestStatus.EMAIL_VALIDATION)<EOL><DEDENT>self.status = RequestStatus.PENDING<EOL><DEDENT>request_confirmed.send(self)<EOL>


def accept(self, message=None, expires_at=None):

with db.session.begin_nested():<EOL><INDENT>if self.status != RequestStatus.PENDING:<EOL><INDENT>raise InvalidRequestStateError(RequestStatus.PENDING)<EOL><DEDENT>self.status = RequestStatus.ACCEPTED<EOL><DEDENT>request_accepted.send(self, message=message, expires_at=expires_at)<EOL>


with db.session.begin_nested():<EOL><INDENT>if self.status != RequestStatus.PENDING:<EOL><INDENT>raise InvalidRequestStateError(RequestStatus.PENDING)<EOL><DEDENT>self.status = RequestStatus.REJECTED<EOL><DEDENT>request_rejected.send(self, message=message)<EOL>


def create_secret_link(self, title, description=None, expires_at=None):

self.link = SecretLink.create(<EOL>title,<EOL>self.receiver,<EOL>extra_data=dict(recid=self.recid),<EOL>description=description,<EOL>expires_at=expires_at,<EOL>)<EOL>return self.link<EOL>


return record.get('<STR_LIT>') == '<STR_LIT>' andrecord.get('<STR_LIT>') andrecord.get('<STR_LIT>', [])<EOL>

Template filter to check if a record is restricted.


return record.get('<STR_LIT>') == '<STR_LIT>' andrecord.get('<STR_LIT>') andrecord.get('<STR_LIT>') > datetime.utcnow().date()<EOL>

Template filter to check if a record is embargoed.


return {'<STR_LIT>': '<STR_LIT>'} in record.get('<STR_LIT>', [])<EOL>

Template filter to check if a record is removed.


def access_request(pid, record, template, **kwargs):

recid = int(pid.pid_value)<EOL>datastore = LocalProxy(<EOL>lambda: current_app.extensions['<STR_LIT>'].datastore)<EOL>if record.get('<STR_LIT>') != '<STR_LIT>' ornot record.get('<STR_LIT>'):<EOL><INDENT>abort(<NUM_LIT>)<EOL><DEDENT>owners = record.get('<STR_LIT>', [])<EOL>record_owners = [datastore.find_user(id=owner_id) for owner_id in owners]<EOL>if not record_owners:<EOL><INDENT>abort(<NUM_LIT>)<EOL><DEDENT>sender = None<EOL>initialdata = dict()<EOL>if current_user.is_authenticated:<EOL><INDENT>sender = current_user<EOL>initialdata['<STR_LIT:email>'] = current_user.email<EOL>if current_user.profile:<EOL><INDENT>initialdata['<STR_LIT>'] = current_user.profile.full_name<EOL><DEDENT><DEDENT>form = AccessRequestForm(formdata=request.form, **initialdata)<EOL>if form.validate_on_submit():<EOL><INDENT>accreq = AccessRequest.create(<EOL>recid=recid,<EOL>receiver=record_owners[<NUM_LIT:0>],<EOL>sender_full_name=form.data['<STR_LIT>'],<EOL>sender_email=form.data['<STR_LIT:email>'],<EOL>justification=form.data['<STR_LIT>'],<EOL>sender=sender<EOL>)<EOL>db.session.commit()<EOL>if accreq.status == RequestStatus.EMAIL_VALIDATION:<EOL><INDENT>flash(_(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"),<EOL>category='<STR_LIT:info>')<EOL><DEDENT>else:<EOL><INDENT>flash(_("<STR_LIT>"), category='<STR_LIT:info>')<EOL><DEDENT>return redirect(url_for('<STR_LIT>', pid_value=recid))<EOL><DEDENT>return render_template(<EOL>template,<EOL>pid=pid,<EOL>record=record,<EOL>form=form,<EOL>owners=record_owners,<EOL>)<EOL>


def confirm(pid, record, template, **kwargs):

recid = int(pid.pid_value)<EOL>token = request.view_args['<STR_LIT>']<EOL>data = EmailConfirmationSerializer.compat_validate_token(token)<EOL>if data is None:<EOL><INDENT>flash(_("<STR_LIT>"), category='<STR_LIT>')<EOL>return redirect(url_for("<STR_LIT>", pid_value=recid))<EOL><DEDENT>r = AccessRequest.query.get(data['<STR_LIT:id>'])<EOL>if not r:<EOL><INDENT>abort(<NUM_LIT>)<EOL><DEDENT>if r.status != RequestStatus.EMAIL_VALIDATION:<EOL><INDENT>abort(<NUM_LIT>)<EOL><DEDENT>r.confirm_email()<EOL>db.session.commit()<EOL>flash(_("<STR_LIT>"), category='<STR_LIT:info>')<EOL>return redirect(url_for("<STR_LIT>", pid_value=recid))<EOL>


result = u'<STR_LIT>'.join(u'<STR_LIT>' % p.replace('<STR_LIT:\n>', '<STR_LIT>')<EOL>for p in _paragraph_re.split(escape(value)))<EOL>if eval_ctx.autoescape:<EOL><INDENT>result = Markup(result)<EOL><DEDENT>return result<EOL>

Template filter to convert newlines to <br>-tags.


query = request.args.get('<STR_LIT>', '<STR_LIT>')<EOL>order = request.args.get('<STR_LIT>', '<STR_LIT>')<EOL>try:<EOL><INDENT>page = int(request.args.get('<STR_LIT>', <NUM_LIT:1>))<EOL>per_page = int(request.args.get('<STR_LIT>', <NUM_LIT:20>))<EOL><DEDENT>except (TypeError, ValueError):<EOL><INDENT>abort(<NUM_LIT>)<EOL><DEDENT>form = DeleteForm(request.form)<EOL>if form.validate_on_submit():<EOL><INDENT>link = SecretLink.query_by_owner(current_user).filter_by(<EOL>id=form.link.data).first()<EOL>if link.revoke():<EOL><INDENT>flash(_("<STR_LIT>"), category='<STR_LIT:success>')<EOL><DEDENT>db.session.commit()<EOL><DEDENT>links = SecretLink.query_by_owner(current_user).filter(<EOL>SecretLink.revoked_at.is_(None)<EOL>)<EOL>if query:<EOL><INDENT>lquery = "<STR_LIT>".format(query)<EOL>links = links.filter(<EOL>SecretLink.title.like(lquery) | SecretLink.description.like(lquery)<EOL>)<EOL><DEDENT>ordering = QueryOrdering(links, ['<STR_LIT:title>', '<STR_LIT>', '<STR_LIT>'], order)<EOL>links = ordering.items()<EOL>requests = AccessRequest.query_by_receiver(current_user).filter_by(<EOL>status=RequestStatus.PENDING).order_by('<STR_LIT>')<EOL>return render_template(<EOL>"<STR_LIT>",<EOL>links_pagination=links.paginate(page, per_page=per_page),<EOL>requests=requests,<EOL>query=query,<EOL>order=ordering,<EOL>get_record=get_record,<EOL>form=DeleteForm(),<EOL>)<EOL>

List pending access requests and shared links.


r = AccessRequest.get_by_receiver(request_id, current_user)<EOL>if not r or r.status != RequestStatus.PENDING:<EOL><INDENT>abort(<NUM_LIT>)<EOL><DEDENT>form = ApprovalForm(request.form)<EOL>if form.validate_on_submit():<EOL><INDENT>if form.accept.data:<EOL><INDENT>r.accept(message=form.data['<STR_LIT:message>'],<EOL>expires_at=form.expires_at.data)<EOL>db.session.commit()<EOL>flash(_("<STR_LIT>"))<EOL>return redirect(url_for("<STR_LIT>"))<EOL><DEDENT>elif form.reject.data:<EOL><INDENT>r.reject(message=form.data['<STR_LIT:message>'])<EOL>db.session.commit()<EOL>flash(_("<STR_LIT>"))<EOL>return redirect(url_for("<STR_LIT>"))<EOL><DEDENT><DEDENT>pid, record = get_record(r.recid)<EOL>return render_template(<EOL>"<STR_LIT>",<EOL>accessrequest=r,<EOL>record=record,<EOL>form=form,<EOL>)<EOL>


import uuid<EOL>from invenio_records.api import Record<EOL>from invenio_pidstore.models import PersistentIdentifier, PIDStatus<EOL>create_test_user()<EOL>indexer = RecordIndexer()<EOL>with db.session.begin_nested():<EOL><INDENT>rec_uuid = uuid.uuid4()<EOL>pid1 = PersistentIdentifier.create(<EOL>'<STR_LIT>', '<STR_LIT:1>', object_type='<STR_LIT>', object_uuid=rec_uuid,<EOL>status=PIDStatus.REGISTERED)<EOL>Record.create({<EOL>'<STR_LIT:title>': '<STR_LIT>',<EOL>'<STR_LIT:description>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT:1>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': [<NUM_LIT:1>, <NUM_LIT:2>],<EOL>'<STR_LIT>': <NUM_LIT:1><EOL>}, id_=rec_uuid)<EOL>indexer.index_by_id(pid1.object_uuid)<EOL><DEDENT>db.session.commit()<EOL>sleep(<NUM_LIT:3>)<EOL>


def _warn_node(self, msg, *args, **kwargs):

if not msg.startswith('<STR_LIT>'):<EOL><INDENT>_warn_node_old(self, msg, *args, **kwargs)<EOL><DEDENT>


def angularDistance(phi1, theta1, phi2, theta2):

<EOL>return arctan( sqrt((cos(theta2)*sin(phi2-phi1))**<NUM_LIT:2> +<EOL>(cos(theta1)*sin(theta2)-sin(theta1)*cos(theta2)*cos(phi2-phi1))**<NUM_LIT:2>) / (sin(theta1)*sin(theta2) +<EOL>cos(phi2-phi1)*cos(theta1)*cos(theta2)) )<EOL>

Calculate the angular distance between pairs of sky coordinates.

Parameters
----------

phi1 : float
    Longitude of first coordinate (radians).
theta1 : float
    Latitude of first coordinate (radians).
phi2 : float
    Longitude of second coordinate (radians).
theta2 : float
    Latitude of second coordinate (radians).

Returns
-------

Angular distance in radians.


self.rotationMatrix=_rotationMatrixMap[desiredTransformation]<EOL>self.transformationStrings=_transformationStringMap[desiredTransformation]<EOL>

Class constructor/initializer. Sets the type of transformation to be used.

Parameters
----------

desiredTransformation - The kind of coordinate transformation that should be provided. For example
Transformations.GAL2ECL


def transformCartesianCoordinates(self, x, y, z):

xrot, yrot, zrot = dot(self.rotationMatrix,[x,y,z])<EOL>return xrot, yrot, zrot<EOL>

Rotates Cartesian coordinates from one reference system to another using the rotation matrix with
which the class was initialized. The inputs  can be scalars or 1-dimensional numpy arrays.

Parameters
----------

x - Value of X-coordinate in original reference system
y - Value of Y-coordinate in original reference system
z - Value of Z-coordinate in original reference system

Returns
-------

xrot - Value of X-coordinate after rotation
yrot - Value of Y-coordinate after rotation
zrot - Value of Z-coordinate after rotation


def transformSkyCoordinates(self, phi, theta):

r=ones_like(phi)<EOL>x, y, z = sphericalToCartesian(r, phi, theta)<EOL>xrot, yrot, zrot = self.transformCartesianCoordinates(x, y, z)<EOL>r, phirot, thetarot = cartesianToSpherical(xrot, yrot, zrot)<EOL>return phirot, thetarot<EOL>

Converts sky coordinates from one reference system to another, making use of the rotation matrix with
which the class was initialized. Inputs can be scalars or 1-dimensional numpy arrays.

Parameters
----------

phi   - Value of the azimuthal angle (right ascension, longitude) in radians.
theta - Value of the elevation angle (declination, latitude) in radians.

Returns
-------

phirot   - Value of the transformed azimuthal angle in radians.
thetarot - Value of the transformed elevation angle in radians.


def transformProperMotions(self, phi, theta, muphistar, mutheta):

c, s = self._getJacobian(phi,theta)<EOL>return c*muphistar+s*mutheta, c*mutheta-s*muphistar<EOL>

Converts proper motions from one reference system to another, using the prescriptions in section
1.5.3 of the Hipparcos Explanatory Volume 1 (equations 1.5.18, 1.5.19).

Parameters
----------

phi       - The longitude-like angle of the position of the source (radians).
theta     - The latitude-like angle of the position of the source (radians).
muphistar - Value of the proper motion in the longitude-like angle, multiplied by cos(latitude).
mutheta   - Value of the proper motion in the latitude-like angle.

Returns
-------

muphistarrot - Value of the transformed proper motion in the longitude-like angle (including the
cos(latitude) factor).
muthetarot   - Value of the transformed proper motion in the latitude-like angle.


def transformSkyCoordinateErrors(self, phi, theta, sigPhiStar, sigTheta, rhoPhiTheta=<NUM_LIT:0>):

if isscalar(rhoPhiTheta) and not isscalar(sigTheta):<EOL><INDENT>rhoPhiTheta=zeros_like(sigTheta)+rhoPhiTheta<EOL><DEDENT>c, s = self._getJacobian(phi,theta)<EOL>cSqr = c*c<EOL>sSqr = s*s<EOL>covar = sigPhiStar*sigTheta*rhoPhiTheta<EOL>varPhiStar = sigPhiStar*sigPhiStar<EOL>varTheta = sigTheta*sigTheta<EOL>varPhiStarRot = cSqr*varPhiStar+sSqr*varTheta+<NUM_LIT>*covar*c*s<EOL>varThetaRot = sSqr*varPhiStar+cSqr*varTheta-<NUM_LIT>*covar*c*s<EOL>covarRot = (cSqr-sSqr)*covar+c*s*(varTheta-varPhiStar)<EOL>return sqrt(varPhiStarRot), sqrt(varThetaRot), covarRot/sqrt(varPhiStarRot*varThetaRot)<EOL>

Converts the sky coordinate errors from one reference system to another, including the covariance
term. Equations (1.5.4) and (1.5.20) from section 1.5 in the Hipparcos Explanatory Volume 1 are used.

Parameters
----------

phi         - The longitude-like angle of the position of the source (radians).
theta       - The latitude-like angle of the position of the source (radians).
sigPhiStar  - Standard error in the longitude-like angle of the position of the source (radians or
              sexagesimal units, including cos(latitude) term)
sigTheta    - Standard error in the latitude-like angle of the position of the source (radians or
              sexagesimal units)

Keywords (optional)
-------------------

rhoPhiTheta - Correlation coefficient of the position errors. Set to zero if this keyword is not
              provided.

Retuns
------

sigPhiRotStar  - The transformed standard error in the longitude-like angle (including
                 cos(latitude) factor)
sigThetaRot    - The transformed standard error in the latitude-like angle.
rhoPhiThetaRot - The transformed correlation coefficient.


def transformProperMotionErrors(self, phi, theta, sigMuPhiStar, sigMuTheta, rhoMuPhiMuTheta=<NUM_LIT:0>):

return self.transformSkyCoordinateErrors(phi, theta, sigMuPhiStar, sigMuTheta,<EOL>rhoPhiTheta=rhoMuPhiMuTheta)<EOL>

Converts the proper motion errors from one reference system to another, including the covariance
term. Equations (1.5.4) and (1.5.20) from section 1.5 in the Hipparcos Explanatory Volume 1 are used.

Parameters
----------

phi             - The longitude-like angle of the position of the source (radians).
theta           - The latitude-like angle of the position of the source (radians).
sigMuPhiStar    - Standard error in the proper motion in the longitude-like direction (including
cos(latitude) factor).
sigMuTheta      - Standard error in the proper motion in the latitude-like direction.

Keywords (optional)
-------------------

rhoMuPhiMuTheta - Correlation coefficient of the proper motion errors. Set to zero if this
                  keyword is not provided.

Retuns
------

sigMuPhiRotStar    - The transformed standard error in the proper motion in the longitude direction
(including cos(latitude) factor).
sigMuThetaRot      - The transformed standard error in the proper motion in the longitude direction.
rhoMuPhiMuThetaRot - The transformed correlation coefficient.


def transformCovarianceMatrix(self, phi, theta, covmat):

c, s = self._getJacobian(phi,theta)<EOL>jacobian = identity(<NUM_LIT:5>)<EOL>jacobian[<NUM_LIT:0>][<NUM_LIT:0>]=c<EOL>jacobian[<NUM_LIT:1>][<NUM_LIT:1>]=c<EOL>jacobian[<NUM_LIT:3>][<NUM_LIT:3>]=c<EOL>jacobian[<NUM_LIT:4>][<NUM_LIT:4>]=c<EOL>jacobian[<NUM_LIT:0>][<NUM_LIT:1>]=s<EOL>jacobian[<NUM_LIT:1>][<NUM_LIT:0>]=-s<EOL>jacobian[<NUM_LIT:3>][<NUM_LIT:4>]=s<EOL>jacobian[<NUM_LIT:4>][<NUM_LIT:3>]=-s<EOL>return dot( dot(jacobian, covmat), jacobian.T )<EOL>

Transform the astrometric covariance matrix to its representation in the new coordinate system.

Parameters
----------

phi       - The longitude-like angle of the position of the source (radians).
theta     - The latitude-like angle of the position of the source (radians).
covmat    - Covariance matrix (5x5) of the astrometric parameters.

Returns
-------

covmat_rot - Covariance matrix in its representation in the new coordinate system.


p, q, r = normalTriad(phi, theta)<EOL>zRot = self.rotationMatrix[<NUM_LIT:2>,:]<EOL>if (p.ndim == <NUM_LIT:2>):<EOL><INDENT>zRotAll = tile(zRot, p.shape[<NUM_LIT:1>]).reshape(p.shape[<NUM_LIT:1>],<NUM_LIT:3>)<EOL>pRot = cross(zRotAll, r.T)<EOL>normPRot = norm(pRot,axis=<NUM_LIT:1>)<EOL>for i in range(pRot.shape[<NUM_LIT:0>]):<EOL><INDENT>pRot[i] = pRot[i]/normPRot[i]<EOL><DEDENT>c = zeros(pRot.shape[<NUM_LIT:0>])<EOL>s = zeros(pRot.shape[<NUM_LIT:0>])<EOL>for i in range(pRot.shape[<NUM_LIT:0>]):<EOL><INDENT>c[i] = dot(pRot[i], p.T[i])<EOL>s[i] = dot(pRot[i], q.T[i])<EOL><DEDENT>return c, s<EOL><DEDENT>else:<EOL><INDENT>pRot = cross(zRot, r.T)<EOL>pRot = pRot/norm(pRot)<EOL>return dot(pRot,p), dot(pRot,q)<EOL><DEDENT>

Calculates the Jacobian for the transformation of the position errors and proper motion errors
between coordinate systems. This Jacobian is also the rotation matrix for the transformation of
proper motions. See section 1.5.3 of the Hipparcos Explanatory Volume 1 (equation 1.5.20). This
matrix has the following form:

    |  c  s |
J = |       |
    | -s  c |

Parameters
----------

phi       - The longitude-like angle of the position of the source (radians).
theta     - The latitude-like angle of the position of the source (radians).

Returns
-------

c, s - The Jacobian matrix elements c and s corresponding to (phi, theta) and the currently
       desired coordinate system transformation.


def propagate_astrometry(self, phi, theta, parallax, muphistar, mutheta, vrad, t0, t1):

t = t1-t0<EOL>p0, q0, r0 = normalTriad(phi, theta)<EOL>pmra0 = muphistar*self.mastorad<EOL>pmdec0 = mutheta*self.mastorad<EOL>plx0 = parallax*self.mastorad<EOL>pmr0 = vrad*parallax/auKmYearPerSec*self.mastorad<EOL>pmtot0sqr = (muphistar**<NUM_LIT:2> + mutheta**<NUM_LIT:2>) * self.mastorad**<NUM_LIT:2><EOL>pmvec0 = pmra0*p0+pmdec0*q0<EOL>f = (<NUM_LIT:1> + <NUM_LIT:2>*pmr0*t + (pmtot0sqr+pmr0**<NUM_LIT:2>)*t**<NUM_LIT:2>)**(-<NUM_LIT:0.5>)<EOL>u = (r0*(<NUM_LIT:1>+pmr0*t) + pmvec0*t)*f<EOL>_, phi1, theta1 = cartesianToSpherical(u[<NUM_LIT:0>], u[<NUM_LIT:1>], u[<NUM_LIT:2>])<EOL>parallax1 = parallax*f<EOL>pmr1 = (pmr0+(pmtot0sqr + pmr0**<NUM_LIT:2>)*t)*f**<NUM_LIT:2><EOL>pmvec1 = (pmvec0*(<NUM_LIT:1>+pmr0*t) - r0*pmr0**<NUM_LIT:2>*t)*f**<NUM_LIT:3><EOL>p1, q1, r1 = normalTriad(phi1, theta1)<EOL>muphistar1 = sum(p1*pmvec1/self.mastorad, axis=<NUM_LIT:0>)<EOL>mutheta1 = sum(q1*pmvec1/self.mastorad, axis =<NUM_LIT:0>)<EOL>murad1 = pmr1/self.mastorad<EOL>return phi1, theta1, parallax1, muphistar1, mutheta1, murad1<EOL>

Propagate the position of a source from the reference epoch t0 to the new epoch t1.

Parameters
----------

phi : float
    Longitude at reference epoch (radians).
theta : float
    Latitude at reference epoch (radians).
parallax : float
    Parallax at the reference epoch (mas).
muphistar : float
    Proper motion in longitude (including cos(latitude) term) at reference epoch (mas/yr).
mutheta : float
    Proper motion in latitude at reference epoch (mas/yr).
vrad : float
    Radial velocity at reference epoch (km/s).
t0 : float
    Reference epoch (Julian years).
t1 : float
    New epoch (Julian years).

Returns
-------

Astrometric parameters, including the "radial proper motion" (NOT the radial velocity), at the new epoch.
phi1, theta1, parallax1, muphistar1, mutheta1, mur1 = epoch_prop_pos(..., t0, t1)


def propagate_pos(self, phi, theta, parallax, muphistar, mutheta, vrad, t0, t1):

phi1, theta1, parallax1, muphistar1, mutheta1, vrad1 = self.epoch_prop_astrometry(phi, theta, parallax, muphistar, mutheta, vrad, t0, t1)<EOL>return phi1, theta1<EOL>

Propagate the position of a source from the reference epoch t0 to the new epoch t1.

Parameters
----------

phi : float
    Longitude at reference epoch (radians).
theta : float
    Latitude at reference epoch (radians).
parallax : float
    Parallax at the reference epoch (mas).
muphistar : float
    Proper motion in longitude (including cos(latitude) term) at reference epoch (mas/yr).
mutheta : float
    Proper motion in latitude at reference epoch (mas/yr).
vrad : float
    Radial velocity at reference epoch (km/s).
t0 : float
    Reference epoch (Julian years).
t1 : float
    New epoch (Julian years).

Returns
-------

Coordinates phi and theta at new epoch (in radians)


def propagate_astrometry_and_covariance_matrix(self, a0, c0, t0, t1):

zero, one, two, three = <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:2>, <NUM_LIT:3><EOL>tau = t1-t0<EOL>p0, q0, r0 = normalTriad(a0[<NUM_LIT:0>], a0[<NUM_LIT:1>])<EOL>par0 = a0[<NUM_LIT:2>]*self.mastorad<EOL>pma0 = a0[<NUM_LIT:3>]*self.mastorad<EOL>pmd0 = a0[<NUM_LIT:4>]*self.mastorad<EOL>pmr0 = a0[<NUM_LIT:5>]*a0[<NUM_LIT:2>]/auKmYearPerSec*self.mastorad<EOL>pmvec0 = pma0*p0+pmd0*q0<EOL>tau2 = tau*tau<EOL>pm02 = pma0**<NUM_LIT:2> + pmd0**<NUM_LIT:2><EOL>w = one + pmr0*tau<EOL>f2 = one/(one + two*pmr0*tau + (pm02+pmr0**<NUM_LIT:2>)*tau2)<EOL>f = sqrt(f2)<EOL>f3 = f2*f<EOL>f4 = f2*f2<EOL>u = (r0*w + pmvec0*tau)*f<EOL>_, ra, dec = cartesianToSpherical(u[<NUM_LIT:0>], u[<NUM_LIT:1>], u[<NUM_LIT:2>])<EOL>par = par0*f<EOL>pmvec = (pmvec0*(one+pmr0*tau) - r0*pmr0**<NUM_LIT:2>*tau)*f3<EOL>pmr = (pmr0+(pm02 + pmr0**<NUM_LIT:2>)*tau)*f2<EOL>p, q, r = normalTriad(ra, dec)<EOL>pma = sum(p*pmvec, axis=<NUM_LIT:0>)<EOL>pmd = sum(q*pmvec, axis =<NUM_LIT:0>)<EOL>a = zeros_like(a0)<EOL>a[<NUM_LIT:0>] = ra<EOL>a[<NUM_LIT:1>] = dec<EOL>a[<NUM_LIT:2>] = par/self.mastorad<EOL>a[<NUM_LIT:3>] = pma/self.mastorad<EOL>a[<NUM_LIT:4>] = pmd/self.mastorad<EOL>a[<NUM_LIT:5>] = pmr/self.mastorad<EOL>pmz = pmvec0*f - three*pmvec*w<EOL>pp0 = sum(p*p0, axis=<NUM_LIT:0>)<EOL>pq0 = sum(p*q0, axis=<NUM_LIT:0>)<EOL>pr0 = sum(p*r0, axis=<NUM_LIT:0>)<EOL>qp0 = sum(q*p0, axis=<NUM_LIT:0>)<EOL>qq0 = sum(q*q0, axis=<NUM_LIT:0>)<EOL>qr0 = sum(q*r0, axis=<NUM_LIT:0>)<EOL>ppmz = sum(p*pmz, axis=<NUM_LIT:0>)<EOL>qpmz = sum(q*pmz, axis=<NUM_LIT:0>)<EOL>J = zeros_like(c0)<EOL>if (c0.ndim==<NUM_LIT:2>):<EOL><INDENT>J = J[newaxis,:,:]<EOL><DEDENT>J[:,<NUM_LIT:0>,<NUM_LIT:0>] = pp0*w*f - pr0*pma0*tau*f<EOL>J[:,<NUM_LIT:0>,<NUM_LIT:1>] = pq0*w*f - pr0*pmd0*tau*f<EOL>J[:,<NUM_LIT:0>,<NUM_LIT:2>] = zero<EOL>J[:,<NUM_LIT:0>,<NUM_LIT:3>] = pp0*tau*f<EOL>J[:,<NUM_LIT:0>,<NUM_LIT:4>] = pq0*tau*f<EOL>J[:,<NUM_LIT:0>,<NUM_LIT:5>] = -pma*tau2<EOL>J[:,<NUM_LIT:1>,<NUM_LIT:0>] = qp0*w*f - qr0*pma0*tau*f<EOL>J[:,<NUM_LIT:1>,<NUM_LIT:1>] = qq0*w*f - qr0*pmd0*tau*f<EOL>J[:,<NUM_LIT:1>,<NUM_LIT:2>] = zero<EOL>J[:,<NUM_LIT:1>,<NUM_LIT:3>] = qp0*tau*f<EOL>J[:,<NUM_LIT:1>,<NUM_LIT:4>] = qq0*tau*f<EOL>J[:,<NUM_LIT:1>,<NUM_LIT:5>] = -pmd*tau2<EOL>J[:,<NUM_LIT:2>,<NUM_LIT:0>] = zero<EOL>J[:,<NUM_LIT:2>,<NUM_LIT:1>] = zero<EOL>J[:,<NUM_LIT:2>,<NUM_LIT:2>] = f<EOL>J[:,<NUM_LIT:2>,<NUM_LIT:3>] = -par*pma0*tau2*f2<EOL>J[:,<NUM_LIT:2>,<NUM_LIT:4>] = -par*pmd0*tau2*f2<EOL>J[:,<NUM_LIT:2>,<NUM_LIT:5>] = -par*w*tau*f2<EOL>J[:,<NUM_LIT:3>,<NUM_LIT:0>] = -pp0*pm02*tau*f3 - pr0*pma0*w*f3<EOL>J[:,<NUM_LIT:3>,<NUM_LIT:1>] = -pq0*pm02*tau*f3 - pr0*pmd0*w*f3<EOL>J[:,<NUM_LIT:3>,<NUM_LIT:2>] = zero<EOL>J[:,<NUM_LIT:3>,<NUM_LIT:3>] = pp0*w*f3 - two*pr0*pma0*tau*f3 - three*pma*pma0*tau2*f2<EOL>J[:,<NUM_LIT:3>,<NUM_LIT:4>] = pq0*w*f3 - two*pr0*pmd0*tau*f3 - three*pma*pmd0*tau2*f2<EOL>J[:,<NUM_LIT:3>,<NUM_LIT:5>] = ppmz*tau*f2<EOL>J[:,<NUM_LIT:4>,<NUM_LIT:0>] = -qp0*pm02*tau*f3 - qr0*pma0*w*f3<EOL>J[:,<NUM_LIT:4>,<NUM_LIT:1>] = -qq0*pm02*tau*f3 - qr0*pmd0*w*f3<EOL>J[:,<NUM_LIT:4>,<NUM_LIT:2>] = zero<EOL>J[:,<NUM_LIT:4>,<NUM_LIT:3>] = qp0*w*f3 - two*qr0*pma0*tau*f3 - three*pmd*pma0*tau2*f2<EOL>J[:,<NUM_LIT:4>,<NUM_LIT:4>] = qq0*w*f3 - two*qr0*pmd0*tau*f3 - three*pmd*pmd0*tau2*f2<EOL>J[:,<NUM_LIT:4>,<NUM_LIT:5>] = qpmz*tau*f2<EOL>J[:,<NUM_LIT:5>,<NUM_LIT:0>] = zero<EOL>J[:,<NUM_LIT:5>,<NUM_LIT:1>] = zero<EOL>J[:,<NUM_LIT:5>,<NUM_LIT:2>] = zero<EOL>J[:,<NUM_LIT:5>,<NUM_LIT:3>] = two*pma0*w*tau*f4<EOL>J[:,<NUM_LIT:5>,<NUM_LIT:4>] = two*pmd0*w*tau*f4<EOL>J[:,<NUM_LIT:5>,<NUM_LIT:5>] = (w**<NUM_LIT:2> - pm02*tau2)*f4<EOL>JT = zeros_like(J)<EOL>for i in range(J.shape[<NUM_LIT:0>]):<EOL><INDENT>JT[i] = J[i].T<EOL><DEDENT>if (c0.ndim==<NUM_LIT:2>):<EOL><INDENT>c = matmul(J,matmul(c0[newaxis,:,:],JT))<EOL><DEDENT>else:<EOL><INDENT>c = matmul(J,matmul(c0,JT))<EOL><DEDENT>return a, squeeze(c)<EOL>

Propagate the covariance matrix of the astrometric parameters and radial proper motion of a
source from epoch t0 to epoch t1.

Code based on the Hipparcos Fortran implementation by Lennart Lindegren.

Parameters
----------

a0 : array_like
    6-element vector: (phi, theta, parallax, muphistar, mutheta, vrad) in units of (radians,
    radians, mas, mas/yr, mas/yr, km/s). Shape of a should be (6,) or (6,N), with N the number of
    sources for which the astrometric parameters are provided.

c0 : array_like
    Covariance matrix stored in a 6x6 element array. This can be constructed from the columns
    listed in the Gaia catalogue. The units are [mas^2, mas^2/yr, mas^2/yr^2] for the various
    elements. Note that the elements in the 6th row and column should be:
    c[6,i]=c[i,6]=c[i,3]*vrad/auKmYearPerSec for i=1,..,5 and
    c[6,6]=c[3,3]*(vrad^2+vrad_error^2)/auKmYearPerSec^2+(parallax*vrad_error/auKmYearPerSec)^2

    Shape of c0 should be (6,6) or (N,6,6).

t0 : float
    Reference epoch (Julian years).

t1 : float
    New epoch (Julian years).

Returns
-------

Astrometric parameters, including the "radial proper motion" (NOT the radial velocity), and
covariance matrix at the new epoch as a 2D matrix with the new variances on the diagional and the
covariance in the off-diagonal elements.


def sphericalToCartesian(r, phi, theta):

ctheta=cos(theta)<EOL>x=r*cos(phi)*ctheta<EOL>y=r*sin(phi)*ctheta<EOL>z=r*sin(theta)<EOL>return x, y, z<EOL>

Convert spherical to Cartesian coordinates. The input can be scalars or 1-dimensional numpy arrays.
Note that the angle coordinates follow the astronomical convention of using elevation (declination,
latitude) rather than its complement (pi/2-elevation), where the latter is commonly used in the
mathematical treatment of spherical coordinates.

Parameters
----------

r     - length of input Cartesian vector.
phi   - longitude-like angle (e.g., right ascension, ecliptic longitude) in radians
theta - latitide-like angle (e.g., declination, ecliptic latitude) in radians

Returns
-------

The Cartesian vector components x, y, z


rCylSq=x*x+y*y<EOL>r=sqrt(rCylSq+z*z)<EOL>if any(r==<NUM_LIT:0.0>):<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>phi = arctan2(y,x)<EOL>phi = where(phi<<NUM_LIT:0.0>, phi+<NUM_LIT:2>*pi, phi)<EOL>return r, phi, arctan2(z,sqrt(rCylSq))<EOL>

Convert Cartesian to spherical coordinates. The input can be scalars or 1-dimensional numpy arrays.
Note that the angle coordinates follow the astronomical convention of using elevation (declination,
latitude) rather than its complement (pi/2-elevation), which is commonly used in the mathematical
treatment of spherical coordinates.

Parameters
----------

x - Cartesian vector component along the X-axis
y - Cartesian vector component along the Y-axis
z - Cartesian vector component along the Z-axis

Returns
-------

The spherical coordinates r=sqrt(x*x+y*y+z*z), longitude phi, latitude theta.

NOTE THAT THE LONGITUDE ANGLE IS BETWEEN 0 AND +2PI. FOR r=0 AN EXCEPTION IS RAISED.


sphi=sin(phi)<EOL>stheta=sin(theta)<EOL>cphi=cos(phi)<EOL>ctheta=cos(theta)<EOL>p=array([-sphi, cphi, zeros_like(phi)])<EOL>q=array([-stheta*cphi, -stheta*sphi, ctheta])<EOL>r=array([ctheta*cphi, ctheta*sphi, stheta])<EOL>return p, q, r<EOL>

Calculate the so-called normal triad [p, q, r] which is associated with a spherical coordinate system .
The three vectors are:

p - The unit tangent vector in the direction of increasing longitudinal angle phi.
q - The unit tangent vector in the direction of increasing latitudinal angle theta.
r - The unit vector toward the point (phi, theta).

Parameters
----------

phi   - longitude-like angle (e.g., right ascension, ecliptic longitude) in radians
theta - latitide-like angle (e.g., declination, ecliptic latitude) in radians

Returns
-------

The normal triad as the vectors p, q, r


if (axis=="<STR_LIT:x>" or axis=="<STR_LIT:X>"):<EOL><INDENT>return array([[<NUM_LIT:1.0>, <NUM_LIT:0.0>, <NUM_LIT:0.0>], [<NUM_LIT:0.0>, cos(rotationAngle), sin(rotationAngle)], [<NUM_LIT:0.0>,<EOL>-sin(rotationAngle), cos(rotationAngle)]])<EOL><DEDENT>elif (axis=="<STR_LIT:y>" or axis=="<STR_LIT:Y>"):<EOL><INDENT>return array([[cos(rotationAngle), <NUM_LIT:0.0>, -sin(rotationAngle)], [<NUM_LIT:0.0>, <NUM_LIT:1.0>, <NUM_LIT:0.0>], [sin(rotationAngle),<EOL><NUM_LIT:0.0>, cos(rotationAngle)]])<EOL><DEDENT>elif (axis=="<STR_LIT:z>" or axis=="<STR_LIT>"):<EOL><INDENT>return array([[cos(rotationAngle), sin(rotationAngle), <NUM_LIT:0.0>], [-sin(rotationAngle),<EOL>cos(rotationAngle), <NUM_LIT:0.0>], [<NUM_LIT:0.0>, <NUM_LIT:0.0>, <NUM_LIT:1.0>]])<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>"+axis+"<STR_LIT:!>")<EOL><DEDENT>

Construct an elementary rotation matrix describing a rotation around the x, y, or z-axis.

Parameters
----------

axis          - Axis around which to rotate ("x", "y", or "z")
rotationAngle - the rotation angle in radians

Returns
-------

The rotation matrix

Example usage
-------------

rotmat = elementaryRotationMatrix("y", pi/6.0)


def phaseSpaceToAstrometry(x, y, z, vx, vy, vz):

u, phi, theta = cartesianToSpherical(x, y, z)<EOL>parallax = _auMasParsec/u<EOL>p, q, r = normalTriad(phi, theta)<EOL>velocitiesArray=array([vx,vy,vz])<EOL>if  isscalar(u):<EOL><INDENT>muphistar=dot(p,velocitiesArray)*parallax/_auKmYearPerSec<EOL>mutheta=dot(q,velocitiesArray)*parallax/_auKmYearPerSec<EOL>vrad=dot(r,velocitiesArray)<EOL><DEDENT>else:<EOL><INDENT>muphistar=zeros_like(parallax)<EOL>mutheta=zeros_like(parallax)<EOL>vrad=zeros_like(parallax)<EOL>for i in range(parallax.size):<EOL><INDENT>muphistar[i]=dot(p[:,i],velocitiesArray[:,i])*parallax[i]/_auKmYearPerSec<EOL>mutheta[i]=dot(q[:,i],velocitiesArray[:,i])*parallax[i]/_auKmYearPerSec<EOL>vrad[i]=dot(r[:,i],velocitiesArray[:,i])<EOL><DEDENT><DEDENT>return phi, theta, parallax, muphistar, mutheta, vrad<EOL>

From the given phase space coordinates calculate the astrometric observables, including the radial
velocity, which here is seen as the sixth astrometric parameter. The phase space coordinates are
assumed to represent barycentric (i.e. centred on the Sun) positions and velocities.

This function has no mechanism to deal with units. The velocity units are always assumed to be km/s,
and the code is set up such that for positions in pc, the return units for the astrometry are radians,
milliarcsec, milliarcsec/year and km/s. For positions in kpc the return units are: radians,
microarcsec, microarcsec/year, and km/s.

NOTE that the doppler factor k=1/(1-vrad/c) is NOT used in the calculations. This is not a problem for
sources moving at typical velocities of Galactic stars.

Parameters
----------

x - The x component of the barycentric position vector (in pc or kpc).
y - The y component of the barycentric position vector (in pc or kpc).
z - The z component of the barycentric position vector (in pc or kpc).
vx - The x component of the barycentric velocity vector (in km/s).
vy - The y component of the barycentric velocity vector (in km/s).
vz - The z component of the barycentric velocity vector (in km/s).

Returns
-------

phi       - The longitude-like angle of the position of the source (radians).
theta     - The latitude-like angle of the position of the source (radians).
parallax  - The parallax of the source (in mas or muas, see above)
muphistar - The proper motion in the longitude-like angle, multiplied by cos(theta) (mas/yr or muas/yr,
see above)
mutheta   - The proper motion in the latitude-like angle (mas/yr or muas/yr, see above)
vrad      - The radial velocity (km/s)


def astrometryToPhaseSpace(phi, theta, parallax, muphistar, mutheta, vrad):

if any(parallax<=<NUM_LIT:0.0>):<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>x, y, z = sphericalToCartesian(_auMasParsec/parallax, phi, theta)<EOL>p, q, r = normalTriad(phi, theta)<EOL>transverseMotionArray = array([muphistar*_auKmYearPerSec/parallax, mutheta*_auKmYearPerSec/parallax,<EOL>vrad])<EOL>if isscalar(parallax):<EOL><INDENT>velocityArray=dot(transpose(array([p, q, r])),transverseMotionArray)<EOL>vx = velocityArray[<NUM_LIT:0>]<EOL>vy = velocityArray[<NUM_LIT:1>]<EOL>vz = velocityArray[<NUM_LIT:2>]<EOL><DEDENT>else:<EOL><INDENT>vx = zeros_like(parallax)<EOL>vy = zeros_like(parallax)<EOL>vz = zeros_like(parallax)<EOL>for i in range(parallax.size):<EOL><INDENT>velocityArray = dot(transpose(array([p[:,i], q[:,i], r[:,i]])), transverseMotionArray[:,i])<EOL>vx[i] = velocityArray[<NUM_LIT:0>]<EOL>vy[i] = velocityArray[<NUM_LIT:1>]<EOL>vz[i] = velocityArray[<NUM_LIT:2>]<EOL><DEDENT><DEDENT>return x, y, z, vx, vy, vz<EOL>

From the input astrometric parameters calculate the phase space coordinates. The output phase space
coordinates represent barycentric (i.e. centred on the Sun) positions and velocities.

This function has no mechanism to deal with units. The code is set up such that for input astrometry
with parallaxes and proper motions in mas and mas/yr, and radial velocities in km/s, the phase space
coordinates are in pc and km/s. For input astrometry with parallaxes and proper motions in muas and
muas/yr, and radial velocities in km/s, the phase space coordinates are in kpc and km/s. Only positive
parallaxes are accepted, an exception is thrown if this condition is not met.

NOTE that the doppler factor k=1/(1-vrad/c) is NOT used in the calculations. This is not a problem for
sources moving at typical velocities of Galactic stars.

THIS FUNCTION SHOULD NOT BE USED WHEN THE PARALLAXES HAVE RELATIVE ERRORS LARGER THAN ABOUT 20 PER CENT
(see http://arxiv.org/abs/1507.02105 for example). For astrometric data with relatively large parallax
errors you should consider doing your analysis in the data space and use forward modelling of some
kind.

Parameters
----------

phi       - The longitude-like angle of the position of the source (radians).
theta     - The latitude-like angle of the position of the source (radians).
parallax  - The parallax of the source (in mas or muas, see above)
muphistar - The proper motion in the longitude-like angle, multiplied by cos(theta) (mas/yr or muas/yr,
see above)
mutheta   - The proper motion in the latitude-like angle (mas/yr or muas/yr, see above)
vrad      - The radial velocity (km/s)

Returns
-------

x - The x component of the barycentric position vector (in pc or kpc).
y - The y component of the barycentric position vector (in pc or kpc).
z - The z component of the barycentric position vector (in pc or kpc).
vx - The x component of the barycentric velocity vector (in km/s).
vy - The y component of the barycentric velocity vector (in km/s).
vz - The z component of the barycentric velocity vector (in km/s).


varDiag1 = rand(num)*<NUM_LIT>+<NUM_LIT:1.0><EOL>varDiag2 = rand(num)*<NUM_LIT>+<NUM_LIT:1.0><EOL>rotAngle = <NUM_LIT>*pi*rand(num)<EOL>sigma1 = zeros(num)<EOL>sigma2 = zeros(num)<EOL>rho12 = zeros(num)<EOL>for i in range(num): <EOL><INDENT>rotationMatrix=array([[cos(rotAngle[i]), sin(rotAngle[i])], [-sin(rotAngle[i]), cos(rotAngle[i])]])<EOL>covMatrix = dot(rotationMatrix,dot(diag([varDiag1[i],varDiag2[i]]),transpose(rotationMatrix)))<EOL>sigma1[i] = sqrt(covMatrix[<NUM_LIT:0>,<NUM_LIT:0>])<EOL>sigma2[i] = sqrt(covMatrix[<NUM_LIT:1>,<NUM_LIT:1>])<EOL>rho12[i] = covMatrix[<NUM_LIT:0>,<NUM_LIT:1>]/(sigma1[i]*sigma2[i])<EOL><DEDENT>return sigma1, sigma2, rho12<EOL>

Generate random covariance matrices through the transformation of diagonal positive definite
matrices.

Parameters
----------

num - Number of matrices to generate.

Returns
-------

sigma1 - Square root of variance of first variable.
sigma2 - Square root of variance of second variable.
rho12  - Correlation coefficient between errors on variables 1 and 2


if allclose(A, A.T, rtol=<NUM_LIT>, atol=<NUM_LIT>):<EOL><INDENT>try:<EOL><INDENT>cholesky(A)<EOL>return True<EOL><DEDENT>except LinAlgError:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

Check if matrix A is positive definite. Code from
https://stackoverflow.com/questions/16266720/find-out-if-matrix-is-positive-definite-with-numpy


xroll=roll(x,<NUM_LIT:1>)<EOL>yroll=roll(y,<NUM_LIT:1>)<EOL>distance=(xroll-x)**<NUM_LIT:2>+(yroll-y)**<NUM_LIT:2><EOL>indexmax=argmax(distance)<EOL>return roll(x,-indexmax), roll(y,-indexmax)<EOL>

This code takes care of ordering the points (x,y), calculated for a sky map parallel or merdian, such
that the drawing code can start at one end of the curve and end at the other (so no artifacts due to
connecting the disjoint ends occur).

Parameters
----------

x - Set of x coordinates
y - Set of y coordinates

Returns
-------

x, y: Order set of (x,y) points