if not kwargs:<EOL><INDENT>raise ValueError("<STR_LIT>".format(<EOL>'<STR_LIT:U+002CU+0020>'.join(self._allowed_args)))<EOL><DEDENT>elif len(kwargs) > <NUM_LIT:1>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>elif set(kwargs.keys()) <= set(self._allowed_args):<EOL><INDENT>_name, _api_arg = list(kwargs.items())[<NUM_LIT:0>]<EOL>name = '<STR_LIT:_>' + _name + '<STR_LIT:_>'<EOL>return TrelloAPI(endpoints=self._endpoints[name],<EOL>name=name,<EOL>apikey=self._apikey,<EOL>parent=self,<EOL>api_arg=_api_arg,<EOL>token=self._token)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>".format(kwargs.keys()))<EOL><DEDENT>

Adds a variable parameter to the API URL.


results = TIMESTRING_RE.findall(text)<EOL>dates = []<EOL>for date in results:<EOL><INDENT>if re.compile('<STR_LIT>', re.I).match(date[<NUM_LIT:0>]):<EOL><INDENT>dates.append((date[<NUM_LIT:0>].strip(), Range(date[<NUM_LIT:0>])))<EOL><DEDENT>else:<EOL><INDENT>dates.append((date[<NUM_LIT:0>].strip(), Date(date[<NUM_LIT:0>])))<EOL><DEDENT><DEDENT>return dates<EOL>

Find all the timestrings within a block of text.

    >>> timestring.findall("once upon a time, about 3 weeks ago, there was a boy whom was born on august 15th at 7:20 am. epic.")
    [
     ('3 weeks ago,', <timestring.Date 2014-02-09 00:00:00 4483019280>),
     ('august 15th at 7:20 am', <timestring.Date 2014-08-15 07:20:00 4483019344>)
    ]


def __init__(self, start, end=None, offset=None, start_of_week=<NUM_LIT:0>, tz=None, verbose=False):

self._dates = []<EOL>pgoffset = None<EOL>if start is None:<EOL><INDENT>raise TimestringInvalid("<STR_LIT>")<EOL><DEDENT>if not isinstance(start, (Date, datetime)):<EOL><INDENT>start = str(start)<EOL><DEDENT>if end and not isinstance(end, (Date, datetime)):<EOL><INDENT>end = str(end)<EOL><DEDENT>if start and end:<EOL><INDENT>"""<STR_LIT>"""<EOL>self._dates = (Date(start, tz=tz), Date(end, tz=tz))<EOL><DEDENT>elif start == '<STR_LIT>':<EOL><INDENT>self._dates = (Date('<STR_LIT>'), Date('<STR_LIT>'))<EOL><DEDENT>elif re.search(r'<STR_LIT>', start):<EOL><INDENT>"""<STR_LIT>"""<EOL>start = re.sub('<STR_LIT>', '<STR_LIT>', start.lower())<EOL>r = tuple(re.split(r'<STR_LIT>', start.strip()))<EOL>self._dates = (Date(r[<NUM_LIT:0>], tz=tz), Date(r[-<NUM_LIT:1>], tz=tz))<EOL><DEDENT>elif re.match(r"<STR_LIT>", start):<EOL><INDENT>"""<STR_LIT>"""<EOL>start, end = tuple(re.sub('<STR_LIT>', '<STR_LIT>', start).split('<STR_LIT:U+002C>'))<EOL>self._dates = (Date(start), Date(end))<EOL><DEDENT>else:<EOL><INDENT>now = datetime.now()<EOL>if re.search(r"<STR_LIT>", start):<EOL><INDENT>pgoffset = re.search(r"<STR_LIT>", start).group() + "<STR_LIT>"<EOL><DEDENT>if tz:<EOL><INDENT>now = now.replace(tzinfo=pytz.timezone(str(tz)))<EOL><DEDENT>res = TIMESTRING_RE.search(start)<EOL>if res:<EOL><INDENT>group = res.groupdict()<EOL>if verbose:<EOL><INDENT>print(dict(map(lambda a: (a, group.get(a)), filter(lambda a: group.get(a), group))))<EOL><DEDENT>if (group.get('<STR_LIT>') or group.get('<STR_LIT>')) is not None:<EOL><INDENT>delta = (group.get('<STR_LIT>') or group.get('<STR_LIT>')).lower()<EOL>start = Date("<STR_LIT>", offset=offset, tz=tz)<EOL>di = "<STR_LIT>" % (str(int(group['<STR_LIT>'] or <NUM_LIT:1>)), delta)<EOL>if group['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>if delta.startswith('<STR_LIT:y>'):<EOL><INDENT>start = Date(datetime(now.year, <NUM_LIT:1>, <NUM_LIT:1>), offset=offset, tz=tz)<EOL><DEDENT>elif delta.startswith('<STR_LIT>'):<EOL><INDENT>start = Date(datetime(now.year, now.month, <NUM_LIT:1>), offset=offset, tz=tz)<EOL><DEDENT>elif delta.startswith('<STR_LIT:w>'):<EOL><INDENT>start = Date("<STR_LIT>", offset=offset, tz=tz) - (str(Date("<STR_LIT>", tz=tz).date.weekday())+'<STR_LIT>')<EOL><DEDENT>elif delta.startswith('<STR_LIT:d>'):<EOL><INDENT>start = Date("<STR_LIT>", offset=offset, tz=tz)<EOL><DEDENT>elif delta.startswith('<STR_LIT:h>'):<EOL><INDENT>start = Date("<STR_LIT>", offset=dict(hour=now.hour+<NUM_LIT:1>), tz=tz)<EOL><DEDENT>elif delta.startswith('<STR_LIT:m>') or delta.startswith('<STR_LIT:s>'):<EOL><INDENT>start = Date("<STR_LIT>", tz=tz)<EOL><DEDENT>else:<EOL><INDENT>raise TimestringInvalid("<STR_LIT>")<EOL><DEDENT>end = start + di<EOL><DEDENT>elif group['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>if int(group['<STR_LIT>'] or <NUM_LIT:1>) > <NUM_LIT:1>:<EOL><INDENT>di = "<STR_LIT>" % (str(int(group['<STR_LIT>'] or <NUM_LIT:1>) - <NUM_LIT:1>), delta)<EOL><DEDENT>end = start + di<EOL><DEDENT>elif group.get('<STR_LIT>') or group['<STR_LIT>'] == '<STR_LIT>' and int(group['<STR_LIT>'] or <NUM_LIT:1>) == <NUM_LIT:1>:<EOL><INDENT>end = start - di<EOL><DEDENT>else:<EOL><INDENT>if not (delta.startswith('<STR_LIT:h>') or delta.startswith('<STR_LIT:m>') or delta.startswith('<STR_LIT:s>')):<EOL><INDENT>start = Range('<STR_LIT>', offset=offset, tz=tz).end<EOL><DEDENT>end = start - di                    <EOL><DEDENT><DEDENT>elif group.get('<STR_LIT>'):<EOL><INDENT>start = Date(start, offset=offset, tz=tz)<EOL>start = start.replace(day=<NUM_LIT:1>)<EOL>end = start + '<STR_LIT>'<EOL><DEDENT>elif group.get('<STR_LIT>'):<EOL><INDENT>start = Date(start, offset=offset, tz=tz)<EOL>start = start.replace(day=<NUM_LIT:1>, month=<NUM_LIT:1>)<EOL>end = start + '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>start = Date(start, offset=offset, tz=tz)<EOL>end = start + '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise TimestringInvalid("<STR_LIT>")<EOL><DEDENT>if end is None:<EOL><INDENT>end = start + '<STR_LIT>'<EOL><DEDENT>if start > end:<EOL><INDENT>start, end = copy(end), copy(start)<EOL><DEDENT>if pgoffset:<EOL><INDENT>start = start - pgoffset<EOL>if end != '<STR_LIT>':<EOL><INDENT>end = end - pgoffset<EOL><DEDENT><DEDENT>self._dates = (start, end)<EOL><DEDENT>if self._dates[<NUM_LIT:0>] > self._dates[<NUM_LIT:1>]:<EOL><INDENT>self._dates = (self._dates[<NUM_LIT:0>], self._dates[<NUM_LIT:1>] + '<STR_LIT>')<EOL><DEDENT>

`start` can be type <class timestring.Date> or <type str>


return abs(int(self[<NUM_LIT:1>].to_unixtime() - self[<NUM_LIT:0>].to_unixtime()))<EOL>

Returns how many `seconds` the `Range` lasts.


if isinstance(other, Range):<EOL><INDENT>start = self.start.replace(tzinfo=other.start.tz) if other.start.tz and self.start.tz is None else self.start<EOL>end = self.end.replace(tzinfo=other.end.tz) if other.end.tz and self.end.tz is None else self.end<EOL>if start == other.start and end == other.end:<EOL><INDENT>return <NUM_LIT:0> <EOL><DEDENT>elif start < other.start:<EOL><INDENT>return -<NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT><DEDENT>elif isinstance(other, Date):<EOL><INDENT>if other.tz and self.start.tz is None:<EOL><INDENT>return <NUM_LIT:0> if other == self.start.replace(tzinfo=other.tz) else -<NUM_LIT:1> if other > self.start.replace(tzinfo=other.start.tz) else <NUM_LIT:1><EOL><DEDENT>return <NUM_LIT:0> if other == self.start else -<NUM_LIT:1> if other > self.start else <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>return self.cmp(Range(other, tz=self.start.tz))<EOL><DEDENT>


if isinstance(other, Date):<EOL><INDENT>if self.start == '<STR_LIT>' and self.end >= other:<EOL><INDENT>return True<EOL><DEDENT>elif self.end == '<STR_LIT>' and self.start <= other:<EOL><INDENT>return True<EOL><DEDENT>elif other == '<STR_LIT>':<EOL><INDENT>return self.start == '<STR_LIT>' or self.end == '<STR_LIT>'<EOL><DEDENT>elif other.tz and self.start.tz is None:<EOL><INDENT>return self.start.replace(tzinfo=other.tz).to_unixtime() <= other.to_unixtime() <= self.end.replace(tzinfo=other.tz).to_unixtime()<EOL><DEDENT>return self.start <= other <= self.end<EOL><DEDENT>elif isinstance(other, Range):<EOL><INDENT>if self.start == '<STR_LIT>':<EOL><INDENT>return other.end <= self.end<EOL><DEDENT>elif self.end == '<STR_LIT>':<EOL><INDENT>return self.start <= other.start<EOL><DEDENT>elif other.start.tz and self.start.tz is None:<EOL><INDENT>return self.start.replace(tzinfo=other.start.tz).to_unixtime() <= other.start.to_unixtime() <= self.end.replace(tzinfo=other.start.tz).to_unixtime()and self.start.replace(tzinfo=other.start.tz).to_unixtime() <= other.end.to_unixtime() <= self.end.replace(tzinfo=other.start.tz).to_unixtime()<EOL><DEDENT>return self.start <= other.start <= self.end and self.start <= other.end <= self.end<EOL><DEDENT>else:<EOL><INDENT>return self.__contains__(Range(other, tz=self.start.tz))<EOL><DEDENT>

*Note: checks Range.start() only*
        Key: self = [], other = {}
            * [---{-}---] => True else False


s, e = copy(self.start), copy(self.end)<EOL>if from_start:<EOL><INDENT>e = s + by<EOL><DEDENT>else:<EOL><INDENT>s = e - by<EOL><DEDENT>return Range(s, e)<EOL>

Cuts this object from_start to the number requestd
        returns new instance


return Range(copy(self.end),<EOL>self.end + self.elapse, tz=self.start.tz)<EOL>

Returns a new instance of self
        times is not supported yet.


return Range(self.start - self.elapse,<EOL>copy(self.start), tz=self.start.tz)<EOL>


if self.date != '<STR_LIT>':<EOL><INDENT>return Date(self.date.replace(**k))<EOL><DEDENT>else:<EOL><INDENT>return Date('<STR_LIT>')<EOL><DEDENT>


if self.date == '<STR_LIT>':<EOL><INDENT>return<EOL><DEDENT>new = copy(self)<EOL>if type(to) in (str, unicode):<EOL><INDENT>to = to.lower()<EOL>res = TIMESTRING_RE.search(to)<EOL>if res:<EOL><INDENT>rgroup = res.groupdict()<EOL>if (rgroup.get('<STR_LIT>') or rgroup.get('<STR_LIT>')):<EOL><INDENT>i = int(text2num(rgroup.get('<STR_LIT>', '<STR_LIT>'))) * (-<NUM_LIT:1> if to.startswith('<STR_LIT:->') else <NUM_LIT:1>)<EOL>delta = (rgroup.get('<STR_LIT>') or rgroup.get('<STR_LIT>')).lower()<EOL>if delta.startswith('<STR_LIT:y>'):<EOL><INDENT>try:<EOL><INDENT>new.date = new.date.replace(year=(new.date.year + i))<EOL><DEDENT>except ValueError:<EOL><INDENT>new.date = new.date + timedelta(days=(<NUM_LIT> * i))<EOL><DEDENT><DEDENT>elif delta.startswith('<STR_LIT>'):<EOL><INDENT>if (new.date.month + i) > <NUM_LIT:12>:<EOL><INDENT>new.date = new.date.replace(month=(i - (i / <NUM_LIT:12>)),<EOL>year=(new.date.year + <NUM_LIT:1> + (i / <NUM_LIT:12>)))<EOL><DEDENT>elif (new.date.month + i) < <NUM_LIT:1>:<EOL><INDENT>new.date = new.date.replace(month=<NUM_LIT:12>, year=(new.date.year - <NUM_LIT:1>))<EOL><DEDENT>else:<EOL><INDENT>new.date = new.date.replace(month=(new.date.month + i))<EOL><DEDENT><DEDENT>elif delta.startswith('<STR_LIT:q>'):<EOL><INDENT>pass<EOL><DEDENT>elif delta.startswith('<STR_LIT:w>'):<EOL><INDENT>new.date = new.date + timedelta(days=(<NUM_LIT:7> * i))<EOL><DEDENT>elif delta.startswith('<STR_LIT:s>'):<EOL><INDENT>new.date = new.date + timedelta(seconds=i)<EOL><DEDENT>else:<EOL><INDENT>new.date = new.date + timedelta(**{('<STR_LIT>' if delta.startswith('<STR_LIT:d>') else '<STR_LIT>' if delta.startswith('<STR_LIT:h>') else '<STR_LIT>' if delta.startswith('<STR_LIT:m>') else '<STR_LIT>'): i})<EOL><DEDENT>return new<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>new.date = new.date + timedelta(seconds=int(to))<EOL>return new<EOL><DEDENT>raise TimestringInvalid('<STR_LIT>')<EOL>

Adjusts the time from kwargs to timedelta
**Will change this object**

return new copy of self


return str(self.date)<EOL>

Returns date in representation of `%x %X` ie `2013-02-17 00:00:00`


if sys.version_info[<NUM_LIT:0>] == <NUM_LIT:2>:<EOL><INDENT>return dct.iteritems()<EOL><DEDENT>return dct.items()<EOL>


if map is None:<EOL><INDENT>map = asyncore.socket_map<EOL><DEDENT>pollster = select.epoll()<EOL>if map:<EOL><INDENT>for fd, obj in iteritems(map):<EOL><INDENT>flags = <NUM_LIT:0><EOL>if obj.readable():<EOL><INDENT>flags |= select.POLLIN | select.POLLPRI<EOL><DEDENT>if obj.writable():<EOL><INDENT>flags |= select.POLLOUT<EOL><DEDENT>if flags:<EOL><INDENT>flags |= select.POLLERR | select.POLLHUP | select.POLLNVAL<EOL>pollster.register(fd, flags)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>r = pollster.poll(timeout)<EOL><DEDENT>except select.error as err:<EOL><INDENT>if err.args[<NUM_LIT:0>] != EINTR:<EOL><INDENT>raise<EOL><DEDENT>r = []<EOL><DEDENT>for fd, flags in r:<EOL><INDENT>obj = map.get(fd)<EOL>if obj is None:<EOL><INDENT>continue<EOL><DEDENT>asyncore.readwrite(obj, flags)<EOL><DEDENT><DEDENT>

A poller which uses epoll(), supported on Linux 2.5.44 and newer

Borrowed from here:
https://github.com/m13253/python-asyncore-epoll/blob/master/asyncore_epoll.py#L200


if hasattr(select, '<STR_LIT>'):<EOL><INDENT>poll_func = epoll_poller<EOL><DEDENT>elif hasattr(select, '<STR_LIT>'):<EOL><INDENT>poll_func = asyncore.poll2<EOL><DEDENT>else:<EOL><INDENT>poll_func = asyncore.poll<EOL><DEDENT>return poll_func<EOL>

Get the best available socket poll function

    :return: poller function


handler_class.ws_handler_class = ws_handler_class<EOL>handler_class.ws_path = ws_path<EOL>httpd = server_class((host, port), RequestHandlerClass=handler_class)<EOL>httpd.set_app(app)<EOL>return httpd<EOL>

Create server instance with an optional WebSocket handler

    For pure WebSocket server ``app`` may be ``None`` but an attempt to access
    any path other than ``ws_path`` will cause server error.

    :param host: hostname or IP
    :type host: str
    :param port: server port
    :type port: int
    :param app: WSGI application
    :param server_class: WSGI server class, defaults to AsyncWsgiServer
    :param handler_class: WSGI handler class, defaults to AsyncWsgiHandler
    :param ws_handler_class: WebSocket hanlder class, defaults to ``None``
    :param ws_path: WebSocket path on the server, defaults to '/ws'
    :type ws_path: str, optional
    :return: initialized server instance


self.iterator = iter(self.result)<EOL>

Get WSGI response iterator for sending in handle_write


if self._map:<EOL><INDENT>self._poll_func(timeout, self._map)<EOL><DEDENT>

Poll active sockets once

This method can be used to allow aborting server polling loop
on some condition.

:param timeout: polling timeout


Start serving HTTP requests

This method blocks the current thread.

:param poll_interval: polling timeout
:return:


pass<EOL>

Called when websocket frame is received.
To access the frame data call self.data.

If the frame is Text then self.data is a unicode object.
If the frame is Binary then self.data is a bytearray object.


pass<EOL>

Called when a websocket client connects to the server.


pass<EOL>

Called when a websocket server gets a Close frame from a client.


def close(self, status=<NUM_LIT:1000>, reason=u'<STR_LIT>'):

try:<EOL><INDENT>if self.closed is False:<EOL><INDENT>close_msg = bytearray()<EOL>close_msg.extend(struct.pack("<STR_LIT>", status))<EOL>if _check_unicode(reason):<EOL><INDENT>close_msg.extend(reason.encode('<STR_LIT:utf-8>'))<EOL><DEDENT>else:<EOL><INDENT>close_msg.extend(reason)<EOL><DEDENT>self._sendMessage(False, CLOSE, close_msg)<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>self.closed = True<EOL><DEDENT>

Send Close frame to the client. The underlying socket is only closed
when the client acknowledges the Close frame.

status is the closing identifier.
reason is the reason for the close.


opcode = BINARY<EOL>if _check_unicode(data):<EOL><INDENT>opcode = TEXT<EOL><DEDENT>self._sendMessage(True, opcode, data)<EOL>

Send the start of a data fragment stream to a websocket client.
Subsequent data should be sent using sendFragment().
A fragment stream is completed when sendFragmentEnd() is called.

If data is a unicode object then the frame is sent as Text.
If the data is a bytearray object then the frame is sent as Binary.


self._sendMessage(True, STREAM, data)<EOL>

see sendFragmentStart()

If data is a unicode object then the frame is sent as Text.
If the data is a bytearray object then the frame is sent as Binary.


self._sendMessage(False, STREAM, data)<EOL>

see sendFragmentEnd()

If data is a unicode object then the frame is sent as Text.
If the data is a bytearray object then the frame is sent as Binary.


opcode = BINARY<EOL>if _check_unicode(data):<EOL><INDENT>opcode = TEXT<EOL><DEDENT>self._sendMessage(False, opcode, data)<EOL>

Send websocket data frame to the client.

If data is a unicode object then the frame is sent as Text.
If the data is a bytearray object then the frame is sent as Binary.


@classmethod<EOL><INDENT>def get(cls, update_dict=None, remove_fields=None):<DEDENT>

cls.demo_course_count += <NUM_LIT:1><EOL>course_copy = copy.deepcopy(cls.DEMO_COURSE)<EOL>if update_dict:<EOL><INDENT>if "<STR_LIT:content>" in update_dict:<EOL><INDENT>course_copy["<STR_LIT:content>"].update(update_dict["<STR_LIT:content>"])<EOL>del update_dict["<STR_LIT:content>"]<EOL><DEDENT>course_copy.update(update_dict)<EOL><DEDENT>course_copy.update({"<STR_LIT:id>": "<STR_LIT>".format(course_copy["<STR_LIT:id>"], cls.demo_course_count)})<EOL>if remove_fields:<EOL><INDENT>for remove_field in remove_fields:<EOL><INDENT>if remove_field in course_copy:<EOL><INDENT>del course_copy[remove_field]<EOL><DEDENT><DEDENT><DEDENT>return course_copy<EOL>


cls.demo_course_count = <NUM_LIT:0><EOL>


searcher.index(doc_type="<STR_LIT>", sources=course_info)<EOL>

Adds course info dictionary to the index


@classmethod<EOL><INDENT>def get_and_index(cls, searcher, update_dict=None, remove_fields=None):<DEDENT>

cls.index(searcher, [cls.get(update_dict, remove_fields)])<EOL>


return isinstance(self.searcher, ElasticSearchEngine)<EOL>

check search engine implementation, to manage cleanup differently


reset mocked tracker and clear logged emits


self.assertFalse(self.mock_tracker.emit.called)<EOL>

Ensures no events were emitted since the last event related assertion


def assert_search_initiated_event(self, search_term, size, page):

initiated_search_call = self.mock_tracker.emit.mock_calls[<NUM_LIT:0>]  <EOL>expected_result = call('<STR_LIT>', {<EOL>"<STR_LIT>": six.text_type(search_term),<EOL>"<STR_LIT>": size,<EOL>"<STR_LIT>": page,<EOL>})<EOL>self.assertEqual(expected_result, initiated_search_call)<EOL>

Ensures an search initiated event was emitted


def assert_results_returned_event(self, search_term, size, page, total):

returned_results_call = self.mock_tracker.emit.mock_calls[<NUM_LIT:1>]  <EOL>expected_result = call('<STR_LIT>', {<EOL>"<STR_LIT>": six.text_type(search_term),<EOL>"<STR_LIT>": size,<EOL>"<STR_LIT>": page,<EOL>"<STR_LIT>": total,<EOL>})<EOL>self.assertEqual(expected_result, returned_results_call)<EOL>

Ensures an results returned event was emitted


def assert_initiated_return_events(self, search_term, size, page, total):

self.assertEqual(self.mock_tracker.emit.call_count, <NUM_LIT:2>)  <EOL>self.assert_search_initiated_event(search_term, size, page)<EOL>self.assert_results_returned_event(search_term, size, page, total)<EOL>

Asserts search initiated and results returned events were emitted


return "<STR_LIT>"<EOL>

additional property that should appear within processed results


if "<STR_LIT>" not in self._results_fields or "<STR_LIT:id>" not in self._results_fields:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return u"<STR_LIT>".format(<EOL>course_id=self._results_fields["<STR_LIT>"],<EOL>location=self._results_fields["<STR_LIT:id>"],<EOL>)<EOL>

Property to display the url for the given location, useful for allowing navigation


return "<STR_LIT>" in self._results_fields<EOL>


address = '<STR_LIT:/>' if course_id is None else '<STR_LIT>'.format(course_id)<EOL>response = Client().post(address, body)<EOL>return getattr(response, "<STR_LIT>", <NUM_LIT>), json.loads(getattr(response, "<STR_LIT:content>", None))<EOL>

Helper method to post the request and process the response


address = '<STR_LIT>'<EOL>response = Client().post(address, body)<EOL>return getattr(response, "<STR_LIT>", <NUM_LIT>), json.loads(getattr(response, "<STR_LIT:content>", None))<EOL>


if self._searcher is None:<EOL><INDENT>self._searcher = SearchEngine.get_search_engine(TEST_INDEX_NAME)<EOL><DEDENT>return self._searcher<EOL>


raise exceptions.ElasticsearchException("<STR_LIT>")<EOL>


if "<STR_LIT:T>" in json_date_string_value:<EOL><INDENT>if "<STR_LIT:.>" in json_date_string_value:<EOL><INDENT>format_string = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>format_string = "<STR_LIT>"<EOL><DEDENT>if json_date_string_value.endswith("<STR_LIT>"):<EOL><INDENT>format_string += "<STR_LIT>"<EOL><DEDENT><DEDENT>else:<EOL><INDENT>format_string = "<STR_LIT>"<EOL><DEDENT>return datetime.strptime(<EOL>json_date_string_value,<EOL>format_string<EOL>)<EOL>

converts json date string to date object


if not isinstance(doc, dict):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if not isinstance(field_name, six.string_types):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>immediate_field, remaining_path = field_name.split('<STR_LIT:.>', <NUM_LIT:1>) if '<STR_LIT:.>' in field_name else (field_name, None)<EOL>field_value = doc.get(immediate_field)<EOL>if isinstance(field_value, dict):<EOL><INDENT>return _find_field(field_value, remaining_path)<EOL><DEDENT>return field_value<EOL>

find the dictionary field corresponding to the . limited name


def _filter_intersection(documents_to_search, dictionary_object, include_blanks=False):

if not dictionary_object:<EOL><INDENT>return documents_to_search<EOL><DEDENT>def value_matches(doc, field_name, field_value):<EOL><INDENT>"""<STR_LIT>"""<EOL>compare_value = _find_field(doc, field_name)<EOL>if compare_value is None:<EOL><INDENT>return include_blanks<EOL><DEDENT>if isinstance(field_value, (DateRange, datetime)):<EOL><INDENT>if isinstance(compare_value, six.string_types):<EOL><INDENT>compare_value = json_date_to_datetime(compare_value)<EOL><DEDENT>field_has_tz_info = False<EOL>if isinstance(field_value, DateRange):<EOL><INDENT>field_has_tz_info = (<EOL>(field_value.lower and field_value.lower.tzinfo is not None) or<EOL>(field_value.upper and field_value.upper.tzinfo is not None)<EOL>)<EOL><DEDENT>else:<EOL><INDENT>field_has_tz_info = field_value.tzinfo is not None<EOL><DEDENT>if not field_has_tz_info:<EOL><INDENT>compare_value = compare_value.replace(tzinfo=None)<EOL><DEDENT>elif compare_value.tzinfo is None:<EOL><INDENT>compare_value = compare_value.replace(tzinfo=pytz.UTC)<EOL><DEDENT><DEDENT>if isinstance(field_value, ValueRange):<EOL><INDENT>return (<EOL>(field_value.lower is None or compare_value >= field_value.lower) and<EOL>(field_value.upper is None or compare_value <= field_value.upper)<EOL>)<EOL><DEDENT>elif _is_iterable(compare_value) and not _is_iterable(field_value):<EOL><INDENT>return any((item == field_value for item in compare_value))<EOL><DEDENT>elif _is_iterable(field_value) and not _is_iterable(compare_value):<EOL><INDENT>return any((item == compare_value for item in field_value))<EOL><DEDENT>elif _is_iterable(compare_value) and _is_iterable(field_value):<EOL><INDENT>return any((six.text_type(item) in field_value for item in compare_value))<EOL><DEDENT>return compare_value == field_value<EOL><DEDENT>filtered_documents = documents_to_search<EOL>for field_name, field_value in dictionary_object.items():<EOL><INDENT>filtered_documents = [d for d in filtered_documents if value_matches(d, field_name, field_value)]<EOL><DEDENT>return filtered_documents<EOL>

Filters out documents that do not match all of the field values within the dictionary_object
If include_blanks is True, then the document is considered a match if the field is not present


def _encode_string(string):<EOL><INDENT>"""<STR_LIT>"""<EOL>if six.PY2:<EOL><INDENT>string = string.encode('<STR_LIT:utf-8>').translate(None, RESERVED_CHARACTERS)<EOL><DEDENT>else:<EOL><INDENT>string = string.translate(string.maketrans('<STR_LIT>', '<STR_LIT>', RESERVED_CHARACTERS))<EOL><DEDENT>return string<EOL><DEDENT>def has_string(dictionary_object, search_string):<EOL><INDENT>"""<STR_LIT>"""<EOL>for name in dictionary_object:<EOL><INDENT>if isinstance(dictionary_object[name], dict):<EOL><INDENT>return has_string(dictionary_object[name], search_string)<EOL><DEDENT>elif dictionary_object[name]:<EOL><INDENT>if search_string.lower() in _encode_string(dictionary_object[name].lower()):<EOL><INDENT>return True<EOL><DEDENT><DEDENT><DEDENT>return False<EOL><DEDENT>search_strings = _encode_string(query_string).split("<STR_LIT:U+0020>")<EOL>documents_to_keep = []<EOL>for search_string in search_strings:<EOL><INDENT>documents_to_keep.extend(<EOL>[d for d in documents_to_search if "<STR_LIT:content>" in d and has_string(d["<STR_LIT:content>"], search_string)]<EOL>)<EOL><DEDENT>return documents_to_keep<EOL>

keep the documents that contain at least one of the search strings provided


for exclude_property in exclude_dictionary:<EOL><INDENT>exclude_values = exclude_dictionary[exclude_property]<EOL>if not isinstance(exclude_values, list):<EOL><INDENT>exclude_values = [exclude_values]<EOL><DEDENT>documents_to_search = [<EOL>document<EOL>for document in documents_to_search<EOL>if document.get(exclude_property) not in exclude_values<EOL>]<EOL><DEDENT>return documents_to_search<EOL>

remove results that have fields that match in the exclude_dictionary


facets = {}<EOL>def process_facet(facet):<EOL><INDENT>"""<STR_LIT>"""<EOL>faceted_documents = [facet_document for facet_document in documents if facet in facet_document]<EOL>terms = {}<EOL>def add_facet_value(facet_value):<EOL><INDENT>"""<STR_LIT>"""<EOL>if isinstance(facet_value, list):<EOL><INDENT>for individual_value in facet_value:<EOL><INDENT>add_facet_value(individual_value)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if facet_value not in terms:<EOL><INDENT>terms[facet_value] = <NUM_LIT:0><EOL><DEDENT>terms[facet_value] += <NUM_LIT:1><EOL><DEDENT><DEDENT>for document in faceted_documents:<EOL><INDENT>add_facet_value(document[facet])<EOL><DEDENT>total = sum([terms[term] for term in terms])<EOL>return total, terms<EOL><DEDENT>for facet in facet_terms:<EOL><INDENT>total, terms = process_facet(facet)<EOL>facets[facet] = {<EOL>"<STR_LIT>": total,<EOL>"<STR_LIT>": terms,<EOL>}<EOL><DEDENT>return facets<EOL>

Calculate the counts for the facets provided:

For each facet, count up the number of hits for each facet value, so
that we can report back the breakdown of how many of each value there
exist. Notice that the total is the total number of facet matches that
we receive - a single document will get counted multiple times in the
total if the facet field is multi-valued:

    e.g. a course may have a value for modes as ["honor", "validated"], and
    so the document will count as 1 towards the honor count, 1 towards the
    validated count, and 2 towards the total. (This may be a little
    surprising but matches the behaviour that elasticsearch presents)


backing_file_name = getattr(settings, "<STR_LIT>", None)<EOL>if cls._file_name_override:<EOL><INDENT>backing_file_name = cls._file_name_override<EOL><DEDENT>if not backing_file_name:<EOL><INDENT>cls._disabled = False<EOL>return None<EOL><DEDENT>if create_if_missing or os.path.exists(backing_file_name):<EOL><INDENT>cls._disabled = False<EOL>return backing_file_name<EOL><DEDENT>cls._disabled = True<EOL>return None<EOL>

return path to test file to use for backing purposes


file_name = cls._backing_file(create_if_missing)<EOL>if file_name:<EOL><INDENT>with open(file_name, "<STR_LIT>") as dict_file:<EOL><INDENT>json.dump(cls._mock_elastic, dict_file, cls=DjangoJSONEncoder)<EOL><DEDENT><DEDENT>

write the index dict to the backing file


file_name = cls._backing_file()<EOL>if file_name and os.path.exists(file_name):<EOL><INDENT>with open(file_name, "<STR_LIT:r>") as dict_file:<EOL><INDENT>cls._mock_elastic = json.load(dict_file)<EOL><DEDENT><DEDENT>

load the index dict from the contents of the backing file


@staticmethod<EOL><INDENT>def _paginate_results(size, from_, raw_results):<DEDENT>

results = raw_results<EOL>if size:<EOL><INDENT>start = <NUM_LIT:0><EOL>if from_ is not None:<EOL><INDENT>start = from_<EOL><DEDENT>results = raw_results[start:start + size]<EOL><DEDENT>return results<EOL>


cls._load_from_file()<EOL>if index_name not in cls._mock_elastic:<EOL><INDENT>cls._mock_elastic[index_name] = {}<EOL>cls._write_to_file()<EOL><DEDENT>return cls._mock_elastic[index_name]<EOL>

load the index, if necessary from the backed file


@classmethod<EOL><INDENT>def load_doc_type(cls, index_name, doc_type):<DEDENT>

index = cls.load_index(index_name)<EOL>if doc_type not in index:<EOL><INDENT>index[doc_type] = []<EOL>cls._write_to_file()<EOL><DEDENT>return index[doc_type]<EOL>

load the documents of type doc_type, if necessary loading from the backed file


@classmethod<EOL><INDENT>def add_documents(cls, index_name, doc_type, sources):<DEDENT>

cls.load_doc_type(index_name, doc_type).extend(sources)<EOL>cls._write_to_file()<EOL>


@classmethod<EOL><INDENT>def remove_documents(cls, index_name, doc_type, doc_ids):<DEDENT>

index = cls.load_index(index_name)<EOL>if doc_type not in index:<EOL><INDENT>return<EOL><DEDENT>index[doc_type] = [d for d in index[doc_type] if "<STR_LIT:id>" not in d or d["<STR_LIT:id>"] not in doc_ids]<EOL>cls._write_to_file()<EOL>

remove documents by id of specific type to index


cls._mock_elastic = {}<EOL>cls._write_to_file()<EOL>

Clean out the dictionary for test resets


if not MockSearchEngine._disabled:<EOL><INDENT>doc_ids = [s["<STR_LIT:id>"] for s in sources if "<STR_LIT:id>" in s]<EOL>MockSearchEngine.remove_documents(self.index_name, doc_type, doc_ids)<EOL>MockSearchEngine.add_documents(self.index_name, doc_type, sources)<EOL><DEDENT>

Add/update documents of given type to the index


if not MockSearchEngine._disabled:<EOL><INDENT>MockSearchEngine.remove_documents(self.index_name, doc_type, doc_ids)<EOL><DEDENT>

Remove documents of type with given ids from the index


if MockSearchEngine._disabled:<EOL><INDENT>return {<EOL>"<STR_LIT>": <NUM_LIT:10>,<EOL>"<STR_LIT>": <NUM_LIT:0>,<EOL>"<STR_LIT>": <NUM_LIT:0>,<EOL>"<STR_LIT>": []<EOL>}<EOL><DEDENT>documents_to_search = []<EOL>if "<STR_LIT>" in kwargs:<EOL><INDENT>documents_to_search = MockSearchEngine.load_doc_type(self.index_name, kwargs["<STR_LIT>"])<EOL><DEDENT>else:<EOL><INDENT>index = MockSearchEngine.load_index(self.index_name)<EOL>for doc_type in index:<EOL><INDENT>documents_to_search.extend(index[doc_type])<EOL><DEDENT><DEDENT>if field_dictionary:<EOL><INDENT>documents_to_search = _filter_intersection(documents_to_search, field_dictionary)<EOL><DEDENT>if filter_dictionary:<EOL><INDENT>documents_to_search = _filter_intersection(documents_to_search, filter_dictionary, True)<EOL><DEDENT>if query_string:<EOL><INDENT>documents_to_search = _process_query_string(documents_to_search, query_string)<EOL><DEDENT>if "<STR_LIT>" in kwargs:<EOL><INDENT>if not exclude_dictionary:<EOL><INDENT>exclude_dictionary = {}<EOL><DEDENT>if "<STR_LIT:id>" not in exclude_dictionary:<EOL><INDENT>exclude_dictionary["<STR_LIT:id>"] = []<EOL><DEDENT>exclude_dictionary["<STR_LIT:id>"].extend(kwargs["<STR_LIT>"])<EOL><DEDENT>if exclude_dictionary:<EOL><INDENT>documents_to_search = _process_exclude_dictionary(documents_to_search, exclude_dictionary)<EOL><DEDENT>def score_documents(documents_to_search):<EOL><INDENT>"""<STR_LIT>"""<EOL>search_results = []<EOL>max_score = <NUM_LIT:0><EOL>while documents_to_search:<EOL><INDENT>current_doc = documents_to_search[<NUM_LIT:0>]<EOL>score = len([d for d in documents_to_search if d == current_doc])<EOL>if score > max_score:<EOL><INDENT>max_score = score<EOL><DEDENT>documents_to_search = [d for d in documents_to_search if d != current_doc]<EOL>data = copy.copy(current_doc)<EOL>search_results.append(<EOL>{<EOL>"<STR_LIT>": score,<EOL>"<STR_LIT:data>": data,<EOL>}<EOL>)<EOL><DEDENT>return search_results, max_score<EOL><DEDENT>search_results, max_score = score_documents(documents_to_search)<EOL>results = MockSearchEngine._paginate_results(<EOL>kwargs["<STR_LIT:size>"] if "<STR_LIT:size>" in kwargs else None,<EOL>kwargs["<STR_LIT>"] if "<STR_LIT>" in kwargs else None,<EOL>sorted(search_results, key=lambda k: k["<STR_LIT>"])<EOL>)<EOL>response = {<EOL>"<STR_LIT>": <NUM_LIT:10>,<EOL>"<STR_LIT>": len(search_results),<EOL>"<STR_LIT>": max_score,<EOL>"<STR_LIT>": results<EOL>}<EOL>if facet_terms:<EOL><INDENT>response["<STR_LIT>"] = _count_facet_values(documents_to_search, facet_terms)<EOL><DEDENT>return response<EOL>

Perform search upon documents within index


return isinstance(self.searcher, ElasticSearchEngine)<EOL>

check search engine implementation, to manage cleanup differently


self.searcher.index("<STR_LIT>", [{"<STR_LIT:id>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>"}])<EOL>self.searcher.index("<STR_LIT>", [{"<STR_LIT:id>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>"}])<EOL>self.searcher.index("<STR_LIT>", [{"<STR_LIT:id>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>"}])<EOL>self.searcher.index("<STR_LIT>", [{"<STR_LIT:id>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>"}])<EOL>self.searcher.index("<STR_LIT>", [{"<STR_LIT:id>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>"}])<EOL>self.searcher.index("<STR_LIT>", [{"<STR_LIT:id>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>"}])<EOL>self.searcher.index("<STR_LIT>", [{"<STR_LIT:id>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>"}])<EOL>


pass<EOL>


initializer = _load_class(getattr(settings, "<STR_LIT>", None), cls)()<EOL>return initializer.initialize(**kwargs)<EOL>

Called from within search handler
Finds desired subclass and calls initialize method


size = <NUM_LIT:20><EOL>page = <NUM_LIT:0><EOL>from_ = <NUM_LIT:0><EOL>if "<STR_LIT>" in request.POST:<EOL><INDENT>size = int(request.POST["<STR_LIT>"])<EOL>max_page_size = getattr(settings, "<STR_LIT>", <NUM_LIT:100>)<EOL>if not (<NUM_LIT:0> < size <= max_page_size):  <EOL><INDENT>raise ValueError(_('<STR_LIT>').format(page_size=size))<EOL><DEDENT>if "<STR_LIT>" in request.POST:<EOL><INDENT>page = int(request.POST["<STR_LIT>"])<EOL>from_ = page * size<EOL><DEDENT><DEDENT>return size, from_, page<EOL>

process pagination requests from request parameter


return {<EOL>field_key: request.POST[field_key]<EOL>for field_key in request.POST<EOL>if field_key in course_discovery_filter_fields()<EOL>}<EOL>

Create separate dictionary of supported filter values provided


<EOL>SearchInitializer.set_search_enviroment(request=request, course_id=course_id)<EOL>results = {<EOL>"<STR_LIT:error>": _("<STR_LIT>")<EOL>}<EOL>status_code = <NUM_LIT><EOL>search_term = request.POST.get("<STR_LIT>", None)<EOL>try:<EOL><INDENT>if not search_term:<EOL><INDENT>raise ValueError(_('<STR_LIT>'))<EOL><DEDENT>size, from_, page = _process_pagination_values(request)<EOL>track.emit(<EOL>'<STR_LIT>',<EOL>{<EOL>"<STR_LIT>": search_term,<EOL>"<STR_LIT>": size,<EOL>"<STR_LIT>": page,<EOL>}<EOL>)<EOL>results = perform_search(<EOL>search_term,<EOL>user=request.user,<EOL>size=size,<EOL>from_=from_,<EOL>course_id=course_id<EOL>)<EOL>status_code = <NUM_LIT:200><EOL>track.emit(<EOL>'<STR_LIT>',<EOL>{<EOL>"<STR_LIT>": search_term,<EOL>"<STR_LIT>": size,<EOL>"<STR_LIT>": page,<EOL>"<STR_LIT>": results["<STR_LIT>"],<EOL>}<EOL>)<EOL><DEDENT>except ValueError as invalid_err:<EOL><INDENT>results = {<EOL>"<STR_LIT:error>": six.text_type(invalid_err)<EOL>}<EOL>log.debug(six.text_type(invalid_err))<EOL><DEDENT>except QueryParseError:<EOL><INDENT>results = {<EOL>"<STR_LIT:error>": _('<STR_LIT>')<EOL>}<EOL><DEDENT>except Exception as err:  <EOL><INDENT>results = {<EOL>"<STR_LIT:error>": _('<STR_LIT>').format(search_string=search_term)<EOL>}<EOL>log.exception(<EOL>'<STR_LIT>',<EOL>search_term,<EOL>request.user.id,<EOL>err<EOL>)<EOL><DEDENT>return JsonResponse(results, status=status_code)<EOL>

Search view for http requests

Args:
    request (required) - django request object
    course_id (optional) - course_id within which to restrict search

Returns:
    http json response with the following fields
        "took" - how many seconds the operation took
        "total" - how many results were found
        "max_score" - maximum score from these results
        "results" - json array of result documents

        or

        "error" - displayable information about an error that occured on the server

POST Params:
    "search_string" (required) - text upon which to search
    "page_size" (optional)- how many results to return per page (defaults to 20, with maximum cutoff at 100)
    "page_index" (optional) - for which page (zero-indexed) to include results (defaults to 0)


results = {<EOL>"<STR_LIT:error>": _("<STR_LIT>")<EOL>}<EOL>status_code = <NUM_LIT><EOL>search_term = request.POST.get("<STR_LIT>", None)<EOL>try:<EOL><INDENT>size, from_, page = _process_pagination_values(request)<EOL>field_dictionary = _process_field_values(request)<EOL>track.emit(<EOL>'<STR_LIT>',<EOL>{<EOL>"<STR_LIT>": search_term,<EOL>"<STR_LIT>": size,<EOL>"<STR_LIT>": page,<EOL>}<EOL>)<EOL>results = course_discovery_search(<EOL>search_term=search_term,<EOL>size=size,<EOL>from_=from_,<EOL>field_dictionary=field_dictionary,<EOL>)<EOL>track.emit(<EOL>'<STR_LIT>',<EOL>{<EOL>"<STR_LIT>": search_term,<EOL>"<STR_LIT>": size,<EOL>"<STR_LIT>": page,<EOL>"<STR_LIT>": results["<STR_LIT>"],<EOL>}<EOL>)<EOL>status_code = <NUM_LIT:200><EOL><DEDENT>except ValueError as invalid_err:<EOL><INDENT>results = {<EOL>"<STR_LIT:error>": six.text_type(invalid_err)<EOL>}<EOL>log.debug(six.text_type(invalid_err))<EOL><DEDENT>except QueryParseError:<EOL><INDENT>results = {<EOL>"<STR_LIT:error>": _('<STR_LIT>')<EOL>}<EOL><DEDENT>except Exception as err:  <EOL><INDENT>results = {<EOL>"<STR_LIT:error>": _('<STR_LIT>').format(search_string=search_term)<EOL>}<EOL>log.exception(<EOL>'<STR_LIT>',<EOL>search_term,<EOL>request.user.id,<EOL>err<EOL>)<EOL><DEDENT>return JsonResponse(results, status=status_code)<EOL>

Search for courses

Args:
    request (required) - django request object

Returns:
    http json response with the following fields
        "took" - how many seconds the operation took
        "total" - how many results were found
        "max_score" - maximum score from these resutls
        "results" - json array of result documents

        or

        "error" - displayable information about an error that occured on the server

POST Params:
    "search_string" (optional) - text with which to search for courses
    "page_size" (optional)- how many results to return per page (defaults to 20, with maximum cutoff at 100)
    "page_index" (optional) - for which page (zero-indexed) to include results (defaults to 0)


return {"<STR_LIT>": DateRange(None, datetime.utcnow())}<EOL>

base implementation which filters via start_date


field_dictionary = {}<EOL>if "<STR_LIT>" in kwargs and kwargs["<STR_LIT>"]:<EOL><INDENT>field_dictionary["<STR_LIT>"] = kwargs["<STR_LIT>"]<EOL><DEDENT>return field_dictionary<EOL>

base implementation which add course if provided


return {}<EOL>

base implementation which excludes nothing


generator = _load_class(getattr(settings, "<STR_LIT>", None), cls)()<EOL>return (<EOL>generator.field_dictionary(**kwargs),<EOL>generator.filter_dictionary(**kwargs),<EOL>generator.exclude_dictionary(**kwargs),<EOL>)<EOL>

Called from within search handler
Finds desired subclass and adds filter information based upon user information


def translate_result(result):<EOL><INDENT>"""<STR_LIT>"""<EOL>translated_result = copy.copy(result)<EOL>data = translated_result.pop("<STR_LIT>")<EOL>translated_result.update({<EOL>"<STR_LIT:data>": data,<EOL>"<STR_LIT>": translated_result["<STR_LIT>"]<EOL>})<EOL>return translated_result<EOL><DEDENT>def translate_facet(result):<EOL><INDENT>"""<STR_LIT>"""<EOL>terms = {term["<STR_LIT>"]: term["<STR_LIT:count>"] for term in result["<STR_LIT>"]}<EOL>return {<EOL>"<STR_LIT>": terms,<EOL>"<STR_LIT>": result["<STR_LIT>"],<EOL>"<STR_LIT>": result["<STR_LIT>"],<EOL>}<EOL><DEDENT>results = [translate_result(hit) for hit in es_response["<STR_LIT>"]["<STR_LIT>"]]<EOL>response = {<EOL>"<STR_LIT>": es_response["<STR_LIT>"],<EOL>"<STR_LIT>": es_response["<STR_LIT>"]["<STR_LIT>"],<EOL>"<STR_LIT>": es_response["<STR_LIT>"]["<STR_LIT>"],<EOL>"<STR_LIT>": results,<EOL>}<EOL>if "<STR_LIT>" in es_response:<EOL><INDENT>response["<STR_LIT>"] = {facet: translate_facet(es_response["<STR_LIT>"][facet]) for facet in es_response["<STR_LIT>"]}<EOL><DEDENT>return response<EOL>

Provide resultset in our desired format from elasticsearch results


filter_field = None<EOL>if isinstance(field_value, ValueRange):<EOL><INDENT>range_values = {}<EOL>if field_value.lower:<EOL><INDENT>range_values.update({"<STR_LIT>": field_value.lower_string})<EOL><DEDENT>if field_value.upper:<EOL><INDENT>range_values.update({"<STR_LIT>": field_value.upper_string})<EOL><DEDENT>filter_field = {<EOL>"<STR_LIT>": {<EOL>field_name: range_values<EOL>}<EOL>}<EOL><DEDENT>elif _is_iterable(field_value):<EOL><INDENT>filter_field = {<EOL>"<STR_LIT>": {<EOL>field_name: field_value<EOL>}<EOL>}<EOL><DEDENT>else:<EOL><INDENT>filter_field = {<EOL>"<STR_LIT>": {<EOL>field_name: field_value<EOL>}<EOL>}<EOL><DEDENT>return filter_field<EOL>

Return field to apply into filter, if an array then use a range, otherwise look for a term match


def field_item(field):<EOL><INDENT>"""<STR_LIT>"""<EOL>return {<EOL>"<STR_LIT>": {<EOL>field: field_dictionary[field]<EOL>}<EOL>}<EOL><DEDENT>return [field_item(field) for field in field_dictionary]<EOL>

We have a field_dictionary - we want to match the values for an elasticsearch "match" query
This is only potentially useful when trying to tune certain search operations


return [_get_filter_field(field, field_value) for field, field_value in field_dictionary.items()]<EOL>

We have a field_dictionary - we match the values using a "term" filter in elasticsearch


def filter_item(field):<EOL><INDENT>"""<STR_LIT>"""<EOL>if filter_dictionary[field] is not None:<EOL><INDENT>return {<EOL>"<STR_LIT>": [<EOL>_get_filter_field(field, filter_dictionary[field]),<EOL>{<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": field<EOL>}<EOL>}<EOL>]<EOL>}<EOL><DEDENT>return {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": field<EOL>}<EOL>}<EOL><DEDENT>return [filter_item(field) for field in filter_dictionary]<EOL>

We have a filter_dictionary - this means that if the field is included
and matches, then we can include, OR if the field is undefined, then we
assume it is safe to include


<EOL>not_properties = []<EOL>for exclude_property in exclude_dictionary:<EOL><INDENT>exclude_values = exclude_dictionary[exclude_property]<EOL>if not isinstance(exclude_values, list):<EOL><INDENT>exclude_values = [exclude_values]<EOL><DEDENT>not_properties.extend([{"<STR_LIT>": {exclude_property: exclude_value}} for exclude_value in exclude_values])<EOL><DEDENT>if not not_properties:<EOL><INDENT>return {}<EOL><DEDENT>return {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": not_properties<EOL>}<EOL>}<EOL>}<EOL>

Based on values in the exclude_dictionary generate a list of term queries that
will filter out unwanted results.


elastic_facets = {}<EOL>for facet in facet_terms:<EOL><INDENT>facet_term = {"<STR_LIT>": facet}<EOL>if facet_terms[facet]:<EOL><INDENT>for facet_option in facet_terms[facet]:<EOL><INDENT>facet_term[facet_option] = facet_terms[facet][facet_option]<EOL><DEDENT><DEDENT>elastic_facets[facet] = {<EOL>"<STR_LIT>": facet_term<EOL>}<EOL><DEDENT>return elastic_facets<EOL>

We have a list of terms with which we return facets


return "<STR_LIT>".format(<EOL>index_name,<EOL>doc_type<EOL>)<EOL>


@classmethod<EOL><INDENT>def get_mappings(cls, index_name, doc_type):<DEDENT>

return cache.get(cls.get_cache_item_name(index_name, doc_type), {})<EOL>


@classmethod<EOL><INDENT>def set_mappings(cls, index_name, doc_type, mappings):<DEDENT>

cache.set(cls.get_cache_item_name(index_name, doc_type), mappings)<EOL>

set new mapped-items structure into cache


indexing_errors_log = []<EOL>for indexing_error in indexing_errors:<EOL><INDENT>indexing_errors_log.append(str(indexing_error))<EOL><DEDENT>raise exceptions.ElasticsearchException('<STR_LIT:U+002CU+0020>'.join(indexing_errors_log))<EOL>

Logs indexing errors and raises a general ElasticSearch Exception


<EOL>mapping = ElasticSearchEngine.get_mappings(self.index_name, doc_type)<EOL>if not mapping:<EOL><INDENT>mapping = self._es.indices.get_mapping(<EOL>index=self.index_name,<EOL>doc_type=doc_type,<EOL>).get(self.index_name, {}).get('<STR_LIT>', {}).get(doc_type, {})<EOL>if mapping:<EOL><INDENT>ElasticSearchEngine.set_mappings(<EOL>self.index_name,<EOL>doc_type,<EOL>mapping<EOL>)<EOL><DEDENT><DEDENT>return mapping<EOL>

Interfaces with the elasticsearch mappings for the index
prevents multiple loading of the same mappings from ES when called more than once

Mappings format in elasticsearch is as follows:
{
   "doc_type": {
      "properties": {
         "nested_property": {
            "properties": {
               "an_analysed_property": {
                  "type": "string"
               },
               "another_analysed_property": {
                  "type": "string"
               }
            }
         },
         "a_not_analysed_property": {
            "type": "string",
            "index": "not_analyzed"
         },
         "a_date_property": {
            "type": "date"
         }
      }
   }
}

We cache the properties of each doc_type, if they are not available, we'll load them again from Elasticsearch


ElasticSearchEngine.set_mappings(self.index_name, doc_type, {})<EOL>

Remove the cached mappings, so that they get loaded from ES next time they are requested


def _check_mappings(self, doc_type, body):

<EOL>exclude_fields = ["<STR_LIT:content>"]<EOL>field_properties = getattr(settings, "<STR_LIT>", {})<EOL>def field_property(field_name, field_value):<EOL><INDENT>"""<STR_LIT>"""<EOL>prop_val = None<EOL>if field_name in field_properties:<EOL><INDENT>prop_val = field_properties[field_name]<EOL><DEDENT>elif isinstance(field_value, dict):<EOL><INDENT>props = {fn: field_property(fn, field_value[fn]) for fn in field_value}<EOL>prop_val = {"<STR_LIT>": props}<EOL><DEDENT>else:<EOL><INDENT>prop_val = {<EOL>"<STR_LIT:type>": "<STR_LIT:string>",<EOL>"<STR_LIT:index>": "<STR_LIT>",<EOL>}<EOL><DEDENT>return prop_val<EOL><DEDENT>new_properties = {<EOL>field: field_property(field, value)<EOL>for field, value in body.items()<EOL>if (field not in exclude_fields) and (field not in self._get_mappings(doc_type).get('<STR_LIT>', {}))<EOL>}<EOL>if new_properties:<EOL><INDENT>self._es.indices.put_mapping(<EOL>index=self.index_name,<EOL>doc_type=doc_type,<EOL>body={<EOL>doc_type: {<EOL>"<STR_LIT>": new_properties,<EOL>}<EOL>}<EOL>)<EOL>self._clear_mapping(doc_type)<EOL><DEDENT>

We desire to index content so that anything we want to be textually searchable(and therefore needing to be
analysed), but the other fields are designed to be filters, and only require an exact match. So, we want to
set up the mappings for these fields as "not_analyzed" - this will allow our filters to work faster because
they only have to work off exact matches


def index(self, doc_type, sources, **kwargs):

try:<EOL><INDENT>actions = []<EOL>for source in sources:<EOL><INDENT>self._check_mappings(doc_type, source)<EOL>id_ = source['<STR_LIT:id>'] if '<STR_LIT:id>' in source else None<EOL>log.debug("<STR_LIT>", doc_type, id_)<EOL>action = {<EOL>"<STR_LIT>": self.index_name,<EOL>"<STR_LIT>": doc_type,<EOL>"<STR_LIT>": id_,<EOL>"<STR_LIT>": source<EOL>}<EOL>actions.append(action)<EOL><DEDENT>_, indexing_errors = bulk(<EOL>self._es,<EOL>actions,<EOL>**kwargs<EOL>)<EOL>if indexing_errors:<EOL><INDENT>ElasticSearchEngine.log_indexing_error(indexing_errors)<EOL><DEDENT><DEDENT>except Exception as ex:<EOL><INDENT>log.exception("<STR_LIT>", str(ex))<EOL>raise<EOL><DEDENT>

Implements call to add documents to the ES index
Note the call to _check_mappings which will setup fields with the desired mappings


def remove(self, doc_type, doc_ids, **kwargs):

try:<EOL><INDENT>actions = []<EOL>for doc_id in doc_ids:<EOL><INDENT>log.debug("<STR_LIT>", doc_type, doc_id)<EOL>action = {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>"<STR_LIT>": self.index_name,<EOL>"<STR_LIT>": doc_type,<EOL>"<STR_LIT>": doc_id<EOL>}<EOL>actions.append(action)<EOL><DEDENT>bulk(self._es, actions, **kwargs)<EOL><DEDENT>except BulkIndexError as ex:<EOL><INDENT>valid_errors = [error for error in ex.errors if error['<STR_LIT>']['<STR_LIT:status>'] != <NUM_LIT>]<EOL>if valid_errors:<EOL><INDENT>log.exception("<STR_LIT>")<EOL>raise<EOL><DEDENT><DEDENT>

Implements call to remove the documents from the index


log.debug("<STR_LIT>", query_string)<EOL>elastic_queries = []<EOL>elastic_filters = []<EOL>if query_string:<EOL><INDENT>if six.PY2:<EOL><INDENT>query_string = query_string.encode('<STR_LIT:utf-8>').translate(None, RESERVED_CHARACTERS)<EOL><DEDENT>else:<EOL><INDENT>query_string = query_string.translate(query_string.maketrans('<STR_LIT>', '<STR_LIT>', RESERVED_CHARACTERS))<EOL><DEDENT>elastic_queries.append({<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": ["<STR_LIT>"],<EOL>"<STR_LIT>": query_string<EOL>}<EOL>})<EOL><DEDENT>if field_dictionary:<EOL><INDENT>if use_field_match:<EOL><INDENT>elastic_queries.extend(_process_field_queries(field_dictionary))<EOL><DEDENT>else:<EOL><INDENT>elastic_filters.extend(_process_field_filters(field_dictionary))<EOL><DEDENT><DEDENT>if filter_dictionary:<EOL><INDENT>elastic_filters.extend(_process_filters(filter_dictionary))<EOL><DEDENT>if exclude_ids:<EOL><INDENT>if not exclude_dictionary:<EOL><INDENT>exclude_dictionary = {}<EOL><DEDENT>if "<STR_LIT>" not in exclude_dictionary:<EOL><INDENT>exclude_dictionary["<STR_LIT>"] = []<EOL><DEDENT>exclude_dictionary["<STR_LIT>"].extend(exclude_ids)<EOL><DEDENT>if exclude_dictionary:<EOL><INDENT>elastic_filters.append(_process_exclude_dictionary(exclude_dictionary))<EOL><DEDENT>query_segment = {<EOL>"<STR_LIT>": {}<EOL>}<EOL>if elastic_queries:<EOL><INDENT>query_segment = {<EOL>"<STR_LIT:bool>": {<EOL>"<STR_LIT>": elastic_queries<EOL>}<EOL>}<EOL><DEDENT>query = query_segment<EOL>if elastic_filters:<EOL><INDENT>filter_segment = {<EOL>"<STR_LIT:bool>": {<EOL>"<STR_LIT>": elastic_filters<EOL>}<EOL>}<EOL>query = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": query_segment,<EOL>"<STR_LIT>": filter_segment,<EOL>}<EOL>}<EOL><DEDENT>body = {"<STR_LIT>": query}<EOL>if facet_terms:<EOL><INDENT>facet_query = _process_facet_terms(facet_terms)<EOL>if facet_query:<EOL><INDENT>body["<STR_LIT>"] = facet_query<EOL><DEDENT><DEDENT>try:<EOL><INDENT>es_response = self._es.search(<EOL>index=self.index_name,<EOL>body=body,<EOL>**kwargs<EOL>)<EOL><DEDENT>except exceptions.ElasticsearchException as ex:<EOL><INDENT>message = six.text_type(ex)<EOL>if '<STR_LIT>' in message:<EOL><INDENT>log.exception("<STR_LIT>", message)<EOL>raise QueryParseError('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>log.exception("<STR_LIT>", str(message))<EOL>raise<EOL><DEDENT><DEDENT>return _translate_hits(es_response)<EOL>

Implements call to search the index for the desired content.

Args:
    query_string (str): the string of values upon which to search within the
    content of the objects within the index

    field_dictionary (dict): dictionary of values which _must_ exist and
    _must_ match in order for the documents to be included in the results

    filter_dictionary (dict): dictionary of values which _must_ match if the
    field exists in order for the documents to be included in the results;
    documents for which the field does not exist may be included in the
    results if they are not otherwise filtered out

    exclude_dictionary(dict): dictionary of values all of which which must
    not match in order for the documents to be included in the results;
    documents which have any of these fields and for which the value matches
    one of the specified values shall be filtered out of the result set

    facet_terms (dict): dictionary of terms to include within search
    facets list - key is the term desired to facet upon, and the value is a
    dictionary of extended information to include. Supported right now is a
    size specification for a cap upon how many facet results to return (can
    be an empty dictionary to use default size for underlying engine):

    e.g.
    {
        "org": {"size": 10},  # only show top 10 organizations
        "modes": {}
    }

    use_field_match (bool): flag to indicate whether to use elastic
    filtering or elastic matching for field matches - this is nothing but a
    potential performance tune for certain queries

    (deprecated) exclude_ids (list): list of id values to exclude from the results -
    useful for finding maches that aren't "one of these"

Returns:
    dict object with results in the desired format
    {
        "took": 3,
        "total": 4,
        "max_score": 2.0123,
        "results": [
            {
                "score": 2.0123,
                "data": {
                    ...
                }
            },
            {
                "score": 0.0983,
                "data": {
                    ...
                }
            }
        ],
        "facets": {
            "org": {
                "total": total_count,
                "other": 1,
                "terms": {
                    "MITx": 25,
                    "HarvardX": 18
                }
            },
            "modes": {
                "total": modes_count,
                "other": 15,
                "terms": {
                    "honor": 58,
                    "verified": 44,
                }
            }
        }
    }

Raises:
    ElasticsearchException when there is a problem with the response from elasticsearch

Example usage:
    .search(
        "find the words within this string",
        {
            "must_have_field": "mast_have_value for must_have_field"
        },
        {

        }
    )


if class_path is None:<EOL><INDENT>return default<EOL><DEDENT>component = class_path.rsplit('<STR_LIT:.>', <NUM_LIT:1>)<EOL>result_processor = getattr(<EOL>importlib.import_module(component[<NUM_LIT:0>]),<EOL>component[<NUM_LIT:1>],<EOL>default<EOL>) if len(component) > <NUM_LIT:1> else default<EOL>return result_processor<EOL>

Loads the class from the class_path string


return isinstance(item, collections.Iterable) and not isinstance(item, six.string_types)<EOL>

Checks if an item is iterable (list, tuple, generator), but not string


return self._upper<EOL>

return class member _upper as a proerty value


return self._lower<EOL>

return class member _lower as a proerty value


return str(self._upper)<EOL>

return string representation of _upper as a proerty value


return str(self._lower)<EOL>


return self._upper.isoformat()<EOL>

use isoformat for _upper date's string format


return self._lower.isoformat()<EOL>

use isoformat for _lower date's string format


return getattr(settings, "<STR_LIT>", DEFAULT_FILTER_FIELDS)<EOL>

look up the desired list of course discovery filter fields


return getattr(settings, "<STR_LIT>", {field: {} for field in course_discovery_filter_fields()})<EOL>

Discovery facets to include, by default we specify each filter field with unspecified size attribute


<EOL>(field_dictionary, filter_dictionary, exclude_dictionary) = SearchFilterGenerator.generate_field_filters(<EOL>user=user,<EOL>course_id=course_id<EOL>)<EOL>searcher = SearchEngine.get_search_engine(getattr(settings, "<STR_LIT>", "<STR_LIT>"))<EOL>if not searcher:<EOL><INDENT>raise NoSearchEngineError("<STR_LIT>")<EOL><DEDENT>results = searcher.search_string(<EOL>search_term,<EOL>field_dictionary=field_dictionary,<EOL>filter_dictionary=filter_dictionary,<EOL>exclude_dictionary=exclude_dictionary,<EOL>size=size,<EOL>from_=from_,<EOL>doc_type="<STR_LIT>",<EOL>)<EOL>for result in results["<STR_LIT>"]:<EOL><INDENT>result["<STR_LIT:data>"] = SearchResultProcessor.process_result(result["<STR_LIT:data>"], search_term, user)<EOL><DEDENT>results["<STR_LIT>"] = len([r for r in results["<STR_LIT>"] if r["<STR_LIT:data>"] is None])<EOL>results["<STR_LIT>"] = [r for r in results["<STR_LIT>"] if r["<STR_LIT:data>"] is not None]<EOL>return results<EOL>

Call the search engine with the appropriate parameters


def course_discovery_search(search_term=None, size=<NUM_LIT:20>, from_=<NUM_LIT:0>, field_dictionary=None):

<EOL>use_search_fields = ["<STR_LIT>"]<EOL>(search_fields, _, exclude_dictionary) = SearchFilterGenerator.generate_field_filters()<EOL>use_field_dictionary = {}<EOL>use_field_dictionary.update({field: search_fields[field] for field in search_fields if field in use_search_fields})<EOL>if field_dictionary:<EOL><INDENT>use_field_dictionary.update(field_dictionary)<EOL><DEDENT>if not getattr(settings, "<STR_LIT>", False):<EOL><INDENT>use_field_dictionary["<STR_LIT>"] = DateRange(None, datetime.utcnow())<EOL><DEDENT>searcher = SearchEngine.get_search_engine(getattr(settings, "<STR_LIT>", "<STR_LIT>"))<EOL>if not searcher:<EOL><INDENT>raise NoSearchEngineError("<STR_LIT>")<EOL><DEDENT>results = searcher.search(<EOL>query_string=search_term,<EOL>doc_type="<STR_LIT>",<EOL>size=size,<EOL>from_=from_,<EOL>field_dictionary=use_field_dictionary,<EOL>filter_dictionary={"<STR_LIT>": DateRange(datetime.utcnow(), None)},<EOL>exclude_dictionary=exclude_dictionary,<EOL>facet_terms=course_discovery_facets(),<EOL>)<EOL>return results<EOL>

Course Discovery activities against the search engine index of course details


raise NotImplementedError<EOL>

This operation is called to add documents of given type to the search index


raise NotImplementedError<EOL>

This operation is called to remove documents of given type from the search index


raise NotImplementedError<EOL>

This operation is called to search for matching documents within the search index


def search_string(self, query_string, **kwargs):

return self.search(query_string=query_string, **kwargs)<EOL>

Helper function when primary search is for a query string


def search_fields(self, field_dictionary, **kwargs):

return self.search(field_dictionary=field_dictionary, **kwargs)<EOL>

Helper function when primary search is for a set of matching fields


search_engine_class = _load_class(getattr(settings, "<STR_LIT>", None), None)<EOL>return search_engine_class(index=index) if search_engine_class else None<EOL>

Returns the desired implementor (defined in settings)


strings = [value for value in six.itervalues(dictionary) if not isinstance(value, dict)]<EOL>for child_dict in [dv for dv in six.itervalues(dictionary) if isinstance(dv, dict)]:<EOL><INDENT>strings.extend(SearchResultProcessor.strings_in_dictionary(child_dict))<EOL><DEDENT>return strings<EOL>

Used by default implementation for finding excerpt


@staticmethod<EOL><INDENT>def find_matches(strings, words, length_hoped):<DEDENT>

lower_words = [w.lower() for w in words]<EOL>def has_match(string):<EOL><INDENT>"""<STR_LIT>"""<EOL>lower_string = string.lower()<EOL>for test_word in lower_words:<EOL><INDENT>if test_word in lower_string:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL><DEDENT>shortened_strings = [textwrap.wrap(s) for s in strings]<EOL>short_string_list = list(chain.from_iterable(shortened_strings))<EOL>matches = [ms for ms in short_string_list if has_match(ms)]<EOL>cumulative_len = <NUM_LIT:0><EOL>break_at = None<EOL>for idx, match in enumerate(matches):<EOL><INDENT>cumulative_len += len(match)<EOL>if cumulative_len >= length_hoped:<EOL><INDENT>break_at = idx<EOL>break<EOL><DEDENT><DEDENT>return matches[<NUM_LIT:0>:break_at]<EOL>


matches = re.finditer(match_word, match_in, re.IGNORECASE)<EOL>for matched_string in set([match.group() for match in matches]):<EOL><INDENT>match_in = match_in.replace(<EOL>matched_string,<EOL>getattr(settings, "<STR_LIT>", u"<STR_LIT>").format(matched_string)<EOL>)<EOL><DEDENT>return match_in<EOL>


return False<EOL>

Override this in a class in order to add in last-chance access checks to the search process
Your application will want to make this decision


for property_name in [p[<NUM_LIT:0>] for p in inspect.getmembers(self.__class__) if isinstance(p[<NUM_LIT:1>], property)]:<EOL><INDENT>self._results_fields[property_name] = getattr(self, property_name, None)<EOL><DEDENT>

Called during post processing of result
Any properties defined in your subclass will get exposed as members of the result json from the search


@classmethod<EOL><INDENT>def process_result(cls, dictionary, match_phrase, user):<DEDENT>

result_processor = _load_class(getattr(settings, "<STR_LIT>", None), cls)<EOL>srp = result_processor(dictionary, match_phrase)<EOL>if srp.should_remove(user):<EOL><INDENT>return None<EOL><DEDENT>try:<EOL><INDENT>srp.add_properties()<EOL><DEDENT>except Exception as ex:  <EOL><INDENT>log.exception("<STR_LIT>",<EOL>json.dumps(dictionary, cls=DjangoJSONEncoder), str(ex))<EOL>return None<EOL><DEDENT>return dictionary<EOL>

Called from within search handler. Finds desired subclass and decides if the
result should be removed and adds properties derived from the result information


if "<STR_LIT:content>" not in self._results_fields:<EOL><INDENT>return None<EOL><DEDENT>match_phrases = [self._match_phrase]<EOL>if six.PY2:<EOL><INDENT>separate_phrases = [<EOL>phrase.decode('<STR_LIT:utf-8>')<EOL>for phrase in shlex.split(self._match_phrase.encode('<STR_LIT:utf-8>'))<EOL>]<EOL><DEDENT>else:<EOL><INDENT>separate_phrases = [<EOL>phrase<EOL>for phrase in shlex.split(self._match_phrase)<EOL>]<EOL><DEDENT>if len(separate_phrases) > <NUM_LIT:1>:<EOL><INDENT>match_phrases.extend(separate_phrases)<EOL><DEDENT>else:<EOL><INDENT>match_phrases = separate_phrases<EOL><DEDENT>matches = SearchResultProcessor.find_matches(<EOL>SearchResultProcessor.strings_in_dictionary(self._results_fields["<STR_LIT:content>"]),<EOL>match_phrases,<EOL>DESIRED_EXCERPT_LENGTH<EOL>)<EOL>excerpt_text = ELLIPSIS.join(matches)<EOL>for match_word in match_phrases:<EOL><INDENT>excerpt_text = SearchResultProcessor.decorate_matches(excerpt_text, match_word)<EOL><DEDENT>return excerpt_text<EOL>

Property to display a useful excerpt representing the matches within the results


def sort(value):<EOL><INDENT>return float(value[<NUM_LIT:1>].get('<STR_LIT:q>', <NUM_LIT:1>))<EOL><DEDENT>return OrderedDict(sorted(_splitHeaders(headers), key=sort, reverse=True))<EOL>

Parse and sort an ``Accept`` header.

The header is sorted according to the ``q`` parameter for each header value.

@rtype: `OrderedDict` mapping `bytes` to `dict`
@return: Mapping of media types to header parameters.


return [cgi.parse_header(value)<EOL>for value in chain.from_iterable(<EOL>s.split('<STR_LIT:U+002C>') for s in headers<EOL>if s)]<EOL>

Split an HTTP header whose components are separated with commas.

Each component is then split on semicolons and the component arguments
converted into a `dict`.

@return: `list` of 2-`tuple` of `bytes`, `dict`
@return: List of header arguments and mapping of component argument names
    to values.


if encoding is None:<EOL><INDENT>encoding = b'<STR_LIT:utf-8>'<EOL><DEDENT>headers = _splitHeaders(<EOL>requestHeaders.getRawHeaders(b'<STR_LIT:Content-Type>', []))<EOL>if headers:<EOL><INDENT>return headers[<NUM_LIT:0>][<NUM_LIT:1>].get(b'<STR_LIT>', encoding)<EOL><DEDENT>return encoding<EOL>

Extract an encoding from a ``Content-Type`` header.

@type  requestHeaders: `twisted.web.http_headers.Headers`
@param requestHeaders: Request headers.

@type  encoding: `bytes`
@param encoding: Default encoding to assume if the ``Content-Type``
    header is lacking one. Defaults to ``UTF-8``.

@rtype: `bytes`
@return: Content encoding.


@wraps(f)<EOL>def _maybe(x, *a, **kw):<EOL><INDENT>if x is None:<EOL><INDENT>return default<EOL><DEDENT>return f(x, *a, **kw)<EOL><DEDENT>return _maybe<EOL>

Create a nil-safe callable decorator.

If the wrapped callable receives ``None`` as its argument, it will return
``None`` immediately.


meth = getattr(resource, '<STR_LIT>' + nativeString(request.method), None)<EOL>if meth is None:<EOL><INDENT>try:<EOL><INDENT>allowedMethods = resource.allowedMethods<EOL><DEDENT>except AttributeError:<EOL><INDENT>allowedMethods = _computeAllowedMethods(resource)<EOL><DEDENT>raise UnsupportedMethod(allowedMethods)<EOL><DEDENT>return meth(request)<EOL>

Render a given resource.

See `IResource.render <twisted:twisted.web.resource.IResource.render>`.


self._wrappedResource = wrappedResource<EOL>Resource.__init__(self)<EOL>

:type  wrappedResource: `ISpinneretResource`
:param wrappedResource: Spinneret resource to wrap in an `IResource`.


if result is None:<EOL><INDENT>return NotFound()<EOL><DEDENT>spinneretResource = ISpinneretResource(result, None)<EOL>if spinneretResource is not None:<EOL><INDENT>return SpinneretResource(spinneretResource)<EOL><DEDENT>renderable = IRenderable(result, None)<EOL>if renderable is not None:<EOL><INDENT>return _RenderableResource(renderable)<EOL><DEDENT>resource = IResource(result, None)<EOL>if resource is not None:<EOL><INDENT>return resource<EOL><DEDENT>if isinstance(result, URLPath):<EOL><INDENT>return Redirect(str(result))<EOL><DEDENT>return result<EOL>

Adapt a result to `IResource`.

Several adaptions are tried they are, in order: ``None``,
`IRenderable <twisted:twisted.web.iweb.IRenderable>`, `IResource
<twisted:twisted.web.resource.IResource>`, and `URLPath
<twisted:twisted.python.urlpath.URLPath>`. Anything else is returned as
is.

A `URLPath <twisted:twisted.python.urlpath.URLPath>` is treated as
a redirect.


def _handleRenderResult(self, request, result):

def _requestFinished(result, cancel):<EOL><INDENT>cancel()<EOL>return result<EOL><DEDENT>if not isinstance(result, Deferred):<EOL><INDENT>result = succeed(result)<EOL><DEDENT>def _whenDone(result):<EOL><INDENT>render = getattr(result, '<STR_LIT>', lambda request: result)<EOL>renderResult = render(request)<EOL>if renderResult != NOT_DONE_YET:<EOL><INDENT>request.write(renderResult)<EOL>request.finish()<EOL><DEDENT>return result<EOL><DEDENT>request.notifyFinish().addBoth(_requestFinished, result.cancel)<EOL>result.addCallback(self._adaptToResource)<EOL>result.addCallback(_whenDone)<EOL>result.addErrback(request.processingFailed)<EOL>return NOT_DONE_YET<EOL>

Handle the result from `IResource.render`.

If the result is a `Deferred` then return `NOT_DONE_YET` and add
a callback to write the result to the request when it arrives.


def __init__(self, handlers, fallback=False):

Resource.__init__(self)<EOL>self._handlers = list(handlers)<EOL>self._fallback = fallback<EOL>self._acceptHandlers = {}<EOL>for handler in self._handlers:<EOL><INDENT>for acceptType in handler.acceptTypes:<EOL><INDENT>if acceptType in self._acceptHandlers:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>' % (acceptType,))<EOL><DEDENT>self._acceptHandlers[acceptType] = handler<EOL><DEDENT><DEDENT>

:type  handlers: ``iterable`` of `INegotiableResource` and either
    `IResource` or `ISpinneretResource`.
:param handlers: Iterable of negotiable resources, either
    `ISpinneretResource` or `IResource`, to use as handlers for
    negotiation.

:type  fallback: `bool`
:param fallback: Fall back to the first handler in the case where
    negotiation fails?


accept = _parseAccept(request.requestHeaders.getRawHeaders('<STR_LIT>'))<EOL>for contentType in accept.keys():<EOL><INDENT>handler = self._acceptHandlers.get(contentType.lower())<EOL>if handler is not None:<EOL><INDENT>return handler, handler.contentType<EOL><DEDENT><DEDENT>if self._fallback:<EOL><INDENT>handler = self._handlers[<NUM_LIT:0>]<EOL>return handler, handler.contentType<EOL><DEDENT>return NotAcceptable(), None<EOL>

Negotiate a handler based on the content types acceptable to the
client.

:rtype: 2-`tuple` of `twisted.web.iweb.IResource` and `bytes`
:return: Pair of a resource and the content type.


Locate another object which can be adapted to `IResource`.

:type  request: `IRequest <twisted:twisted.web.iweb.IRequest>`
:param request: Request.

:type  segments: ``sequence`` of `bytes`
:param segments: Sequence of strings giving the remaining query
    segments to resolve.

:rtype: 2-`tuple` of `IResource`, `IRenderable` or `URLPath` and
    a ``sequence`` of `bytes`
:return: Pair of an `IResource`, `IRenderable` or `URLPath` and
    a sequence of the remaining path segments to be process, or
    a `Deferred` containing the aforementioned result.


return Raises(<EOL>AfterPreprocessing(<EOL>lambda x: x[<NUM_LIT:1>],<EOL>MatchesAll(IsInstance(exc_type), matcher)))<EOL>

Match an exception type and a user-provided matcher against the exception
instance.


request = InMemoryRequest(segments)<EOL>child = getChildForRequest(resource, request)<EOL>request.render(child)<EOL>return request<EOL>

Locate and render a child resource.

@type  resource: `IResource`
@param resource: Resource to locate the child resource on.

@type  segments: `list` of `bytes`
@param segments: Path segments.

@return: Request.


return isSequenceType(x) and not isinstance(x, (bytes, unicode))<EOL>

Is this a ``sequence`` type that isn't also a ``string`` type?


def _one(result):<EOL><INDENT>if _isSequenceTypeNotText(result) and len(result) > n:<EOL><INDENT>return func(result[n])<EOL><DEDENT>return None<EOL><DEDENT>return maybe(_one)<EOL>

Create a callable that applies ``func`` to a value in a sequence.

If the value is not a sequence or is an empty sequence then ``None`` is
returned.

:type  func: `callable`
:param func: Callable to be applied to each result.

:type  n: `int`
:param n: Index of the value to apply ``func`` to.


def _many(result):<EOL><INDENT>if _isSequenceTypeNotText(result):<EOL><INDENT>return map(func, result)<EOL><DEDENT>return []<EOL><DEDENT>return maybe(_many, default=[])<EOL>

Create a callable that applies ``func`` to every value in a sequence.

If the value is not a sequence then an empty list is returned.

:type  func: `callable`
:param func: Callable to be applied to the first result.


if encoding is None:<EOL><INDENT>encoding = '<STR_LIT:utf-8>'<EOL><DEDENT>if isinstance(value, bytes):<EOL><INDENT>return value.decode(encoding)<EOL><DEDENT>elif isinstance(value, unicode):<EOL><INDENT>return value<EOL><DEDENT>return None<EOL>

Parse a value as text.

:type  value: `unicode` or `bytes`
:param value: Text value to parse

:type  encoding: `bytes`
:param encoding: Encoding to treat ``bytes`` values as, defaults to
    ``utf-8``.

:rtype: `unicode`
:return: Parsed text or ``None`` if ``value`` is neither `bytes` nor
    `unicode`.


def Integer(value, base=<NUM_LIT:10>, encoding=None):

try:<EOL><INDENT>return int(Text(value, encoding), base)<EOL><DEDENT>except (TypeError, ValueError):<EOL><INDENT>return None<EOL><DEDENT>

Parse a value as an integer.

:type  value: `unicode` or `bytes`
:param value: Text value to parse

:type  base: `unicode` or `bytes`
:param base: Base to assume ``value`` is specified in.

:type  encoding: `bytes`
:param encoding: Encoding to treat ``bytes`` values as, defaults to
    ``utf-8``.

:rtype: `int`
:return: Parsed integer or ``None`` if ``value`` could not be parsed as an
    integer.


try:<EOL><INDENT>return float(Text(value, encoding))<EOL><DEDENT>except (TypeError, ValueError):<EOL><INDENT>return None<EOL><DEDENT>

Parse a value as a floating point number.

:type  value: `unicode` or `bytes`
:param value: Text value to parse.

:type  encoding: `bytes`
:param encoding: Encoding to treat `bytes` values as, defaults to
    ``utf-8``.

:rtype: `float`
:return: Parsed float or ``None`` if ``value`` could not be parsed as a
    float.


def Boolean(value, true=(u'<STR_LIT:yes>', u'<STR_LIT:1>', u'<STR_LIT:true>'), false=(u'<STR_LIT>', u'<STR_LIT:0>', u'<STR_LIT:false>'),<EOL>encoding=None):

value = Text(value, encoding)<EOL>if value is not None:<EOL><INDENT>value = value.lower().strip()<EOL><DEDENT>if value in true:<EOL><INDENT>return True<EOL><DEDENT>elif value in false:<EOL><INDENT>return False<EOL><DEDENT>return None<EOL>

Parse a value as a boolean.

:type  value: `unicode` or `bytes`
:param value: Text value to parse.

:type  true: `tuple` of `unicode`
:param true: Values to compare, ignoring case, for ``True`` values.

:type  false: `tuple` of `unicode`
:param false: Values to compare, ignoring case, for ``False`` values.

:type  encoding: `bytes`
:param encoding: Encoding to treat `bytes` values as, defaults to
    ``utf-8``.

:rtype: `bool`
:return: Parsed boolean or ``None`` if ``value`` did not match ``true`` or
    ``false`` values.


def Delimited(value, parser=Text, delimiter=u'<STR_LIT:U+002C>', encoding=None):

value = Text(value, encoding)<EOL>if value is None or value == u'<STR_LIT>':<EOL><INDENT>return []<EOL><DEDENT>return map(parser, value.split(delimiter))<EOL>

Parse a value as a delimited list.

:type  value: `unicode` or `bytes`
:param value: Text value to parse.

:type  parser: `callable` taking a `unicode` parameter
:param parser: Callable to map over the delimited text values.

:type  delimiter: `unicode`
:param delimiter: Delimiter text.

:type  encoding: `bytes`
:param encoding: Encoding to treat `bytes` values as, defaults to
    ``utf-8``.

:rtype: `list`
:return: List of parsed values.


def Timestamp(value, _divisor=<NUM_LIT:1.>, tz=UTC, encoding=None):

value = Float(value, encoding)<EOL>if value is not None:<EOL><INDENT>value = value / _divisor<EOL>return datetime.fromtimestamp(value, tz)<EOL><DEDENT>return None<EOL>

Parse a value as a POSIX timestamp in seconds.

:type  value: `unicode` or `bytes`
:param value: Text value to parse, which should be the number of seconds
    since the epoch.

:type  _divisor: `float`
:param _divisor: Number to divide the value by.

:type  tz: `tzinfo`
:param tz: Timezone, defaults to UTC.

:type  encoding: `bytes`
:param encoding: Encoding to treat `bytes` values as, defaults to
    ``utf-8``.

:rtype: `datetime.datetime`
:return: Parsed datetime or ``None`` if ``value`` could not be parsed.


return Timestamp(value, _divisor=<NUM_LIT>, encoding=encoding)<EOL>

Parse a value as a POSIX timestamp in milliseconds.

:type  value: `unicode` or `bytes`
:param value: Text value to parse, which should be the number of
    milliseconds since the epoch.

:type  encoding: `bytes`
:param encoding: Encoding to treat `bytes` values as, defaults to
    ``utf-8``.

:rtype: `datetime.datetime`
:return: Parsed datetime or ``None`` if ``value`` could not be parsed.


return dict(<EOL>(key, parser(query.get(key, [])))<EOL>for key, parser in expected.items())<EOL>

Parse query parameters.

:type  expected: `dict` mapping `bytes` to `callable`
:param expected: Mapping of query argument names to argument parsing
    callables.

:type  query: `dict` mapping `bytes` to `list` of `bytes`
:param query: Mapping of query argument names to lists of argument values,
    this is the form that Twisted Web's `IRequest.args
    <twisted:twisted.web.iweb.IRequest.args>` value takes.

:rtype: `dict` mapping `bytes` to `object`
:return: Mapping of query argument names to parsed argument values.


def _match(request, value):<EOL><INDENT>return name, query.Text(<EOL>value,<EOL>encoding=contentEncoding(request.requestHeaders, encoding))<EOL><DEDENT>return _match<EOL>

Match a route parameter.

`Any` is a synonym for `Text`.

:type  name: `bytes`
:param name: Route parameter name.

:type  encoding: `bytes`
:param encoding: Default encoding to assume if the ``Content-Type``
    header is lacking one.

:return: ``callable`` suitable for use with `route` or `subroute`.


def Integer(name, base=<NUM_LIT:10>, encoding=None):

def _match(request, value):<EOL><INDENT>return name, query.Integer(<EOL>value,<EOL>base=base,<EOL>encoding=contentEncoding(request.requestHeaders, encoding))<EOL><DEDENT>return _match<EOL>

Match an integer route parameter.

:type  name: `bytes`
:param name: Route parameter name.

:type  base: `int`
:param base: Base to interpret the value in.

:type  encoding: `bytes`
:param encoding: Default encoding to assume if the ``Content-Type``
    header is lacking one.

:return: ``callable`` suitable for use with `route` or `subroute`.


def _matchRoute(components, request, segments, partialMatching):

if len(components) == <NUM_LIT:1> and isinstance(components[<NUM_LIT:0>], bytes):<EOL><INDENT>components = components[<NUM_LIT:0>]<EOL>if components[:<NUM_LIT:1>] == '<STR_LIT:/>':<EOL><INDENT>components = components[<NUM_LIT:1>:]<EOL><DEDENT>components = components.split('<STR_LIT:/>')<EOL><DEDENT>results = OrderedDict()<EOL>NO_MATCH = None, segments<EOL>remaining = list(segments)<EOL>if len(segments) == len(components) == <NUM_LIT:0>:<EOL><INDENT>return results, remaining<EOL><DEDENT>for us, them in izip_longest(components, segments):<EOL><INDENT>if us is None:<EOL><INDENT>if partialMatching:<EOL><INDENT>break<EOL><DEDENT>else:<EOL><INDENT>return NO_MATCH<EOL><DEDENT><DEDENT>elif them is None:<EOL><INDENT>return NO_MATCH<EOL><DEDENT>if callable(us):<EOL><INDENT>name, match = us(request, them)<EOL>if match is None:<EOL><INDENT>return NO_MATCH<EOL><DEDENT>results[name] = match<EOL><DEDENT>elif us != them:<EOL><INDENT>return NO_MATCH<EOL><DEDENT>remaining.pop(<NUM_LIT:0>)<EOL><DEDENT>return results, remaining<EOL>

Match a request path against our path components.

The path components are always matched relative to their parent is in the
resource hierarchy, in other words it is only possible to match URIs nested
more deeply than the parent resource.

:type  components: ``iterable`` of `bytes` or `callable`
:param components: Iterable of path components, to match against the
    request, either static strings or dynamic parameters. As a convenience,
    a single `bytes` component containing ``/`` may be given instead of
    manually separating the components. If no components are given the null
    route is matched, this is the case where ``segments`` is empty.

:type  segments: ``sequence`` of `bytes`
:param segments: Sequence of path segments, from the request, to match
    against.

:type  partialMatching: `bool`
:param partialMatching: Allow partial matching against the request path?

:rtype: 2-`tuple` of `dict` keyed on `bytes` and `list` of `bytes`
:return: Pair of parameter results, mapping parameter names to processed
    values, and a list of the remaining request path segments. If there is
    no route match the result will be ``None`` and the original request path
    segments.


return partial(_matchRoute, components, partialMatching=False)<EOL>

Match a request path exactly.

The path components are always matched relative to their parent is in the
resource hierarchy, in other words it is only possible to match URIs nested
more deeply than the parent resource.

:type  components: ``iterable`` of `bytes` or `callable`
:param components: Iterable of path components, to match against the
    request, either static strings or dynamic parameters. As a convenience,
    a single `bytes` component containing ``/`` may be given instead of
    manually separating the components. If no components are given the null
    route is matched, this is the case where ``segments`` is empty.

:rtype: 2-`tuple` of `dict` keyed on `bytes` and `list` of `bytes`
:return: Pair of parameter results, mapping parameter names to processed
    values, and a list of the remaining request path segments. If there is
    no route match the result will be ``None`` and the original request
    path segments.


return partial(_matchRoute, components, partialMatching=True)<EOL>

Partially match a request path exactly.

The path components are always matched relative to their parent is in the
resource hierarchy, in other words it is only possible to match URIs nested
more deeply than the parent resource.

If there are more request path segments than components the match may still
be successful, the remaining path segments are returned in the second part
of the result.

:type  components: ``iterable`` of `bytes` or `callable`
:param components: Iterable of path components, to match against the
    request, either static strings or dynamic parameters. As a convenience,
    a single `bytes` component containing ``/`` may be given instead of
    manually separating the components. If no components are given the null
    route is matched, this is the case where ``segments`` is empty.

:rtype: 2-`tuple` of `dict` keyed on `bytes` and `list` of `bytes`
:return: Pair of parameter results, mapping parameter names to processed
    values, and a list of the remaining request path segments. If there is
    no route match the result will be ``None`` and the original request
    path segments.


return wraps(f)(<EOL>lambda *a, **kw: getattr(f(*a, **kw), routerAttribute).resource())<EOL>

Decorate a router-producing callable to instead produce a resource.

This simply produces a new callable that invokes the original callable, and
calls ``resource`` on the ``routerAttribute``.

If the router producer has multiple routers the attribute can be altered to
choose the appropriate one, for example:

.. code-block:: python

    class _ComplexRouter(object):
        router = Router()
        privateRouter = Router()

        @router.route('/')
        def publicRoot(self, request, params):
            return SomethingPublic(...)

        @privateRouter.route('/')
        def privateRoot(self, request, params):
            return SomethingPrivate(...)

    PublicResource = routedResource(_ComplexRouter)
    PrivateResource = routedResource(_ComplexRouter, 'privateRouter')

:type  f: ``callable``
:param f: Callable producing an object with a `Router` attribute, for
    example, a type.

:type  routerAttribute: `str`
:param routerAttribute: Name of the `Router` attribute on the result of
    calling ``f``.

:rtype: `callable`
:return: Callable producing an `IResource`.


self._obj = obj<EOL>self._routes = routes<EOL>

:param obj: Parent object containing the route handler.

:type  routes: `list` of 3-`tuple` containing `bytes`, `callable`,
    `callable`
:param routes: List of 3-tuple containing the route handler name, the
    route handler function and the matcher function.


def _matchRoute(self, request, segments):

for name, meth, route in self._routes:<EOL><INDENT>matches, remaining = route(request, segments)<EOL>if matches is not None:<EOL><INDENT>return meth(self._obj, request, matches), remaining<EOL><DEDENT><DEDENT>return None, segments<EOL>

Find a route handler that matches the request path and invoke it.


router = type(self)()<EOL>router._routes = list(self._routes)<EOL>router._self = obj<EOL>return router<EOL>

Create a new `Router` instance, with it's own set of routes, for
``obj``.


self._routes.append((f.func_name, f, matcher))<EOL>

Add a route handler and matcher to the collection of possible routes.


return SpinneretResource(_RouterResource(self._self, self._routes))<EOL>

Create an `IResource <twisted:twisted.web.resource.IResource>` that
will perform URL routing.


def _factory(f):<EOL><INDENT>self._addRoute(f, route(*components))<EOL>return f<EOL><DEDENT>return _factory<EOL>

See `txspinneret.route.route`.

This decorator can be stacked with itself to specify multiple routes
with a single handler.


def _factory(f):<EOL><INDENT>self._addRoute(f, subroute(*components))<EOL>return f<EOL><DEDENT>return _factory<EOL>

See `txspinneret.route.subroute`.

This decorator can be stacked with itself to specify multiple routes
with a single handler.


version_module_path = os.path.join(<EOL>os.path.dirname(__file__), "<STR_LIT>", "<STR_LIT>")<EOL>with open(version_module_path) as version_module:<EOL><INDENT>exec(version_module.read())<EOL><DEDENT>return locals()["<STR_LIT>"]<EOL>

Get the version from version module without importing more than
necessary.


with open(path) as f:<EOL><INDENT>return f.read()<EOL><DEDENT>


def t(*args):<EOL><INDENT>raise _TailCall(f, args, uid)<EOL><DEDENT>t.C = f<EOL>return t<EOL>

This is the "callable" version of the continuation, which sould only
be accessible from the inside of the function to be continued. An
attribute called "C" can be used in order to get back the public
version of the continuation (for passing the continuation to another
function).


if len(c): keys, k = zip(*c.items())<EOL>else: keys, k = tuple([]), tuple([])<EOL>def d(f):<EOL><INDENT>return C(<EOL>lambda kself, *conts:<EOL>lambda *args:<EOL>f(*args, self=kself, **dict(zip(keys, conts)))) (*k)<EOL><DEDENT>return d<EOL>

A decorator for defining tail-call optimized functions.

Example
-------

    @with_continuations()
    def factorial(n, k, self=None):
        return self(n-1, k*n) if n > 1 else k

    @with_continuations()
    def identity(x, self=None):
        return x

    @with_continuations(out=identity)
    def factorial2(n, k, self=None, out=None):
        return self(n-1, k*n) if n > 1 else out(k)

    print(factorial(7,1))
    print(factorial2(7,1))


@http_connection(<NUM_LIT>)<EOL>def make_get_request(url, params, headers, connection):

timeout = getattr(connection, '<STR_LIT>')<EOL>response = connection.get(url, params=params, headers=headers, timeout=timeout)<EOL>if response.ok or response.status_code == <NUM_LIT>:<EOL><INDENT>return response.json() if response.content else None<EOL><DEDENT>else:<EOL><INDENT>response.raise_for_status()<EOL><DEDENT>

Helper function that makes an HTTP GET request to the given firebase
endpoint. Timeout is 60 seconds.
`url`: The full URL of the firebase endpoint (DSN appended.)
`params`: Python dict that is appended to the URL like a querystring.
`headers`: Python dict. HTTP request headers.
`connection`: Predefined HTTP connection instance. If not given, it
is supplied by the `decorators.http_connection` function.

The returning value is a Python dict deserialized by the JSON decoder. However,
if the status code is not 2x or 403, an requests.HTTPError is raised.

connection = connection_pool.get_available_connection()
response = make_get_request('http://firebase.localhost/users', {'print': silent'},
                            {'X_FIREBASE_SOMETHING': 'Hi'}, connection)
response => {'1': 'John Doe', '2': 'Jane Doe'}


@http_connection(<NUM_LIT>)<EOL>def make_put_request(url, data, params, headers, connection):

timeout = getattr(connection, '<STR_LIT>')<EOL>response = connection.put(url, data=data, params=params, headers=headers,<EOL>timeout=timeout)<EOL>if response.ok or response.status_code == <NUM_LIT>:<EOL><INDENT>return response.json() if response.content else None<EOL><DEDENT>else:<EOL><INDENT>response.raise_for_status()<EOL><DEDENT>

Helper function that makes an HTTP PUT request to the given firebase
endpoint. Timeout is 60 seconds.
`url`: The full URL of the firebase endpoint (DSN appended.)
`data`: JSON serializable dict that will be stored in the remote storage.
`params`: Python dict that is appended to the URL like a querystring.
`headers`: Python dict. HTTP request headers.
`connection`: Predefined HTTP connection instance. If not given, it
is supplied by the `decorators.http_connection` function.

The returning value is a Python dict deserialized by the JSON decoder. However,
if the status code is not 2x or 403, an requests.HTTPError is raised.

connection = connection_pool.get_available_connection()
response = make_put_request('http://firebase.localhost/users',
                            '{"1": "Ozgur Vatansever"}',
                            {'X_FIREBASE_SOMETHING': 'Hi'}, connection)
response => {'1': 'Ozgur Vatansever'} or {'error': 'Permission denied.'}


@http_connection(<NUM_LIT>)<EOL>def make_post_request(url, data, params, headers, connection):

timeout = getattr(connection, '<STR_LIT>')<EOL>response = connection.post(url, data=data, params=params, headers=headers,<EOL>timeout=timeout)<EOL>if response.ok or response.status_code == <NUM_LIT>:<EOL><INDENT>return response.json() if response.content else None<EOL><DEDENT>else:<EOL><INDENT>response.raise_for_status()<EOL><DEDENT>

Helper function that makes an HTTP POST request to the given firebase
endpoint. Timeout is 60 seconds.
`url`: The full URL of the firebase endpoint (DSN appended.)
`data`: JSON serializable dict that will be stored in the remote storage.
`params`: Python dict that is appended to the URL like a querystring.
`headers`: Python dict. HTTP request headers.
`connection`: Predefined HTTP connection instance. If not given, it
is supplied by the `decorators.http_connection` function.

The returning value is a Python dict deserialized by the JSON decoder. However,
if the status code is not 2x or 403, an requests.HTTPError is raised.

connection = connection_pool.get_available_connection()
response = make_put_request('http://firebase.localhost/users/',
   '{"Ozgur Vatansever"}', {'X_FIREBASE_SOMETHING': 'Hi'}, connection)
response => {u'name': u'-Inw6zol_2f5ThHwVcSe'} or {'error': 'Permission denied.'}


@http_connection(<NUM_LIT>)<EOL>def make_patch_request(url, data, params, headers, connection):

timeout = getattr(connection, '<STR_LIT>')<EOL>response = connection.patch(url, data=data, params=params, headers=headers,<EOL>timeout=timeout)<EOL>if response.ok or response.status_code == <NUM_LIT>:<EOL><INDENT>return response.json() if response.content else None<EOL><DEDENT>else:<EOL><INDENT>response.raise_for_status()<EOL><DEDENT>

Helper function that makes an HTTP PATCH request to the given firebase
endpoint. Timeout is 60 seconds.
`url`: The full URL of the firebase endpoint (DSN appended.)
`data`: JSON serializable dict that will be stored in the remote storage.
`params`: Python dict that is appended to the URL like a querystring.
`headers`: Python dict. HTTP request headers.
`connection`: Predefined HTTP connection instance. If not given, it
is supplied by the `decorators.http_connection` function.

The returning value is a Python dict deserialized by the JSON decoder. However,
if the status code is not 2x or 403, an requests.HTTPError is raised.

connection = connection_pool.get_available_connection()
response = make_put_request('http://firebase.localhost/users/1',
   '{"Ozgur Vatansever"}', {'X_FIREBASE_SOMETHING': 'Hi'}, connection)
response => {'Ozgur Vatansever'} or {'error': 'Permission denied.'}


@http_connection(<NUM_LIT>)<EOL>def make_delete_request(url, params, headers, connection):

timeout = getattr(connection, '<STR_LIT>')<EOL>response = connection.delete(url, params=params, headers=headers, timeout=timeout)<EOL>if response.ok or response.status_code == <NUM_LIT>:<EOL><INDENT>return response.json() if response.content else None<EOL><DEDENT>else:<EOL><INDENT>response.raise_for_status()<EOL><DEDENT>

Helper function that makes an HTTP DELETE request to the given firebase
endpoint. Timeout is 60 seconds.
`url`: The full URL of the firebase endpoint (DSN appended.)
`params`: Python dict that is appended to the URL like a querystring.
`headers`: Python dict. HTTP request headers.
`connection`: Predefined HTTP connection instance. If not given, it
is supplied by the `decorators.http_connection` function.

The returning value is NULL. However, if the status code is not 2x or 403,
an requests.HTTPError is raised.

connection = connection_pool.get_available_connection()
response = make_put_request('http://firebase.localhost/users/1',
                            {'X_FIREBASE_SOMETHING': 'Hi'}, connection)
response => NULL or {'error': 'Permission denied.'}


token = self.authenticator.create_token(self.extra)<EOL>user_id = self.extra.get('<STR_LIT:id>')<EOL>return FirebaseUser(self.email, token, self.provider, user_id)<EOL>

Method that gets the authenticated user. The returning user has
the token, email and the provider data.


def _build_endpoint_url(self, url, name=None):

if not url.endswith(self.URL_SEPERATOR):<EOL><INDENT>url = url + self.URL_SEPERATOR<EOL><DEDENT>if name is None:<EOL><INDENT>name = '<STR_LIT>'<EOL><DEDENT>return '<STR_LIT>' % (urlparse.urljoin(self.dsn, url), name,<EOL>self.NAME_EXTENSION)<EOL>

Method that constructs a full url with the given url and the
snapshot name.

Example:
full_url = _build_endpoint_url('/users', '1')
full_url => 'http://firebase.localhost/users/1.json'


def _authenticate(self, params, headers):

if self.authentication:<EOL><INDENT>user = self.authentication.get_user()<EOL>params.update({'<STR_LIT>': user.firebase_auth_token})<EOL>headers.update(self.authentication.authenticator.HEADERS)<EOL><DEDENT>

Method that simply adjusts authentication credentials for the
request.
`params` is the querystring of the request.
`headers` is the header of the request.

If auth instance is not provided to this class, this method simply
returns without doing anything.


@http_connection(<NUM_LIT>)<EOL><INDENT>def get(self, url, name, params=None, headers=None, connection=None):<DEDENT>

if name is None: name = '<STR_LIT>'<EOL>params = params or {}<EOL>headers = headers or {}<EOL>endpoint = self._build_endpoint_url(url, name)<EOL>self._authenticate(params, headers)<EOL>return make_get_request(endpoint, params, headers, connection=connection)<EOL>


def get_async(self, url, name, callback=None, params=None, headers=None):

if name is None: name = '<STR_LIT>'<EOL>params = params or {}<EOL>headers = headers or {}<EOL>endpoint = self._build_endpoint_url(url, name)<EOL>self._authenticate(params, headers)<EOL>process_pool.apply_async(make_get_request,<EOL>args=(endpoint, params, headers), callback=callback)<EOL>

Asynchronous GET request with the process pool.


@http_connection(<NUM_LIT>)<EOL><INDENT>def put(self, url, name, data, params=None, headers=None, connection=None):<DEDENT>

assert name, '<STR_LIT>'<EOL>params = params or {}<EOL>headers = headers or {}<EOL>endpoint = self._build_endpoint_url(url, name)<EOL>self._authenticate(params, headers)<EOL>data = json.dumps(data, cls=JSONEncoder)<EOL>return make_put_request(endpoint, data, params, headers,<EOL>connection=connection)<EOL>

Synchronous PUT request. There will be no returning output from
the server, because the request will be made with ``silent``
parameter. ``data`` must be a JSONable value.


def put_async(self, url, name, data, callback=None, params=None, headers=None):

if name is None: name = '<STR_LIT>'<EOL>params = params or {}<EOL>headers = headers or {}<EOL>endpoint = self._build_endpoint_url(url, name)<EOL>self._authenticate(params, headers)<EOL>data = json.dumps(data, cls=JSONEncoder)<EOL>process_pool.apply_async(make_put_request,<EOL>args=(endpoint, data, params, headers),<EOL>callback=callback)<EOL>

Asynchronous PUT request with the process pool.


@http_connection(<NUM_LIT>)<EOL><INDENT>def post(self, url, data, params=None, headers=None, connection=None):<DEDENT>

params = params or {}<EOL>headers = headers or {}<EOL>endpoint = self._build_endpoint_url(url, None)<EOL>self._authenticate(params, headers)<EOL>data = json.dumps(data, cls=JSONEncoder)<EOL>return make_post_request(endpoint, data, params, headers,<EOL>connection=connection)<EOL>

Synchronous POST request. ``data`` must be a JSONable value.


def post_async(self, url, data, callback=None, params=None, headers=None):

params = params or {}<EOL>headers = headers or {}<EOL>endpoint = self._build_endpoint_url(url, None)<EOL>self._authenticate(params, headers)<EOL>data = json.dumps(data, cls=JSONEncoder)<EOL>process_pool.apply_async(make_post_request,<EOL>args=(endpoint, data, params, headers),<EOL>callback=callback)<EOL>

Asynchronous POST request with the process pool.


@http_connection(<NUM_LIT>)<EOL><INDENT>def patch(self, url, data, params=None, headers=None, connection=None):<DEDENT>

params = params or {}<EOL>headers = headers or {}<EOL>endpoint = self._build_endpoint_url(url, None)<EOL>self._authenticate(params, headers)<EOL>data = json.dumps(data, cls=JSONEncoder)<EOL>return make_patch_request(endpoint, data, params, headers,<EOL>connection=connection)<EOL>


def patch_async(self, url, data, callback=None, params=None, headers=None):

params = params or {}<EOL>headers = headers or {}<EOL>endpoint = self._build_endpoint_url(url, None)<EOL>self._authenticate(params, headers)<EOL>data = json.dumps(data, cls=JSONEncoder)<EOL>process_pool.apply_async(make_patch_request,<EOL>args=(endpoint, data, params, headers),<EOL>callback=callback)<EOL>

Asynchronous PATCH request with the process pool.


@http_connection(<NUM_LIT>)<EOL><INDENT>def delete(self, url, name, params=None, headers=None, connection=None):<DEDENT>

if not name: name = '<STR_LIT>'<EOL>params = params or {}<EOL>headers = headers or {}<EOL>endpoint = self._build_endpoint_url(url, name)<EOL>self._authenticate(params, headers)<EOL>return make_delete_request(endpoint, params, headers, connection=connection)<EOL>

Synchronous DELETE request. ``data`` must be a JSONable value.


def delete_async(self, url, name, callback=None, params=None, headers=None):

if not name: name = '<STR_LIT>'<EOL>params = params or {}<EOL>headers = headers or {}<EOL>endpoint = self._build_endpoint_url(url, name)<EOL>self._authenticate(params, headers)<EOL>process_pool.apply_async(make_delete_request,<EOL>args=(endpoint, params, headers), callback=callback)<EOL>

Asynchronous DELETE request with the process pool.


def create_token(self, data, options=None):

if not options:<EOL><INDENT>options = {}<EOL><DEDENT>options.update({'<STR_LIT>': self.admin, '<STR_LIT>': self.debug})<EOL>claims = self._create_options_claims(options)<EOL>claims['<STR_LIT:v>'] = self.TOKEN_VERSION<EOL>claims['<STR_LIT>'] = int(time.mktime(time.gmtime()))<EOL>claims['<STR_LIT:d>'] = data<EOL>return self._encode_token(self.secret, claims)<EOL>

Generates a secure authentication token.

Our token format follows the JSON Web Token (JWT) standard:
header.claims.signature

Where:
1) 'header' is a stringified, base64-encoded JSON object containing version and algorithm information.
2) 'claims' is a stringified, base64-encoded JSON object containing a set of claims:
    Library-generated claims:
    'iat' -> The issued at time in seconds since the epoch as a number
    'd' -> The arbitrary JSON object supplied by the user.
    User-supplied claims (these are all optional):
    'exp' (optional) -> The expiration time of this token, as a number of seconds since the epoch.
    'nbf' (optional) -> The 'not before' time before which the token should be rejected (seconds since the epoch)
    'admin' (optional) -> If set to true, this client will bypass all security rules (use this to authenticate servers)
    'debug' (optional) -> 'set to true to make this client receive debug information about security rule execution.
    'simulate' (optional, internal-only for now) -> Set to true to neuter all API operations (listens / puts
               will run security rules but not actually write or return data).
3) A signature that proves the validity of this token (see: http://tools.ietf.org/html/draft-ietf-jose-json-web-signature-07)

For base64-encoding we use URL-safe base64 encoding. This ensures that the entire token is URL-safe
and could, for instance, be placed as a query argument without any encoding (and this is what the JWT spec requires).

Args:
    data - a json serializable object of data to be included in the token
    options - An optional dictionary of additional claims for the token. Possible keys include:
        a) 'expires' -- A timestamp (as a number of seconds since the epoch) denoting a time after which
            this token should no longer be valid.
        b) 'notBefore' -- A timestamp (as a number of seconds since the epoch) denoting a time before
            which this token should be rejected by the server.
        c) 'admin' -- Set to true to bypass all security rules (use this for your trusted servers).
        d) 'debug' -- Set to true to enable debug mode (so you can see the results of Rules API operations)
        e) 'simulate' -- (internal-only for now) Set to true to neuter all API operations (listens / puts
                        will run security rules but not actually write or return data)
Returns:
    A signed Firebase Authentication Token
Raises:
    ValueError: if an invalid key is specified in options


process_pool.close()<EOL>process_pool.join()<EOL>process_pool.terminate()<EOL>

Clean up function that closes and terminates the process pool
defined in the ``async`` file.