dhcp_req = (<EOL>self.gen_ether_ip_unicast() /<EOL>self.gen_udp() /<EOL>self.gen_bootp_unicast() /<EOL>DHCP(options=[<EOL>("<STR_LIT>", "<STR_LIT>"),<EOL>("<STR_LIT>", mac2str(self.client_mac)),<EOL>("<STR_LIT>", self.prl),<EOL>"<STR_LIT:end>"])<EOL>)<EOL>logger.debug('<STR_LIT>', dhcp_req.summary())<EOL>return dhcp_req<EOL>

Generate DHCP REQUEST unicast packet.

Same comments as in gen_request apply.


dhcp_decline = (<EOL>self.gen_ether_ip() /<EOL>self.gen_udp() /<EOL>self.gen_bootp() /<EOL>DHCP(options=[<EOL>("<STR_LIT>", "<STR_LIT>"),<EOL>("<STR_LIT>", self.server_ip),<EOL>("<STR_LIT>", self.client_ip),<EOL>"<STR_LIT:end>"])<EOL>)<EOL>logger.debug('<STR_LIT>')<EOL>logger.debug(dhcp_decline.summary())<EOL>return dhcp_decline<EOL>

Generate DHCP decline packet (broadcast).

[:rfc:`7844#section-3.`] ::

    MUST contain the Message Type option,
    MUST contain the Server Identifier option,
    MUST contain the Requested IP address option;

.. note:: currently not being used.


dhcp_release = (<EOL>self.gen_ether_ip() /<EOL>self.gen_udp() /<EOL>self.gen_bootp() /<EOL>DHCP(options=[<EOL>("<STR_LIT>", "<STR_LIT>"),<EOL>("<STR_LIT>", mac2str(self.client_mac)),<EOL>("<STR_LIT>", self.server_ip),<EOL>"<STR_LIT:end>"])<EOL>)<EOL>logger.debug('<STR_LIT>')<EOL>logger.debug(dhcp_release.summary())<EOL>return dhcp_release<EOL>

Generate DHCP release packet (broadcast?).

[:rfc:`7844#section-3.`] ::

    MUST contain the Message Type option and
    MUST contain the Server Identifier option,

.. note:: currently not being used.


dhcp_inform = (<EOL>self.gen_ether_ip_unicast() /<EOL>self.gen_udp() /<EOL>self.gen_bootp_unicast() /<EOL>DHCP(options=[<EOL>("<STR_LIT>", "<STR_LIT>"),<EOL>("<STR_LIT>", mac2str(self.client_mac)),<EOL>"<STR_LIT:end>"])<EOL>)<EOL>logger.debug('<STR_LIT>')<EOL>logger.debug(dhcp_inform.summary())<EOL>return dhcp_inform<EOL>

Generate DHCP inform packet (unicast).

[:rfc:`7844#section-3.`] ::

    MUST contain the Message Type option,

.. note:: currently not being used.


<EOL>assert attrs_dict['<STR_LIT>']<EOL>assert attrs_dict['<STR_LIT:address>']<EOL>ipn = IPNetwork(attrs_dict['<STR_LIT:address>'] + '<STR_LIT:/>' +<EOL>attrs_dict['<STR_LIT>'])<EOL>if attrs_dict.get('<STR_LIT>') is None:<EOL><INDENT>attrs_dict['<STR_LIT>'] = self.server_ip<EOL><DEDENT>if attrs_dict.get('<STR_LIT>') is None:<EOL><INDENT>attrs_dict['<STR_LIT>'] = attrs_dict['<STR_LIT>']<EOL><DEDENT>ripn = IPNetwork(attrs_dict['<STR_LIT>'] + '<STR_LIT:/>' +<EOL>attrs_dict['<STR_LIT>'])<EOL>assert ripn.network == ipn.network<EOL>attrs_dict['<STR_LIT>'] = str(ipn.prefixlen)<EOL>attrs_dict['<STR_LIT>'] = str(ipn.network)<EOL>if attrs_dict.get('<STR_LIT>') is None:<EOL><INDENT>attrs_dict['<STR_LIT>'] = str(ipn.broadcast)<EOL><DEDENT>if attrs_dict.get('<STR_LIT>') is None:<EOL><INDENT>attrs_dict['<STR_LIT>'] = attrs_dict['<STR_LIT>']<EOL><DEDENT>if attrs_dict.get('<STR_LIT>') is None:<EOL><INDENT>attrs_dict['<STR_LIT>'] = attrs_dict['<STR_LIT>']<EOL><DEDENT>logger.debug('<STR_LIT>')<EOL>return attrs_dict<EOL>

Generate network mask in CIDR format and subnet.

        Validate the given arguments. Otherwise AddrFormatError exception
        will be raised and catched in the FSM.


def handle_offer_ack(self, pkt, time_sent_request=None):

attrs_dict = dict()<EOL>for opt in pkt[DHCP].options:<EOL><INDENT>if isinstance(opt, tuple) and opt[<NUM_LIT:0>] in DHCP_OFFER_OPTIONS:<EOL><INDENT>v = opt[<NUM_LIT:1>] if len(opt[<NUM_LIT:1>:]) < <NUM_LIT:2> else '<STR_LIT:U+0020>'.join(opt[<NUM_LIT:1>:])<EOL>v = str(v.decode('<STR_LIT:utf8>')) if isinstance(v, bytes) else str(v)<EOL>attrs_dict[opt[<NUM_LIT:0>]] = v<EOL><DEDENT><DEDENT>attrs_dict.update({<EOL>"<STR_LIT>": self.iface,<EOL>"<STR_LIT:address>": pkt[BOOTP].yiaddr,<EOL>"<STR_LIT>": pkt[BOOTP].siaddr,<EOL>})<EOL>self.gen_check_lease_attrs(attrs_dict)<EOL>logger.debug('<STR_LIT>')<EOL>logger.debug('<STR_LIT>', attrs_dict)<EOL>lease = DHCPCAPLease(**attrs_dict)<EOL>return lease<EOL>

Create a lease object with the values in OFFER/ACK packet.


logger.debug("<STR_LIT>")<EOL>logger.debug('<STR_LIT>')<EOL>self.lease = self.handle_offer_ack(pkt)<EOL>


def handle_ack(self, pkt, time_sent_request):

logger.debug("<STR_LIT>")<EOL>logger.debug('<STR_LIT>')<EOL>self.server_mac = pkt[Ether].src<EOL>self.server_ip = pkt[IP].src<EOL>self.server_port = pkt[UDP].sport<EOL>event = DHCP_EVENTS['<STR_LIT>']<EOL>lease = self.handle_offer_ack(pkt, time_sent_request)<EOL>lease.set_times(time_sent_request)<EOL>if self.lease is not None:<EOL><INDENT>if (self.lease.address != lease.address or<EOL>self.lease.subnet_mask != lease.subnet_mask or<EOL>self.lease.router != lease.router):<EOL><INDENT>event = DHCP_EVENTS['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>event = DHCP_EVENTS['<STR_LIT>']<EOL><DEDENT><DEDENT>logger.debug('<STR_LIT>')<EOL>self.lease = lease<EOL>self.client_ip = self.lease.address<EOL>self.event = event<EOL>return event<EOL>


return value.isnumeric()<EOL>


return any(char.isnumeric() for char in value)<EOL>

Check to see if the string contains a number.


return tuple(fauxfactory.gen_integer(<NUM_LIT:0>, <NUM_LIT:255>) for _ in range(number))<EOL>


return args, kwargs<EOL>

generic function that return args and kwargs


return tuple(fauxfactory.gen_alpha(length=length) for _ in range(number))<EOL>

function that return a tuple of generated alpha string


return {<EOL>'<STR_LIT:name>': fauxfactory.gen_alpha(length=<NUM_LIT:12>),<EOL>'<STR_LIT>': fauxfactory.gen_integer(min_value=<NUM_LIT:12>, max_value=<NUM_LIT:100>)<EOL>}<EOL>


return (<EOL>fauxfactory.gen_alpha(length=<NUM_LIT:12>),<EOL>fauxfactory.gen_integer(min_value=<NUM_LIT:12>, max_value=<NUM_LIT:100>)<EOL>)<EOL>

Generate a random person record in a tuple (name, age).


for _ in range(items):<EOL><INDENT>yield fauxfactory.gen_alpha(length=length)<EOL><DEDENT>

Generate alpha string value at each iteration.


yield '<STR_LIT:foo>'<EOL>yield [<NUM_LIT:1>, <NUM_LIT:2>, <NUM_LIT:3>]<EOL>

Returns different values: first, a string 'foo'; second iteration, a
    list of integers.


filename = os.path.join(*paths)<EOL>with codecs.open(filename, mode='<STR_LIT:r>', encoding='<STR_LIT:utf-8>') as handle:<EOL><INDENT>return handle.read()<EOL><DEDENT>

Build a file path from *paths* and return the contents.


return [<EOL>'<STR_LIT>'.format(func_name, idx)<EOL>for idx<EOL>in range(len(data))<EOL>]<EOL>


for key in metafunc.function.__dict__:<EOL><INDENT>if key.lower().startswith('<STR_LIT>'):<EOL><INDENT>return getattr(metafunc.function, key)<EOL><DEDENT><DEDENT>

Extract the name of the function being called.


usage_message = (<EOL>'<STR_LIT>'<EOL>)<EOL>if len(args) < <NUM_LIT:2>:<EOL><INDENT>raise pytest.UsageError(<EOL>'<STR_LIT>'.format(usage_message)<EOL>)<EOL><DEDENT>items, callable_function = args[<NUM_LIT:0>:<NUM_LIT:2>]<EOL>if not isinstance(items, int):<EOL><INDENT>raise pytest.UsageError(<EOL>'<STR_LIT>'.format(<EOL>type(items), items))<EOL><DEDENT>if items < <NUM_LIT:1>:<EOL><INDENT>raise pytest.UsageError(<EOL>'<STR_LIT>'.format(<EOL>items))<EOL><DEDENT>if not callable(callable_function):<EOL><INDENT>raise pytest.UsageError(<EOL>'<STR_LIT>'.format(<EOL>type(callable_function), callable_function))<EOL><DEDENT>return faux_callable(items, callable_function, *args[<NUM_LIT:2>:], **kwargs)<EOL>


usage_message = '<STR_LIT>'<EOL>if len(args) == <NUM_LIT:0>:<EOL><INDENT>raise pytest.UsageError(<EOL>'<STR_LIT>'.format(usage_message)<EOL>)<EOL><DEDENT>for index, arg in enumerate(args):<EOL><INDENT>if not isgenerator(arg):<EOL><INDENT>raise pytest.UsageError(<EOL>'<STR_LIT>'<EOL>.format(index, usage_message)<EOL>)<EOL><DEDENT><DEDENT>return faux_generator(*args)<EOL>


<EOL>if len(args) == <NUM_LIT:0>:<EOL><INDENT>args = (<NUM_LIT:1>, None)<EOL><DEDENT>elif len(args) == <NUM_LIT:1>:<EOL><INDENT>if isinstance(args[<NUM_LIT:0>], int):<EOL><INDENT>args = (args[<NUM_LIT:0>], None)<EOL><DEDENT>elif isinstance(args[<NUM_LIT:0>], str):<EOL><INDENT>if args[<NUM_LIT:0>] in STRING_TYPES:<EOL><INDENT>args = (<NUM_LIT:1>, args[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>raise pytest.UsageError(<EOL>'<STR_LIT>'.format(args[<NUM_LIT:0>])<EOL>)<EOL><DEDENT><DEDENT><DEDENT>items, str_type = args[<NUM_LIT:0>:<NUM_LIT:2>]<EOL>if items < <NUM_LIT:1>:<EOL><INDENT>raise pytest.UsageError(<EOL>'<STR_LIT>'.format(<EOL>items))<EOL><DEDENT>return faux_string(items, str_type, *args[<NUM_LIT:2>:], **kwargs)<EOL>


def faux_callable(items, callable_func, *args, **kwargs):

if items is None:<EOL><INDENT>items = <NUM_LIT:1><EOL><DEDENT>for _ in range(items):<EOL><INDENT>yield callable_func(*args, **kwargs)<EOL><DEDENT>

Generate new values from callable object.


return chain.from_iterable(args)<EOL>

Generate values from generators passed as arguments.


def faux_string(items, str_type=None, *args, **kwargs):

item = <NUM_LIT:0><EOL>if not str_type:<EOL><INDENT>str_type = fauxfactory.gen_choice(STRING_TYPES)<EOL><DEDENT>if not isinstance(str_type, list):<EOL><INDENT>str_type = [str_type]<EOL><DEDENT>str_cycle = cycle(str_type)<EOL>length = kwargs.get('<STR_LIT>', None)<EOL>if not length:<EOL><INDENT>length = [None]<EOL><DEDENT>if not isinstance(length, list):<EOL><INDENT>length = [length]<EOL><DEDENT>length_cycle = cycle(length)<EOL>while item < items:<EOL><INDENT>str_type = next(str_cycle)<EOL>kwargs['<STR_LIT>'] = next(length_cycle)<EOL>yield fauxfactory.gen_string(str_type, *args, **kwargs)<EOL>item += <NUM_LIT:1><EOL><DEDENT>


if not hasattr(metric_class, '<STR_LIT:label>'):<EOL><INDENT>raise ImproperlyConfigured("<STR_LIT>" % metric_class.__name__)<EOL><DEDENT>if not hasattr(metric_class, '<STR_LIT>'):<EOL><INDENT>raise ImproperlyConfigured("<STR_LIT>" % metric_class.__name__)<EOL><DEDENT>


stats = []<EOL>for gadget in self._registry.values():<EOL><INDENT>for s in gadget.stats:<EOL><INDENT>if s not in stats:<EOL><INDENT>stats.append(s)<EOL><DEDENT><DEDENT><DEDENT>return stats<EOL>

Returns all of the active statistics for the gadgets currently registered.


if gadget in self._registry:<EOL><INDENT>raise AlreadyRegistered<EOL><DEDENT>else:<EOL><INDENT>self._registry.append(gadget)<EOL><DEDENT>

Registers a gadget object.
If a gadget is already registered, this will raise AlreadyRegistered.


gadgets = maintenance.ensure_list(gadgets)<EOL>for gadget in gadgets:<EOL><INDENT>while gadget in self._registry:<EOL><INDENT>self._registry.remove(gadget)<EOL><DEDENT><DEDENT>

Unregisters the specified gadget(s) if it/they has/have already been registered.
"gadgets" can be a single class or a tuple/list of classes to unregister.


<EOL>context = {<EOL>'<STR_LIT>': self._registry,<EOL>'<STR_LIT>': self.columns,<EOL>'<STR_LIT>': self.rows,<EOL>'<STR_LIT>': <NUM_LIT:100> - self.columns * <NUM_LIT:2>,<EOL>'<STR_LIT>': <NUM_LIT:100> - self.rows * <NUM_LIT:2>,<EOL>}<EOL>context.update(kwargs)<EOL>return context<EOL>


return '<STR_LIT:U+0020>'.join(re.findall(r'<STR_LIT>', cls.__name__)).capitalize()<EOL>


if verbose:<EOL><INDENT>print(_("<STR_LIT>") % {'<STR_LIT:class>': cls.get_label()})<EOL><DEDENT>start_datetime = None<EOL>end_datetime = None<EOL>latest_stat = cls.get_latest(frequency)<EOL>today = datetime.strptime(datetime.now().strftime("<STR_LIT>"), "<STR_LIT>") <EOL>now = datetime.now()<EOL>if cls.cumulative:<EOL><INDENT>if frequency == settings.STATISTIC_FREQUENCY_HOURLY:<EOL><INDENT>start_datetime = datetime.strptime(now.strftime("<STR_LIT>"), "<STR_LIT>")<EOL><DEDENT>elif frequency == settings.STATISTIC_FREQUENCY_DAILY:<EOL><INDENT>start_datetime = today<EOL><DEDENT>elif frequency == settings.STATISTIC_FREQUENCY_WEEKLY:<EOL><INDENT>start_datetime = datetime.strptime(today.strftime("<STR_LIT>"), "<STR_LIT>")<EOL><DEDENT>elif frequency == settings.STATISTIC_FREQUENCY_MONTHLY:<EOL><INDENT>start_datetime = datetime.strptime(today.strftime("<STR_LIT>"), "<STR_LIT>")<EOL><DEDENT>stat, created = cls.objects.get_or_create(date_time=start_datetime, frequency=frequency)<EOL>stat.cumulative_count = cls.get_cumulative()<EOL>stat.count = (stat.cumulative_count-latest_stat.cumulative_count) if latest_stat else stat.cumulative_count<EOL><DEDENT>else:<EOL><INDENT>start_datetime = cls.get_start_datetime() if latest_stat is None else latest_stat.date_time<EOL>if frequency == settings.STATISTIC_FREQUENCY_HOURLY:<EOL><INDENT>start_datetime = datetime.strptime(start_datetime.strftime("<STR_LIT>"), "<STR_LIT>")<EOL>end_datetime = start_datetime+timedelta(hours=<NUM_LIT:1>)<EOL><DEDENT>elif frequency == settings.STATISTIC_FREQUENCY_DAILY:<EOL><INDENT>start_datetime = datetime.strptime(start_datetime.strftime("<STR_LIT>"), "<STR_LIT>")<EOL>end_datetime = start_datetime+timedelta(days=<NUM_LIT:1>)<EOL><DEDENT>elif frequency == settings.STATISTIC_FREQUENCY_WEEKLY:<EOL><INDENT>start_datetime = datetime.strptime(start_datetime.strftime("<STR_LIT>"), "<STR_LIT>")-timedelta(days=<NUM_LIT:7>)<EOL>end_datetime = start_datetime+timedelta(days=<NUM_LIT:7>)<EOL><DEDENT>elif frequency == settings.STATISTIC_FREQUENCY_MONTHLY:<EOL><INDENT>start_datetime = datetime.strptime(start_datetime.strftime("<STR_LIT>"), "<STR_LIT>")<EOL>end_datetime = datetime.strptime((start_datetime+timedelta(days=<NUM_LIT>)).strftime("<STR_LIT>"), "<STR_LIT>")<EOL><DEDENT>while start_datetime < now:<EOL><INDENT>count = cls.get_count(start_datetime, end_datetime)<EOL>cumulative_count = <NUM_LIT:0><EOL>if isinstance(count, tuple):<EOL><INDENT>cumulative_count = count[<NUM_LIT:1>]<EOL>count = count[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>cumulative_count = (latest_stat.cumulative_count+count) if latest_stat else count<EOL><DEDENT>stat, created = cls.objects.get_or_create(date_time=start_datetime, frequency=frequency)<EOL>stat.count = count<EOL>stat.cumulative_count = cumulative_count<EOL>stat.save()<EOL>latest_stat = stat<EOL>start_datetime = end_datetime<EOL>if frequency == settings.STATISTIC_FREQUENCY_HOURLY:<EOL><INDENT>end_datetime += timedelta(hours=<NUM_LIT:1>)<EOL><DEDENT>elif frequency == settings.STATISTIC_FREQUENCY_DAILY:<EOL><INDENT>end_datetime += timedelta(days=<NUM_LIT:1>)<EOL><DEDENT>elif frequency == settings.STATISTIC_FREQUENCY_WEEKLY:<EOL><INDENT>end_datetime += timedelta(days=<NUM_LIT:7>)<EOL><DEDENT>elif frequency == settings.STATISTIC_FREQUENCY_MONTHLY:<EOL><INDENT>end_datetime = datetime.strptime((start_datetime+timedelta(days=<NUM_LIT>)).strftime("<STR_LIT>"), "<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>

Runs the calculator for this type of statistic.


@classmethod<EOL><INDENT>def get_count(cls, start_datetime, end_datetime):<DEDENT>

raise NotImplementedError("<STR_LIT>" % cls.__name__)<EOL>

Must calculate the number of statistics between the two
specified date/times. These date/times are passed from the
calculator functions depending on the type of calculation
being performed.

Results must be returned for date >= start_datetime and
date < end_datetime.


Must return cumulative count for statistic.


return settings.STATISTIC_DEFAULT_START_DATETIME<EOL>

Must return a date/time object indicating when the earliest
data available for this metric occurred.


try:<EOL><INDENT>return cls.objects.filter(frequency=frequency).order_by('<STR_LIT>')[<NUM_LIT:0>]<EOL><DEDENT>except IndexError:<EOL><INDENT>return None<EOL><DEDENT>

Returns the earliest instance of this statistic.


metric = Metric.objects.get(uid=uid)<EOL>frequency = request.GET.get('<STR_LIT>', settings.STATISTIC_FREQUENCY_DAILY)<EOL>response = HttpResponse(mimetype='<STR_LIT>')<EOL>response['<STR_LIT>'] = '<STR_LIT>' % (uid, datetime.datetime.now().strftime("<STR_LIT>"))<EOL>writer = csv.writer(response)<EOL>writer.writerow([_('<STR_LIT>'), _('<STR_LIT>'), _('<STR_LIT>')])<EOL>for stat in metric.statistics.filter(frequency=frequency).order_by('<STR_LIT>'):<EOL><INDENT>writer.writerow([stat.date_time.strftime(settings.CSV_DATETIME_FORMAT), stat.count, stat.cumulative_count])<EOL><DEDENT>return response<EOL>

Returns a CSV dump of all of the specified metric's counts
and cumulative counts.


self._registry[gadget.id] = gadget<EOL>


from django.conf import settings<EOL>from django.utils.importlib import import_module<EOL>from django.utils.module_loading import module_has_submodule<EOL>for app in settings.INSTALLED_APPS:<EOL><INDENT>mod = import_module(app)<EOL>try:<EOL><INDENT>import_module('<STR_LIT>' % app)<EOL><DEDENT>except:<EOL><INDENT>if module_has_submodule(mod, '<STR_LIT>'):<EOL><INDENT>raise<EOL><DEDENT><DEDENT><DEDENT>

Auto-discover INSTALLED_APPS gadgets.py modules and fail silently when
not present. This forces an import on them to register any gadgets they
may want.


for s in get_statistic_models():<EOL><INDENT>print(s.__name__, "<STR_LIT>" % s)<EOL><DEDENT>


return list(v) if getattr(v, '<STR_LIT>', False) else [v]<EOL>

Makes sure that the given value is a list.


if stat_name == '<STR_LIT>':<EOL><INDENT>return get_statistic_models()<EOL><DEDENT>for stat in get_statistic_models():<EOL><INDENT>if stat.__name__ == stat_name:<EOL><INDENT>return stat<EOL><DEDENT><DEDENT>raise Exception(_("<STR_LIT>") % {'<STR_LIT>': stat_name})<EOL>

Fetches a statistics based on the given class name. Does a look-up
in the gadgets' registered statistics to find the specified one.


stats = ensure_list(stat)<EOL>frequencies = ensure_list(frequencies)<EOL>for stat in stats:<EOL><INDENT>for f in frequencies:<EOL><INDENT>print("<STR_LIT>" % (stat.__name__, settings.STATISTIC_FREQUENCY_DICT[f]))<EOL>stat.calculate(f)<EOL><DEDENT><DEDENT>

Calculates all of the metrics associated with the registered gadgets.


def reset_statistics(stat, frequencies, reset_cumulative, recalculate=False):

stats = ensure_list(stat)<EOL>frequencies = ensure_list(frequencies)<EOL>for s in stats:<EOL><INDENT>for f in frequencies:<EOL><INDENT>if not s.cumulative or reset_cumulative:<EOL><INDENT>print("<STR_LIT>" % (s.__name__, settings.STATISTIC_FREQUENCY_DICT[f]))<EOL>s.objects.filter(frequency=f).delete()<EOL><DEDENT>elif s.cumulative and not reset_cumulative:<EOL><INDENT>print("<STR_LIT>" % s.__name__)<EOL><DEDENT><DEDENT><DEDENT>if recalculate:<EOL><INDENT>print("<STR_LIT>")<EOL>calculate_statistics(stats, frequencies)<EOL><DEDENT>

Resets the specified statistic's data (deletes it) for the given
frequency/ies.


frequency = kwargs['<STR_LIT>']<EOL>frequencies = settings.STATISTIC_FREQUENCY_ALL if frequency == '<STR_LIT:a>' else (frequency.split('<STR_LIT:U+002C>') if '<STR_LIT:U+002C>' in frequency else [frequency])<EOL>if kwargs['<STR_LIT:list>']:<EOL><INDENT>maintenance.list_statistics()<EOL><DEDENT>elif kwargs['<STR_LIT>']:<EOL><INDENT>maintenance.calculate_statistics(maintenance.get_statistic_by_name(kwargs['<STR_LIT>']), frequencies)<EOL><DEDENT>elif kwargs['<STR_LIT>']:<EOL><INDENT>maintenance.reset_statistics(maintenance.get_statistic_by_name(kwargs['<STR_LIT>']), frequencies, kwargs['<STR_LIT>'])<EOL><DEDENT>elif kwargs['<STR_LIT>']:<EOL><INDENT>maintenance.reset_statistics(maintenance.get_statistic_by_name(kwargs['<STR_LIT>']), frequencies, kwargs['<STR_LIT>'], True)<EOL><DEDENT>

Command handler for the "metrics" command.


def get_GET_array(request, var_name, fail_silently=True):

vals = request.GET.getlist(var_name)<EOL>if not vals:<EOL><INDENT>if fail_silently:<EOL><INDENT>return []<EOL><DEDENT>else:<EOL><INDENT>raise Exception(_("<STR_LIT>") % {'<STR_LIT>': var_name})<EOL><DEDENT><DEDENT>return vals<EOL>

Returns the GET array's contents for the specified variable.


def get_GET_bool(request, var_name, default=True):

val = request.GET.get(var_name, default)<EOL>if isinstance(val, str) or isinstance(val, str):<EOL><INDENT>val = True if val[<NUM_LIT:0>] == '<STR_LIT:t>' else False<EOL><DEDENT>return val<EOL>

Tries to extract a boolean variable from the specified request.


colour = settings.GECKOBOARD_COLOURS[get_next_colour.cur_colour]<EOL>get_next_colour.cur_colour += <NUM_LIT:1><EOL>if get_next_colour.cur_colour >= len(settings.GECKOBOARD_COLOURS):<EOL><INDENT>get_next_colour.cur_colour = <NUM_LIT:0><EOL><DEDENT>return colour<EOL>

Gets the next colour in the Geckoboard colour list.


def get_gecko_params(request, uid=None, days_back=<NUM_LIT:0>, cumulative=True,<EOL>frequency=settings.STATISTIC_FREQUENCY_DAILY, min_val=<NUM_LIT:0>, max_val=<NUM_LIT:100>,<EOL>chart_type='<STR_LIT>', percentage='<STR_LIT>', sort=False):

return {<EOL>'<STR_LIT>'  : int(request.GET.get('<STR_LIT>', days_back)),<EOL>'<STR_LIT>'        : request.GET.get('<STR_LIT>', uid),<EOL>'<STR_LIT>'       : get_GET_array(request, '<STR_LIT>'),<EOL>'<STR_LIT>' : get_GET_bool(request, '<STR_LIT>', cumulative),<EOL>'<STR_LIT>'  : request.GET.get('<STR_LIT>', frequency),<EOL>'<STR_LIT>'        : request.GET.get('<STR_LIT>', min_val),<EOL>'<STR_LIT>'        : request.GET.get('<STR_LIT>', max_val),<EOL>'<STR_LIT:type>'       : request.GET.get('<STR_LIT:type>', chart_type),<EOL>'<STR_LIT>' : request.GET.get('<STR_LIT>', percentage),<EOL>'<STR_LIT>'       : get_GET_bool(request, '<STR_LIT>', sort),<EOL>}<EOL>

Returns the default GET parameters for a particular Geckoboard
view request.


params = get_gecko_params(request, days_back=<NUM_LIT:7>)<EOL>metric = Metric.objects.get(uid=params['<STR_LIT>'])<EOL>try:<EOL><INDENT>latest_stat = metric.statistics.filter(frequency=params['<STR_LIT>']).order_by('<STR_LIT>')[<NUM_LIT:0>]<EOL><DEDENT>except IndexError:<EOL><INDENT>return (<NUM_LIT:0>, <NUM_LIT:0>)<EOL><DEDENT>try:<EOL><INDENT>prev_stat = metric.statistics.filter(frequency=params['<STR_LIT>'],<EOL>date_time__lte=latest_stat.date_time-timedelta(days=params['<STR_LIT>'])).order_by('<STR_LIT>')[<NUM_LIT:0>]<EOL><DEDENT>except IndexError:<EOL><INDENT>return (latest_stat.cumulative_count, <NUM_LIT:0>) if params['<STR_LIT>'] else (latest_stat.count, <NUM_LIT:0>)<EOL><DEDENT>return (latest_stat.cumulative_count, prev_stat.cumulative_count) if params['<STR_LIT>'] else (latest_stat.count, prev_stat.count)<EOL>

Returns a number widget for the specified metric's cumulative total.


params = get_gecko_params(request)<EOL>print(params['<STR_LIT>'])<EOL>max_date = datetime.now()-timedelta(days=params['<STR_LIT>'])<EOL>metrics = Metric.objects.filter(uid__in=params['<STR_LIT>'])<EOL>results = [(metric.latest_count(frequency=params['<STR_LIT>'], count=not params['<STR_LIT>'],<EOL>cumulative=params['<STR_LIT>'], max_date=max_date), metric.title) for metric in metrics]<EOL>return tuple(results)<EOL>

Searches the GET variables for metric UIDs, and displays
them in a RAG widget.


params = get_gecko_params(request, cumulative=True)<EOL>from_date = datetime.now()-timedelta(days=params['<STR_LIT>'])<EOL>metrics = Metric.objects.filter(uid__in=params['<STR_LIT>'])<EOL>results = [(metric.latest_count(frequency=params['<STR_LIT>'], count=not params['<STR_LIT>'],<EOL>cumulative=params['<STR_LIT>']), metric.title, get_next_colour()) for metric in metrics]<EOL>return tuple(results)<EOL>

Shows a pie chart of the metrics in the uids[] GET variable array.


params = get_gecko_params(request, cumulative=False, days_back=<NUM_LIT:7>)<EOL>metric = Metric.objects.get(uid=params['<STR_LIT>'])<EOL>start_date = datetime.now()-timedelta(days=params['<STR_LIT>'])<EOL>stats = [s for s in metric.statistics.filter(frequency=params['<STR_LIT>'],<EOL>date_time__gte=start_date).order_by('<STR_LIT>')]<EOL>if len(stats) == <NUM_LIT:0>:<EOL><INDENT>raise Exception(_("<STR_LIT>") % {'<STR_LIT>': params['<STR_LIT>']})<EOL><DEDENT>dates = [stats[<NUM_LIT:0>].date_time]<EOL>if len(stats) >= <NUM_LIT:3>:<EOL><INDENT>mid = len(stats)/<NUM_LIT:2><EOL>if not mid:<EOL><INDENT>mid = <NUM_LIT:1><EOL><DEDENT>dates.extend([stats[mid].date_time, stats[-<NUM_LIT:1>].date_time])<EOL><DEDENT>elif len(stats) == <NUM_LIT:2>:<EOL><INDENT>dates.extend([stats[-<NUM_LIT:1>].date_time])<EOL><DEDENT>return (<EOL>[s.count for s in stats],<EOL>dates,<EOL>metric.title,<EOL>)<EOL>

Returns the data for a line chart for the specified metric.


params = get_gecko_params(request, cumulative=True)<EOL>metric = Metric.objects.get(uid=params['<STR_LIT>'])<EOL>return (metric.latest_count(frequency=params['<STR_LIT>'], count=not params['<STR_LIT>'],<EOL>cumulative=params['<STR_LIT>']), params['<STR_LIT>'], params['<STR_LIT>'])<EOL>

Returns a Geck-o-Meter control for the specified metric.


<EOL>params = get_gecko_params(request, cumulative=True)<EOL>metrics = Metric.objects.filter(uid__in=params['<STR_LIT>'])<EOL>items = [(metric.latest_count(frequency=params['<STR_LIT>'], count=not params['<STR_LIT>'],<EOL>cumulative=params['<STR_LIT>']), metric.title) for metric in metrics]<EOL>return {<EOL>'<STR_LIT>'     : items,<EOL>'<STR_LIT:type>'      : params['<STR_LIT:type>'],<EOL>'<STR_LIT>': params['<STR_LIT>'],<EOL>'<STR_LIT>'      : params['<STR_LIT>'],<EOL>}<EOL>

Returns a funnel chart for the metrics specified in the GET variables.


def __init__(self, application, options={}):

self.application = application<EOL>self.usage = None<EOL>self.callable = None<EOL>self.options = options<EOL>self.prog = None<EOL>self.do_load_config()<EOL>

Construct the Application. Default gUnicorn configuration is loaded


cfg = {}<EOL>for k, v in self.options.items():<EOL><INDENT>if k.lower() in self.cfg.settings and v is not None:<EOL><INDENT>cfg[k.lower()] = v<EOL><DEDENT><DEDENT>return cfg<EOL>


if isinstance(self.application, str):<EOL><INDENT>return util.import_app(self.application)<EOL><DEDENT>else:<EOL><INDENT>return self.application<EOL><DEDENT>

Attempt an import of the specified application


self.app = WSGIApp(app, options)<EOL>


self.app.run()<EOL>


if not hasattr(app, '<STR_LIT>'):<EOL><INDENT>app.extensions = {}<EOL><DEDENT>if '<STR_LIT>' in app.extensions:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>app.extensions['<STR_LIT>'] = self<EOL>self.app = app<EOL>if '<STR_LIT>' not in app.config:<EOL><INDENT>with app.test_request_context():<EOL><INDENT>app.wsgi_app = WhiteNoise(app.wsgi_app, root=url_for('<STR_LIT>', filename='<STR_LIT>')[<NUM_LIT:1>:])<EOL><DEDENT><DEDENT>self.cache = Cache(app, config={'<STR_LIT>': app.config.get("<STR_LIT>", '<STR_LIT>')})<EOL>@app.before_request<EOL>def before_request_callback():<EOL><INDENT>request.start_time = maya.now()<EOL><DEDENT>@app.after_request<EOL>def after_request_callback(response):<EOL><INDENT>if '<STR_LIT>' not in current_app.config:<EOL><INDENT>response.headers['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>if '<STR_LIT>' not in current_app.config:<EOL><INDENT>response.headers['<STR_LIT>'] = maya.now().epoch - request.start_time.epoch<EOL><DEDENT>return response<EOL><DEDENT>@app.route('<STR_LIT>')<EOL>def favicon():<EOL><INDENT>return redirect(url_for('<STR_LIT>', filename='<STR_LIT>'), code=<NUM_LIT>)<EOL><DEDENT>

Initializes the Flask application with Common.


def serve(self, workers=None, **kwargs):

if self.app.debug:<EOL><INDENT>print(crayons.yellow('<STR_LIT>'))<EOL>self.app.run()<EOL><DEDENT>else:<EOL><INDENT>print(crayons.yellow('<STR_LIT>'))<EOL>server = GunicornServer(<EOL>self.app, workers=workers or number_of_gunicorn_workers(),<EOL>worker_class='<STR_LIT>', **kwargs)<EOL>server.run()<EOL><DEDENT>


body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>httpretty.register_uri(httpretty.GET,<EOL>DOCKERHUB_RESPOSITORY_URL,<EOL>body=body, status=<NUM_LIT:200>)<EOL>


@classmethod<EOL><INDENT>def routes(cls, version="<STR_LIT>", empty=False, response_num=<NUM_LIT:200>):<DEDENT>

assert(version in TESTED_VERSIONS)<EOL>if version == "<STR_LIT>":<EOL><INDENT>mediawiki_siteinfo = read_file('<STR_LIT>')<EOL><DEDENT>elif version == "<STR_LIT>":<EOL><INDENT>mediawiki_siteinfo = read_file('<STR_LIT>')<EOL><DEDENT>mediawiki_namespaces = read_file('<STR_LIT>')<EOL>mediawiki_pages_allrevisions = read_file('<STR_LIT>')<EOL>if empty:<EOL><INDENT>mediawiki_pages_allrevisions = read_file('<STR_LIT>')<EOL><DEDENT>mediawiki_pages_all = read_file('<STR_LIT>')<EOL>if empty:<EOL><INDENT>mediawiki_pages_all = read_file('<STR_LIT>')<EOL><DEDENT>mediawiki_pages_recent_changes = read_file('<STR_LIT>')<EOL>mediawiki_page_476583 = read_file('<STR_LIT>')<EOL>mediawiki_page_592384 = read_file('<STR_LIT>')<EOL>mediawiki_page_476589 = read_file('<STR_LIT>')<EOL>mediawiki_page_476590 = read_file('<STR_LIT>')<EOL>def request_callback(method, uri, headers):<EOL><INDENT>params = urllib.parse.parse_qs(urllib.parse.urlparse(uri).query)<EOL>if '<STR_LIT>' in params and '<STR_LIT>' in params['<STR_LIT>']:<EOL><INDENT>body = mediawiki_siteinfo<EOL>if '<STR_LIT>' in params:<EOL><INDENT>body = mediawiki_namespaces<EOL><DEDENT><DEDENT>elif '<STR_LIT:list>' in params:<EOL><INDENT>if '<STR_LIT>' in params['<STR_LIT:list>']:<EOL><INDENT>body = mediawiki_pages_all<EOL><DEDENT>elif '<STR_LIT>' in params['<STR_LIT:list>']:<EOL><INDENT>body = mediawiki_pages_recent_changes<EOL><DEDENT>elif '<STR_LIT>' in params['<STR_LIT:list>']:<EOL><INDENT>body = mediawiki_pages_allrevisions<EOL><DEDENT><DEDENT>elif '<STR_LIT>' in params:<EOL><INDENT>if params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>body = mediawiki_page_476583<EOL><DEDENT>elif params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>body = mediawiki_page_592384<EOL><DEDENT>elif params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>body = mediawiki_page_476589<EOL><DEDENT>elif params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>body = mediawiki_page_476590<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT>HTTPServer.requests_http.append(httpretty.last_request())<EOL>return (response_num, headers, body)<EOL><DEDENT>httpretty.register_uri(httpretty.GET,<EOL>MEDIAWIKI_API,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>

Configure in http the routes to be served


body_no_hits = read_file('<STR_LIT>', '<STR_LIT:r>')<EOL>body_bitergia = read_file('<STR_LIT>', '<STR_LIT:r>')<EOL>body_grimoirelab = read_file('<STR_LIT>', '<STR_LIT:r>')<EOL>if no_hits:<EOL><INDENT>httpretty.register_uri(httpretty.GET,<EOL>GOOGLE_SEARCH_URL,<EOL>params={'<STR_LIT:q>': '<STR_LIT>'},<EOL>body=body_no_hits, status=status)<EOL>return<EOL><DEDENT>httpretty.register_uri(httpretty.GET,<EOL>GOOGLE_SEARCH_URL,<EOL>params={'<STR_LIT:q>': '<STR_LIT>'},<EOL>body=body_bitergia, status=status)<EOL>httpretty.register_uri(httpretty.GET,<EOL>GOOGLE_SEARCH_URL,<EOL>params={'<STR_LIT:q>': '<STR_LIT>'},<EOL>body=body_grimoirelab, status=status)<EOL>


with open(filepath) as fout:<EOL><INDENT>buff = None<EOL>for line in fout.readlines():<EOL><INDENT>if line.startswith('<STR_LIT>'):<EOL><INDENT>buff = line<EOL><DEDENT>elif line.startswith('<STR_LIT>'):<EOL><INDENT>buff += line<EOL>obj = json.loads(buff)<EOL>yield obj<EOL><DEDENT>else:<EOL><INDENT>buff += line<EOL><DEDENT><DEDENT><DEDENT>

Transforms the output of a BackendCommand into json objects


http_requests = []<EOL>events_bodies = [<EOL>read_file('<STR_LIT>', '<STR_LIT:rb>'),<EOL>read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>]<EOL>events_range_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>events_empty_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>event_comments_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>event_rsvps_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>def request_callback(method, uri, headers, too_many_requests=False):<EOL><INDENT>last_request = httpretty.last_request()<EOL>if uri.startswith(MEETUP_EVENT_1_COMMENTS_URL):<EOL><INDENT>body = event_comments_body<EOL><DEDENT>elif uri.startswith(MEETUP_EVENT_2_COMMENTS_URL):<EOL><INDENT>body = event_comments_body<EOL><DEDENT>elif uri.startswith(MEETUP_EVENT_3_COMMENTS_URL):<EOL><INDENT>body = event_comments_body<EOL><DEDENT>elif uri.startswith(MEETUP_EVENT_1_RSVPS_URL):<EOL><INDENT>body = event_rsvps_body<EOL><DEDENT>elif uri.startswith(MEETUP_EVENT_2_RSVPS_URL):<EOL><INDENT>body = event_rsvps_body<EOL><DEDENT>elif uri.startswith(MEETUP_EVENT_3_RSVPS_URL):<EOL><INDENT>body = event_rsvps_body<EOL><DEDENT>elif uri.startswith(MEETUP_EVENTS_URL):<EOL><INDENT>params = last_request.querystring<EOL>scroll = params.get('<STR_LIT>', None)<EOL>if scroll and scroll[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>body = events_bodies[-<NUM_LIT:1>]<EOL><DEDENT>elif scroll and scroll[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>body = events_range_body<EOL><DEDENT>elif scroll and scroll[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>body = events_empty_body<EOL><DEDENT>else:<EOL><INDENT>body = events_bodies.pop(<NUM_LIT:0>)<EOL>if events_bodies:<EOL><INDENT>headers['<STR_LIT>'] = '<STR_LIT:<>' + MEETUP_EVENTS_URL + '<STR_LIT>'<EOL><DEDENT>if rate_limit != -<NUM_LIT:1>:<EOL><INDENT>headers['<STR_LIT>'] = str(rate_limit)<EOL><DEDENT>if reset_rate_limit != -<NUM_LIT:1>:<EOL><INDENT>headers['<STR_LIT>'] = str(reset_rate_limit)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT>if rate_limit == -<NUM_LIT:1>:<EOL><INDENT>headers['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>if reset_rate_limit == -<NUM_LIT:1>:<EOL><INDENT>headers['<STR_LIT>'] = '<STR_LIT:0>'<EOL><DEDENT>http_requests.append(last_request)<EOL>return (<NUM_LIT:200>, headers, body)<EOL><DEDENT>httpretty.register_uri(httpretty.GET,<EOL>MEETUP_EVENTS_URL,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>for _ in range(<NUM_LIT:2>)<EOL>])<EOL>for url in MEETUP_COMMENTS_URL:<EOL><INDENT>httpretty.register_uri(httpretty.GET,<EOL>url,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL><DEDENT>for url in MEETUP_RSVPS_URL:<EOL><INDENT>httpretty.register_uri(httpretty.GET,<EOL>url,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL><DEDENT>return http_requests<EOL>


http_requests = []<EOL>body_msgs = read_file('<STR_LIT>')<EOL>body_msgs_next = read_file('<STR_LIT>')<EOL>body_msgs_empty = read_file('<STR_LIT>')<EOL>def request_callback(method, uri, headers):<EOL><INDENT>params = urllib.parse.parse_qs(urllib.parse.urlparse(uri).query)<EOL>if '<STR_LIT>' in params and params['<STR_LIT>'] == ['<STR_LIT>']:<EOL><INDENT>body = body_msgs_next<EOL><DEDENT>elif '<STR_LIT>' in params and params['<STR_LIT>'] == ['<STR_LIT>']:<EOL><INDENT>body = body_msgs_empty<EOL><DEDENT>else:<EOL><INDENT>body = body_msgs<EOL><DEDENT>http_requests.append(httpretty.last_request())<EOL>return (<NUM_LIT:200>, headers, body)<EOL><DEDENT>httpretty.register_uri(httpretty.GET,<EOL>TELEGRAM_UPDATES_URL,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>return http_requests<EOL>


http_requests = []<EOL>channel_error = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>channel_empty = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>channel_info = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>conversation_members_1 = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>conversation_members_2 = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>channel_history = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>channel_history_next = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>channel_history_date = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>user_U0001 = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>user_U0002 = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>user_U0003 = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>def request_callback(method, uri, headers):<EOL><INDENT>last_request = httpretty.last_request()<EOL>params = last_request.querystring<EOL>status = <NUM_LIT:200><EOL>if uri.startswith(SLACK_CHANNEL_INFO_URL):<EOL><INDENT>body = channel_info<EOL><DEDENT>elif uri.startswith(SLACK_CHANNEL_HISTORY_URL):<EOL><INDENT>if params['<STR_LIT>'][<NUM_LIT:0>] != '<STR_LIT>':<EOL><INDENT>body = channel_error<EOL><DEDENT>elif '<STR_LIT>' not in params:<EOL><INDENT>body = channel_history<EOL><DEDENT>elif (params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT:1>' and<EOL>params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>'):<EOL><INDENT>body = channel_history_next<EOL><DEDENT>elif (params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT:0>' and<EOL>params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>'):<EOL><INDENT>body = channel_history<EOL><DEDENT>elif (params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT:0>' and<EOL>params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>'):<EOL><INDENT>body = channel_history_next<EOL><DEDENT>elif (params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>' and<EOL>params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>'):<EOL><INDENT>body = channel_history_date<EOL><DEDENT>elif (params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>' and<EOL>params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>'):<EOL><INDENT>body = channel_empty<EOL><DEDENT><DEDENT>elif uri.startswith(SLACK_USER_INFO_URL):<EOL><INDENT>if params['<STR_LIT:user>'][<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>body = user_U0001<EOL><DEDENT>elif params['<STR_LIT:user>'][<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>body = user_U0002<EOL><DEDENT>else:<EOL><INDENT>body = user_U0003<EOL><DEDENT><DEDENT>elif uri.startswith(SLACK_CONVERSATION_MEMBERS):<EOL><INDENT>if '<STR_LIT>' not in params:<EOL><INDENT>body = conversation_members_1<EOL><DEDENT>else:<EOL><INDENT>body = conversation_members_2<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT>http_requests.append(last_request)<EOL>return (status, headers, body)<EOL><DEDENT>httpretty.register_uri(httpretty.GET,<EOL>SLACK_CHANNEL_INFO_URL,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>httpretty.register_uri(httpretty.GET,<EOL>SLACK_CHANNEL_HISTORY_URL,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>for _ in range(<NUM_LIT:1>)<EOL>])<EOL>httpretty.register_uri(httpretty.GET,<EOL>SLACK_USER_INFO_URL,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>httpretty.register_uri(httpretty.GET,<EOL>SLACK_CONVERSATION_MEMBERS,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>return http_requests<EOL>


cmd = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>gitlog = subprocess.check_output(cmd, stderr=subprocess.STDOUT,<EOL>cwd=gitpath,<EOL>env={'<STR_LIT>': '<STR_LIT:C>', '<STR_LIT>': '<STR_LIT>'})<EOL>commits = gitlog.strip(b'<STR_LIT:\n>').split(b'<STR_LIT:\n>')<EOL>return len(commits)<EOL>

Get the number of commits counting the entries on the log


cmd = ['<STR_LIT>', '<STR_LIT>']<EOL>gitrefs = subprocess.check_output(cmd, stderr=subprocess.STDOUT,<EOL>cwd=gitpath,<EOL>env={'<STR_LIT>': '<STR_LIT:C>', '<STR_LIT>': '<STR_LIT>'})<EOL>outs = gitrefs.decode('<STR_LIT:utf-8>', errors='<STR_LIT>').rstrip()<EOL>refs = {}<EOL>for line in outs.split('<STR_LIT:\n>'):<EOL><INDENT>data = line.split('<STR_LIT:U+0020>')<EOL>refs[data[<NUM_LIT:1>]] = data[<NUM_LIT:0>]<EOL><DEDENT>return refs<EOL>


http_requests = []<EOL>body_contents = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>body_contents_next = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>body_contents_empty = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>body_content_1_v1 = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>body_content_1_v2 = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>body_content_1_v3 = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>body_content_2 = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>body_content_att = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>def request_callback(method, uri, headers):<EOL><INDENT>status_code = STATUS_CODE_SUCCESS<EOL>if uri.startswith(CONFLUENCE_CONTENTS_URL):<EOL><INDENT>params = urllib.parse.parse_qs(urllib.parse.urlparse(uri).query)<EOL>if '<STR_LIT:start>' in params and params['<STR_LIT:start>'] == ['<STR_LIT:2>']:<EOL><INDENT>body = body_contents_next<EOL><DEDENT>elif '<STR_LIT:start>' in params and params['<STR_LIT:start>'] == ['<STR_LIT:3>']:<EOL><INDENT>body = body_contents_empty<EOL><DEDENT>elif params['<STR_LIT>'][<NUM_LIT:0>].startswith("<STR_LIT>"):<EOL><INDENT>body = body_contents_empty<EOL><DEDENT>else:<EOL><INDENT>body = body_contents<EOL><DEDENT><DEDENT>elif uri.startswith(CONFLUENCE_HISTORICAL_CONTENT_1):<EOL><INDENT>params = urllib.parse.parse_qs(urllib.parse.urlparse(uri).query)<EOL>if params['<STR_LIT:version>'] == ['<STR_LIT:1>']:<EOL><INDENT>body = body_content_1_v1<EOL><DEDENT>elif params['<STR_LIT:version>'] == ['<STR_LIT:2>']:<EOL><INDENT>body = body_content_1_v2<EOL><DEDENT>else:<EOL><INDENT>body = body_content_1_v3<EOL><DEDENT><DEDENT>elif uri.startswith(CONFLUENCE_HISTORICAL_CONTENT_2):<EOL><INDENT>body = body_content_2<EOL>if not_handle_status_code:<EOL><INDENT>status_code = STATUS_CODE_NOT_HANDLED<EOL><DEDENT><DEDENT>elif uri.startswith(CONFLUENCE_HISTORICAL_CONTENT_ATT):<EOL><INDENT>body = body_content_att<EOL><DEDENT>else:<EOL><INDENT>raise Exception<EOL><DEDENT>http_requests.append(httpretty.last_request())<EOL>return status_code, headers, body<EOL><DEDENT>httpretty.register_uri(httpretty.GET,<EOL>CONFLUENCE_CONTENTS_URL,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>httpretty.register_uri(httpretty.GET,<EOL>CONFLUENCE_HISTORICAL_CONTENT_1,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>httpretty.register_uri(httpretty.GET,<EOL>CONFLUENCE_HISTORICAL_CONTENT_2,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>httpretty.register_uri(httpretty.GET,<EOL>CONFLUENCE_HISTORICAL_CONTENT_ATT,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>return http_requests<EOL>


http_requests = []<EOL>issues_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>issues_next_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>issues_empty_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>issue_2_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>issue_5_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>issue_9_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>issue_7311_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>user_3_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>user_4_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>user_24_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>user_25_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>def request_callback(method, uri, headers):<EOL><INDENT>last_request = httpretty.last_request()<EOL>params = last_request.querystring<EOL>status = <NUM_LIT:200><EOL>if uri.startswith(REDMINE_ISSUES_URL):<EOL><INDENT>if (params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>' and<EOL>params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT:0>'):<EOL><INDENT>body = issues_body<EOL><DEDENT>elif (params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>' and<EOL>params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT:3>'):<EOL><INDENT>body = issues_next_body<EOL><DEDENT>elif (params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT>' and<EOL>params['<STR_LIT>'][<NUM_LIT:0>] == '<STR_LIT:0>'):<EOL><INDENT>body = issues_next_body<EOL><DEDENT>else:<EOL><INDENT>body = issues_empty_body<EOL><DEDENT><DEDENT>elif uri.startswith(REDMINE_ISSUE_2_URL):<EOL><INDENT>body = issue_2_body<EOL><DEDENT>elif uri.startswith(REDMINE_ISSUE_5_URL):<EOL><INDENT>body = issue_5_body<EOL><DEDENT>elif uri.startswith(REDMINE_ISSUE_9_URL):<EOL><INDENT>body = issue_9_body<EOL><DEDENT>elif uri.startswith(REDMINE_ISSUE_7311_URL):<EOL><INDENT>body = issue_7311_body<EOL><DEDENT>elif uri.startswith(REDMINE_USER_3_URL):<EOL><INDENT>body = user_3_body<EOL><DEDENT>elif uri.startswith(REDMINE_USER_4_URL):<EOL><INDENT>body = user_4_body<EOL><DEDENT>elif uri.startswith(REDMINE_USER_24_URL):<EOL><INDENT>body = user_24_body<EOL><DEDENT>elif uri.startswith(REDMINE_USER_25_URL):<EOL><INDENT>body = user_25_body<EOL><DEDENT>elif uri.startswith(REDMINE_NOT_FOUND_USER_URL):<EOL><INDENT>body = "<STR_LIT>"<EOL>status = <NUM_LIT><EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT>http_requests.append(last_request)<EOL>return (status, headers, body)<EOL><DEDENT>for url in REDMINE_URL_LIST:<EOL><INDENT>httpretty.register_uri(httpretty.GET,<EOL>url,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL><DEDENT>return http_requests<EOL>


http_requests = []<EOL>channel_posts = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>channel_posts_next = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>channel_posts_empty = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>user_sduenas = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>user_valcos = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>full_response = [<EOL>channel_posts, channel_posts_next, channel_posts_empty<EOL>]<EOL>def request_callback(method, uri, headers):<EOL><INDENT>last_request = httpretty.last_request()<EOL>params = last_request.querystring<EOL>status = <NUM_LIT:200><EOL>if uri.startswith(MATTERMOST_USER_SDUENAS):<EOL><INDENT>body = user_sduenas<EOL><DEDENT>elif uri.startswith(MATTERMOST_USER_VALCOS):<EOL><INDENT>body = user_valcos<EOL><DEDENT>elif uri.startswith(MATTERMOST_CHANNEL_POSTS):<EOL><INDENT>if '<STR_LIT>' not in params:<EOL><INDENT>page = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>page = int(params['<STR_LIT>'][<NUM_LIT:0>])<EOL><DEDENT>body = full_response[page]<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>http_requests.append(last_request)<EOL>return (status, headers, body)<EOL><DEDENT>httpretty.register_uri(httpretty.GET,<EOL>MATTERMOST_CHANNEL_POSTS,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>httpretty.register_uri(httpretty.GET,<EOL>MATTERMOST_USER_SDUENAS,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>for _ in range(<NUM_LIT:1>)<EOL>])<EOL>httpretty.register_uri(httpretty.GET,<EOL>MATTERMOST_USER_VALCOS,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>return http_requests<EOL>


def setup_http_server(no_tweets=False, rate_limit=None, reset_rate_limit=None, status=<NUM_LIT:200>):

headers = {}<EOL>headers['<STR_LIT>'] = '<STR_LIT>' if not rate_limit else rate_limit<EOL>headers['<STR_LIT>'] = '<STR_LIT>' if not reset_rate_limit else reset_rate_limit<EOL>if no_tweets:<EOL><INDENT>tweets_page_empty = read_file('<STR_LIT>')<EOL>httpretty.register_uri(httpretty.GET,<EOL>TWITTER_API_URL +<EOL>"<STR_LIT>",<EOL>body=tweets_page_empty,<EOL>status=<NUM_LIT:200>,<EOL>forcing_headers=headers)<EOL>return<EOL><DEDENT>tweets_page_1 = read_file('<STR_LIT>')<EOL>tweets_page_2 = read_file('<STR_LIT>')<EOL>tweets_page_3 = read_file('<STR_LIT>')<EOL>httpretty.register_uri(httpretty.GET,<EOL>TWITTER_API_URL +<EOL>"<STR_LIT>",<EOL>body=tweets_page_3,<EOL>status=status,<EOL>forcing_headers=headers)<EOL>httpretty.register_uri(httpretty.GET,<EOL>TWITTER_API_URL +<EOL>"<STR_LIT>",<EOL>body=tweets_page_1,<EOL>status=status,<EOL>forcing_headers=headers)<EOL>httpretty.register_uri(httpretty.GET,<EOL>TWITTER_API_URL +<EOL>"<STR_LIT>",<EOL>body=tweets_page_2,<EOL>status=status,<EOL>forcing_headers=headers)<EOL>


http_requests = []<EOL>error_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>tasks_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>tasks_next_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>tasks_empty_body = read_file('<STR_LIT>')<EOL>tasks_trans_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>tasks_trans_next_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>users_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>jane_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>janes_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>jdoe_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>jrae_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>jsmith_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>phids_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>herald_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>bugreport_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>teamdevel_body = read_file('<STR_LIT>', '<STR_LIT:rb>')<EOL>phids_users = {<EOL>'<STR_LIT>': jane_body,<EOL>'<STR_LIT>': janes_body,<EOL>'<STR_LIT>': jdoe_body,<EOL>'<STR_LIT>': jrae_body,<EOL>'<STR_LIT>': jsmith_body<EOL>}<EOL>phids = {<EOL>'<STR_LIT>': herald_body,<EOL>'<STR_LIT>': bugreport_body,<EOL>'<STR_LIT>': teamdevel_body<EOL>}<EOL>def request_callback(method, uri, headers):<EOL><INDENT>last_request = httpretty.last_request()<EOL>params = json.loads(last_request.parsed_body['<STR_LIT>'][<NUM_LIT:0>])<EOL>if uri == PHABRICATOR_TASKS_URL:<EOL><INDENT>if params['<STR_LIT>']['<STR_LIT>'] == <NUM_LIT>:<EOL><INDENT>body = tasks_next_body<EOL><DEDENT>elif params['<STR_LIT>']['<STR_LIT>'] == <NUM_LIT>:<EOL><INDENT>body = tasks_empty_body<EOL><DEDENT>elif '<STR_LIT>' not in params:<EOL><INDENT>body = tasks_body<EOL><DEDENT>else:<EOL><INDENT>body = tasks_next_body<EOL><DEDENT><DEDENT>elif uri == PHABRICATOR_TRANSACTIONS_URL:<EOL><INDENT>if <NUM_LIT> in params['<STR_LIT>']:<EOL><INDENT>body = tasks_trans_body<EOL><DEDENT>else:<EOL><INDENT>body = tasks_trans_next_body<EOL><DEDENT><DEDENT>elif uri == PHABRICATOR_USERS_URL:<EOL><INDENT>if len(params['<STR_LIT>']) == <NUM_LIT:4>:<EOL><INDENT>body = users_body<EOL><DEDENT>else:<EOL><INDENT>body = phids_users[params['<STR_LIT>'][<NUM_LIT:0>]]<EOL><DEDENT><DEDENT>elif uri == PHABRICATOR_PHIDS_URL:<EOL><INDENT>if len(params['<STR_LIT>']) == <NUM_LIT:2>:<EOL><INDENT>body = phids_body<EOL><DEDENT>else:<EOL><INDENT>body = phids[params['<STR_LIT>'][<NUM_LIT:0>]]<EOL><DEDENT><DEDENT>elif uri == PHABRICATOR_API_ERROR_URL:<EOL><INDENT>body = error_body<EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT>http_requests.append(last_request)<EOL>return (<NUM_LIT:200>, headers, body)<EOL><DEDENT>httpretty.register_uri(httpretty.POST,<EOL>PHABRICATOR_TASKS_URL,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>httpretty.register_uri(httpretty.POST,<EOL>PHABRICATOR_TRANSACTIONS_URL,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>httpretty.register_uri(httpretty.POST,<EOL>PHABRICATOR_USERS_URL,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>httpretty.register_uri(httpretty.POST,<EOL>PHABRICATOR_PHIDS_URL,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>httpretty.register_uri(httpretty.POST,<EOL>PHABRICATOR_API_ERROR_URL,<EOL>responses=[<EOL>httpretty.Response(body=request_callback)<EOL>])<EOL>return http_requests<EOL>


responses = {CMD_VERSION: VERSION_214,<EOL>CMD_REVIEWS_1: REVIEWS_PAGE_1,<EOL>CMD_REVIEWS_2: REVIEWS_PAGE_2,<EOL>CMD_REVIEWS_3: REVIEWS_PAGE_3}<EOL>cmd = args[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>data = read_file(responses[cmd], '<STR_LIT:rb>')<EOL><DEDENT>except Exception:<EOL><INDENT>data = None<EOL><DEDENT>return data<EOL>


data = read_file(VERSION_UNKNOWN, '<STR_LIT:rb>')<EOL>return data<EOL>


return None<EOL>


def fetch(self, url, payload=None, headers=None, method=GET, stream=False, verify=True):

if self.from_archive:<EOL><INDENT>response = self._fetch_from_archive(url, payload, headers)<EOL><DEDENT>else:<EOL><INDENT>response = self._fetch_from_remote(url, payload, headers, method, stream, verify)<EOL><DEDENT>return response<EOL>

Fetch the data from a given URL.

        :param url: link to the resource
        :param payload: payload of the request
        :param headers: headers of the request
        :param method: type of request call (GET or POST)
        :param stream: defer downloading the response body until the response content is available
        :param verify: verifying the SSL certificate

        :returns a response object


@staticmethod<EOL><INDENT>def sanitize_for_archive(url, headers, payload):<DEDENT>

return url, headers, payload<EOL>

Sanitize the URL, headers and payload of a HTTP request before storing/retrieving items.
        By default, this method does not modify url, headers and payload. The modifications take
        place within the specific backends that redefine the sanitize_for_archive.

        :param: url: HTTP url request
        :param: headers: HTTP headers request
        :param: payload: HTTP payload request

        :returns url, headers and payload sanitized


Create a http session and initialize the retry object.


if self.session:<EOL><INDENT>self.session.keep_alive = False<EOL><DEDENT>


self.rate_limit = None<EOL>self.rate_limit_reset_ts = None<EOL>self.sleep_for_rate = sleep_for_rate<EOL>self.rate_limit_header = rate_limit_header<EOL>self.rate_limit_reset_header = rate_limit_reset_header<EOL>if min_rate_to_sleep > self.MAX_RATE_LIMIT:<EOL><INDENT>msg = "<STR_LIT>"<EOL>msg += "<STR_LIT>"<EOL>msg += "<STR_LIT>"<EOL>self.min_rate_to_sleep = self.MAX_RATE_LIMIT<EOL>logger.warning(msg, min_rate_to_sleep, self.MAX_RATE_LIMIT)<EOL><DEDENT>else:<EOL><INDENT>self.min_rate_to_sleep = min_rate_to_sleep<EOL><DEDENT>

Setup the rate limit handler.

        :param sleep_for_rate: sleep until rate limit is reset
        :param min_rate_to_sleep: minimun rate needed to make the fecthing process sleep
        :param rate_limit_header: header from where extract the rate limit data
        :param rate_limit_reset_header: header from where extract the rate limit reset data


if self.rate_limit is not None and self.rate_limit <= self.min_rate_to_sleep:<EOL><INDENT>seconds_to_reset = self.calculate_time_to_reset()<EOL>if seconds_to_reset < <NUM_LIT:0>:<EOL><INDENT>logger.warning("<STR_LIT>")<EOL>seconds_to_reset = <NUM_LIT:0><EOL><DEDENT>cause = "<STR_LIT>"<EOL>if self.sleep_for_rate:<EOL><INDENT>logger.info("<STR_LIT>", cause, seconds_to_reset)<EOL>time.sleep(seconds_to_reset)<EOL><DEDENT>else:<EOL><INDENT>raise RateLimitError(cause=cause, seconds_to_reset=seconds_to_reset)<EOL><DEDENT><DEDENT>

The fetching process sleeps until the rate limit is restored or
           raises a RateLimitError exception if sleep_for_rate flag is disabled.


raise NotImplementedError<EOL>

Calculate the seconds to reset the token requests.


if self.rate_limit_header in response.headers:<EOL><INDENT>self.rate_limit = int(response.headers[self.rate_limit_header])<EOL>logger.debug("<STR_LIT>", self.rate_limit)<EOL><DEDENT>else:<EOL><INDENT>self.rate_limit = None<EOL><DEDENT>if self.rate_limit_reset_header in response.headers:<EOL><INDENT>self.rate_limit_reset_ts = int(response.headers[self.rate_limit_reset_header])<EOL>logger.debug("<STR_LIT>", self.calculate_time_to_reset())<EOL><DEDENT>else:<EOL><INDENT>self.rate_limit_reset_ts = None<EOL><DEDENT>

Update the rate limit and the time to reset
        from the response headers.

        :param: response: the response object


def init_metadata(self, origin, backend_name, backend_version,<EOL>category, backend_params):

created_on = datetime_to_utc(datetime_utcnow())<EOL>created_on_dumped = created_on.isoformat()<EOL>backend_params_dumped = pickle.dumps(backend_params, <NUM_LIT:0>)<EOL>metadata = (origin, backend_name, backend_version, category,<EOL>backend_params_dumped, created_on_dumped,)<EOL>try:<EOL><INDENT>cursor = self._db.cursor()<EOL>insert_stmt = "<STR_LIT>" + self.METADATA_TABLE + "<STR_LIT:U+0020>""<STR_LIT>""<STR_LIT>""<STR_LIT>"<EOL>cursor.execute(insert_stmt, metadata)<EOL>self._db.commit()<EOL>cursor.close()<EOL><DEDENT>except sqlite3.DatabaseError as e:<EOL><INDENT>msg = "<STR_LIT>" % str(e)<EOL>raise ArchiveError(cause=msg)<EOL><DEDENT>self.origin = origin<EOL>self.backend_name = backend_name<EOL>self.backend_version = backend_version<EOL>self.category = category<EOL>self.backend_params = backend_params<EOL>self.created_on = created_on<EOL>logger.debug("<STR_LIT>",<EOL>self.archive_path, metadata)<EOL>

Init metadata information.

        Metatada is composed by basic information needed to identify
        where archived data came from and how it can be retrieved
        and built into Perceval items.

        :param: origin: identifier of the repository
        :param: backend_name: name of the backend
        :param: backend_version: version of the backend
        :param: category: category of the items fetched
        :param: backend_params: dict representation of the fetch parameters

        raises ArchiveError: when an error occurs initializing the metadata


def store(self, uri, payload, headers, data):

hashcode = self.make_hashcode(uri, payload, headers)<EOL>payload_dump = pickle.dumps(payload, <NUM_LIT:0>)<EOL>headers_dump = pickle.dumps(headers, <NUM_LIT:0>)<EOL>data_dump = pickle.dumps(data, <NUM_LIT:0>)<EOL>logger.debug("<STR_LIT>",<EOL>hashcode, uri, payload, headers, self.archive_path)<EOL>try:<EOL><INDENT>cursor = self._db.cursor()<EOL>insert_stmt = "<STR_LIT>" + self.ARCHIVE_TABLE + "<STR_LIT>""<STR_LIT>""<STR_LIT>"<EOL>cursor.execute(insert_stmt, (None, hashcode, uri,<EOL>payload_dump, headers_dump, data_dump))<EOL>self._db.commit()<EOL>cursor.close()<EOL><DEDENT>except sqlite3.IntegrityError as e:<EOL><INDENT>msg = "<STR_LIT>" % hashcode<EOL>raise ArchiveError(cause=msg)<EOL><DEDENT>except sqlite3.DatabaseError as e:<EOL><INDENT>msg = "<STR_LIT>" % str(e)<EOL>raise ArchiveError(cause=msg)<EOL><DEDENT>logger.debug("<STR_LIT>", hashcode, self.archive_path)<EOL>

Store a raw item in this archive.

        The method will store `data` content in this archive. The unique
        identifier for that item will be generated using the rest of the
        parameters.

        :param uri: request URI
        :param payload: request payload
        :param headers: request headers
        :param data: data to store in this archive

        :raises ArchiveError: when an error occurs storing the given data


def retrieve(self, uri, payload, headers):

hashcode = self.make_hashcode(uri, payload, headers)<EOL>logger.debug("<STR_LIT>",<EOL>hashcode, uri, payload, headers, self.archive_path)<EOL>self._db.row_factory = sqlite3.Row<EOL>try:<EOL><INDENT>cursor = self._db.cursor()<EOL>select_stmt = "<STR_LIT>""<STR_LIT>" + self.ARCHIVE_TABLE + "<STR_LIT:U+0020>""<STR_LIT>"<EOL>cursor.execute(select_stmt, (hashcode,))<EOL>row = cursor.fetchone()<EOL>cursor.close()<EOL><DEDENT>except sqlite3.DatabaseError as e:<EOL><INDENT>msg = "<STR_LIT>" % str(e)<EOL>raise ArchiveError(cause=msg)<EOL><DEDENT>if row:<EOL><INDENT>found = pickle.loads(row['<STR_LIT:data>'])<EOL><DEDENT>else:<EOL><INDENT>msg = "<STR_LIT>" % (hashcode, self.archive_path)<EOL>raise ArchiveError(cause=msg)<EOL><DEDENT>return found<EOL>

Retrieve a raw item from the archive.

        The method will return the `data` content corresponding to the
        hascode derived from the given parameters.

        :param uri: request URI
        :param payload: request payload
        :param headers: request headers

        :returns: the archived data

        :raises ArchiveError: when an error occurs retrieving data


if os.path.exists(archive_path):<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise ArchiveError(cause=msg % (archive_path))<EOL><DEDENT>conn = sqlite3.connect(archive_path)<EOL>cursor = conn.cursor()<EOL>cursor.execute(cls.METADATA_CREATE_STMT)<EOL>cursor.execute(cls.ARCHIVE_CREATE_STMT)<EOL>conn.commit()<EOL>cursor.close()<EOL>conn.close()<EOL>logger.debug("<STR_LIT>", archive_path)<EOL>archive = cls(archive_path)<EOL>logger.debug("<STR_LIT>", archive_path)<EOL>return archive<EOL>

Create a brand new archive.

         Call this method to create a new and empty archive. It will initialize
         the storage file in the path defined by `archive_path`.

        :param archive_path: absolute path where the archive file will be created

        :raises ArchiveError: when the archive file already exists


@staticmethod<EOL><INDENT>def make_hashcode(uri, payload, headers):<DEDENT>

def dict_to_json_str(data):<EOL><INDENT>return json.dumps(data, sort_keys=True)<EOL><DEDENT>content = '<STR_LIT::>'.join([uri, dict_to_json_str(payload), dict_to_json_str(headers)])<EOL>hashcode = hashlib.sha1(content.encode('<STR_LIT:utf-8>'))<EOL>return hashcode.hexdigest()<EOL>

Generate a SHA1 based on the given arguments.

        Hashcodes created by this method will used as unique identifiers
        for the raw items or resources stored by this archive.

        :param uri: URI to the resource
        :param payload: payload of the request needed to fetch the resource
        :param headers: headers of the request needed to fetch the resource

        :returns: a SHA1 hash code


nentries = self._count_table_rows(self.ARCHIVE_TABLE)<EOL>nmetadata = self._count_table_rows(self.METADATA_TABLE)<EOL>if nmetadata > <NUM_LIT:1>:<EOL><INDENT>msg = "<STR_LIT>" % (self.archive_path)<EOL>raise ArchiveError(cause=msg)<EOL><DEDENT>if nmetadata == <NUM_LIT:0> and nentries > <NUM_LIT:0>:<EOL><INDENT>msg = "<STR_LIT>" % (self.archive_path)<EOL>raise ArchiveError(cause=msg)<EOL><DEDENT>logger.debug("<STR_LIT>",<EOL>self.archive_path, nentries, nmetadata)<EOL>

Check whether the archive is valid or not.

        This method will check if tables were created and if they
        contain valid data.


logger.debug("<STR_LIT>", self.archive_path)<EOL>cursor = self._db.cursor()<EOL>select_stmt = "<STR_LIT>""<STR_LIT>""<STR_LIT>" + self.METADATA_TABLE + "<STR_LIT:U+0020>""<STR_LIT>"<EOL>cursor.execute(select_stmt)<EOL>row = cursor.fetchone()<EOL>cursor.close()<EOL>if row:<EOL><INDENT>self.origin = row[<NUM_LIT:0>]<EOL>self.backend_name = row[<NUM_LIT:1>]<EOL>self.backend_version = row[<NUM_LIT:2>]<EOL>self.category = row[<NUM_LIT:3>]<EOL>self.backend_params = pickle.loads(row[<NUM_LIT:4>])<EOL>self.created_on = str_to_datetime(row[<NUM_LIT:5>])<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>", self.archive_path)<EOL><DEDENT>logger.debug("<STR_LIT>", self.archive_path)<EOL>


cursor = self._db.cursor()<EOL>select_stmt = "<STR_LIT>" + table_name<EOL>try:<EOL><INDENT>cursor.execute(select_stmt)<EOL>row = cursor.fetchone()<EOL><DEDENT>except sqlite3.DatabaseError as e:<EOL><INDENT>msg = "<STR_LIT>" % str(e)<EOL>raise ArchiveError(cause=msg)<EOL><DEDENT>finally:<EOL><INDENT>cursor.close()<EOL><DEDENT>return row[<NUM_LIT:0>]<EOL>


hashcode = uuid.uuid4().hex<EOL>archive_dir = os.path.join(self.dirpath, hashcode[<NUM_LIT:0>:<NUM_LIT:2>])<EOL>archive_name = hashcode[<NUM_LIT:2>:] + self.STORAGE_EXT<EOL>archive_path = os.path.join(archive_dir, archive_name)<EOL>if not os.path.exists(archive_dir):<EOL><INDENT>os.makedirs(archive_dir)<EOL><DEDENT>try:<EOL><INDENT>archive = Archive.create(archive_path)<EOL><DEDENT>except ArchiveError as e:<EOL><INDENT>raise ArchiveManagerError(cause=str(e))<EOL><DEDENT>return archive<EOL>

Create a new archive.

        The method creates in the filesystem a brand new archive with
        a random SHA1 as its name. The first byte of the hashcode will
        be the name of the subdirectory; the remaining bytes, the
        archive name.

        :returns: a new `Archive` object

        :raises ArchiveManagerError: when an error occurs creating the
            new archive


try:<EOL><INDENT>Archive(archive_path)<EOL><DEDENT>except ArchiveError as e:<EOL><INDENT>raise ArchiveManagerError(cause=str(e))<EOL><DEDENT>os.remove(archive_path)<EOL>

Remove an archive.

        This method deletes from the filesystem the archive stored
        in `archive_path`.

        :param archive_path: path to the archive

        :raises ArchiveManangerError: when an error occurs removing the
            archive


def search(self, origin, backend_name, category, archived_after):

archives = self._search_archives(origin, backend_name,<EOL>category, archived_after)<EOL>archives = [(fp, date) for fp, date in archives]<EOL>archives = [fp for fp, _ in sorted(archives, key=lambda x: x[<NUM_LIT:1>])]<EOL>return archives<EOL>

Search archives.

        Get the archives which store data based on the given parameters.
        These parameters define which the origin was (`origin`), how data
        was fetched (`backend_name`) and data type ('category').
        Only those archives created on or after `archived_after` will be
        returned.

        The method returns a list with the file paths to those archives.
        The list is sorted by the date of creation of each archive.

        :param origin: data origin
        :param backend_name: backed used to fetch data
        :param category: type of the items fetched by the backend
        :param archived_after: get archives created on or after this date

        :returns: a list with archive names which match the search criteria


def _search_archives(self, origin, backend_name, category, archived_after):

for archive_path in self._search_files():<EOL><INDENT>try:<EOL><INDENT>archive = Archive(archive_path)<EOL><DEDENT>except ArchiveError:<EOL><INDENT>continue<EOL><DEDENT>match = archive.origin == origin andarchive.backend_name == backend_name andarchive.category == category andarchive.created_on >= archived_after<EOL>if not match:<EOL><INDENT>continue<EOL><DEDENT>yield archive_path, archive.created_on<EOL><DEDENT>


for root, _, files in os.walk(self.dirpath):<EOL><INDENT>for filename in files:<EOL><INDENT>location = os.path.join(root, filename)<EOL>yield location<EOL><DEDENT><DEDENT>

Retrieve the file paths stored under the base path.


def fetch(self, category=CATEGORY_QUESTION, from_date=DEFAULT_DATETIME):

if not from_date:<EOL><INDENT>from_date = DEFAULT_DATETIME<EOL><DEDENT>kwargs = {'<STR_LIT>': from_date}<EOL>items = super().fetch(category, **kwargs)<EOL>return items<EOL>

Fetch the questions/answers from the repository.

        The method retrieves, from an Askbot site, the questions and answers
        updated since the given date.

        :param category: the category of items to fetch
        :param from_date: obtain questions/answers updated since this date

        :returns: a generator of items


def fetch_items(self, category, **kwargs):

from_date = datetime_to_utc(kwargs['<STR_LIT>']).timestamp()<EOL>questions_groups = self.client.get_api_questions(AskbotClient.API_QUESTIONS)<EOL>for questions in questions_groups:<EOL><INDENT>for question in questions['<STR_LIT>']:<EOL><INDENT>updated_at = int(question['<STR_LIT>'])<EOL>if updated_at > from_date:<EOL><INDENT>html_question = self.__fetch_question(question)<EOL>if not html_question:<EOL><INDENT>continue<EOL><DEDENT>logger.debug("<STR_LIT>", question['<STR_LIT:id>'])<EOL>comments = self.__fetch_comments(question)<EOL>question_obj = self.__build_question(html_question, question, comments)<EOL>question.update(question_obj)<EOL>yield question<EOL><DEDENT><DEDENT><DEDENT>

Fetch the questions

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items


return True<EOL>

Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming


return True<EOL>

Returns whether it supports archiving items on the fetch process.

        :returns: this backend supports items archive


return CATEGORY_QUESTION<EOL>

Extracts the category from an Askbot item.

        This backend only generates one type of item which is
        'question'.


return str(item['<STR_LIT:id>'])<EOL>

Extracts the identifier from an Askbot question item.


return float(item['<STR_LIT>'])<EOL>

Extracts the update time from an Askbot item.

        The timestamp is extracted from 'last_activity_at' field.
        This date is a UNIX timestamp but needs to be converted to
        a float value.

        :param item: item generated by the backend

        :returns: a UNIX timestamp


return AskbotClient(self.url, self.archive, from_archive)<EOL>


html_question_items = []<EOL>npages = <NUM_LIT:1><EOL>next_request = True<EOL>while next_request:<EOL><INDENT>try:<EOL><INDENT>html_question = self.client.get_html_question(question['<STR_LIT:id>'], npages)<EOL>html_question_items.append(html_question)<EOL>tpages = self.ab_parser.parse_number_of_html_pages(html_question)<EOL>if npages == tpages:<EOL><INDENT>next_request = False<EOL><DEDENT>npages = npages + <NUM_LIT:1><EOL><DEDENT>except requests.exceptions.TooManyRedirects as e:<EOL><INDENT>logger.warning("<STR_LIT>", e, question['<STR_LIT:id>'])<EOL>next_request = False<EOL><DEDENT><DEDENT>return html_question_items<EOL>

Fetch an Askbot HTML question body.

        The method fetchs the HTML question retrieving the
        question body of the item question received

        :param question: item with the question itself

        :returns: a list of HTML page/s for the question


comments = {}<EOL>comments[question['<STR_LIT:id>']] = json.loads(self.client.get_comments(question['<STR_LIT:id>']))<EOL>for object_id in question['<STR_LIT>']:<EOL><INDENT>comments[object_id] = json.loads(self.client.get_comments(object_id))<EOL><DEDENT>return comments<EOL>

Fetch all the comments of an Askbot question and answers.

        The method fetchs the list of every comment existing in a question and
        its answers.

        :param question: item with the question itself

        :returns: a list of comments with the ids as hashes


@staticmethod<EOL><INDENT>def __build_question(html_question, question, comments):<DEDENT>

question_object = {}<EOL>question_container = AskbotParser.parse_question_container(html_question[<NUM_LIT:0>])<EOL>question_object.update(question_container)<EOL>if comments[int(question['<STR_LIT:id>'])]:<EOL><INDENT>question_object['<STR_LIT>'] = comments[int(question['<STR_LIT:id>'])]<EOL><DEDENT>answers = []<EOL>for page in html_question:<EOL><INDENT>answers.extend(AskbotParser.parse_answers(page))<EOL><DEDENT>if len(answers) != <NUM_LIT:0>:<EOL><INDENT>question_object['<STR_LIT>'] = answers<EOL>for answer in question_object['<STR_LIT>']:<EOL><INDENT>if comments[int(answer['<STR_LIT:id>'])]:<EOL><INDENT>answer['<STR_LIT>'] = comments[int(answer['<STR_LIT:id>'])]<EOL><DEDENT><DEDENT><DEDENT>return question_object<EOL>

Build an Askbot HTML response.

        The method puts together all the information regarding a question

        :param html_question: array of HTML raw pages
        :param question: question object from the API
        :param comments: list of comments to add

        :returns: a dict item with the parsed question information


npages = <NUM_LIT:1><EOL>next_request = True<EOL>path = urijoin(self.base_url, path)<EOL>while next_request:<EOL><INDENT>try:<EOL><INDENT>params = {<EOL>'<STR_LIT>': npages,<EOL>'<STR_LIT>': self.ORDER_API<EOL>}<EOL>response = self.fetch(path, payload=params)<EOL>whole_page = response.text<EOL>raw_questions = json.loads(whole_page)<EOL>tpages = raw_questions['<STR_LIT>']<EOL>logger.debug("<STR_LIT>",<EOL>self.base_url, npages, tpages)<EOL>if npages == tpages:<EOL><INDENT>next_request = False<EOL><DEDENT>npages = npages + <NUM_LIT:1><EOL>yield raw_questions<EOL><DEDENT>except requests.exceptions.TooManyRedirects as e:<EOL><INDENT>logger.warning("<STR_LIT>", e, path)<EOL>next_request = False<EOL><DEDENT><DEDENT>

Retrieve a question page using the API.

        :param page: page to retrieve


def get_html_question(self, question_id, page=<NUM_LIT:1>):

path = urijoin(self.base_url, self.HTML_QUESTION, question_id)<EOL>params = {<EOL>'<STR_LIT>': page,<EOL>'<STR_LIT>': self.ORDER_HTML<EOL>}<EOL>response = self.fetch(path, payload=params)<EOL>return response.text<EOL>

Retrieve a raw HTML question and all it's information.

        :param question_id: question identifier
        :param page: page to retrieve


path = urijoin(self.base_url, self.COMMENTS if self._use_new_urls else self.COMMENTS_OLD)<EOL>params = {<EOL>'<STR_LIT>': post_id,<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': <NUM_LIT:0><EOL>}<EOL>headers = {'<STR_LIT>': '<STR_LIT>'}<EOL>try:<EOL><INDENT>response = self.fetch(path, payload=params, headers=headers)<EOL>raw = response.text<EOL><DEDENT>except requests.exceptions.HTTPError as ex:<EOL><INDENT>if ex.response.status_code == <NUM_LIT>:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>self._use_new_urls = False<EOL>path = urijoin(self.base_url, self.COMMENTS_OLD)<EOL>response = self.fetch(path, payload=params, headers=headers)<EOL>raw = response.text<EOL><DEDENT>elif ex.response.status_code == <NUM_LIT>:<EOL><INDENT>logger.warning("<STR_LIT>", ex)<EOL>raw = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>raise ex<EOL><DEDENT><DEDENT>return raw<EOL>

Retrieve a list of comments by a given id.

        :param object_id: object identifiere


container_info = {}<EOL>bs_question = bs4.BeautifulSoup(html_question, "<STR_LIT>")<EOL>question = AskbotParser._find_question_container(bs_question)<EOL>container = question.select("<STR_LIT>")<EOL>created = container[<NUM_LIT:0>]<EOL>container_info['<STR_LIT>'] = AskbotParser.parse_user_info(created)<EOL>try:<EOL><INDENT>container[<NUM_LIT:1>]<EOL><DEDENT>except IndexError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>updated = container[<NUM_LIT:1>]<EOL>if AskbotParser.parse_user_info(updated):<EOL><INDENT>container_info['<STR_LIT>'] = AskbotParser.parse_user_info(updated)<EOL><DEDENT><DEDENT>return container_info<EOL>

Parse the question info container of a given HTML question.

        The method parses the information available in the question information
        container. The container can have up to 2 elements: the first one
        contains the information related with the user who generated the question
        and the date (if any). The second one contains the date of the updated,
        and the user who updated it (if not the same who generated the question).

        :param html_question: raw HTML question element

        :returns: an object with the parsed information


def parse_answer_container(update_info):<EOL><INDENT>"""<STR_LIT>"""<EOL>container_info = {}<EOL>created = update_info[<NUM_LIT:0>]<EOL>answered_at = created.abbr.attrs["<STR_LIT:title>"]<EOL>container_info['<STR_LIT>'] = str(str_to_datetime(answered_at).timestamp())<EOL>container_info['<STR_LIT>'] = AskbotParser.parse_user_info(created)<EOL>try:<EOL><INDENT>update_info[<NUM_LIT:1>]<EOL><DEDENT>except IndexError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>updated = update_info[<NUM_LIT:1>]<EOL>updated_at = updated.abbr.attrs["<STR_LIT:title>"]<EOL>container_info['<STR_LIT>'] = str(str_to_datetime(updated_at).timestamp())<EOL>if AskbotParser.parse_user_info(updated):<EOL><INDENT>container_info['<STR_LIT>'] = AskbotParser.parse_user_info(updated)<EOL><DEDENT><DEDENT>return container_info<EOL><DEDENT>answer_list = []<EOL>bs_question = bs4.BeautifulSoup(html_question, "<STR_LIT>")<EOL>bs_answers = bs_question.select("<STR_LIT>")<EOL>for bs_answer in bs_answers:<EOL><INDENT>answer_id = bs_answer.attrs["<STR_LIT>"]<EOL>votes_element = bs_answer.select("<STR_LIT>")[<NUM_LIT:0>].text<EOL>accepted_answer = bs_answer.select("<STR_LIT>")[<NUM_LIT:0>].get('<STR_LIT:title>').endswith("<STR_LIT>")<EOL>body = bs_answer.select("<STR_LIT>")<EOL>update_info = body[<NUM_LIT:0>].select("<STR_LIT>")<EOL>answer_container = parse_answer_container(update_info)<EOL>body[<NUM_LIT:0>].div.extract().select("<STR_LIT>")<EOL>body = body[<NUM_LIT:0>].get_text(strip=True)<EOL>answer = {'<STR_LIT:id>': answer_id,<EOL>'<STR_LIT>': votes_element,<EOL>'<STR_LIT>': body,<EOL>'<STR_LIT>': accepted_answer<EOL>}<EOL>answer.update(answer_container)<EOL>answer_list.append(answer)<EOL><DEDENT>return answer_list<EOL>

Parse the answers of a given HTML question.

        The method parses the answers related with a given HTML question,
        as well as all the comments related to the answer.

        :param html_question: raw HTML question element

        :returns: a list with the answers


bs_question = bs4.BeautifulSoup(html_question, "<STR_LIT>")<EOL>try:<EOL><INDENT>bs_question.select('<STR_LIT>')[<NUM_LIT:0>]<EOL><DEDENT>except IndexError:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>return int(bs_question.select('<STR_LIT>')[<NUM_LIT:0>].attrs['<STR_LIT>'])<EOL><DEDENT>

Parse number of answer pages to paginate over them.

        :param html_question: raw HTML question element

        :returns: an integer with the number of pages


user_info = {}<EOL>if update_info.select("<STR_LIT>"):<EOL><INDENT>elements = update_info.select("<STR_LIT>")[<NUM_LIT:0>].find_all("<STR_LIT:a>")<EOL>href = elements[<NUM_LIT:0>].attrs["<STR_LIT>"]<EOL>user_info['<STR_LIT:id>'] = re.search(r'<STR_LIT>', href).group(<NUM_LIT:0>)<EOL>user_info['<STR_LIT:username>'] = elements[<NUM_LIT:0>].text<EOL>user_info['<STR_LIT>'] = update_info.select('<STR_LIT>')[<NUM_LIT:0>].text<EOL>user_info['<STR_LIT>'] = update_info.select("<STR_LIT>")[<NUM_LIT:0>].attrs["<STR_LIT:title>"]<EOL>try:<EOL><INDENT>elements[<NUM_LIT:1>]<EOL><DEDENT>except IndexError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>user_info['<STR_LIT>'] = elements[<NUM_LIT:1>].attrs["<STR_LIT>"]<EOL><DEDENT>if update_info.select("<STR_LIT>"):<EOL><INDENT>flag = update_info.select("<STR_LIT>")[<NUM_LIT:0>].attrs["<STR_LIT>"]<EOL>user_info['<STR_LIT>'] = re.sub("<STR_LIT>", "<STR_LIT>", flag)<EOL><DEDENT><DEDENT>return user_info<EOL>

Parse the user information of a given HTML container.

        The method parses all the available user information in the container.
        If the class "user-info" exists, the method will get all the available
        information in the container. If not, if a class "tip" exists, it will be
        a wiki post with no user associated. Else, it can be an empty container.

        :param update_info: beautiful soup answer container element

        :returns: an object with the parsed information


parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,<EOL>from_date=True,<EOL>archive=True)<EOL>parser.parser.add_argument('<STR_LIT:url>',<EOL>help="<STR_LIT>")<EOL>return parser<EOL>


def fetch(self, category=CATEGORY_MESSAGE, offset=DEFAULT_OFFSET, chats=None):

if not offset:<EOL><INDENT>offset = DEFAULT_OFFSET<EOL><DEDENT>kwargs = {"<STR_LIT>": offset, "<STR_LIT>": chats}<EOL>items = super().fetch(category, **kwargs)<EOL>return items<EOL>

Fetch the messages the bot can read from the server.

        The method retrieves, from the Telegram server, the messages
        sent with an offset equal or greater than the given.

        A list of chats, groups and channels identifiers can be set
        using the parameter `chats`. When it is set, only those
        messages sent to any of these will be returned. An empty list
        will return no messages.

        :param category: the category of items to fetch
        :param offset: obtain messages from this offset
        :param chats: list of chat names used to filter messages

        :returns: a generator of messages

        :raises ValueError: when `chats` is an empty list


offset = kwargs['<STR_LIT>']<EOL>chats = kwargs['<STR_LIT>']<EOL>logger.info("<STR_LIT>",<EOL>self.bot, offset)<EOL>if chats is not None:<EOL><INDENT>if len(chats) == <NUM_LIT:0>:<EOL><INDENT>logger.warning("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>logger.info("<STR_LIT>",<EOL>'<STR_LIT:[>' + '<STR_LIT:U+002C>'.join(str(ch_id) for ch_id in chats) + '<STR_LIT:]>')<EOL><DEDENT><DEDENT>nmsgs = <NUM_LIT:0><EOL>while True:<EOL><INDENT>raw_json = self.client.updates(offset=offset)<EOL>messages = [msg for msg in self.parse_messages(raw_json)]<EOL>if len(messages) == <NUM_LIT:0>:<EOL><INDENT>break<EOL><DEDENT>for msg in messages:<EOL><INDENT>offset = max(msg['<STR_LIT>'], offset)<EOL>if not self._filter_message_by_chats(msg, chats):<EOL><INDENT>logger.debug("<STR_LIT>",<EOL>msg['<STR_LIT:message>']['<STR_LIT>'])<EOL>continue<EOL><DEDENT>yield msg<EOL>nmsgs += <NUM_LIT:1><EOL><DEDENT>offset += <NUM_LIT:1><EOL><DEDENT>logger.info("<STR_LIT>",<EOL>nmsgs)<EOL>

Fetch the messages

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items


def metadata(self, item, filter_classified=False):

item = super().metadata(item, filter_classified=filter_classified)<EOL>item['<STR_LIT>'] = item['<STR_LIT:data>']['<STR_LIT>']<EOL>return item<EOL>

Telegram metadata.

        The method takes an item and overrides the `metadata` information
        to add extra information related to Telegram.

        Currently, it adds the 'offset' keyword.

        :param item: an item fetched by a backend
        :param filter_classified: sets if classified fields were filtered


return True<EOL>

Returns whether it supports archiving items on the fetch process.

        :returns: this backend supports items archive


return True<EOL>

Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming


return str(item['<STR_LIT:message>']['<STR_LIT>'])<EOL>

Extracts the identifier from a Telegram item.


ts = item['<STR_LIT:message>']['<STR_LIT:date>']<EOL>ts = float(ts)<EOL>return ts<EOL>

Extracts and coverts the update time from a Telegram item.

        The timestamp is extracted from 'date' field that is inside
        of 'message' dict. This date is converted to UNIX timestamp
        format.

        :param item: item generated by the backend

        :returns: a UNIX timestamp


return CATEGORY_MESSAGE<EOL>

Extracts the category from a Telegram item.

        This backend only generates one type of item which is
        'message'.


result = json.loads(raw_json)<EOL>messages = result['<STR_LIT:result>']<EOL>for msg in messages:<EOL><INDENT>yield msg<EOL><DEDENT>

Parse a Telegram JSON messages list.

        The method parses the JSON stream and returns an iterator of
        dictionaries. Each one of this, contains a Telegram message.

        :param raw_json: JSON string to parse

        :returns: a generator of parsed messages


return TelegramBotClient(self.bot_token, self.archive, from_archive)<EOL>


def _filter_message_by_chats(self, message, chats):

if chats is None:<EOL><INDENT>return True<EOL><DEDENT>chat_id = message['<STR_LIT:message>']['<STR_LIT>']['<STR_LIT:id>']<EOL>return chat_id in chats<EOL>

Check if a message can be filtered based in a list of chats.

        This method returns `True` when the message was sent to a chat
        of the given list. It also returns `True` when chats is `None`.

        :param message: Telegram message
        :param chats: list of chat, groups and channels identifiers

        :returns: `True` when the message can be filtered; otherwise,
            it returns `False`


aliases = {<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>}<EOL>parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,<EOL>offset=True,<EOL>token_auth=True,<EOL>archive=True,<EOL>aliases=aliases)<EOL>action = parser.parser._option_string_actions['<STR_LIT>']<EOL>action.required = True<EOL>group = parser.parser.add_argument_group('<STR_LIT>')<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>nargs='<STR_LIT:+>', type=int, default=None,<EOL>help="<STR_LIT>")<EOL>parser.parser.add_argument('<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>return parser<EOL>


params = {}<EOL>if offset:<EOL><INDENT>params[self.OFFSET] = offset<EOL><DEDENT>response = self._call(self.UPDATES_METHOD, params)<EOL>return response<EOL>

Fetch the messages that a bot can read.

        When the `offset` is given it will retrieve all the messages
        that are greater or equal to that offset. Take into account
        that, due to how the API works, all previous messages will
        be removed from the server.

        :param offset: fetch the messages starting on this offset


url = re.sub('<STR_LIT>', '<STR_LIT>', url)<EOL>return url, headers, payload<EOL>

Sanitize URL of a HTTP request by removing the token information
        before storing/retrieving archived items

        :param: url: HTTP url request
        :param: headers: HTTP headers request
        :param: payload: HTTP payload request

        :returns the sanitized url, plus the headers and payload


url = self.base_url % {'<STR_LIT>': self.bot_token, '<STR_LIT>': method}<EOL>logger.debug("<STR_LIT>",<EOL>method, str(params))<EOL>r = self.fetch(url, payload=params)<EOL>return r.text<EOL>

Retrive the given resource.

        :param resource: resource to retrieve
        :param params: dict with the HTTP parameters needed to retrieve
            the given resource


from_date = kwargs['<STR_LIT>']<EOL>to_date = kwargs['<STR_LIT>']<EOL>branches = kwargs['<STR_LIT>']<EOL>latest_items = kwargs['<STR_LIT>']<EOL>no_update = kwargs['<STR_LIT>']<EOL>ncommits = <NUM_LIT:0><EOL>try:<EOL><INDENT>if os.path.isfile(self.gitpath):<EOL><INDENT>commits = self.__fetch_from_log()<EOL><DEDENT>else:<EOL><INDENT>commits = self.__fetch_from_repo(from_date, to_date, branches,<EOL>latest_items, no_update)<EOL><DEDENT>for commit in commits:<EOL><INDENT>yield commit<EOL>ncommits += <NUM_LIT:1><EOL><DEDENT><DEDENT>except EmptyRepositoryError:<EOL><INDENT>pass<EOL><DEDENT>logger.info("<STR_LIT>",<EOL>ncommits)<EOL>

Fetch the commits

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items


return False<EOL>

Returns whether it supports archiving items on the fetch process.

        :returns: this backend does not support items archive


return True<EOL>


return item['<STR_LIT>']<EOL>

Extracts the identifier from a Git item.


ts = item['<STR_LIT>']<EOL>ts = str_to_datetime(ts)<EOL>return ts.timestamp()<EOL>

Extracts the update time from a Git item.

        The timestamp used is extracted from 'CommitDate' field.
        This date is converted to UNIX timestamp format taking into
        account the timezone of the date.

        :param item: item generated by the backend

        :returns: a UNIX timestamp


return CATEGORY_COMMIT<EOL>

Extracts the category from a Git item.

        This backend only generates one type of item which is
        'commit'.


with open(filepath, '<STR_LIT:r>', errors='<STR_LIT>',<EOL>newline=os.linesep) as f:<EOL><INDENT>parser = GitParser(f)<EOL>for commit in parser.parse():<EOL><INDENT>yield commit<EOL><DEDENT><DEDENT>

Parse a Git log file.

        The method parses the Git log file and returns an iterator of
        dictionaries. Each one of this, contains a commit.

        :param filepath: path to the log file

        :returns: a generator of parsed commits

        :raises ParseError: raised when the format of the Git log file
            is invalid
        :raises OSError: raised when an error occurs reading the
            given file


parser = GitParser(iterator)<EOL>for commit in parser.parse():<EOL><INDENT>yield commit<EOL><DEDENT>

Parse a Git log obtained from an iterator.

        The method parses the Git log fetched from an iterator, where
        each item is a line of the log. It returns and iterator of
        dictionaries. Each dictionary contains a commit.

        :param iterator: iterator of Git log lines

        :raises ParseError: raised when the format of the Git log
            is invalid


if self.parsed_args.git_log:<EOL><INDENT>git_path = self.parsed_args.git_log<EOL><DEDENT>elif not self.parsed_args.git_path:<EOL><INDENT>base_path = os.path.expanduser('<STR_LIT>')<EOL>processed_uri = self.parsed_args.uri.lstrip('<STR_LIT:/>')<EOL>git_path = os.path.join(base_path, processed_uri) + '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>git_path = self.parsed_args.git_path<EOL><DEDENT>setattr(self.parsed_args, '<STR_LIT>', git_path)<EOL>


parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,<EOL>from_date=True,<EOL>to_date=True)<EOL>group = parser.parser.add_argument_group('<STR_LIT>')<EOL>group.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>nargs='<STR_LIT:+>', type=str, default=None,<EOL>help="<STR_LIT>")<EOL>exgroup = group.add_mutually_exclusive_group()<EOL>exgroup.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>exgroup.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>exgroup_fetch = group.add_mutually_exclusive_group()<EOL>exgroup_fetch.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>exgroup_fetch.add_argument('<STR_LIT>', dest='<STR_LIT>',<EOL>action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>parser.parser.add_argument('<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>return parser<EOL>


for line in self.stream:<EOL><INDENT>line = line.rstrip('<STR_LIT:\n>')<EOL>parsed = False<EOL>self.nline += <NUM_LIT:1><EOL>while not parsed:<EOL><INDENT>parsed = self.handlers[self.state](line)<EOL>if self.state == self.COMMIT and self.commit:<EOL><INDENT>commit = self._build_commit()<EOL>logger.debug("<STR_LIT>", commit['<STR_LIT>'])<EOL>yield commit<EOL><DEDENT><DEDENT><DEDENT>if self.commit:<EOL><INDENT>commit = self._build_commit()<EOL>logger.debug("<STR_LIT>", commit['<STR_LIT>'])<EOL>yield commit<EOL><DEDENT>


i = f.find('<STR_LIT:{>')<EOL>j = f.find('<STR_LIT:}>')<EOL>if i > -<NUM_LIT:1> and j > -<NUM_LIT:1>:<EOL><INDENT>prefix = f[<NUM_LIT:0>:i]<EOL>inner = f[i + <NUM_LIT:1>:f.find('<STR_LIT>', i)]<EOL>suffix = f[j + <NUM_LIT:1>:]<EOL>return prefix + inner + suffix<EOL><DEDENT>elif '<STR_LIT>' in f:<EOL><INDENT>return f.split('<STR_LIT>')[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return f<EOL><DEDENT>

Get the old filepath of a moved/renamed file.

        Moved or renamed files can be found in the log with any of the
        next patterns:
          'old_name => new_name'
          '{old_prefix => new_prefix}/name'
          'name/{old_suffix => new_suffix}'

        This method returns the filepath before the file was moved or
        renamed.


@classmethod<EOL><INDENT>def clone(cls, uri, dirpath):<DEDENT>

cmd = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', uri, dirpath]<EOL>env = {<EOL>'<STR_LIT>': '<STR_LIT:C>',<EOL>'<STR_LIT>': os.getenv('<STR_LIT>', '<STR_LIT>')<EOL>}<EOL>cls._exec(cmd, env=env)<EOL>logger.debug("<STR_LIT>",<EOL>uri, dirpath)<EOL>return cls(uri, dirpath)<EOL>

Clone a Git repository.

        Make a bare copy of the repository stored in `uri` into `dirpath`.
        The repository would be either local or remote.

        :param uri: URI of the repository
        :param dirtpath: directory where the repository will be cloned

        :returns: a `GitRepository` class having cloned the repository

        :raises RepositoryError: when an error occurs cloning the given
            repository


cmd_count = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>outs = self._exec(cmd_count, cwd=self.dirpath, env=self.gitenv)<EOL>outs = outs.decode('<STR_LIT:utf-8>', errors='<STR_LIT>').rstrip()<EOL>try:<EOL><INDENT>cobjs = {k: v for k, v in (x.split('<STR_LIT>') for x in outs.split('<STR_LIT:\n>'))}<EOL>nobjs = int(cobjs['<STR_LIT:count>']) + int(cobjs['<STR_LIT>'])<EOL><DEDENT>except KeyError as e:<EOL><INDENT>error = "<STR_LIT>"% e.args[<NUM_LIT:0>]<EOL>raise RepositoryError(cause=error)<EOL><DEDENT>except ValueError as e:<EOL><INDENT>error = "<STR_LIT>" % str(e)<EOL>raise RepositoryError(cause=error)<EOL><DEDENT>logger.debug("<STR_LIT>",<EOL>self.uri, str(nobjs))<EOL>return nobjs<EOL>

Count the objects of a repository.

        The method returns the total number of objects (packed and unpacked)
        available on the repository.

        :raises RepositoryError: when an error occurs counting the objects
            of a repository


cmd_sym = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>try:<EOL><INDENT>self._exec(cmd_sym, cwd=self.dirpath, env=self.gitenv)<EOL><DEDENT>except RepositoryError as e:<EOL><INDENT>if e.msg.find("<STR_LIT>") == -<NUM_LIT:1>:<EOL><INDENT>raise e<EOL><DEDENT>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

Check if the repo is in a detached state.

        The repository is in a detached state when HEAD is not a symbolic
        reference.

        :returns: whether the repository is detached or not

        :raises RepositoryError: when an error occurs checking the state
            of the repository


return self.count_objects() == <NUM_LIT:0><EOL>

Determines whether the repository is empty or not.

        Returns `True` when the repository is empty. Under the hood,
        it checks the number of objects on the repository. When
        this number is 0, the repositoy is empty.

        :raises RepositoryError: when an error occurs accessing the
            repository


cmd_update = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>self._exec(cmd_update, cwd=self.dirpath, env=self.gitenv)<EOL>logger.debug("<STR_LIT>",<EOL>self.uri, self.dirpath)<EOL>

Update repository from its remote.

        Calling this method, the repository will be synchronized with
        the remote repository using 'fetch' command for 'heads' refs.
        Any commit stored in the local copy will be removed; refs
        will be overwritten.

        :raises RepositoryError: when an error occurs updating the
            repository


pack_name, refs = self._fetch_pack()<EOL>if pack_name:<EOL><INDENT>commits = self._read_commits_from_pack(pack_name)<EOL><DEDENT>else:<EOL><INDENT>commits = []<EOL>logger.debug("<STR_LIT>",<EOL>self.uri, self.dirpath)<EOL><DEDENT>self._update_references(refs)<EOL>logger.debug("<STR_LIT>",<EOL>self.uri, self.dirpath)<EOL>return commits<EOL>

Keep the repository in sync.

        This method will synchronize the repository with its 'origin',
        fetching newest objects and updating references. It uses low
        level commands which allow to keep track of which things
        have changed in the repository.

        The method also returns a list of hashes related to the new
        commits fetched during the process.

        :returns: list of new commits

        :raises RepositoryError: when an error occurs synchronizing
            the repository


if self.is_empty():<EOL><INDENT>logger.warning("<STR_LIT>",<EOL>self.uri)<EOL>raise EmptyRepositoryError(repository=self.uri)<EOL><DEDENT>cmd_rev_list = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>if branches is None:<EOL><INDENT>cmd_rev_list.extend(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>elif len(branches) == <NUM_LIT:0>:<EOL><INDENT>cmd_rev_list.extend(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>branches = ['<STR_LIT>' + branch for branch in branches]<EOL>cmd_rev_list.extend(branches)<EOL><DEDENT>for line in self._exec_nb(cmd_rev_list, cwd=self.dirpath, env=self.gitenv):<EOL><INDENT>yield line.rstrip('<STR_LIT:\n>')<EOL><DEDENT>logger.debug("<STR_LIT>",<EOL>self.uri, self.dirpath)<EOL>

Read the list commits from the repository

        The list of branches is a list of strings, with the names of the
        branches to fetch. If the list of branches is empty, no commit
        is fetched. If the list of branches is None, all commits
        for all branches will be fetched.

        The method returns the Git rev-list of the repository using the
        following options:

            git rev-list --topo-order

        :param branches: names of branches to fetch from (default: None)

        :raises EmptyRepositoryError: when the repository is empty and
            the action cannot be performed
        :raises RepositoryError: when an error occurs executing the command


def log(self, from_date=None, to_date=None, branches=None, encoding='<STR_LIT:utf-8>'):

if self.is_empty():<EOL><INDENT>logger.warning("<STR_LIT>",<EOL>self.uri)<EOL>raise EmptyRepositoryError(repository=self.uri)<EOL><DEDENT>cmd_log = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>cmd_log.extend(self.GIT_PRETTY_OUTPUT_OPTS)<EOL>if from_date:<EOL><INDENT>dt = from_date.strftime("<STR_LIT>")<EOL>cmd_log.append('<STR_LIT>' + dt)<EOL><DEDENT>if to_date:<EOL><INDENT>dt = to_date.strftime("<STR_LIT>")<EOL>cmd_log.append('<STR_LIT>' + dt)<EOL><DEDENT>if branches is None:<EOL><INDENT>cmd_log.extend(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>elif len(branches) == <NUM_LIT:0>:<EOL><INDENT>cmd_log.append('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>branches = ['<STR_LIT>' + branch for branch in branches]<EOL>cmd_log.extend(branches)<EOL><DEDENT>for line in self._exec_nb(cmd_log, cwd=self.dirpath, env=self.gitenv):<EOL><INDENT>yield line<EOL><DEDENT>logger.debug("<STR_LIT>",<EOL>self.uri, self.dirpath)<EOL>

Read the commit log from the repository.

        The method returns the Git log of the repository using the
        following options:

            git log --raw --numstat --pretty=fuller --decorate=full
                --all --reverse --topo-order --parents -M -C -c
                --remotes=origin

        When `from_date` is given, it gets the commits equal or older
        than that date. This date is given in a datetime object.

        The list of branches is a list of strings, with the names of the
        branches to fetch. If the list of branches is empty, no commit
        is fetched. If the list of branches is None, all commits
        for all branches will be fetched.

        :param from_date: fetch commits newer than a specific
            date (inclusive)
        :param branches: names of branches to fetch from (default: None)
        :param encoding: encode the log using this format

        :returns: a generator where each item is a line from the log

        :raises EmptyRepositoryError: when the repository is empty and
            the action cannot be performed
        :raises RepositoryError: when an error occurs fetching the log


def show(self, commits=None, encoding='<STR_LIT:utf-8>'):

if self.is_empty():<EOL><INDENT>logger.warning("<STR_LIT>",<EOL>self.uri)<EOL>raise EmptyRepositoryError(repository=self.uri)<EOL><DEDENT>if commits is None:<EOL><INDENT>commits = []<EOL><DEDENT>cmd_show = ['<STR_LIT>', '<STR_LIT>']<EOL>cmd_show.extend(self.GIT_PRETTY_OUTPUT_OPTS)<EOL>cmd_show.extend(commits)<EOL>for line in self._exec_nb(cmd_show, cwd=self.dirpath, env=self.gitenv):<EOL><INDENT>yield line<EOL><DEDENT>logger.debug("<STR_LIT>",<EOL>self.uri, self.dirpath)<EOL>

Show the data of a set of commits.

        The method returns the output of Git show command for a
        set of commits using the following options:

            git show --raw --numstat --pretty=fuller --decorate=full
                --parents -M -C -c [<commit>...<commit>]

        When the list of commits is empty, the command will return
        data about the last commit, like the default behaviour of
        `git show`.

        :param commits: list of commits to show data
        :param encoding: encode the output using this format

        :returns: a generator where each item is a line from the show output

        :raises EmptyRepositoryError: when the repository is empty and
            the action cannot be performed
        :raises RepositoryError: when an error occurs fetching the show output


def prepare_refs(refs):<EOL><INDENT>return [ref.hash.encode('<STR_LIT:utf-8>') for ref in refs<EOL>if not ref.refname.endswith('<STR_LIT>')]<EOL><DEDENT>def determine_wants(refs):<EOL><INDENT>remote_refs = prepare_refs(self._discover_refs(remote=True))<EOL>local_refs = prepare_refs(self._discover_refs())<EOL>wants = [ref for ref in remote_refs if ref not in local_refs]<EOL>return wants<EOL><DEDENT>client, repo_path = dulwich.client.get_transport_and_path(self.uri)<EOL>repo = dulwich.repo.Repo(self.dirpath)<EOL>fd = io.BytesIO()<EOL>local_refs = self._discover_refs()<EOL>graph_walker = _GraphWalker(local_refs)<EOL>result = client.fetch_pack(repo_path,<EOL>determine_wants,<EOL>graph_walker,<EOL>fd.write)<EOL>refs = [GitRef(ref_hash.decode('<STR_LIT:utf-8>'), ref_name.decode('<STR_LIT:utf-8>'))<EOL>for ref_name, ref_hash in result.refs.items()]<EOL>if len(fd.getvalue()) > <NUM_LIT:0>:<EOL><INDENT>fd.seek(<NUM_LIT:0>)<EOL>pack = repo.object_store.add_thin_pack(fd.read, None)<EOL>pack_name = pack.name().decode('<STR_LIT:utf-8>')<EOL><DEDENT>else:<EOL><INDENT>pack_name = None<EOL><DEDENT>return (pack_name, refs)<EOL>


filepath = '<STR_LIT>' + packet_name<EOL>cmd_verify_pack = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', filepath]<EOL>outs = self._exec(cmd_verify_pack, cwd=self.dirpath, env=self.gitenv)<EOL>outs = outs.decode('<STR_LIT:utf-8>', errors='<STR_LIT>').rstrip()<EOL>lines = [line.split('<STR_LIT:U+0020>') for line in outs.split('<STR_LIT:\n>')]<EOL>commits = [parts[<NUM_LIT:0>] for parts in lines if parts[<NUM_LIT:1>] == '<STR_LIT>']<EOL>commits.reverse()<EOL>return commits<EOL>