<EOL>if nparr.dtype.itemsize not in (<NUM_LIT:1>, <NUM_LIT:2>, <NUM_LIT:4>):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if len(nparr.shape) == <NUM_LIT:1>:<EOL><INDENT>arrays = [nparr]<EOL><DEDENT>elif len(nparr.shape) == <NUM_LIT:2>:<EOL><INDENT>arrays = [nparr[i,:] for i in range(nparr.shape[<NUM_LIT:0>])]<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>interleaved = np.vstack(arrays).reshape((-<NUM_LIT:1>,), order='<STR_LIT:F>')<EOL>dubseg = pydub.AudioSegment(interleaved.tobytes(),<EOL>frame_rate=framerate,<EOL>sample_width=interleaved.dtype.itemsize,<EOL>channels=len(interleaved.shape)<EOL>)<EOL>return AudioSegment(dubseg, "<STR_LIT>")<EOL>

Returns an AudioSegment created from the given numpy array.

The numpy array must have shape = (num_samples, num_channels).

:param nparr: The numpy array to create an AudioSegment from.
:returns: An AudioSegment created from the given array.


seg = pydub.AudioSegment.silent(duration=duration, frame_rate=frame_rate)<EOL>return AudioSegment(seg, "<STR_LIT>")<EOL>

Creates an AudioSegment object of the specified duration/frame_rate filled with digital silence.

:param duration: The duration of the returned object in ms.
:param frame_rate: The samples per second of the returned object.
:returns: AudioSegment object filled with pure digital silence.


arr = self.to_numpy_array()<EOL>if len(arr) == <NUM_LIT:0>:<EOL><INDENT>return <NUM_LIT:0.0><EOL><DEDENT>else:<EOL><INDENT>rms = self.rms<EOL>ratio = rms / P_REF_PCM<EOL>return <NUM_LIT> * np.log10(ratio + <NUM_LIT>)<EOL><DEDENT>

Sound Pressure Level - defined as 20 * log10(p/p0),
where p is the RMS of the sound wave in Pascals and p0 is
20 micro Pascals.

Since we would need to know calibration information about the
microphone used to record the sound in order to transform
the PCM values of this audiosegment into Pascals, we can't really
give an accurate SPL measurement.

However, we can give a reasonable guess that can certainly be used
to compare two sounds taken from the same microphone set up.

Be wary about using this to compare sounds taken under different recording
conditions however, except as a simple approximation.

Returns a scalar float representing the dB SPL of this audiosegment.


def filter_bank(self, lower_bound_hz=<NUM_LIT:50>, upper_bound_hz=<NUM_LIT>, nfilters=<NUM_LIT>, mode='<STR_LIT>'):

<EOL>data = self.to_numpy_array()<EOL>if mode.lower() == '<STR_LIT>':<EOL><INDENT>frequencies = librosa.core.mel_frequencies(n_mels=nfilters, fmin=lower_bound_hz, fmax=upper_bound_hz)<EOL><DEDENT>elif mode.lower() == '<STR_LIT>':<EOL><INDENT>frequencies = np.linspace(lower_bound_hz, upper_bound_hz, num=nfilters, endpoint=True)<EOL><DEDENT>elif mode.lower() == '<STR_LIT>':<EOL><INDENT>start = np.log10(lower_bound_hz)<EOL>stop = np.log10(upper_bound_hz)<EOL>frequencies = np.logspace(start, stop, num=nfilters, endpoint=True, base=<NUM_LIT>)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>".format(mode))<EOL><DEDENT>rows = [filters.bandpass_filter(data, freq*<NUM_LIT>, freq*<NUM_LIT>, self.frame_rate) for freq in frequencies]<EOL>rows = np.array(rows)<EOL>spect = np.vstack(rows)<EOL>return spect, frequencies<EOL>

Returns a numpy array of shape (nfilters, nsamples), where each
row of data is the result of bandpass filtering the audiosegment
around a particular frequency. The frequencies are
spaced from `lower_bound_hz` to `upper_bound_hz` and are returned with
the np array. The particular spacing of the frequencies depends on `mode`,
which can be either: 'linear', 'mel', or 'log'.

.. note:: This method is an approximation of a gammatone filterbank
          until I get around to writing an actual gammatone filterbank
          function.

.. code-block:: python

    # Example usage
    import audiosegment
    import matplotlib.pyplot as plt
    import numpy as np

    def visualize(spect, frequencies, title=""):
        # Visualize the result of calling seg.filter_bank() for any number of filters
        i = 0
        for freq, (index, row) in zip(frequencies[::-1], enumerate(spect[::-1, :])):
            plt.subplot(spect.shape[0], 1, index + 1)
            if i == 0:
                plt.title(title)
                i += 1
            plt.ylabel("{0:.0f}".format(freq))
            plt.plot(row)
        plt.show()

    seg = audiosegment.from_file("some_audio.wav").resample(sample_rate_Hz=24000, sample_width=2, channels=1)
    spec, frequencies = seg.filter_bank(nfilters=5)
    visualize(spec, frequencies)

.. image:: images/filter_bank.png

:param lower_bound_hz:  The lower bound of the frequencies to use in the bandpass filters.
:param upper_bound_hz:  The upper bound of the frequencies to use in the bandpass filters.
:param nfilters:        The number of filters to apply. This will determine which frequencies
                        are used as well, as they are interpolated between
                        `lower_bound_hz` and `upper_bound_hz` based on `mode`.
:param mode:            The way the frequencies are spaced. Options are: `linear`, in which case
                        the frequencies are linearly interpolated between `lower_bound_hz` and
                        `upper_bound_hz`, `mel`, in which case the mel frequencies are used,
                        or `log`, in which case they are log-10 spaced.
:returns:               A numpy array of the form (nfilters, nsamples), where each row is the
                        audiosegment, bandpass-filtered around a particular frequency,
                        and the list of frequencies. I.e., returns (spec, freqs).


def auditory_scene_analysis(self, debug=False, debugplot=False):

normalized = self.normalize_spl_by_average(db=<NUM_LIT>)<EOL>def printd(*args, **kwargs):<EOL><INDENT>if debug:<EOL><INDENT>print(*args, **kwargs)<EOL><DEDENT><DEDENT>printd("<STR_LIT>")<EOL>spect, frequencies = normalized.filter_bank(nfilters=<NUM_LIT>)  <EOL>printd("<STR_LIT>")<EOL>with warnings.catch_warnings():  <EOL><INDENT>warnings.simplefilter("<STR_LIT:ignore>")<EOL>spect[spect < <NUM_LIT:0>] = <NUM_LIT:0><EOL><DEDENT>printd("<STR_LIT>")<EOL>low_boundary = <NUM_LIT:30><EOL>order = <NUM_LIT:6><EOL>spect = np.apply_along_axis(filters.lowpass_filter, <NUM_LIT:1>, spect, low_boundary, self.frame_rate, order)<EOL>printd("<STR_LIT>")<EOL>downsample_freq_hz = <NUM_LIT><EOL>if self.frame_rate > downsample_freq_hz:<EOL><INDENT>step = int(round(self.frame_rate / downsample_freq_hz))<EOL>spect = spect[:, ::step]<EOL><DEDENT>scales = [(<NUM_LIT:6>, <NUM_LIT:1>/<NUM_LIT:4>), (<NUM_LIT:6>, <NUM_LIT:1>/<NUM_LIT>), (<NUM_LIT:1>/<NUM_LIT:2>, <NUM_LIT:1>/<NUM_LIT>)]<EOL>gaussian = lambda x, mu, sig: np.exp(-np.power(x - mu, <NUM_LIT>) / (<NUM_LIT:2> * np.power(sig, <NUM_LIT>)))<EOL>gaussian_kernel = lambda sig: gaussian(np.linspace(-<NUM_LIT:10>, <NUM_LIT:10>, len(frequencies) / <NUM_LIT:2>), <NUM_LIT:0>, sig)<EOL>spectrograms = []<EOL>printd("<STR_LIT>")<EOL>for sc, st in scales:<EOL><INDENT>printd("<STR_LIT>", sc, st)<EOL>printd("<STR_LIT>")<EOL>time_smoothed = np.apply_along_axis(filters.lowpass_filter, <NUM_LIT:1>, spect, <NUM_LIT:1>/st, downsample_freq_hz, <NUM_LIT:6>)<EOL>freq_smoothed = np.apply_along_axis(np.convolve, <NUM_LIT:0>, time_smoothed, gaussian_kernel(sc), '<STR_LIT>')<EOL>printd("<STR_LIT>")<EOL>with warnings.catch_warnings():<EOL><INDENT>warnings.simplefilter("<STR_LIT:ignore>")<EOL>freq_smoothed[freq_smoothed > <NUM_LIT>] = <NUM_LIT><EOL>freq_smoothed[freq_smoothed < -<NUM_LIT>] = -<NUM_LIT><EOL><DEDENT>spectrograms.append(freq_smoothed)<EOL><DEDENT>segmasks = []<EOL>printd("<STR_LIT>")<EOL>for spect, (sc, st) in zip(spectrograms, scales):<EOL><INDENT>printd("<STR_LIT>", sc, st)<EOL>printd("<STR_LIT>")<EOL>onsets, gradients = asa._compute_peaks_or_valleys_of_first_derivative(spect)<EOL>printd("<STR_LIT>")<EOL>offsets, _ = asa._compute_peaks_or_valleys_of_first_derivative(spect, do_peaks=False)<EOL>printd("<STR_LIT>")<EOL>offsets = asa._correlate_onsets_and_offsets(onsets, offsets, gradients)<EOL>printd("<STR_LIT>")<EOL>onset_fronts = asa._form_onset_offset_fronts(onsets, sample_rate_hz=downsample_freq_hz, threshold_ms=<NUM_LIT:20>)<EOL>offset_fronts = asa._form_onset_offset_fronts(offsets, sample_rate_hz=downsample_freq_hz, threshold_ms=<NUM_LIT:20>)<EOL>printd("<STR_LIT>")<EOL>asa._break_poorly_matched_fronts(onset_fronts)<EOL>printd("<STR_LIT>")<EOL>segmentation_mask = asa._match_fronts(onset_fronts, offset_fronts, onsets, offsets, debug=debug)<EOL>segmasks.append(segmentation_mask)<EOL>break  <EOL><DEDENT>finished_segmentation_mask = segmasks[<NUM_LIT:0>]<EOL>if debugplot:<EOL><INDENT>asa.visualize_segmentation_mask(finished_segmentation_mask, spect, frequencies)<EOL><DEDENT>times = np.arange(<NUM_LIT:2> * downsample_freq_hz * len(self) / MS_PER_S)<EOL>printd("<STR_LIT>", times.shape, finished_segmentation_mask.shape[<NUM_LIT:1>])<EOL>nsamples_for_each_fft = <NUM_LIT:2> * finished_segmentation_mask.shape[<NUM_LIT:0>]<EOL>printd("<STR_LIT>")<EOL>stft_frequencies, stft_times, stft = signal.stft(self.to_numpy_array(), self.frame_rate, nperseg=nsamples_for_each_fft)<EOL>printd("<STR_LIT>", stft.shape)<EOL>printd("<STR_LIT>", stft_frequencies.shape)<EOL>printd("<STR_LIT>", stft_times.shape)<EOL>if stft_frequencies.shape[<NUM_LIT:0>] > finished_segmentation_mask.shape[<NUM_LIT:0>]:<EOL><INDENT>stft_frequencies = stft_frequencies[:finished_segmentation_mask.shape[<NUM_LIT:0>]]<EOL>stft = stft[:stft_frequencies.shape[<NUM_LIT:0>], :]<EOL><DEDENT>finished_segmentation_mask, times, stft, stft_times = asa._downsample_one_or_the_other(stft, stft_times, finished_segmentation_mask, times)<EOL>printd("<STR_LIT>", stft.shape)<EOL>printd("<STR_LIT>", stft_frequencies.shape)<EOL>printd("<STR_LIT>", stft_times.shape)<EOL>printd("<STR_LIT>", finished_segmentation_mask.shape)<EOL>printd("<STR_LIT>", finished_segmentation_mask.shape, "<STR_LIT>", (stft_frequencies.shape[<NUM_LIT:0>], stft_times.shape[<NUM_LIT:0>]))<EOL>finished_segmentation_mask = asa._map_segmentation_mask_to_stft_domain(finished_segmentation_mask, times, frequencies, stft_times, stft_frequencies)<EOL>printd("<STR_LIT>")<EOL>masks = asa._separate_masks(finished_segmentation_mask)<EOL>printd("<STR_LIT>", len(masks))<EOL>if len(masks) == <NUM_LIT:0>:<EOL><INDENT>clone = from_numpy_array(self.to_numpy_array(), self.frame_rate)<EOL>return [clone]<EOL><DEDENT>try:<EOL><INDENT>ncpus = multiprocessing.cpu_count()<EOL><DEDENT>except NotImplementedError:<EOL><INDENT>ncpus = <NUM_LIT:2><EOL><DEDENT>ncpus = len(masks) if len(masks) < ncpus else ncpus<EOL>chunks = np.array_split(masks, ncpus)<EOL>assert len(chunks) == ncpus<EOL>queue = multiprocessing.Queue()<EOL>printd("<STR_LIT>".format(ncpus, len(masks)))<EOL>for i in range(ncpus):<EOL><INDENT>p = multiprocessing.Process(target=asa._asa_task,<EOL>args=(queue, chunks[i], stft, self.sample_width, self.frame_rate, nsamples_for_each_fft),<EOL>daemon=True)<EOL>p.start()<EOL><DEDENT>results = []<EOL>dones = []<EOL>while len(dones) < ncpus:<EOL><INDENT>item = queue.get()<EOL>if type(item) == str and item == "<STR_LIT>":<EOL><INDENT>dones.append(item)<EOL><DEDENT>else:<EOL><INDENT>wav = from_numpy_array(item, self.frame_rate)<EOL>results.append(wav)<EOL><DEDENT><DEDENT>return results<EOL>

Algorithm based on paper: Auditory Segmentation Based on Onset and Offset Analysis,
by Hu and Wang, 2007.

Returns a list of AudioSegments, each of which is all the sound during this AudioSegment's duration from
a particular source. That is, if there are several overlapping sounds in this AudioSegment, this
method will return one AudioSegment object for each of those sounds. At least, that's the idea.

Current version is very much in alpha, and while it shows promise, will require quite a bit more
tuning before it can really claim to work.

:param debug:       If `True` will print out debug outputs along the way. Useful if you want to see why it is
                    taking so long.
:param debugplot:   If `True` will use Matplotlib to plot the resulting spectrogram masks in Mel frequency scale.
:returns:           List of AudioSegment objects, each of which is from a particular sound source.


assert self.frame_rate in (<NUM_LIT>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT>), "<STR_LIT>"<EOL>assert self.sample_width == <NUM_LIT:2>, "<STR_LIT>"<EOL>assert self.channels == <NUM_LIT:1>, "<STR_LIT>"<EOL>class model_class:<EOL><INDENT>def __init__(self, aggressiveness):<EOL><INDENT>self.v = webrtcvad.Vad(int(aggressiveness))<EOL><DEDENT>def predict(self, vector):<EOL><INDENT>if self.v.is_speech(vector.raw_data, vector.frame_rate):<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT><DEDENT><DEDENT>model = model_class(aggressiveness=<NUM_LIT:2>)<EOL>pyesno = <NUM_LIT>  <EOL>pnoyes = <NUM_LIT>  <EOL>p_realyes_outputyes = <NUM_LIT>  <EOL>p_realyes_outputno  = <NUM_LIT>  <EOL>p_yes_raw = prob_detect_voice<EOL>filtered = self.detect_event(model=model,<EOL>ms_per_input=<NUM_LIT:20>,<EOL>transition_matrix=(pyesno, pnoyes),<EOL>model_stats=(p_realyes_outputyes, p_realyes_outputno),<EOL>event_length_s=<NUM_LIT>,<EOL>prob_raw_yes=p_yes_raw)<EOL>ret = []<EOL>for tup in filtered:<EOL><INDENT>t = ('<STR_LIT:v>', tup[<NUM_LIT:1>]) if tup[<NUM_LIT:0>] == '<STR_LIT:y>' else ('<STR_LIT:u>', tup[<NUM_LIT:1>])<EOL>ret.append(t)<EOL><DEDENT>return ret<EOL>

Returns self as a list of tuples:
[('v', voiced segment), ('u', unvoiced segment), (etc.)]

The overall order of the AudioSegment is preserved.

:param prob_detect_voice: The raw probability that any random 20ms window of the audio file
                          contains voice.
:returns: The described list.


def dice(self, seconds, zero_pad=False):

try:<EOL><INDENT>total_s = sum(seconds)<EOL>if not (self.duration_seconds <= total_s + <NUM_LIT:1> and self.duration_seconds >= total_s - <NUM_LIT:1>):<EOL><INDENT>raise ValueError("<STR_LIT>" % (total_s, self.duration_seconds))<EOL><DEDENT>starts = []<EOL>stops = []<EOL>time_ms = <NUM_LIT:0><EOL>for dur in seconds:<EOL><INDENT>starts.append(time_ms)<EOL>time_ms += dur * MS_PER_S<EOL>stops.append(time_ms)<EOL><DEDENT>zero_pad = False<EOL><DEDENT>except TypeError:<EOL><INDENT>starts = range(<NUM_LIT:0>, int(round(self.duration_seconds * MS_PER_S)), int(round(seconds * MS_PER_S)))<EOL>stops = (min(self.duration_seconds * MS_PER_S, start + seconds * MS_PER_S) for start in starts)<EOL><DEDENT>outs = [self[start:stop] for start, stop in zip(starts, stops)]<EOL>out_lens = [out.duration_seconds for out in outs]<EOL>if zero_pad and not (out_lens[-<NUM_LIT:1>] <= seconds * MS_PER_S + <NUM_LIT:1> and out_lens[-<NUM_LIT:1>] >= seconds * MS_PER_S - <NUM_LIT:1>):<EOL><INDENT>num_zeros = self.frame_rate * (seconds * MS_PER_S - out_lens[-<NUM_LIT:1>])<EOL>outs[-<NUM_LIT:1>] = outs[-<NUM_LIT:1>].zero_extend(num_samples=num_zeros)<EOL><DEDENT>return outs<EOL>

Cuts the AudioSegment into `seconds` segments (at most). So for example, if seconds=10,
this will return a list of AudioSegments, in order, where each one is at most 10 seconds
long. If `zero_pad` is True, the last item AudioSegment object will be zero padded to result
in `seconds` seconds.

:param seconds: The length of each segment in seconds. Can be either a float/int, in which case
                `self.duration_seconds` / `seconds` are made, each of `seconds` length, or a
                list-like can be given, in which case the given list must sum to
                `self.duration_seconds` and each segment is specified by the list - e.g.
                the 9th AudioSegment in the returned list will be `seconds[8]` seconds long.
:param zero_pad: Whether to zero_pad the final segment if necessary. Ignored if `seconds` is
                 a list-like.
:returns: A list of AudioSegments, each of which is the appropriate number of seconds long.
:raises: ValueError if a list-like is given for `seconds` and the list's durations do not sum
         to `self.duration_seconds`.


def detect_event(self, model, ms_per_input, transition_matrix, model_stats, event_length_s,<EOL>start_as_yes=False, prob_raw_yes=<NUM_LIT:0.5>):

if ms_per_input < <NUM_LIT:0> or ms_per_input / MS_PER_S > self.duration_seconds:<EOL><INDENT>raise ValueError("<STR_LIT>""<STR_LIT>" + str(ms_per_input))<EOL><DEDENT>elif not hasattr(transition_matrix, "<STR_LIT>") or len(transition_matrix) != <NUM_LIT:2>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>elif not hasattr(model_stats, "<STR_LIT>") or len(model_stats) != <NUM_LIT:2>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>elif any([True for prob in transition_matrix if prob > <NUM_LIT:1.0> or prob < <NUM_LIT:0.0>]):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>elif any([True for prob in model_stats if prob > <NUM_LIT:1.0> or prob < <NUM_LIT:0.0>]):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>elif prob_raw_yes > <NUM_LIT:1.0> or prob_raw_yes < <NUM_LIT:0.0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>filter_indices = [yes_or_no for yes_or_no in detect._get_filter_indices(self,<EOL>start_as_yes,<EOL>prob_raw_yes,<EOL>ms_per_input,<EOL>model,<EOL>transition_matrix,<EOL>model_stats)]<EOL>ret = detect._homogeneity_filter(filter_indices, window_size=int(round(<NUM_LIT> * MS_PER_S / ms_per_input)))<EOL>ret = detect._group_filter_values(self, ret, ms_per_input)<EOL>real_ret = []<EOL>for i, (this_yesno, next_timestamp) in enumerate(ret):<EOL><INDENT>if i > <NUM_LIT:0>:<EOL><INDENT>_next_yesno, timestamp = ret[i - <NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>timestamp = <NUM_LIT:0><EOL><DEDENT>ms_per_s = <NUM_LIT:1000><EOL>data = self[timestamp * ms_per_s:next_timestamp * ms_per_s].raw_data<EOL>seg = AudioSegment(pydub.AudioSegment(data=data, sample_width=self.sample_width,<EOL>frame_rate=self.frame_rate, channels=self.channels), self.name)<EOL>real_ret.append((this_yesno, seg))<EOL><DEDENT>return real_ret<EOL>

A list of tuples of the form [('n', AudioSegment), ('y', AudioSegment), etc.] is returned, where tuples
of the form ('n', AudioSegment) are the segments of sound where the event was not detected,
while ('y', AudioSegment) tuples were the segments of sound where the event was detected.

.. code-block:: python

    # Example usage
    import audiosegment
    import keras
    import keras.models
    import numpy as np
    import sys

    class Model:
        def __init__(self, modelpath):
            self.model = keras.models.load_model(modelpath)

        def predict(self, seg):
            _bins, fft_vals = seg.fft()
            fft_vals = np.abs(fft_vals) / len(fft_vals)
            predicted_np_form = self.model.predict(np.array([fft_vals]), batch_size=1)
            prediction_as_int = int(round(predicted_np_form[0][0]))
            return prediction_as_int

    modelpath = sys.argv[1]
    wavpath = sys.argv[2]
    model = Model(modelpath)
    seg = audiosegment.from_file(wavpath).resample(sample_rate_Hz=32000, sample_width=2, channels=1)
    pyes_to_no = 0.3  # The probability of one 30 ms sample being an event, and the next one not
    pno_to_yes = 0.2  # The probability of one 30 ms sample not being an event, and the next one yes
    ptrue_pos_rate = 0.8  # The true positive rate (probability of a predicted yes being right)
    pfalse_neg_rate = 0.3  # The false negative rate (probability of a predicted no being wrong)
    raw_prob = 0.7  # The raw probability of seeing the event in any random 30 ms slice of this file
    events = seg.detect_event(model, ms_per_input=30, transition_matrix=[pyes_to_no, pno_to_yes],
                              model_stats=[ptrue_pos_rate, pfalse_neg_rate], event_length_s=0.25,
                              prob_raw_yes=raw_prob)
    nos = [event[1] for event in events if event[0] == 'n']
    yeses = [event[1] for event in events if event[0] == 'y']
    if len(nos) > 1:
        notdetected = nos[0].reduce(nos[1:])
        notdetected.export("notdetected.wav", format="WAV")
    if len(yeses) > 1:
        detected = yeses[0].reduce(yeses[1:])
        detected.export("detected.wav", format="WAV")

:param model:               The model. The model must have a predict() function which takes an AudioSegment
                            of `ms_per_input` number of ms and which outputs 1 if the audio event is detected
                            in that input, and 0 if not. Make sure to resample the AudioSegment to the right
                            values before calling this function on it.

:param ms_per_input:        The number of ms of AudioSegment to be fed into the model at a time. If this does not
                            come out even, the last AudioSegment will be zero-padded.

:param transition_matrix:   An iterable of the form: [p(yes->no), p(no->yes)]. That is, the probability of moving
                            from a 'yes' state to a 'no' state and the probability of vice versa.

:param model_stats:         An iterable of the form: [p(reality=1|output=1), p(reality=1|output=0)]. That is,
                            the probability of the ground truth really being a 1, given that the model output a 1,
                            and the probability of the ground truth being a 1, given that the model output a 0.

:param event_length_s:      The typical duration of the event you are looking for in seconds (can be a float).

:param start_as_yes:        If True, the first `ms_per_input` will be in the 'y' category. Otherwise it will be
                            in the 'n' category.

:param prob_raw_yes:        The raw probability of finding the event in any given `ms_per_input` vector.

:returns:                   A list of tuples of the form [('n', AudioSegment), ('y', AudioSegment), etc.],
                            where over the course of the list, the AudioSegment in tuple 3 picks up
                            where the one in tuple 2 left off.

:raises:                    ValueError if `ms_per_input` is negative or larger than the number of ms in this
                            AudioSegment; if `transition_matrix` or `model_stats` do not have a __len__ attribute
                            or are not length 2; if the values in `transition_matrix` or `model_stats` are not
                            in the closed interval [0.0, 1.0].


def _execute_sox_cmd(self, cmd, console_output=False):

on_windows = platform.system().lower() == "<STR_LIT>"<EOL>def _get_random_tmp_file():<EOL><INDENT>if on_windows:<EOL><INDENT>rand_string = "<STR_LIT>".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(<NUM_LIT:8>))<EOL>tmp = self.name + "<STR_LIT:_>" + rand_string<EOL>WinTempFile = collections.namedtuple("<STR_LIT>", "<STR_LIT:name>")<EOL>tmp = WinTempFile(tmp)<EOL><DEDENT>else:<EOL><INDENT>tmp = tempfile.NamedTemporaryFile()<EOL><DEDENT>return tmp<EOL><DEDENT>tmp = _get_random_tmp_file()<EOL>othertmp = _get_random_tmp_file()<EOL>self.export(tmp.name, format="<STR_LIT>")<EOL>stdout = stderr = subprocess.PIPE if console_output else subprocess.DEVNULL<EOL>command = cmd.format(inputfile=tmp.name, outputfile=othertmp.name)<EOL>res = subprocess.call(command.split('<STR_LIT:U+0020>'), stdout=stdout, stderr=stderr)<EOL>assert res == <NUM_LIT:0>, "<STR_LIT>"<EOL>other = AudioSegment(pydub.AudioSegment.from_wav(othertmp.name), self.name)<EOL>if on_windows:<EOL><INDENT>os.remove(tmp.name)<EOL>os.remove(othertmp.name)<EOL><DEDENT>else:<EOL><INDENT>tmp.close()<EOL>othertmp.close()<EOL><DEDENT>return other<EOL>

Executes a Sox command in a platform-independent manner.

`cmd` must be a format string that includes {inputfile} and {outputfile}.


def filter_silence(self, duration_s=<NUM_LIT:1>, threshold_percentage=<NUM_LIT:1>, console_output=False):

command = "<STR_LIT>"+ str(threshold_percentage) + "<STR_LIT>" + str(float(duration_s)) + "<STR_LIT:U+0020>" + str(threshold_percentage) + "<STR_LIT:%>"<EOL>try:<EOL><INDENT>result = self._execute_sox_cmd(command)<EOL><DEDENT>except pydub.exceptions.CouldntDecodeError:<EOL><INDENT>warnings.warn("<STR_LIT>", stacklevel=<NUM_LIT:2>)<EOL>result = AudioSegment(self.seg, self.name)<EOL><DEDENT>return result<EOL>

Returns a copy of this AudioSegment, but whose silence has been removed.

.. note:: This method requires that you have the program 'sox' installed.

.. warning:: This method uses the program 'sox' to perform the task. While this is very fast for a single
             function call, the IO may add up for large numbers of AudioSegment objects.

:param duration_s: The number of seconds of "silence" that must be present in a row to
                   be stripped.
:param threshold_percentage: Silence is defined as any samples whose absolute value is below
                             `threshold_percentage * max(abs(samples in this segment))`.
:param console_output: If True, will pipe all sox output to the console.
:returns: A copy of this AudioSegment, but whose silence has been removed.


def fft(self, start_s=None, duration_s=None, start_sample=None, num_samples=None, zero_pad=False):

if start_s is not None and start_sample is not None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if duration_s is not None and num_samples is not None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if start_s is None and start_sample is None:<EOL><INDENT>start_sample = <NUM_LIT:0><EOL><DEDENT>if duration_s is None and num_samples is None:<EOL><INDENT>num_samples = len(self.get_array_of_samples()) - int(start_sample)<EOL><DEDENT>if duration_s is not None:<EOL><INDENT>num_samples = int(round(duration_s * self.frame_rate))<EOL><DEDENT>if start_s is not None:<EOL><INDENT>start_sample = int(round(start_s * self.frame_rate))<EOL><DEDENT>end_sample = start_sample + num_samples  <EOL>if end_sample > len(self.get_array_of_samples()) and not zero_pad:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>elif end_sample > len(self.get_array_of_samples()) and zero_pad:<EOL><INDENT>arr = np.array(self.get_array_of_samples())<EOL>zeros = np.zeros(end_sample - len(arr))<EOL>arr = np.append(arr, zeros)<EOL><DEDENT>else:<EOL><INDENT>arr = np.array(self.get_array_of_samples())<EOL><DEDENT>audioslice = np.array(arr[start_sample:end_sample])<EOL>fft_result = np.fft.fft(audioslice)[range(int(round(num_samples/<NUM_LIT:2>)) + <NUM_LIT:1>)]<EOL>step_size = self.frame_rate / num_samples<EOL>bins = np.arange(<NUM_LIT:0>, int(round(num_samples/<NUM_LIT:2>)) + <NUM_LIT:1>, <NUM_LIT:1.0>) * step_size<EOL>return bins, fft_result<EOL>

Transforms the indicated slice of the AudioSegment into the frequency domain and returns the bins
and the values.

If neither `start_s` or `start_sample` is specified, the first sample of the slice will be the first sample
of the AudioSegment.

If neither `duration_s` or `num_samples` is specified, the slice will be from the specified start
to the end of the segment.

.. code-block:: python

    # Example for plotting the FFT using this function
    import matplotlib.pyplot as plt
    import numpy as np

    seg = audiosegment.from_file("furelise.wav")
    # Just take the first 3 seconds
    hist_bins, hist_vals = seg[1:3000].fft()
    hist_vals_real_normed = np.abs(hist_vals) / len(hist_vals)
    plt.plot(hist_bins / 1000, hist_vals_real_normed)
    plt.xlabel("kHz")
    plt.ylabel("dB")
    plt.show()

.. image:: images/fft.png

:param start_s: The start time in seconds. If this is specified, you cannot specify `start_sample`.
:param duration_s: The duration of the slice in seconds. If this is specified, you cannot specify `num_samples`.
:param start_sample: The zero-based index of the first sample to include in the slice.
                     If this is specified, you cannot specify `start_s`.
:param num_samples: The number of samples to include in the slice. If this is specified, you cannot
                    specify `duration_s`.
:param zero_pad: If True and the combination of start and duration result in running off the end of
                 the AudioSegment, the end is zero padded to prevent this.
:returns: np.ndarray of frequencies in Hz, np.ndarray of amount of each frequency
:raises: ValueError If `start_s` and `start_sample` are both specified and/or if both `duration_s` and
                    `num_samples` are specified.


def generate_frames(self, frame_duration_ms, zero_pad=True):

Frame = collections.namedtuple("<STR_LIT>", "<STR_LIT>")<EOL>bytes_per_frame = int(self.frame_rate * (frame_duration_ms / <NUM_LIT:1000>) * self.sample_width)<EOL>offset = <NUM_LIT:0>  <EOL>timestamp = <NUM_LIT:0.0>  <EOL>frame_duration_s = (bytes_per_frame / self.frame_rate) / self.sample_width<EOL>while offset + bytes_per_frame < len(self.raw_data):<EOL><INDENT>yield Frame(self.raw_data[offset:offset + bytes_per_frame], timestamp, frame_duration_s)<EOL>timestamp += frame_duration_s<EOL>offset += bytes_per_frame<EOL><DEDENT>if zero_pad:<EOL><INDENT>rest = self.raw_data[offset:]<EOL>zeros = bytes(bytes_per_frame - len(rest))<EOL>yield Frame(rest + zeros, timestamp, frame_duration_s)<EOL><DEDENT>

Yields self's data in chunks of frame_duration_ms.

This function adapted from pywebrtc's example [https://github.com/wiseman/py-webrtcvad/blob/master/example.py].

:param frame_duration_ms: The length of each frame in ms.
:param zero_pad: Whether or not to zero pad the end of the AudioSegment object to get all
                 the audio data out as frames. If not, there may be a part at the end
                 of the Segment that is cut off (the part will be <= `frame_duration_ms` in length).
:returns: A Frame object with properties 'bytes (the data)', 'timestamp (start time)', and 'duration'.


def generate_frames_as_segments(self, frame_duration_ms, zero_pad=True):

for frame in self.generate_frames(frame_duration_ms, zero_pad=zero_pad):<EOL><INDENT>seg = AudioSegment(pydub.AudioSegment(data=frame.bytes, sample_width=self.sample_width,<EOL>frame_rate=self.frame_rate, channels=self.channels), self.name)<EOL>yield seg, frame.timestamp<EOL><DEDENT>

Does the same thing as `generate_frames`, but yields tuples of (AudioSegment, timestamp) instead of Frames.


hist_bins, hist_vals = self.fft()<EOL>hist_vals_real_normed = np.abs(hist_vals) / len(hist_vals)<EOL>f_characteristic = hist_bins[np.argmax(hist_vals_real_normed)]<EOL>threshold_fc = <NUM_LIT> - (<NUM_LIT> * f_characteristic) + (<NUM_LIT> * f_characteristic ** <NUM_LIT:2>) - (<NUM_LIT> * f_characteristic ** <NUM_LIT:3>)<EOL>return self.spl >= threshold_fc<EOL>

Returns an estimate of whether this AudioSegment is mostly human audible or not.
This is done by taking an FFT of the segment and checking if the SPL of the segment
falls below the function `f(x) = 40.11453 - 0.01683697x + 1.406211e-6x^2 - 2.371512e-11x^3`,
where x is the most characteristic frequency in the FFT.

Note that this method is essentially trying to determine if the estimated SPL of the segment
falls below the threshold of human hearing, which changes over frequency. If you graph the
threshold over different frequencies, you get what is called an audiogram. The equation above
is derived as a curve that tries to fit a typical audiogram, specifically as found in
Hearing Thresholds by Yost and Killion, 1997 (see https://www.etymotic.com/media/publications/erl-0096-1997.pdf).

Sources of error are: 1) The SPL of an AudioSegment is merely an approximation; 2) this curve
is not a perfect fit, and besides, it is only an approximation of a typical audiogram; 3) the
algorithm uses a characteristic frequency, which is only really going to be a thing for
short segments or for segments which are dominated by a single frequency.

:returns: `True` if we estimate that this sound is mostly human audible. `False` if we think
          it is not.


arr = self.to_numpy_array().copy()<EOL>if len(arr) == <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>def rms(x):<EOL><INDENT>return np.sqrt(np.mean(np.square(x)))<EOL><DEDENT>desired_rms = P_REF_PCM * ((<NUM_LIT:10> ** (db/<NUM_LIT>)) - <NUM_LIT>)<EOL>max_ntries = <NUM_LIT:50><EOL>res_rms = <NUM_LIT:0.0><EOL>ntries = <NUM_LIT:0><EOL>factor = <NUM_LIT:0.1><EOL>left = <NUM_LIT:0.0><EOL>right = desired_rms<EOL>while (ntries < max_ntries) and not util.isclose(res_rms, desired_rms, abs_tol=<NUM_LIT:0.1>):<EOL><INDENT>res_rms = rms(arr * factor)<EOL>if res_rms < desired_rms:<EOL><INDENT>left = factor<EOL><DEDENT>else:<EOL><INDENT>right = factor<EOL><DEDENT>factor = <NUM_LIT:0.5> * (left + right)<EOL>ntries += <NUM_LIT:1><EOL><DEDENT>dtype_dict = {<NUM_LIT:1>: np.int8, <NUM_LIT:2>: np.int16, <NUM_LIT:4>: np.int32}<EOL>dtype = dtype_dict[self.sample_width]<EOL>new_seg = from_numpy_array(np.array(arr * factor, dtype=dtype), self.frame_rate)<EOL>return new_seg<EOL>

Normalize the values in the AudioSegment so that its `spl` property
gives `db`.

.. note:: This method is currently broken - it returns an AudioSegment whose
          values are much smaller than reasonable, yet which yield an SPL value
          that equals the given `db`. Such an AudioSegment will not be serializable
          as a WAV file, which will also break any method that relies on SOX.
          I may remove this method in the future, since the SPL of an AudioSegment is
          pretty questionable to begin with.

:param db: The decibels to normalize average to.
:returns: A new AudioSegment object whose values are changed so that their
          average is `db`.
:raises: ValueError if there are no samples in this AudioSegment.


ret = AudioSegment(self.seg, self.name)<EOL>selfdata = [self.seg._data]<EOL>otherdata = [o.seg._data for o in others]<EOL>ret.seg._data = b'<STR_LIT>'.join(selfdata + otherdata)<EOL>return ret<EOL>

Reduces others into this one by concatenating all the others onto this one and
returning the result. Does not modify self, instead, makes a copy and returns that.

:param others: The other AudioSegment objects to append to this one.
:returns: The concatenated result.


def resample(self, sample_rate_Hz=None, sample_width=None, channels=None, console_output=False):

if sample_rate_Hz is None:<EOL><INDENT>sample_rate_Hz = self.frame_rate<EOL><DEDENT>if sample_width is None:<EOL><INDENT>sample_width = self.sample_width<EOL><DEDENT>if channels is None:<EOL><INDENT>channels = self.channels<EOL><DEDENT>command = "<STR_LIT>" + str(sample_width * <NUM_LIT:8>) + "<STR_LIT>" + str(sample_rate_Hz)+ "<STR_LIT>" + str(channels)<EOL>return self._execute_sox_cmd(command, console_output=console_output)<EOL>

Returns a new AudioSegment whose data is the same as this one, but which has been resampled to the
specified characteristics. Any parameter left None will be unchanged.

.. note:: This method requires that you have the program 'sox' installed.

.. warning:: This method uses the program 'sox' to perform the task. While this is very fast for a single
             function call, the IO may add up for large numbers of AudioSegment objects.

:param sample_rate_Hz: The new sample rate in Hz.
:param sample_width: The new sample width in bytes, so sample_width=2 would correspond to 16 bit (2 byte) width.
:param channels: The new number of channels.
:param console_output: Will print the output of sox to the console if True.
:returns: The newly sampled AudioSegment.


return {'<STR_LIT:name>': self.name, '<STR_LIT>': self.seg}<EOL>

Serializes into a dict for the pickle protocol.

:returns: The dict to pickle.


Deserializes from a dict for the pickle protocol.

:param d: The dict to unpickle from.


d = self.__getstate__()<EOL>return pickle.dumps({<EOL>'<STR_LIT:name>': d['<STR_LIT:name>'],<EOL>'<STR_LIT>': pickle.dumps(d['<STR_LIT>'], protocol=-<NUM_LIT:1>),<EOL>}, protocol=-<NUM_LIT:1>)<EOL>

Serializes into a bytestring.

:returns: An object of type Bytes.


def spectrogram(self, start_s=None, duration_s=None, start_sample=None, num_samples=None,<EOL>window_length_s=None, window_length_samples=None, overlap=<NUM_LIT:0.5>, window=('<STR_LIT>', <NUM_LIT>)):

if start_s is not None and start_sample is not None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if duration_s is not None and num_samples is not None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if window_length_s is not None and window_length_samples is not None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if window_length_s is None and window_length_samples is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if start_s is None and start_sample is None:<EOL><INDENT>start_sample = <NUM_LIT:0><EOL><DEDENT>elif start_s is not None:<EOL><INDENT>start_sample = int(round(start_s * self.frame_rate))<EOL><DEDENT>if duration_s is None and num_samples is None:<EOL><INDENT>num_samples = len(self.get_array_of_samples()) - int(start_sample)<EOL><DEDENT>elif duration_s is not None:<EOL><INDENT>num_samples = int(round(duration_s * self.frame_rate))<EOL><DEDENT>if window_length_s is not None:<EOL><INDENT>window_length_samples = int(round(window_length_s * self.frame_rate))<EOL><DEDENT>if start_sample + num_samples > len(self.get_array_of_samples()):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>arr = self.to_numpy_array()[start_sample:start_sample+num_samples]<EOL>fs, ts, sxx = signal.spectrogram(arr, self.frame_rate, scaling='<STR_LIT>', nperseg=window_length_samples,<EOL>noverlap=int(round(overlap * window_length_samples)),<EOL>mode='<STR_LIT>', window=window)<EOL>return fs, ts, sxx<EOL>

Does a series of FFTs from `start_s` or `start_sample` for `duration_s` or `num_samples`.
Effectively, transforms a slice of the AudioSegment into the frequency domain across different
time bins.

.. code-block:: python

    # Example for plotting a spectrogram using this function
    import audiosegment
    import matplotlib.pyplot as plt

    #...
    seg = audiosegment.from_file("somebodytalking.wav")
    freqs, times, amplitudes = seg.spectrogram(window_length_s=0.03, overlap=0.5)
    amplitudes = 10 * np.log10(amplitudes + 1e-9)

    # Plot
    plt.pcolormesh(times, freqs, amplitudes)
    plt.xlabel("Time in Seconds")
    plt.ylabel("Frequency in Hz")
    plt.show()

.. image:: images/spectrogram.png

:param start_s: The start time. Starts at the beginning if neither this nor `start_sample` is specified.
:param duration_s: The duration of the spectrogram in seconds. Goes to the end if neither this nor
                   `num_samples` is specified.
:param start_sample: The index of the first sample to use. Starts at the beginning if neither this nor
                     `start_s` is specified.
:param num_samples: The number of samples in the spectrogram. Goes to the end if neither this nor
                    `duration_s` is specified.
:param window_length_s: The length of each FFT in seconds. If the total number of samples in the spectrogram
                        is not a multiple of the window length in samples, the last window will be zero-padded.
:param window_length_samples: The length of each FFT in number of samples. If the total number of samples in the
                        spectrogram is not a multiple of the window length in samples, the last window will
                        be zero-padded.
:param overlap: The fraction of each window to overlap.
:param window: See Scipy's spectrogram-function_.
               This parameter is passed as-is directly into the Scipy spectrogram function. It's documentation is reproduced here:
               Desired window to use. If window is a string or tuple, it is passed to get_window to generate the window values,
               which are DFT-even by default. See get_window for a list of windows and required parameters.
               If window is array_like it will be used directly as the window and its length must be
               `window_length_samples`.
               Defaults to a Tukey window with shape parameter of 0.25.
:returns: Three np.ndarrays: The frequency values in Hz (the y-axis in a spectrogram), the time values starting
          at start time and then increasing by `duration_s` each step (the x-axis in a spectrogram), and
          the dB of each time/frequency bin as a 2D array of shape [len(frequency values), len(duration)].
:raises ValueError: If `start_s` and `start_sample` are both specified, if `duration_s` and `num_samples` are both
                    specified, if the first window's duration plus start time lead to running off the end
                    of the AudioSegment, or if `window_length_s` and `window_length_samples` are either
                    both specified or if they are both not specified.

.. _spectrogram-function: https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.spectrogram.html


dtype_dict = {<EOL><NUM_LIT:1>: np.int8,<EOL><NUM_LIT:2>: np.int16,<EOL><NUM_LIT:4>: np.int32<EOL>}<EOL>dtype = dtype_dict[self.sample_width]<EOL>return np.array(self.get_array_of_samples(), dtype=dtype)<EOL>

Convenience function for `np.array(self.get_array_of_samples())` while
keeping the appropriate dtype.


def zero_extend(self, duration_s=None, num_samples=None):

if duration_s is not None and num_samples is not None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>elif duration_s is not None:<EOL><INDENT>num_samples = self.frame_rate * duration_s<EOL><DEDENT>seg = AudioSegment(self.seg, self.name)<EOL>zeros = silent(duration=num_samples / self.frame_rate, frame_rate=self.frame_rate)<EOL>return zeros.overlay(seg)<EOL>

Adds a number of zeros (digital silence) to the AudioSegment (returning a new one).

:param duration_s: The number of seconds of zeros to add. If this is specified, `num_samples` must be None.
:param num_samples: The number of zeros to add. If this is specified, `duration_s` must be None.
:returns: A new AudioSegment object that has been zero extended.
:raises: ValueError if duration_s and num_samples are both specified.


def isclose(a, b, *, rel_tol=<NUM_LIT>, abs_tol=<NUM_LIT:0.0>):

try:<EOL><INDENT>return math.isclose(a, b, rel_tol=rel_tol, abs_tol=abs_tol)<EOL><DEDENT>except AttributeError:<EOL><INDENT>if (rel_tol < <NUM_LIT:0.0>) or (abs_tol < <NUM_LIT:0.0>):<EOL><INDENT>raise ValueError("<STR_LIT>".format(rel_tol, abs_tol))<EOL><DEDENT>if math.isnan(a) or math.isnan(b):<EOL><INDENT>return False  <EOL><DEDENT>if (a == b):<EOL><INDENT>return True<EOL><DEDENT>if math.isinf(a) or math.isinf(b):<EOL><INDENT>return False  <EOL><DEDENT>diff = abs(a - b)<EOL>return (diff <= rel_tol * abs(b)) or (diff <= rel_tol * abs(a)) or (diff <= abs_tol)<EOL><DEDENT>

Python 3.4 does not have math.isclose, so we need to steal it and add it here.


def _get_filter_indices(seg, start_as_yes, prob_raw_yes, ms_per_input, model, transition_matrix, model_stats):

filter_triggered = <NUM_LIT:1> if start_as_yes else <NUM_LIT:0><EOL>prob_raw_no = <NUM_LIT:1.0> - prob_raw_yes<EOL>for segment, _timestamp in seg.generate_frames_as_segments(ms_per_input):<EOL><INDENT>yield filter_triggered<EOL>observation = int(round(model.predict(segment)))<EOL>assert observation == <NUM_LIT:1> or observation == <NUM_LIT:0>, "<STR_LIT>"+ str(observation)<EOL>prob_hyp_yes_given_last_hyp = <NUM_LIT:1.0> - transition_matrix[<NUM_LIT:0>] if filter_triggered else transition_matrix[<NUM_LIT:1>]<EOL>prob_hyp_no_given_last_hyp  = transition_matrix[<NUM_LIT:0>] if filter_triggered else <NUM_LIT:1.0> - transition_matrix[<NUM_LIT:1>]<EOL>prob_hyp_yes_given_data = model_stats[<NUM_LIT:0>] if observation == <NUM_LIT:1> else model_stats[<NUM_LIT:1>]<EOL>prob_hyp_no_given_data = <NUM_LIT:1.0> - model_stats[<NUM_LIT:0>] if observation == <NUM_LIT:1> else <NUM_LIT:1.0> - model_stats[<NUM_LIT:1>]<EOL>hypothesis_yes = prob_raw_yes * prob_hyp_yes_given_last_hyp * prob_hyp_yes_given_data<EOL>hypothesis_no  = prob_raw_no * prob_hyp_no_given_last_hyp  * prob_hyp_no_given_data<EOL>distribution = [<NUM_LIT:1> for i in range(int(round(hypothesis_yes * <NUM_LIT:100>)))]<EOL>distribution.extend([<NUM_LIT:0> for i in range(int(round(hypothesis_no * <NUM_LIT:100>)))])<EOL>random.shuffle(distribution)<EOL>filter_triggered = random.choice(distribution)<EOL><DEDENT>

Runs a Markov Decision Process over the given `seg` in chunks of `ms_per_input`, yielding `True` if
this `ms_per_input` chunk has been classified as positive (1) and `False` if this chunk has been
classified as negative (0).

:param seg:                 The AudioSegment to apply this algorithm to.
:param start_as_yes:        If True, the first `ms_per_input` chunk will be classified as positive.
:param prob_raw_yes:        The raw probability of finding the event in any given independently sampled `ms_per_input`.
:param ms_per_input:        The number of ms of AudioSegment to be fed into the model at a time.
:param model:               The model, which must hava predict() function, which takes an AudioSegment of `ms_per_input`
                            number of ms and which outputs 1 if the audio event is detected in that input, 0 if not.
:param transition_matrix:   An iterable of the form: [p(yes->no), p(no->yes)].
:param model_stats:         An iterable of the form: [p(reality=1|output=1), p(reality=1|output=0)].
:yields:                    `True` if the event has been classified in this chunk, `False` otherwise.


def _group_filter_values(seg, filter_indices, ms_per_input):

ret = []<EOL>for filter_value, (_segment, timestamp) in zip(filter_indices, seg.generate_frames_as_segments(ms_per_input)):<EOL><INDENT>if filter_value == <NUM_LIT:1>:<EOL><INDENT>if len(ret) > <NUM_LIT:0> and ret[-<NUM_LIT:1>][<NUM_LIT:0>] == '<STR_LIT:n>':<EOL><INDENT>ret.append(['<STR_LIT:y>', timestamp])  <EOL><DEDENT>elif len(ret) > <NUM_LIT:0> and ret[-<NUM_LIT:1>][<NUM_LIT:0>] == '<STR_LIT:y>':<EOL><INDENT>ret[-<NUM_LIT:1>][<NUM_LIT:1>] = timestamp  <EOL><DEDENT>else:<EOL><INDENT>ret.append(['<STR_LIT:y>', timestamp])  <EOL><DEDENT><DEDENT>else:<EOL><INDENT>if len(ret) > <NUM_LIT:0> and ret[-<NUM_LIT:1>][<NUM_LIT:0>] == '<STR_LIT:n>':<EOL><INDENT>ret[-<NUM_LIT:1>][<NUM_LIT:1>] = timestamp<EOL><DEDENT>elif len(ret) > <NUM_LIT:0> and ret[-<NUM_LIT:1>][<NUM_LIT:0>] == '<STR_LIT:y>':<EOL><INDENT>ret.append(['<STR_LIT:n>', timestamp])<EOL><DEDENT>else:<EOL><INDENT>ret.append(['<STR_LIT:n>', timestamp])<EOL><DEDENT><DEDENT><DEDENT>return ret<EOL>

Takes a list of 1s and 0s and returns a list of tuples of the form:
['y/n', timestamp].


<EOL>k = window_size<EOL>i = k<EOL>while i <= len(ls) - k:<EOL><INDENT>window = [ls[i + j] for j in range(k)]<EOL>mode = <NUM_LIT:1> if sum(window) >= k / <NUM_LIT:2> else <NUM_LIT:0><EOL>for j in range(k):<EOL><INDENT>ls[i+j] = mode<EOL><DEDENT>i += k<EOL><DEDENT>return ls<EOL>

Takes `ls` (a list of 1s and 0s) and smoothes it so that adjacent values are more likely
to be the same.

:param ls:          A list of 1s and 0s to smooth.
:param window_size: How large the smoothing kernel is.
:returns:           A list of 1s and 0s, but smoother.


<EOL>gradient = np.nan_to_num(np.apply_along_axis(np.gradient, <NUM_LIT:1>, s), copy=False)<EOL>threshold = np.squeeze(np.nanmean(gradient, axis=<NUM_LIT:1>) + np.nanstd(gradient, axis=<NUM_LIT:1>))<EOL>half_window = <NUM_LIT:4><EOL>if do_peaks:<EOL><INDENT>indexes = [signal.argrelextrema(gradient[i, :], np.greater, order=half_window)[<NUM_LIT:0>] for i in range(gradient.shape[<NUM_LIT:0>])]<EOL><DEDENT>else:<EOL><INDENT>indexes = [signal.argrelextrema(gradient[i, :], np.less, order=half_window)[<NUM_LIT:0>] for i in range(gradient.shape[<NUM_LIT:0>])]<EOL><DEDENT>extrema = np.zeros(s.shape)<EOL>for row_index, index_array in enumerate(indexes):<EOL><INDENT>for col_index in index_array:<EOL><INDENT>if do_peaks and (gradient[row_index, col_index] > threshold[row_index]):<EOL><INDENT>extrema[row_index, col_index] = <NUM_LIT:1><EOL><DEDENT>elif not do_peaks:<EOL><INDENT>extrema[row_index, col_index] = <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>return extrema, gradient<EOL>

Takes a spectrogram and returns a 2D array of the form:

0 0 0 1 0 0 1 0 0 0 1   <-- Frequency 0
0 0 1 0 0 0 0 0 0 1 0   <-- Frequency 1
0 0 0 0 0 0 1 0 1 0 0   <-- Frequency 2
*** Time axis *******

Where a 1 means that the value in that time bin in the spectrogram corresponds to
a peak/valley in the first derivative.

This function is used as part of the ASA algorithm and is not meant to be used publicly.


def _correlate_onsets_and_offsets(onsets, offsets, gradients):

<EOL>for freq_index, (ons, offs) in enumerate(zip(onsets[:, :], offsets[:, :])):<EOL><INDENT>indexes_of_all_ones = np.reshape(np.where(ons == <NUM_LIT:1>), (-<NUM_LIT:1>,))<EOL>last_idx = indexes_of_all_ones[<NUM_LIT:0>]<EOL>offs[<NUM_LIT:0>:last_idx + <NUM_LIT:1>] = <NUM_LIT:0><EOL>if len(indexes_of_all_ones > <NUM_LIT:1>):<EOL><INDENT>for next_idx in indexes_of_all_ones[<NUM_LIT:1>:]:<EOL><INDENT>offset_choices = offs[last_idx:next_idx]<EOL>offset_choice_indexes = np.where(offset_choices == <NUM_LIT:1>)<EOL>if not np.any(offset_choices):<EOL><INDENT>continue<EOL><DEDENT>assert np.any(offset_choices), "<STR_LIT>".format(last_idx, next_idx)<EOL>offset_choice_indexes = np.reshape(last_idx + offset_choice_indexes, (-<NUM_LIT:1>,))<EOL>assert np.all(offsets[freq_index, offset_choice_indexes])<EOL>gradient_values = gradients[freq_index, offset_choice_indexes]<EOL>index_of_largest_from_gradient_values = np.where(gradient_values == np.min(gradient_values))[<NUM_LIT:0>]<EOL>index_of_largest_offset_choice = offset_choice_indexes[index_of_largest_from_gradient_values]<EOL>assert offsets[freq_index, index_of_largest_offset_choice] == <NUM_LIT:1><EOL>offsets[freq_index, offset_choice_indexes] = <NUM_LIT:0><EOL>offsets[freq_index, index_of_largest_offset_choice] = <NUM_LIT:1><EOL>last_idx = next_idx<EOL><DEDENT><DEDENT>else:<EOL><INDENT>offsets[freq_index, :] = <NUM_LIT:0><EOL>offsets[freq_index, -<NUM_LIT:1>] = <NUM_LIT:1><EOL><DEDENT><DEDENT>return offsets<EOL>

Takes an array of onsets and an array of offsets, of the shape [nfrequencies, nsamples], where
each item in these arrays is either a 0 (not an on/offset) or a 1 (a possible on/offset).

This function returns a new offsets array, where there is a one-to-one correlation between
onsets and offsets, such that each onset has exactly one offset that occurs after it in
the time domain (the second dimension of the array).

The gradients array is used to decide which offset to use in the case of multiple possibilities.


def _form_onset_offset_fronts(ons_or_offs, sample_rate_hz, threshold_ms=<NUM_LIT:20>):

threshold_s = threshold_ms / <NUM_LIT:1000><EOL>threshold_samples = sample_rate_hz * threshold_s<EOL>ons_or_offs = np.copy(ons_or_offs)<EOL>claimed = []<EOL>this_id = <NUM_LIT:2><EOL>for frequency_index, row in enumerate(ons_or_offs[:, :]):<EOL><INDENT>ones = np.reshape(np.where(row == <NUM_LIT:1>), (-<NUM_LIT:1>,))<EOL>for top_level_frequency_one_index in ones:<EOL><INDENT>claimed.append((frequency_index, top_level_frequency_one_index))<EOL>found_a_front = False<EOL>for other_frequency_index, other_row in enumerate(ons_or_offs[frequency_index + <NUM_LIT:1>:, :], start=frequency_index + <NUM_LIT:1>):<EOL><INDENT>upper_limit_index = top_level_frequency_one_index + threshold_samples<EOL>lower_limit_index = top_level_frequency_one_index - threshold_samples<EOL>other_ones = np.reshape(np.where(other_row == <NUM_LIT:1>), (-<NUM_LIT:1>,))  <EOL>tmp = np.reshape(np.where((other_ones >= lower_limit_index)  <EOL>& (other_ones <= upper_limit_index)), (-<NUM_LIT:1>,))<EOL>other_ones = other_ones[tmp]  <EOL>if len(other_ones) > <NUM_LIT:0>:<EOL><INDENT>unclaimed_idx = other_ones[<NUM_LIT:0>]  <EOL>claimed.append((other_frequency_index, unclaimed_idx))<EOL><DEDENT>elif len(claimed) < <NUM_LIT:3>:<EOL><INDENT>ons_or_offs[frequency_index, top_level_frequency_one_index] = <NUM_LIT:0><EOL>claimed = []<EOL>break  <EOL><DEDENT>elif len(claimed) >= <NUM_LIT:3>:<EOL><INDENT>found_a_front = True<EOL>claimed_as_indexes = tuple(np.array(claimed).T)<EOL>ons_or_offs[claimed_as_indexes] = this_id<EOL>this_id += <NUM_LIT:1><EOL>claimed = []<EOL>break  <EOL><DEDENT><DEDENT>if len(claimed) >= <NUM_LIT:3>:<EOL><INDENT>claimed_as_indexes = tuple(np.array(claimed).T)<EOL>ons_or_offs[claimed_as_indexes] = this_id<EOL>this_id += <NUM_LIT:1><EOL>claimed = []<EOL><DEDENT>elif found_a_front:<EOL><INDENT>this_id += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>ons_or_offs[frequency_index, top_level_frequency_one_index] = <NUM_LIT:0><EOL>claimed = []<EOL><DEDENT><DEDENT><DEDENT>return ons_or_offs<EOL>

Takes an array of onsets or offsets (shape = [nfrequencies, nsamples], where a 1 corresponds to an on/offset,
and samples are 0 otherwise), and returns a new array of the same shape, where each 1 has been replaced by
either a 0, if the on/offset has been discarded, or a non-zero positive integer, such that
each front within the array has a unique ID - for example, all 2s in the array will be the front for on/offset
front 2, and all the 15s will be the front for on/offset front 15, etc.

Due to implementation details, there will be no 1 IDs.


def _lookup_offset_by_onset_idx(onset_idx, onsets, offsets):

assert len(onset_idx) == <NUM_LIT:2>, "<STR_LIT>"<EOL>frequency_idx, sample_idx = onset_idx<EOL>offset_sample_idxs = np.reshape(np.where(offsets[frequency_idx, :] == <NUM_LIT:1>), (-<NUM_LIT:1>,))<EOL>offset_sample_idxs = offset_sample_idxs[offset_sample_idxs > sample_idx]<EOL>if len(offset_sample_idxs) == <NUM_LIT:0>:<EOL><INDENT>chosen_offset_sample_idx = offsets.shape[<NUM_LIT:1>] - <NUM_LIT:1><EOL>assert offsets[frequency_idx, chosen_offset_sample_idx] == <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>chosen_offset_sample_idx = offset_sample_idxs[<NUM_LIT:0>]<EOL>assert offsets[frequency_idx, chosen_offset_sample_idx] != <NUM_LIT:0><EOL><DEDENT>return frequency_idx, chosen_offset_sample_idx<EOL>

Takes an onset index (freq, sample) and returns the offset index (freq, sample)
such that frequency index is the same, and sample index is the minimum of all
offsets ocurring after the given onset. If there are no offsets after the given
onset in that frequency channel, the final sample in that channel is returned.


if id == -<NUM_LIT:1>:<EOL><INDENT>freq_idxs = np.arange(fronts.shape[<NUM_LIT:0>], dtype=np.int64)<EOL>sample_idxs = np.ones(len(freq_idxs), dtype=np.int64) * (fronts.shape[<NUM_LIT:1>] - <NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>freq_idxs, sample_idxs = np.where(fronts == id)<EOL><DEDENT>return [(f, i) for f, i in zip(freq_idxs, sample_idxs)]<EOL>

Return a list of tuples of the form (frequency_idx, sample_idx),
corresponding to all the indexes of the given front.


def _choose_front_id_from_candidates(candidate_offset_front_ids, offset_fronts, offsets_corresponding_to_onsets):

noverlaps = []  <EOL>for offset_front_id in candidate_offset_front_ids:<EOL><INDENT>offset_front_f_idxs, offset_front_s_idxs = np.where(offset_fronts == offset_front_id)<EOL>offset_front_idxs = [(f, i) for f, i in zip(offset_front_f_idxs, offset_front_s_idxs)]<EOL>noverlap_this_id = len(set(offset_front_idxs).symmetric_difference(set(offsets_corresponding_to_onsets)))<EOL>noverlaps.append((noverlap_this_id, offset_front_id))<EOL><DEDENT>_overlapped, chosen_offset_front_id = max(noverlaps, key=lambda t: t[<NUM_LIT:0>])<EOL>return int(chosen_offset_front_id)<EOL>

Returns a front ID which is the id of the offset front that contains the most overlap
with offsets that correspond to the given onset front ID.


<EOL>offset_front_ids = [i for i in np.unique(offset_fronts) if i != <NUM_LIT:0>]<EOL>best_id_so_far = -<NUM_LIT:1><EOL>closest_offset_sample_idx = sys.maxsize<EOL>for offset_front_id in offset_front_ids:<EOL><INDENT>offset_front_idxs = _get_front_idxs_from_id(offset_fronts, offset_front_id)<EOL>offset_front_sample_idxs = [s for _f, s in offset_front_idxs]<EOL>min_sample_idx = min(offset_front_sample_idxs)<EOL>if min_sample_idx > onset_sample_idx and min_sample_idx < closest_offset_sample_idx:<EOL><INDENT>closest_offset_sample_idx = min_sample_idx<EOL>best_id_so_far = offset_front_id<EOL><DEDENT><DEDENT>assert best_id_so_far > <NUM_LIT:1> or best_id_so_far == -<NUM_LIT:1><EOL>return best_id_so_far<EOL>

Returns the offset_front_id which corresponds to the offset front which occurs
first entirely after the given onset sample_idx.


def _get_offset_front_id_after_onset_front(onset_front_id, onset_fronts, offset_fronts):

<EOL>onset_idxs = _get_front_idxs_from_id(onset_fronts, onset_front_id)<EOL>onset_sample_idxs = [s for _f, s in onset_idxs]<EOL>latest_onset_in_front = max(onset_sample_idxs)<EOL>offset_front_id_after_this_onset_front = _get_offset_front_id_after_onset_sample_idx(latest_onset_in_front, offset_fronts)<EOL>return int(offset_front_id_after_this_onset_front)<EOL>

Get the ID corresponding to the offset which occurs first after the given onset_front_id.
By `first` I mean the front which contains the offset which is closest to the latest point
in the onset front. By `after`, I mean that the offset must contain only offsets which
occur after the latest onset in the onset front.

If there is no appropriate offset front, the id returned is -1.


def _match_offset_front_id_to_onset_front_id(onset_front_id, onset_fronts, offset_fronts, onsets, offsets):

<EOL>onset_idxs = _get_front_idxs_from_id(onset_fronts, onset_front_id)<EOL>offset_idxs = [_lookup_offset_by_onset_idx(i, onsets, offsets) for i in onset_idxs]<EOL>candidate_offset_front_ids = set([int(offset_fronts[f, i]) for f, i in offset_idxs])<EOL>candidate_offset_front_ids = [id for id in candidate_offset_front_ids if id != <NUM_LIT:0>]<EOL>if candidate_offset_front_ids:<EOL><INDENT>chosen_offset_front_id = _choose_front_id_from_candidates(candidate_offset_front_ids, offset_fronts, offset_idxs)<EOL><DEDENT>else:<EOL><INDENT>chosen_offset_front_id = _get_offset_front_id_after_onset_front(onset_front_id, onset_fronts, offset_fronts)<EOL><DEDENT>return chosen_offset_front_id<EOL>

Find all offset fronts which are composed of at least one offset which corresponds to one of the onsets in the
given onset front.
The offset front which contains the most of such offsets is the match.
If there are no such offset fronts, return -1.


last_f = None<EOL>ls = []<EOL>for f, s in front:<EOL><INDENT>if last_f is not None and f != last_f + <NUM_LIT:1>:<EOL><INDENT>yield ls<EOL>ls = []<EOL><DEDENT>ls.append((f, s))<EOL>last_f = f<EOL><DEDENT>yield ls<EOL>

Yields lists of the form [(f, s), (f, s)], one at a time from the given front (which is a list of the same form),
such that each list yielded is consecutive in frequency.


def _get_consecutive_and_overlapping_fronts(onset_fronts, offset_fronts, onset_front_id, offset_front_id):

<EOL>onset_front = _get_front_idxs_from_id(onset_fronts, onset_front_id)<EOL>offset_front = _get_front_idxs_from_id(offset_fronts, offset_front_id)<EOL>consecutive_portions_of_onset_front = [c for c in _get_consecutive_portions_of_front(onset_front)]<EOL>for consecutive_portion_of_onset_front in consecutive_portions_of_onset_front:<EOL><INDENT>onset_front_frequency_indexes = [f for f, _ in consecutive_portion_of_onset_front]<EOL>overlapping_offset_front = [(f, s) for f, s in offset_front if f in onset_front_frequency_indexes]<EOL>for consecutive_portion_of_offset_front in _get_consecutive_portions_of_front(overlapping_offset_front):<EOL><INDENT>if consecutive_portion_of_offset_front:<EOL><INDENT>return consecutive_portion_of_onset_front, consecutive_portion_of_offset_front<EOL><DEDENT><DEDENT><DEDENT>return [], []<EOL>

Gets an onset_front and an offset_front such that they both occupy at least some of the same
frequency channels, then returns the portion of each that overlaps with the other.


def _update_segmentation_mask(segmentation_mask, onset_fronts, offset_fronts, onset_front_id, offset_front_id_most_overlap):

<EOL>onset_front_overlap, offset_front_overlap = _get_consecutive_and_overlapping_fronts(onset_fronts, offset_fronts, onset_front_id, offset_front_id_most_overlap)<EOL>onset_front = _get_front_idxs_from_id(onset_fronts, onset_front_id)<EOL>offset_front = _get_front_idxs_from_id(offset_fronts, offset_front_id_most_overlap)<EOL>msg = "<STR_LIT>".format(<EOL>onset_front, offset_front, onset_front_overlap, offset_front_overlap<EOL>)<EOL>assert onset_front_overlap, msg<EOL>assert offset_front_overlap, msg<EOL>onset_front = onset_front_overlap<EOL>offset_front = offset_front_overlap<EOL>flow_on, _slow_on = onset_front[<NUM_LIT:0>]<EOL>fhigh_on, _shigh_on = onset_front[-<NUM_LIT:1>]<EOL>flow_off, _slow_off = offset_front[<NUM_LIT:0>]<EOL>fhigh_off, _shigh_off = offset_front[-<NUM_LIT:1>]<EOL>flow = max(flow_on, flow_off)<EOL>fhigh = min(fhigh_on, fhigh_off)<EOL>for fidx, _freqchan in enumerate(segmentation_mask[flow:fhigh + <NUM_LIT:1>, :], start=flow):<EOL><INDENT>assert fidx >= flow, "<STR_LIT>".format(fidx, flow)<EOL>assert (fidx - flow) < len(onset_front), "<STR_LIT>".format(<EOL>fidx, flow, len(onset_front), onset_front<EOL>)<EOL>assert (fidx - flow) < len(offset_front), "<STR_LIT>".format(<EOL>fidx, flow, len(offset_front), offset_front<EOL>)<EOL>_, beg = onset_front[fidx - flow]<EOL>_, end = offset_front[fidx - flow]<EOL>if beg > end:<EOL><INDENT>end, beg = beg, end<EOL><DEDENT>assert end >= beg<EOL>segmentation_mask[fidx, beg:end + <NUM_LIT:1>] = onset_front_id<EOL>onset_fronts[fidx, (beg + <NUM_LIT:1>):(end + <NUM_LIT:1>)] = <NUM_LIT:0><EOL>offset_fronts[fidx, (beg + <NUM_LIT:1>):(end + <NUM_LIT:1>)] = <NUM_LIT:0><EOL><DEDENT>nfreqs_used_in_onset_front = (fidx - flow) + <NUM_LIT:1><EOL>indexes = np.arange(flow, fhigh + <NUM_LIT:1>, <NUM_LIT:1>, dtype=np.int64)<EOL>onset_front_sample_idxs_across_freqs = np.array([s for _, s in onset_front])<EOL>onset_front_sample_idxs_across_freqs_up_to_break = onset_front_sample_idxs_across_freqs[:nfreqs_used_in_onset_front]<EOL>offset_front_sample_idxs_across_freqs = np.array([s for _, s in offset_front])<EOL>offset_front_sample_idxs_across_freqs_up_to_break = offset_front_sample_idxs_across_freqs[:nfreqs_used_in_onset_front]<EOL>offset_fronts[indexes[:nfreqs_used_in_onset_front], offset_front_sample_idxs_across_freqs_up_to_break] = <NUM_LIT:0><EOL>onset_fronts[indexes[:nfreqs_used_in_onset_front], onset_front_sample_idxs_across_freqs_up_to_break] = <NUM_LIT:0><EOL>whole_onset_front_matched = onset_front_id not in np.unique(onset_fronts)<EOL>return whole_onset_front_matched<EOL>

Returns an updated segmentation mask such that the input `segmentation_mask` has been updated by segmenting between
`onset_front_id` and `offset_front_id`, as found in `onset_fronts` and `offset_fronts`, respectively.

This function also returns the onset_fronts and offset_fronts matrices, updated so that any fronts that are of
less than 3 channels wide are removed.

This function also returns a boolean value indicating whether the onset channel went to completion.

Specifically, segments by doing the following:

- Going across frequencies in the onset_front,
- add the segment mask ID (the onset front ID) to all samples between the onset_front and the offset_front,
  if the offset_front is in that frequency.

Possible scenarios:

Fronts line up completely:

::

    |   |       S S S
    |   |  =>   S S S
    |   |       S S S
    |   |       S S S

Onset front starts before offset front:

::

    |           |
    |   |       S S S
    |   |  =>   S S S
    |   |       S S S

Onset front ends after offset front:

::

    |   |       S S S
    |   |  =>   S S S
    |   |       S S S
    |           |

Onset front starts before and ends after offset front:

::

    |           |
    |   |  =>   S S S
    |   |       S S S
    |           |

The above three options in reverse:

::

        |       |S S|           |
    |S S|       |S S|       |S S|
    |S S|       |S S|       |S S|
    |S S|           |           |

There is one last scenario:

::

    |   |
    \   /
     \ /
     / \
    |   |

Where the offset and onset fronts cross one another. If this happens, we simply
reverse the indices and accept:

::

    |sss|
    \sss/
     \s/
     /s\
    |sss|

The other option would be to destroy the offset front from the crossover point on, and
then search for a new offset front for the rest of the onset front.


fidx, sidx = index<EOL>id = front[fidx, sidx]<EOL>if id == <NUM_LIT:0>:<EOL><INDENT>return -<NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>return id<EOL><DEDENT>

Returns the front ID found in `front` at the given `index`.

:param front:               An onset or offset front array of shape [nfrequencies, nsamples]
:index:                     A tuple of the form (frequency index, sample index)
:returns:                   The ID of the front or -1 if not found in `front` and the item at `onsets_or_offsets[index]`
                            is not a 1.


yielded_so_far = set()<EOL>for row in onset_fronts:<EOL><INDENT>for id in row:<EOL><INDENT>if id != <NUM_LIT:0> and id not in yielded_so_far:<EOL><INDENT>yield id<EOL>yielded_so_far.add(id)<EOL><DEDENT><DEDENT><DEDENT>

Yields one onset front ID at a time until they are gone. All the onset fronts from a
frequency channel are yielded, then all of the next channel's, etc., though one at a time.


def _get_corresponding_offsets(onset_fronts, onset_front_id, onsets, offsets):

corresponding_offsets = []<EOL>for index in _get_front_idxs_from_id(onset_fronts, onset_front_id):<EOL><INDENT>offset_fidx, offset_sidx = _lookup_offset_by_onset_idx(index, onsets, offsets)<EOL>corresponding_offsets.append((offset_fidx, offset_sidx))<EOL><DEDENT>return corresponding_offsets<EOL>

Gets the offsets that occur as close as possible to the onsets in the given onset-front.


all_offset_fronts_of_interest = []<EOL>ids_ntimes_seen = {}<EOL>for offset_index in corresponding_offsets:<EOL><INDENT>offset_id = _front_id_from_idx(offset_fronts, offset_index)<EOL>if offset_id not in ids_ntimes_seen:<EOL><INDENT>offset_front_idxs = _get_front_idxs_from_id(offset_fronts, offset_id)<EOL>all_offset_fronts_of_interest.append(offset_front_idxs)<EOL>ids_ntimes_seen[offset_id] = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>ids_ntimes_seen[offset_id] += <NUM_LIT:1><EOL><DEDENT><DEDENT>return all_offset_fronts_of_interest, ids_ntimes_seen<EOL>

Returns all the offset fronts that are composed of at least one of the given offset indexes.
Also returns a dict of the form {offset_front_id: ntimes saw}


fidxs, sidxs = np.where((segmentation_mask != fronts) & (segmentation_mask != <NUM_LIT:0>) & (fronts != <NUM_LIT:0>))<EOL>fronts[fidxs, sidxs] = <NUM_LIT:0><EOL>

Removes all points in the fronts that overlap with the segmentation mask.


def _match_fronts(onset_fronts, offset_fronts, onsets, offsets, debug=False):

def printd(*args, **kwargs):<EOL><INDENT>if debug:<EOL><INDENT>print(*args, **kwargs)<EOL><DEDENT><DEDENT>onset_fronts = np.copy(onset_fronts)<EOL>offset_fronts = np.copy(offset_fronts)<EOL>onsets = np.copy(onsets)<EOL>offsets = np.copy(offsets)<EOL>segmentation_mask = np.zeros_like(onset_fronts)<EOL>resulting_onset_fronts = np.copy(onset_fronts)<EOL>printd("<STR_LIT>")<EOL>for onset_front_id in _get_front_ids_one_at_a_time(onset_fronts):<EOL><INDENT>printd("<STR_LIT>", int(onset_front_id))<EOL>front_is_complete = False<EOL>while not front_is_complete:<EOL><INDENT>corresponding_offsets = _get_corresponding_offsets(resulting_onset_fronts, onset_front_id, onsets, offsets)<EOL>if not corresponding_offsets:<EOL><INDENT>break<EOL><DEDENT>_all_offset_fronts_of_interest, ids_ntimes_seen = _get_all_offset_fronts_from_offsets(offset_fronts, corresponding_offsets)<EOL>ntimes_seen_sorted = sorted([(k, v) for k, v in ids_ntimes_seen.items()], key=lambda tup: (-<NUM_LIT:1> * tup[<NUM_LIT:1>], tup[<NUM_LIT:0>]))<EOL>assert len(ntimes_seen_sorted) > <NUM_LIT:0>, "<STR_LIT>"<EOL>offset_front_id, _ntimes_seen = ntimes_seen_sorted[<NUM_LIT:0>]<EOL>if offset_front_id == -<NUM_LIT:1> and len(ntimes_seen_sorted) > <NUM_LIT:1>:<EOL><INDENT>offset_front_id, _ntimes_seen = ntimes_seen_sorted[<NUM_LIT:1>]<EOL><DEDENT>offset_front_id_most_overlap = offset_front_id<EOL>front_is_complete = _update_segmentation_mask(segmentation_mask,<EOL>resulting_onset_fronts,<EOL>offset_fronts,<EOL>onset_front_id,<EOL>offset_front_id_most_overlap)<EOL>_remove_overlaps(segmentation_mask, resulting_onset_fronts)<EOL>_remove_overlaps(segmentation_mask, offset_fronts)<EOL><DEDENT><DEDENT>return segmentation_mask<EOL>

Returns a segmentation mask, which looks like this:
frequency 1: 0 0 4 4 4 4 4 0 0 5 5 5
frequency 2: 0 4 4 4 4 4 0 0 0 0 5 5
frequency 3: 0 4 4 4 4 4 4 4 5 5 5 5

That is, each item in the array is either a 0 (not part of a segment) or a positive
integer which indicates which segment the sample in that frequency band belongs to.


ids = np.unique(fronts)<EOL>for id in ids:<EOL><INDENT>if id == <NUM_LIT:0> or id == -<NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT>front = _get_front_idxs_from_id(fronts, id)<EOL>if len(front) < size:<EOL><INDENT>indexes = ([f for f, _ in front], [s for _, s in front])<EOL>fronts[indexes] = <NUM_LIT:0><EOL><DEDENT><DEDENT>

Removes all fronts from `fronts` which are strictly smaller than
`size` consecutive frequencies in length.


def _break_poorly_matched_fronts(fronts, threshold=<NUM_LIT:0.1>, threshold_overlap_samples=<NUM_LIT:3>):

assert threshold_overlap_samples > <NUM_LIT:0>, "<STR_LIT>"<EOL>breaks_after = {}<EOL>for front_id in _get_front_ids_one_at_a_time(fronts):<EOL><INDENT>front = _get_front_idxs_from_id(fronts, front_id)<EOL>for i, (f, s) in enumerate(front):<EOL><INDENT>if i < len(front) - <NUM_LIT:1>:<EOL><INDENT>next_f, next_s = front[i + <NUM_LIT:1>]<EOL>low_s = min(s, next_s)<EOL>high_s = max(s, next_s)<EOL>sig_this_f = fronts[f, low_s:high_s]<EOL>sig_next_f = fronts[next_f, low_s:high_s]<EOL>assert len(sig_next_f) == len(sig_this_f)<EOL>if len(sig_next_f) > threshold_overlap_samples:<EOL><INDENT>correlation = signal.correlate(sig_this_f, sig_next_f, mode='<STR_LIT>')<EOL>assert len(correlation) > <NUM_LIT:0><EOL>correlation = correlation / max(correlation + <NUM_LIT>)<EOL>similarity = np.sum(correlation) / len(correlation)<EOL>if similarity < threshold:<EOL><INDENT>if front_id in breaks_after:<EOL><INDENT>breaks_after[front_id].append((f, s))<EOL><DEDENT>else:<EOL><INDENT>breaks_after[front_id] = []<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>taken_ids = sorted(np.unique(fronts))<EOL>next_id = taken_ids[-<NUM_LIT:1>] + <NUM_LIT:1><EOL>for id in breaks_after.keys():<EOL><INDENT>for f, s in breaks_after[id]:<EOL><INDENT>fidxs, sidxs = np.where(fronts == id)<EOL>idxs_greater_than_f = [fidx for fidx in fidxs if fidx > f]<EOL>start = len(sidxs) - len(idxs_greater_than_f)<EOL>indexes = (idxs_greater_than_f, sidxs[start:])<EOL>fronts[indexes] = next_id<EOL>next_id += <NUM_LIT:1><EOL><DEDENT><DEDENT>_remove_fronts_that_are_too_small(fronts, <NUM_LIT:3>)<EOL>

For each onset front, for each frequency in that front, break the onset front if the signals
between this frequency's onset and the next frequency's onset are not similar enough.

Specifically:
If we have the following two frequency channels, and the two O's are part of the same onset front,

::

    [ . O . . . . . . . . . . ]
    [ . . . . O . . . . . . . ]

We compare the signals x and y:

::

    [ . x x x x . . . . . . . ]
    [ . y y y y . . . . . . . ]

And if they are not sufficiently similar (via a DSP correlation algorithm), we break the onset
front between these two channels.

Once this is done, remove any onset fronts that are less than 3 channels wide.


def _update_segmentation_mask_if_overlap(toupdate, other, id, otherid):

<EOL>yourmask = other == otherid<EOL>mymask = toupdate == id<EOL>overlap_exists = np.any(yourmask & mymask)<EOL>if not overlap_exists:<EOL><INDENT>return<EOL><DEDENT>yourfidxs, yoursidxs = np.where(other == otherid)<EOL>toupdate[yourfidxs, yoursidxs] = id<EOL>

Merges the segments specified by `id` (found in `toupdate`) and `otherid`
(found in `other`) if they overlap at all. Updates `toupdate` accordingly.


<EOL>lsf1, lss1 = seg1<EOL>lsf2, lss2 = seg2<EOL>for i, f1 in enumerate(lsf1):<EOL><INDENT>for j, f2 in enumerate(lsf2):<EOL><INDENT>if f1 <= f2 + <NUM_LIT:1> and f1 >= f2 - <NUM_LIT:1>:<EOL><INDENT>if lss1[i] <= lss2[j] + <NUM_LIT:1> and lss1[i] >= lss2[j] - <NUM_LIT:1>:<EOL><INDENT>return True<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return False<EOL>

Checks if seg1 and seg2 are adjacent at any point. Each is a tuple of the form
(fidxs, sidxs).


mask_ids = [id for id in np.unique(mask) if id != <NUM_LIT:0>]<EOL>for id in mask_ids:<EOL><INDENT>myfidxs, mysidxs = np.where(mask == id)<EOL>for other in mask_ids:  <EOL><INDENT>if id == other:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>other_fidxs, other_sidxs = np.where(mask == other)<EOL>if _segments_are_adjacent((myfidxs, mysidxs), (other_fidxs, other_sidxs)):<EOL><INDENT>mask[other_fidxs, other_sidxs] = id<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Merges all segments in `mask` which are touching.


if len(segmasks) == <NUM_LIT:1>:<EOL><INDENT>return segmasks<EOL><DEDENT>assert len(segmasks) > <NUM_LIT:0>, "<STR_LIT>"<EOL>coarse_mask = np.copy(segmasks[<NUM_LIT:0>])<EOL>mask_ids = [id for id in np.unique(coarse_mask) if id != <NUM_LIT:0>]<EOL>for id in mask_ids:<EOL><INDENT>for mask in segmasks[<NUM_LIT:1>:]:<EOL><INDENT>finer_ids = [i for i in np.unique(mask) if i != <NUM_LIT:0>]<EOL>for finer_id in finer_ids:<EOL><INDENT>_update_segmentation_mask_if_overlap(coarse_mask, mask, id, finer_id)<EOL><DEDENT><DEDENT><DEDENT>return coarse_mask<EOL>

`segmasks` should be in sorted order of [coarsest, ..., finest].

Integrates the given list of segmentation masks together to form one segmentation mask
by having each segment subsume ones that exist in the finer masks.


try:<EOL><INDENT>ncpus = multiprocessing.cpu_count()<EOL><DEDENT>except NotImplementedError:<EOL><INDENT>ncpus = <NUM_LIT:2><EOL><DEDENT>with multiprocessing.Pool(processes=ncpus) as pool:<EOL><INDENT>mask_ids = [id for id in np.unique(mask) if id != <NUM_LIT:0>]<EOL>thresholds = [threshold * mask.size for _ in range(len(mask_ids))]<EOL>masks = [mask for _ in range(len(mask_ids))]<EOL>ms = pool.starmap(_separate_masks_task, zip(mask_ids, thresholds, masks))<EOL><DEDENT>return [m for m in ms if m is not None]<EOL>

Returns a list of segmentation masks each of the same dimension as the input one,
but where they each have exactly one segment in them and all other samples in them
are zeroed.

Only bothers to return segments that are larger in total area than `threshold * mask.size`.


def _downsample_one_or_the_other(mask, mask_indexes, stft, stft_indexes):

assert len(mask.shape) == <NUM_LIT:2>, "<STR_LIT>".format(len(mask.shape))<EOL>assert len(stft.shape) == <NUM_LIT:2>, "<STR_LIT>".format(len(stft.shape))<EOL>if mask.shape[<NUM_LIT:1>] > stft.shape[<NUM_LIT:1>]:<EOL><INDENT>downsample_factor = mask.shape[<NUM_LIT:1>] / stft.shape[<NUM_LIT:1>]<EOL>indexes = _get_downsampled_indexes(mask, downsample_factor)<EOL>mask = mask[:, indexes]<EOL>mask_indexes = np.array(indexes)<EOL><DEDENT>elif mask.shape[<NUM_LIT:1>] < stft.shape[<NUM_LIT:1>]:<EOL><INDENT>downsample_factor = stft.shape[<NUM_LIT:1>] / mask.shape[<NUM_LIT:1>]<EOL>indexes = _get_downsampled_indexes(stft, downsample_factor)<EOL>stft = stft[:, indexes]<EOL>stft_indexes = np.array(indexes)<EOL><DEDENT>return mask, mask_indexes, stft, stft_indexes<EOL>

Takes the given `mask` and `stft`, which must be matrices of shape `frequencies, times`
and downsamples one of them into the other one's times, so that the time dimensions
are equal. Leaves the frequency dimension untouched.


def _map_segmentation_mask_to_stft_domain(mask, times, frequencies, stft_times, stft_frequencies):

assert mask.shape == (frequencies.shape[<NUM_LIT:0>], times.shape[<NUM_LIT:0>]), "<STR_LIT>".format(<EOL>times.shape, frequencies.shape, mask.shape<EOL>)<EOL>result = np.zeros((stft_frequencies.shape[<NUM_LIT:0>], stft_times.shape[<NUM_LIT:0>]))<EOL>if len(stft_times) > len(times):<EOL><INDENT>all_j = [j for j in range(len(stft_times))]<EOL>idxs  = [int(i) for i in np.linspace(<NUM_LIT:0>, len(times) - <NUM_LIT:1>, num=len(stft_times))]<EOL>all_i = [all_j[idx] for idx in idxs]<EOL><DEDENT>else:<EOL><INDENT>all_i = [i for i in range(len(times))]<EOL>idxs  = [int(i) for i in np.linspace(<NUM_LIT:0>, len(stft_times) - <NUM_LIT:1>, num=len(times))]<EOL>all_j = [all_i[idx] for idx in idxs]<EOL><DEDENT>for i, j in zip(all_i, all_j):<EOL><INDENT>result[:, j] = np.interp(stft_frequencies, frequencies, mask[:, i])<EOL><DEDENT>return result<EOL>

Maps the given `mask`, which is in domain (`frequencies`, `times`) to the new domain (`stft_frequencies`, `stft_times`)
and returns the result.


def _asa_task(q, masks, stft, sample_width, frame_rate, nsamples_for_each_fft):

<EOL>for mask in masks:<EOL><INDENT>mask = np.where(mask > <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>)<EOL><DEDENT>masks = [mask * stft for mask in masks]<EOL>nparrs = []<EOL>dtype_dict = {<NUM_LIT:1>: np.int8, <NUM_LIT:2>: np.int16, <NUM_LIT:4>: np.int32}<EOL>dtype = dtype_dict[sample_width]<EOL>for m in masks:<EOL><INDENT>_times, nparr = signal.istft(m, frame_rate, nperseg=nsamples_for_each_fft)<EOL>nparr = nparr.astype(dtype)<EOL>nparrs.append(nparr)<EOL><DEDENT>for m in nparrs:<EOL><INDENT>q.put(m)<EOL><DEDENT>q.put("<STR_LIT>")<EOL>

Worker for the ASA algorithm's multiprocessing step.


def bandpass_filter(data, low, high, fs, order=<NUM_LIT:5>):

nyq = <NUM_LIT:0.5> * fs<EOL>low = low / nyq<EOL>high = high / nyq<EOL>b, a = signal.butter(order, [low, high], btype='<STR_LIT>')<EOL>y = signal.lfilter(b, a, data)<EOL>return y<EOL>

Does a bandpass filter over the given data.

:param data: The data (numpy array) to be filtered.
:param low: The low cutoff in Hz.
:param high: The high cutoff in Hz.
:param fs: The sample rate (in Hz) of the data.
:param order: The order of the filter. The higher the order, the tighter the roll-off.
:returns: Filtered data (numpy array).


def lowpass_filter(data, cutoff, fs, order=<NUM_LIT:5>):

nyq = <NUM_LIT:0.5> * fs<EOL>normal_cutoff = cutoff / nyq<EOL>b, a = signal.butter(order, normal_cutoff, btype='<STR_LIT>', analog=False)<EOL>y = signal.lfilter(b, a, data)<EOL>return y<EOL>

Does a lowpass filter over the given data.

:param data: The data (numpy array) to be filtered.
:param cutoff: The high cutoff in Hz.
:param fs: The sample rate in Hz of the data.
:param order: The order of the filter. The higher the order, the tighter the roll-off.
:returns: Filtered data (numpy array).


class_name = inflection.underscore(top_cls.__name__).lower()<EOL>def entity_name(cls):<EOL><INDENT>return class_name<EOL><DEDENT>top_cls.entity_name = classmethod(entity_name)<EOL>return top_cls<EOL>

Assign an entity name based on the class immediately inhering from Base.

This is needed because we don't want
entity names to come from any class that simply inherits our classes,
just the ones in our module.

For example, if you create a class Project2 that exists outside of
kalibro_client and inherits from Project, it's entity name should still
be Project.


def __init__(self,<EOL>url: str,<EOL>method: str = '<STR_LIT:GET>',<EOL>*,<EOL>callback=None,<EOL>encoding: Optional[str] = None,<EOL>headers: dict = None,<EOL>metadata: dict = None,<EOL>request_config: dict = None,<EOL>request_session=None,<EOL>**kwargs):

self.url = url<EOL>self.method = method.upper()<EOL>if self.method not in self.METHOD:<EOL><INDENT>raise InvalidRequestMethod(<EOL>f'<STR_LIT>'<EOL>)<EOL><DEDENT>self.callback = callback<EOL>self.encoding = encoding<EOL>self.headers = headers or {}<EOL>self.metadata = metadata or {}<EOL>self.request_session = request_session<EOL>self.request_config = self.REQUEST_CONFIG if request_config is None else request_config<EOL>self.ssl = kwargs.pop('<STR_LIT>', False)<EOL>self.kwargs = kwargs<EOL>self.close_request_session = False<EOL>self.logger = get_logger(name=self.name)<EOL>self.retry_times = self.request_config.get('<STR_LIT>', <NUM_LIT:3>)<EOL>


if self.request_config.get('<STR_LIT>', <NUM_LIT:0>) > <NUM_LIT:0>:<EOL><INDENT>await asyncio.sleep(self.request_config['<STR_LIT>'])<EOL><DEDENT>timeout = self.request_config.get('<STR_LIT>', <NUM_LIT:10>)<EOL>try:<EOL><INDENT>async with async_timeout.timeout(timeout):<EOL><INDENT>resp = await self._make_request()<EOL><DEDENT>try:<EOL><INDENT>resp_data = await resp.text(encoding=self.encoding)<EOL><DEDENT>except UnicodeDecodeError:<EOL><INDENT>resp_data = await resp.read()<EOL><DEDENT>response = Response(<EOL>url=self.url,<EOL>method=self.method,<EOL>encoding=resp.get_encoding(),<EOL>html=resp_data,<EOL>metadata=self.metadata,<EOL>cookies=resp.cookies,<EOL>headers=resp.headers,<EOL>history=resp.history,<EOL>status=resp.status,<EOL>aws_json=resp.json,<EOL>aws_text=resp.text,<EOL>aws_read=resp.read)<EOL>aws_valid_response = self.request_config.get('<STR_LIT>')<EOL>if aws_valid_response and iscoroutinefunction(aws_valid_response):<EOL><INDENT>response = await aws_valid_response(response)<EOL><DEDENT>if response.ok:<EOL><INDENT>return response<EOL><DEDENT>else:<EOL><INDENT>return await self._retry(error_msg='<STR_LIT>')<EOL><DEDENT><DEDENT>except asyncio.TimeoutError:<EOL><INDENT>return await self._retry(error_msg='<STR_LIT>')<EOL><DEDENT>except Exception as e:<EOL><INDENT>return await self._retry(error_msg=e)<EOL><DEDENT>finally:<EOL><INDENT>await self._close_request_session()<EOL><DEDENT>

Fetch all the information by using aiohttp


middleware = args[<NUM_LIT:0>]<EOL>@wraps(middleware)<EOL>def register_middleware(*args, **kwargs):<EOL><INDENT>self.request_middleware.append(middleware)<EOL>return middleware<EOL><DEDENT>return register_middleware()<EOL>

Define a Decorate to be called before a request.
eg: @middleware.request


middleware = args[<NUM_LIT:0>]<EOL>@wraps(middleware)<EOL>def register_middleware(*args, **kwargs):<EOL><INDENT>self.response_middleware.appendleft(middleware)<EOL>return middleware<EOL><DEDENT>return register_middleware()<EOL>

Define a Decorate to be called after a response.
eg: @middleware.response


async def _run_spider_hook(self, hook_func):

if callable(hook_func):<EOL><INDENT>try:<EOL><INDENT>aws_hook_func = hook_func(weakref.proxy(self))<EOL>if isawaitable(aws_hook_func):<EOL><INDENT>await aws_hook_func<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>self.logger.error(f'<STR_LIT>')<EOL><DEDENT><DEDENT>

Run hook before/after spider start crawling
:param hook_func: aws function
:return:


async def process_failed_response(self, request, response):

pass<EOL>

Corresponding processing for the failed response
:param request: Request
:param response: Response
:return:


async def process_succeed_response(self, request, response):

pass<EOL>

Corresponding processing for the succeed response
:param request: Request
:param response: Response
:return:


pass<EOL>

Corresponding processing for the Item type
:param item: Item
:return:


async def process_callback_result(self, callback_result):

callback_result_name = type(callback_result).__name__<EOL>process_func_name = self.callback_result_map.get(<EOL>callback_result_name, '<STR_LIT>')<EOL>process_func = getattr(self, process_func_name, None)<EOL>if process_func is not None:<EOL><INDENT>await process_func(callback_result)<EOL><DEDENT>else:<EOL><INDENT>raise InvalidCallbackResult(<EOL>f'<STR_LIT>'<EOL>)<EOL><DEDENT>

Corresponding processing for the invalid callback result
:param item:
:return:


def __init__(self,<EOL>middleware: typing.Union[typing.Iterable, Middleware] = None,<EOL>loop=None,<EOL>is_async_start: bool = False):

if not self.start_urls or not isinstance(self.start_urls,<EOL>collections.Iterable):<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>self.callback_result_map = self.callback_result_map or {}<EOL>self.request_config = self.request_config or {}<EOL>self.headers = self.headers or {}<EOL>self.metadata = self.metadata or {}<EOL>self.kwargs = self.kwargs or {}<EOL>self.request_config = self.request_config or {}<EOL>self.is_async_start = is_async_start<EOL>self.loop = loop<EOL>asyncio.set_event_loop(self.loop)<EOL>if isinstance(middleware, list):<EOL><INDENT>self.middleware = reduce(lambda x, y: x + y, middleware)<EOL><DEDENT>else:<EOL><INDENT>self.middleware = middleware or Middleware()<EOL><DEDENT>self.request_queue = asyncio.Queue()<EOL>self.sem = asyncio.Semaphore(self.concurrency)<EOL>

Init spider object.
:param middleware: a list of or a single Middleware
:param loop:
:param is_async_start:


@classmethod<EOL><INDENT>async def async_start(<EOL>cls,<EOL>middleware: typing.Union[typing.Iterable, Middleware] = None,<EOL>loop=None,<EOL>after_start=None,<EOL>before_stop=None,<EOL>**kwargs):<DEDENT>

loop = loop or asyncio.get_event_loop()<EOL>spider_ins = cls(middleware=middleware, loop=loop, is_async_start=True)<EOL>await spider_ins._start(<EOL>after_start=after_start, before_stop=before_stop)<EOL>

Start an async spider
:param middleware: customize middleware or a list of middleware
:param loop:
:param after_start: hook
:param before_stop: hook
:return:


loop = loop or asyncio.new_event_loop()<EOL>spider_ins = cls(middleware=middleware, loop=loop)<EOL>spider_ins.loop.run_until_complete(<EOL>spider_ins._start(<EOL>after_start=after_start, before_stop=before_stop)<EOL>)<EOL>spider_ins.loop.run_until_complete(<EOL>spider_ins.loop.shutdown_asyncgens())<EOL>if close_event_loop:<EOL><INDENT>spider_ins.loop.close()<EOL><DEDENT>

Start a spider
:param after_start: hook
:param before_stop: hook
:param middleware: customize middleware or a list of middleware
:param loop: event loop
:param close_event_loop: bool
:return:


async def handle_callback(self, aws_callback: typing.Coroutine, response):

callback_result = None<EOL>try:<EOL><INDENT>callback_result = await aws_callback<EOL><DEDENT>except NothingMatchedError as e:<EOL><INDENT>self.logger.error(f'<STR_LIT>')<EOL><DEDENT>except Exception as e:<EOL><INDENT>self.logger.error(f'<STR_LIT>')<EOL><DEDENT>return callback_result, response<EOL>


async def handle_request(self, request: Request<EOL>) -> typing.Tuple[AsyncGeneratorType, Response]:

callback_result, response = None, None<EOL>await self._run_request_middleware(request)<EOL>try:<EOL><INDENT>callback_result, response = await request.fetch_callback(self.sem)<EOL><DEDENT>except NotImplementedParseError as e:<EOL><INDENT>self.logger.error(e)<EOL><DEDENT>except NothingMatchedError as e:<EOL><INDENT>self.logger.error(f'<STR_LIT>')<EOL><DEDENT>except Exception as e:<EOL><INDENT>self.logger.error(f'<STR_LIT>')<EOL><DEDENT>await self._run_response_middleware(request, response)<EOL>await self._process_response(request=request, response=response)<EOL>return callback_result, response<EOL>

Wrap request with middleware.
:param request:
:return:


async def multiple_request(self, urls, is_gather=False, **kwargs):

if is_gather:<EOL><INDENT>resp_results = await asyncio.gather(<EOL>*[<EOL>self.handle_request(self.request(url=url, **kwargs))<EOL>for url in urls<EOL>],<EOL>return_exceptions=True)<EOL>for index, task_result in enumerate(resp_results):<EOL><INDENT>if not isinstance(task_result, RuntimeError) and task_result:<EOL><INDENT>_, response = task_result<EOL>response.index = index<EOL>yield response<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for index, url in enumerate(urls):<EOL><INDENT>_, response = await self.handle_request(<EOL>self.request(url=url, **kwargs))<EOL>response.index = index<EOL>yield response<EOL><DEDENT><DEDENT>


raise NotImplementedParseError('<STR_LIT>')<EOL>

Used for subclasses, directly parse the responses corresponding with start_urls
:param response: Response
:return:


def request(self,<EOL>url: str,<EOL>method: str = '<STR_LIT:GET>',<EOL>*,<EOL>callback=None,<EOL>encoding: typing.Optional[str] = None,<EOL>headers: dict = None,<EOL>metadata: dict = None,<EOL>request_config: dict = None,<EOL>request_session=None,<EOL>**kwargs):

headers = headers or {}<EOL>metadata = metadata or {}<EOL>request_config = request_config or {}<EOL>request_session = request_session or self.request_session<EOL>headers.update(self.headers.copy())<EOL>request_config.update(self.request_config.copy())<EOL>kwargs.update(self.kwargs.copy())<EOL>return Request(<EOL>url=url,<EOL>method=method,<EOL>callback=callback,<EOL>encoding=encoding,<EOL>headers=headers,<EOL>metadata=metadata,<EOL>request_config=request_config,<EOL>request_session=request_session,<EOL>**kwargs)<EOL>


for url in self.start_urls:<EOL><INDENT>request_ins = self.request(<EOL>url=url, callback=self.parse, metadata=self.metadata)<EOL>self.request_queue.put_nowait(self.handle_request(request_ins))<EOL><DEDENT>workers = [<EOL>asyncio.ensure_future(self.start_worker())<EOL>for i in range(self.worker_numbers)<EOL>]<EOL>for worker in workers:<EOL><INDENT>self.logger.info(f"<STR_LIT>")<EOL><DEDENT>await self.request_queue.join()<EOL>if not self.is_async_start:<EOL><INDENT>await self.stop(SIGINT)<EOL><DEDENT>else:<EOL><INDENT>await self._cancel_tasks()<EOL><DEDENT>


self.logger.info(f'<STR_LIT>')<EOL>await self._cancel_tasks()<EOL>self.loop.stop()<EOL>

Finish all running tasks, cancel remaining tasks, then stop loop.
:param _signal:
:return:


def __init__(self, default: str = '<STR_LIT>', many: bool = False):

self.default = default<EOL>self.many = many<EOL>

Init BaseField class
url: http://lxml.de/index.html
:param default: default value
:param many: if there are many fields in one page


def __init__(self,<EOL>css_select: str = None,<EOL>xpath_select: str = None,<EOL>default: str = None,<EOL>many: bool = False):

super(_LxmlElementField, self).__init__(default=default, many=many)<EOL>self.css_select = css_select<EOL>self.xpath_select = xpath_select<EOL>

:param css_select: css select http://lxml.de/cssselect.html
:param xpath_select: http://www.w3school.com.cn/xpath/index.asp
:param default: inherit
:param many: inherit


if not match:<EOL><INDENT>if self.default:<EOL><INDENT>return self.default<EOL><DEDENT>else:<EOL><INDENT>raise NothingMatchedError(<EOL>f"<STR_LIT>"<EOL>f"<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>string = match.group()<EOL>groups = match.groups()<EOL>group_dict = match.groupdict()<EOL>if group_dict:<EOL><INDENT>return group_dict<EOL><DEDENT>if groups:<EOL><INDENT>return groups[<NUM_LIT:0>] if len(groups) == <NUM_LIT:1> else groups<EOL><DEDENT>return string<EOL><DEDENT>

If there is a group dict, return the dict;
    even if there's only one value in the dict, return a dictionary;
If there is a group in match, return the group;
    if there is only one value in the group, return the value;
if there has no group, return the whole matched string;
if there are many groups, return a tuple;
:param match:
:return:


async def json(self,<EOL>*,<EOL>encoding: str = None,<EOL>loads: JSONDecoder = DEFAULT_JSON_DECODER,<EOL>content_type: Optional[str] = '<STR_LIT:application/json>') -> Any:

return await self._aws_json(<EOL>encoding=encoding, loads=loads, content_type=content_type)<EOL>


return await self._aws_read()<EOL>


async def text(self,<EOL>*,<EOL>encoding: Optional[str] = None,<EOL>errors: str = '<STR_LIT:strict>') -> str:

return await self._aws_text(encoding=encoding, errors=errors)<EOL>


return str(value).strip()<EOL>


def django_boolean_icon(field_val, alt_text=None, title=None):

<EOL>BOOLEAN_MAPPING = {True: '<STR_LIT:yes>', False: '<STR_LIT>', None: '<STR_LIT>'}<EOL>alt_text = alt_text or BOOLEAN_MAPPING[field_val]<EOL>if title is not None:<EOL><INDENT>title = '<STR_LIT>' % title<EOL><DEDENT>else:<EOL><INDENT>title = '<STR_LIT>'<EOL><DEDENT>return mark_safe('<STR_LIT>' %<EOL>(settings.STATIC_URL, BOOLEAN_MAPPING[field_val], alt_text, title))<EOL>

Return HTML code for a nice representation of true/false.


all_nodes = {}<EOL>for p_id, parent_id in cls.objects.order_by(cls._mptt_meta.tree_id_attr, cls._mptt_meta.left_attr).values_list(<EOL>"<STR_LIT>", "<STR_LIT>" % cls._mptt_meta.parent_attr<EOL>):<EOL><INDENT>all_nodes[p_id] = []<EOL>if parent_id:<EOL><INDENT>if parent_id not in all_nodes:<EOL><INDENT>all_nodes[parent_id] = []<EOL><DEDENT>all_nodes[parent_id].append(p_id)<EOL><DEDENT><DEDENT>return all_nodes<EOL>

Build an in-memory representation of the item tree, trying to keep
database accesses down to a minimum. The returned dictionary looks like
this (as json dump):

    {"6": [7, 8, 10]
     "7": [12],
     "8": [],
     ...
     }


def ajax_editable_boolean_cell(item, attr, text='<STR_LIT>', override=None):

if text:<EOL><INDENT>text = '<STR_LIT>' % str(text)<EOL><DEDENT>if override is not None:<EOL><INDENT>a = [django_boolean_icon(override, text), text]<EOL><DEDENT>else:<EOL><INDENT>value = getattr(item, attr)<EOL>a = [<EOL>'<STR_LIT>',<EOL>value and '<STR_LIT>' or '<STR_LIT>',<EOL>'<STR_LIT>' % (item.id, attr),<EOL>'<STR_LIT>',<EOL>text,<EOL>]<EOL><DEDENT>a.insert(<NUM_LIT:0>, '<STR_LIT>' % ( attr, item.id ))<EOL>a.append('<STR_LIT>')<EOL>return str('<STR_LIT>'.join(a))<EOL>

Generate a html snippet for showing a boolean value on the admin page.
Item is an object, attr is the attribute name we should display. Text
is an optional explanatory text to be included in the output.

This function will emit code to produce a checkbox input with its state
corresponding to the item.attr attribute if no override value is passed.
This input is wired to run a JS ajax updater to toggle the value.

If override is passed in, ignores the attr attribute and returns a
static image for the override boolean with no user interaction possible
(useful for "disabled and you can't change it" situations).


def _fn(self, item):<EOL><INDENT>return ajax_editable_boolean_cell(item, attr)<EOL><DEDENT>_fn.allow_tags = True<EOL>_fn.short_description = short_description<EOL>_fn.editable_boolean_field = attr<EOL>return _fn<EOL>

Convenience function: Assign the return value of this method to a variable
of your ModelAdmin class and put the variable name into list_display.

Example::

    class MyTreeEditor(TreeEditor):
        list_display = ('__unicode__', 'active_toggle')

        active_toggle = ajax_editable_boolean('active', _('is active'))


if hasattr(self, '<STR_LIT>'):<EOL><INDENT>return<EOL><DEDENT>self._ajax_editable_booleans = {}<EOL>for field in self.list_display:<EOL><INDENT>item = getattr(self.__class__, field, None)<EOL>if not item:<EOL><INDENT>continue<EOL><DEDENT>attr = getattr(item, '<STR_LIT>', None)<EOL>if attr:<EOL><INDENT>def _fn(self, instance):<EOL><INDENT>return [ajax_editable_boolean_cell(instance, _fn.attr)]<EOL><DEDENT>_fn.attr = attr<EOL>result_func = getattr(item, '<STR_LIT>', _fn)<EOL>self._ajax_editable_booleans[attr] = result_func<EOL><DEDENT><DEDENT>

Collect all fields marked as editable booleans. We do not
want the user to be able to edit arbitrary fields by crafting
an AJAX request by hand.


pass<EOL>

Refresh information used to show the changelist tree structure such as
inherited active/inactive states etc.

XXX: This is somewhat hacky, but since it's an internal method, so be it.


try:<EOL><INDENT>item_id = int(request.POST.get('<STR_LIT>', None))<EOL>attr = str(request.POST.get('<STR_LIT>', None))<EOL><DEDENT>except:<EOL><INDENT>return HttpResponseBadRequest("<STR_LIT>")<EOL><DEDENT>if not request.user.is_staff:<EOL><INDENT>logging.warning("<STR_LIT>", request.user,<EOL>attr, item_id)<EOL>return HttpResponseForbidden("<STR_LIT>")<EOL><DEDENT>self._collect_editable_booleans()<EOL>if attr not in self._ajax_editable_booleans:<EOL><INDENT>return HttpResponseBadRequest("<STR_LIT>" % attr)<EOL><DEDENT>try:<EOL><INDENT>obj = self.model._default_manager.get(pk=item_id)<EOL><DEDENT>except self.model.DoesNotExist:<EOL><INDENT>return HttpResponseNotFound("<STR_LIT>")<EOL><DEDENT>can_change = False<EOL>if hasattr(obj, "<STR_LIT>") and obj.user_can(request.user, change_page=True):<EOL><INDENT>can_change = True<EOL><DEDENT>else:<EOL><INDENT>can_change = self.has_change_permission(request, obj=obj)<EOL><DEDENT>if not can_change:<EOL><INDENT>logging.warning("<STR_LIT>", request.user, attr, item_id)<EOL>return HttpResponseForbidden("<STR_LIT>")<EOL><DEDENT>logging.info("<STR_LIT>", request.user, attr, obj)<EOL>try:<EOL><INDENT>before_data = self._ajax_editable_booleans[attr](self, obj)<EOL>setattr(obj, attr, not getattr(obj, attr))<EOL>obj.save()<EOL>self._refresh_changelist_caches() <EOL>data = self._ajax_editable_booleans[attr](self, obj)<EOL><DEDENT>except Exception:<EOL><INDENT>logging.exception("<STR_LIT>", attr, obj)<EOL>return HttpResponseServerError("<STR_LIT>" % (attr, obj))<EOL><DEDENT>d = []<EOL>for a, b in zip(before_data, data):<EOL><INDENT>if a != b:<EOL><INDENT>d.append(b)<EOL><DEDENT><DEDENT>return HttpResponse(json.dumps(d), mimetype="<STR_LIT:application/json>")<EOL>


def changelist_view(self, request, extra_context=None, *args, **kwargs):

if '<STR_LIT>' not in self.list_display:<EOL><INDENT>self.list_display.append('<STR_LIT>')<EOL><DEDENT>if request.is_ajax():<EOL><INDENT>cmd = request.POST.get('<STR_LIT>')<EOL>if cmd == '<STR_LIT>':<EOL><INDENT>return self._toggle_boolean(request)<EOL><DEDENT>elif cmd == '<STR_LIT>':<EOL><INDENT>return self._move_node(request)<EOL><DEDENT>else:<EOL><INDENT>return HttpResponseBadRequest('<STR_LIT>')<EOL><DEDENT><DEDENT>self._refresh_changelist_caches()<EOL>extra_context = extra_context or {}<EOL>extra_context['<STR_LIT>'] = settings.STATIC_URL<EOL>extra_context['<STR_LIT>'] = settings.JQUERY_LIB<EOL>extra_context['<STR_LIT>'] = settings.JQUERYUI_LIB<EOL>extra_context['<STR_LIT>'] = mark_safe(json.dumps(<EOL>_build_tree_structure(self.model)))<EOL>return super(TreeEditor, self).changelist_view(request, extra_context, *args, **kwargs)<EOL>

Handle the changelist view, the django view for the model instances
change list/actions page.


def has_change_permission(self, request, obj=None):

if settings.TREE_EDITOR_OBJECT_PERMISSIONS:<EOL><INDENT>opts = self.opts<EOL>r = request.user.has_perm(opts.app_label + '<STR_LIT:.>' + opts.get_change_permission(), obj)<EOL><DEDENT>else:<EOL><INDENT>r = True<EOL><DEDENT>return r and super(TreeEditor, self).has_change_permission(request, obj)<EOL>

Implement a lookup for object level permissions. Basically the same as
ModelAdmin.has_change_permission, but also passes the obj parameter in.


def has_delete_permission(self, request, obj=None):

if settings.TREE_EDITOR_OBJECT_PERMISSIONS:<EOL><INDENT>opts = self.opts<EOL>r = request.user.has_perm(opts.app_label + '<STR_LIT:.>' + opts.get_delete_permission(), obj)<EOL><DEDENT>else:<EOL><INDENT>r = True<EOL><DEDENT>return r and super(TreeEditor, self).has_delete_permission(request, obj)<EOL>

Implement a lookup for object level permissions. Basically the same as
ModelAdmin.has_delete_permission, but also passes the obj parameter in.


<EOL>repo_dir = dirname(os.path.abspath(__file__))<EOL>target_dir = dirname(path)<EOL>copy_tree(join(repo_dir, "<STR_LIT>"), join(target_dir, "<STR_LIT>"))<EOL>copy_tree(join(repo_dir, "<STR_LIT>"), join(target_dir, "<STR_LIT>"))<EOL>copy_tree(join(repo_dir, "<STR_LIT>"), join(target_dir, "<STR_LIT>"))<EOL>copy_tree(join(repo_dir, "<STR_LIT>"), join(target_dir, "<STR_LIT>"))<EOL>shutil.copy(join(repo_dir, "<STR_LIT>"),<EOL>join(target_dir, "<STR_LIT>"))<EOL>shutil.copy(join(repo_dir, "<STR_LIT>"),<EOL>join(target_dir, "<STR_LIT>"))<EOL>

Parameters
----------
path

Returns
-------