def plot_drawdown_underwater(returns, ax=None, **kwargs):

if ax is None:<EOL><INDENT>ax = plt.gca()<EOL><DEDENT>y_axis_formatter = FuncFormatter(utils.percentage)<EOL>ax.yaxis.set_major_formatter(FuncFormatter(y_axis_formatter))<EOL>df_cum_rets = ep.cum_returns(returns, starting_value=<NUM_LIT:1.0>)<EOL>running_max = np.maximum.accumulate(df_cum_rets)<EOL>underwater = -<NUM_LIT:100> * ((running_max - df_cum_rets) / running_max)<EOL>(underwater).plot(ax=ax, kind='<STR_LIT>', color='<STR_LIT>', alpha=<NUM_LIT>, **kwargs)<EOL>ax.set_ylabel('<STR_LIT>')<EOL>ax.set_title('<STR_LIT>')<EOL>ax.set_xlabel('<STR_LIT>')<EOL>return ax<EOL>

Plots how far underwaterr returns are over time, or plots current
drawdown vs. date.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in tears.create_full_tear_sheet.
ax : matplotlib.Axes, optional
    Axes upon which to plot.
**kwargs, optional
    Passed to plotting function.

Returns
-------
ax : matplotlib.Axes
    The axes that were plotted on.


def plot_monthly_returns_dist(returns, ax=None, **kwargs):

if ax is None:<EOL><INDENT>ax = plt.gca()<EOL><DEDENT>x_axis_formatter = FuncFormatter(utils.percentage)<EOL>ax.xaxis.set_major_formatter(FuncFormatter(x_axis_formatter))<EOL>ax.tick_params(axis='<STR_LIT:x>', which='<STR_LIT>')<EOL>monthly_ret_table = ep.aggregate_returns(returns, '<STR_LIT>')<EOL>ax.hist(<EOL><NUM_LIT:100> * monthly_ret_table,<EOL>color='<STR_LIT>',<EOL>alpha=<NUM_LIT>,<EOL>bins=<NUM_LIT:20>,<EOL>**kwargs)<EOL>ax.axvline(<EOL><NUM_LIT:100> * monthly_ret_table.mean(),<EOL>color='<STR_LIT>',<EOL>linestyle='<STR_LIT>',<EOL>lw=<NUM_LIT:4>,<EOL>alpha=<NUM_LIT:1.0>)<EOL>ax.axvline(<NUM_LIT:0.0>, color='<STR_LIT>', linestyle='<STR_LIT:->', lw=<NUM_LIT:3>, alpha=<NUM_LIT>)<EOL>ax.legend(['<STR_LIT>'], frameon=True, framealpha=<NUM_LIT:0.5>)<EOL>ax.set_ylabel('<STR_LIT>')<EOL>ax.set_xlabel('<STR_LIT>')<EOL>ax.set_title("<STR_LIT>")<EOL>return ax<EOL>

Plots a distribution of monthly returns.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in tears.create_full_tear_sheet.
ax : matplotlib.Axes, optional
    Axes upon which to plot.
**kwargs, optional
    Passed to plotting function.

Returns
-------
ax : matplotlib.Axes
    The axes that were plotted on.


def plot_rolling_volatility(returns, factor_returns=None,<EOL>rolling_window=APPROX_BDAYS_PER_MONTH * <NUM_LIT:6>,<EOL>legend_loc='<STR_LIT>', ax=None, **kwargs):

if ax is None:<EOL><INDENT>ax = plt.gca()<EOL><DEDENT>y_axis_formatter = FuncFormatter(utils.two_dec_places)<EOL>ax.yaxis.set_major_formatter(FuncFormatter(y_axis_formatter))<EOL>rolling_vol_ts = timeseries.rolling_volatility(<EOL>returns, rolling_window)<EOL>rolling_vol_ts.plot(alpha=<NUM_LIT>, lw=<NUM_LIT:3>, color='<STR_LIT>', ax=ax,<EOL>**kwargs)<EOL>if factor_returns is not None:<EOL><INDENT>rolling_vol_ts_factor = timeseries.rolling_volatility(<EOL>factor_returns, rolling_window)<EOL>rolling_vol_ts_factor.plot(alpha=<NUM_LIT>, lw=<NUM_LIT:3>, color='<STR_LIT>', ax=ax,<EOL>**kwargs)<EOL><DEDENT>ax.set_title('<STR_LIT>')<EOL>ax.axhline(<EOL>rolling_vol_ts.mean(),<EOL>color='<STR_LIT>',<EOL>linestyle='<STR_LIT>',<EOL>lw=<NUM_LIT:3>)<EOL>ax.axhline(<NUM_LIT:0.0>, color='<STR_LIT>', linestyle='<STR_LIT:->', lw=<NUM_LIT:2>)<EOL>ax.set_ylabel('<STR_LIT>')<EOL>ax.set_xlabel('<STR_LIT>')<EOL>if factor_returns is None:<EOL><INDENT>ax.legend(['<STR_LIT>', '<STR_LIT>'],<EOL>loc=legend_loc, frameon=True, framealpha=<NUM_LIT:0.5>)<EOL><DEDENT>else:<EOL><INDENT>ax.legend(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'],<EOL>loc=legend_loc, frameon=True, framealpha=<NUM_LIT:0.5>)<EOL><DEDENT>return ax<EOL>

Plots the rolling volatility versus date.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in tears.create_full_tear_sheet.
factor_returns : pd.Series, optional
    Daily noncumulative returns of the benchmark factor to which betas are
    computed. Usually a benchmark such as market returns.
     - This is in the same style as returns.
rolling_window : int, optional
    The days window over which to compute the volatility.
legend_loc : matplotlib.loc, optional
    The location of the legend on the plot.
ax : matplotlib.Axes, optional
    Axes upon which to plot.
**kwargs, optional
    Passed to plotting function.

Returns
-------
ax : matplotlib.Axes
    The axes that were plotted on.


def plot_exposures(returns, positions, ax=None, **kwargs):

if ax is None:<EOL><INDENT>ax = plt.gca()<EOL><DEDENT>pos_no_cash = positions.drop('<STR_LIT>', axis=<NUM_LIT:1>)<EOL>l_exp = pos_no_cash[pos_no_cash > <NUM_LIT:0>].sum(axis=<NUM_LIT:1>) / positions.sum(axis=<NUM_LIT:1>)<EOL>s_exp = pos_no_cash[pos_no_cash < <NUM_LIT:0>].sum(axis=<NUM_LIT:1>) / positions.sum(axis=<NUM_LIT:1>)<EOL>net_exp = pos_no_cash.sum(axis=<NUM_LIT:1>) / positions.sum(axis=<NUM_LIT:1>)<EOL>ax.fill_between(l_exp.index,<EOL><NUM_LIT:0>,<EOL>l_exp.values,<EOL>label='<STR_LIT>', color='<STR_LIT>', alpha=<NUM_LIT:0.5>)<EOL>ax.fill_between(s_exp.index,<EOL><NUM_LIT:0>,<EOL>s_exp.values,<EOL>label='<STR_LIT>', color='<STR_LIT>', alpha=<NUM_LIT:0.5>)<EOL>ax.plot(net_exp.index, net_exp.values,<EOL>label='<STR_LIT>', color='<STR_LIT>', linestyle='<STR_LIT>')<EOL>ax.set_xlim((returns.index[<NUM_LIT:0>], returns.index[-<NUM_LIT:1>]))<EOL>ax.set_title("<STR_LIT>")<EOL>ax.set_ylabel('<STR_LIT>')<EOL>ax.legend(loc='<STR_LIT>', frameon=True, framealpha=<NUM_LIT:0.5>)<EOL>ax.set_xlabel('<STR_LIT>')<EOL>return ax<EOL>

Plots a cake chart of the long and short exposure.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in tears.create_full_tear_sheet.
positions_alloc : pd.DataFrame
    Portfolio allocation of positions. See
    pos.get_percent_alloc.
ax : matplotlib.Axes, optional
    Axes upon which to plot.
**kwargs, optional
    Passed to plotting function.

Returns
-------
ax : matplotlib.Axes
    The axes that were plotted on.


def plot_daily_turnover_hist(transactions, positions,<EOL>ax=None, **kwargs):

if ax is None:<EOL><INDENT>ax = plt.gca()<EOL><DEDENT>turnover = txn.get_turnover(positions, transactions)<EOL>sns.distplot(turnover, ax=ax, **kwargs)<EOL>ax.set_title('<STR_LIT>')<EOL>ax.set_xlabel('<STR_LIT>')<EOL>return ax<EOL>

Plots a histogram of daily turnover rates.

Parameters
----------
transactions : pd.DataFrame
    Prices and amounts of executed trades. One row per trade.
     - See full explanation in tears.create_full_tear_sheet.
positions : pd.DataFrame
    Daily net position values.
     - See full explanation in tears.create_full_tear_sheet.
ax : matplotlib.Axes, optional
    Axes upon which to plot.
**kwargs, optional
    Passed to seaborn plotting function.

Returns
-------
ax : matplotlib.Axes
    The axes that were plotted on.


pos_wo_cash = positions.drop('<STR_LIT>', axis=<NUM_LIT:1>)<EOL>longs = pos_wo_cash[pos_wo_cash > <NUM_LIT:0>].sum(axis=<NUM_LIT:1>).fillna(<NUM_LIT:0>)<EOL>shorts = pos_wo_cash[pos_wo_cash < <NUM_LIT:0>].sum(axis=<NUM_LIT:1>).fillna(<NUM_LIT:0>)<EOL>cash = positions.cash<EOL>net_liquidation = longs + shorts + cash<EOL>df_pos = pd.DataFrame({'<STR_LIT>': longs.divide(net_liquidation, axis='<STR_LIT:index>'),<EOL>'<STR_LIT>': shorts.divide(net_liquidation,<EOL>axis='<STR_LIT:index>')})<EOL>df_pos['<STR_LIT>'] = df_pos['<STR_LIT>'] + df_pos['<STR_LIT>']<EOL>return df_pos<EOL>

Determines the long and short allocations in a portfolio.

Parameters
----------
positions : pd.DataFrame
    The positions that the strategy takes over time.

Returns
-------
df_long_short : pd.DataFrame
    Long and short allocations as a decimal
    percentage of the total net liquidation


cash = positions['<STR_LIT>']<EOL>positions = positions.drop('<STR_LIT>', axis=<NUM_LIT:1>)<EOL>unmapped_pos = np.setdiff1d(positions.columns.values,<EOL>list(symbol_sector_map.keys()))<EOL>if len(unmapped_pos) > <NUM_LIT:0>:<EOL><INDENT>warn_message = """<STR_LIT>""".format(<EOL>"<STR_LIT:U+002CU+0020>".join(map(str, unmapped_pos)))<EOL>warnings.warn(warn_message, UserWarning)<EOL><DEDENT>sector_exp = positions.groupby(<EOL>by=symbol_sector_map, axis=<NUM_LIT:1>).sum()<EOL>sector_exp['<STR_LIT>'] = cash<EOL>return sector_exp<EOL>

Sum position exposures by sector.

Parameters
----------
positions : pd.DataFrame
    Contains position values or amounts.
    - Example
        index         'AAPL'         'MSFT'        'CHK'        cash
        2004-01-09    13939.380     -15012.993    -403.870      1477.483
        2004-01-12    14492.630     -18624.870    142.630       3989.610
        2004-01-13    -13853.280    13653.640     -100.980      100.000
symbol_sector_map : dict or pd.Series
    Security identifier to sector mapping.
    Security ids as keys/index, sectors as values.
    - Example:
        {'AAPL' : 'Technology'
         'MSFT' : 'Technology'
         'CHK' : 'Natural Resources'}

Returns
-------
sector_exp : pd.DataFrame
    Sectors and their allocations.
    - Example:
        index         'Technology'    'Natural Resources' cash
        2004-01-09    -1073.613       -403.870            1477.4830
        2004-01-12    -4132.240       142.630             3989.6100
        2004-01-13    -199.640        -100.980            100.0000


expos = get_percent_alloc(positions)<EOL>expos = expos.drop('<STR_LIT>', axis=<NUM_LIT:1>)<EOL>longs = expos.where(expos.applymap(lambda x: x > <NUM_LIT:0>))<EOL>shorts = expos.where(expos.applymap(lambda x: x < <NUM_LIT:0>))<EOL>alloc_summary = pd.DataFrame()<EOL>alloc_summary['<STR_LIT>'] = longs.max(axis=<NUM_LIT:1>)<EOL>alloc_summary['<STR_LIT>'] = longs.median(axis=<NUM_LIT:1>)<EOL>alloc_summary['<STR_LIT>'] = shorts.median(axis=<NUM_LIT:1>)<EOL>alloc_summary['<STR_LIT>'] = shorts.min(axis=<NUM_LIT:1>)<EOL>return alloc_summary<EOL>

Finds the max and median long and short position concentrations
in each time period specified by the index of positions.

Parameters
----------
positions : pd.DataFrame
    The positions that the strategy takes over time.

Returns
-------
pd.DataFrame
    Columns are max long, max short, median long, and median short
    position concentrations. Rows are timeperiods.


def model_returns_normal(data, samples=<NUM_LIT>, progressbar=True):

with pm.Model() as model:<EOL><INDENT>mu = pm.Normal('<STR_LIT>', mu=<NUM_LIT:0>, sd=<NUM_LIT>, testval=data.mean())<EOL>sigma = pm.HalfCauchy('<STR_LIT>', beta=<NUM_LIT:1>, testval=data.std())<EOL>returns = pm.Normal('<STR_LIT>', mu=mu, sd=sigma, observed=data)<EOL>pm.Deterministic(<EOL>'<STR_LIT>',<EOL>returns.distribution.variance**<NUM_LIT> *<EOL>np.sqrt(<NUM_LIT>))<EOL>pm.Deterministic(<EOL>'<STR_LIT>',<EOL>returns.distribution.mean /<EOL>returns.distribution.variance**<NUM_LIT> *<EOL>np.sqrt(<NUM_LIT>))<EOL>trace = pm.sample(samples, progressbar=progressbar)<EOL><DEDENT>return model, trace<EOL>

Run Bayesian model assuming returns are normally distributed.

Parameters
----------
returns : pandas.Series
    Series of simple returns of an algorithm or stock.
samples : int (optional)
    Number of posterior samples to draw.

Returns
-------
model : pymc.Model object
    PyMC3 model containing all random variables.
trace : pymc3.sampling.BaseTrace object
    A PyMC3 trace object that contains samples for each parameter
    of the posterior.


def model_returns_t_alpha_beta(data, bmark, samples=<NUM_LIT>, progressbar=True):

data_bmark = pd.concat([data, bmark], axis=<NUM_LIT:1>).dropna()<EOL>with pm.Model() as model:<EOL><INDENT>sigma = pm.HalfCauchy(<EOL>'<STR_LIT>',<EOL>beta=<NUM_LIT:1>)<EOL>nu = pm.Exponential('<STR_LIT>', <NUM_LIT:1.> / <NUM_LIT>)<EOL>X = data_bmark.iloc[:, <NUM_LIT:1>]<EOL>y = data_bmark.iloc[:, <NUM_LIT:0>]<EOL>alpha_reg = pm.Normal('<STR_LIT>', mu=<NUM_LIT:0>, sd=<NUM_LIT>)<EOL>beta_reg = pm.Normal('<STR_LIT>', mu=<NUM_LIT:0>, sd=<NUM_LIT:1>)<EOL>mu_reg = alpha_reg + beta_reg * X<EOL>pm.StudentT('<STR_LIT>',<EOL>nu=nu + <NUM_LIT:2>,<EOL>mu=mu_reg,<EOL>sd=sigma,<EOL>observed=y)<EOL>trace = pm.sample(samples, progressbar=progressbar)<EOL><DEDENT>return model, trace<EOL>

Run Bayesian alpha-beta-model with T distributed returns.

This model estimates intercept (alpha) and slope (beta) of two
return sets. Usually, these will be algorithm returns and
benchmark returns (e.g. S&P500). The data is assumed to be T
distributed and thus is robust to outliers and takes tail events
into account.  If a pandas.DataFrame is passed as a benchmark, then
multiple linear regression is used to estimate alpha and beta.

Parameters
----------
returns : pandas.Series
    Series of simple returns of an algorithm or stock.
bmark : pandas.DataFrame
    DataFrame of benchmark returns (e.g., S&P500) or risk factors (e.g.,
    Fama-French SMB, HML, and UMD).
    If bmark has more recent returns than returns_train, these dates
    will be treated as missing values and predictions will be
    generated for them taking market correlations into account.
samples : int (optional)
    Number of posterior samples to draw.

Returns
-------
model : pymc.Model object
    PyMC3 model containing all random variables.
trace : pymc3.sampling.BaseTrace object
    A PyMC3 trace object that contains samples for each parameter
    of the posterior.


def plot_stoch_vol(data, trace=None, ax=None):

if trace is None:<EOL><INDENT>trace = model_stoch_vol(data)<EOL><DEDENT>if ax is None:<EOL><INDENT>fig, ax = plt.subplots(figsize=(<NUM_LIT:15>, <NUM_LIT:8>))<EOL><DEDENT>data.abs().plot(ax=ax)<EOL>ax.plot(data.index, np.exp(trace['<STR_LIT:s>', ::<NUM_LIT:30>].T), '<STR_LIT:r>', alpha=<NUM_LIT>)<EOL>ax.set(title='<STR_LIT>', xlabel='<STR_LIT>', ylabel='<STR_LIT>')<EOL>ax.legend(['<STR_LIT>', '<STR_LIT>'],<EOL>frameon=True, framealpha=<NUM_LIT:0.5>)<EOL>return ax<EOL>

Generate plot for stochastic volatility model.

Parameters
----------
data : pandas.Series
    Returns to model.
trace : pymc3.sampling.BaseTrace object, optional
    trace as returned by model_stoch_vol
    If not passed, sample from model.
ax : matplotlib.axes object, optional
    Plot into axes object

Returns
-------
ax object

See Also
--------
model_stoch_vol : run stochastic volatility model


def model_best(y1, y2, samples=<NUM_LIT:1000>, progressbar=True):

y = np.concatenate((y1, y2))<EOL>mu_m = np.mean(y)<EOL>mu_p = <NUM_LIT> * <NUM_LIT:1> / np.std(y)**<NUM_LIT:2><EOL>sigma_low = np.std(y) / <NUM_LIT:1000><EOL>sigma_high = np.std(y) * <NUM_LIT:1000><EOL>with pm.Model() as model:<EOL><INDENT>group1_mean = pm.Normal('<STR_LIT>', mu=mu_m, tau=mu_p,<EOL>testval=y1.mean())<EOL>group2_mean = pm.Normal('<STR_LIT>', mu=mu_m, tau=mu_p,<EOL>testval=y2.mean())<EOL>group1_std = pm.Uniform('<STR_LIT>', lower=sigma_low,<EOL>upper=sigma_high, testval=y1.std())<EOL>group2_std = pm.Uniform('<STR_LIT>', lower=sigma_low,<EOL>upper=sigma_high, testval=y2.std())<EOL>nu = pm.Exponential('<STR_LIT>', <NUM_LIT:1> / <NUM_LIT>, testval=<NUM_LIT>) + <NUM_LIT><EOL>returns_group1 = pm.StudentT('<STR_LIT>', nu=nu, mu=group1_mean,<EOL>lam=group1_std**-<NUM_LIT:2>, observed=y1)<EOL>returns_group2 = pm.StudentT('<STR_LIT>', nu=nu, mu=group2_mean,<EOL>lam=group2_std**-<NUM_LIT:2>, observed=y2)<EOL>diff_of_means = pm.Deterministic('<STR_LIT>',<EOL>group2_mean - group1_mean)<EOL>pm.Deterministic('<STR_LIT>',<EOL>group2_std - group1_std)<EOL>pm.Deterministic('<STR_LIT>', diff_of_means /<EOL>pm.math.sqrt((group1_std**<NUM_LIT:2> +<EOL>group2_std**<NUM_LIT:2>) / <NUM_LIT:2>))<EOL>pm.Deterministic('<STR_LIT>',<EOL>returns_group1.distribution.variance**<NUM_LIT> *<EOL>np.sqrt(<NUM_LIT>))<EOL>pm.Deterministic('<STR_LIT>',<EOL>returns_group2.distribution.variance**<NUM_LIT> *<EOL>np.sqrt(<NUM_LIT>))<EOL>pm.Deterministic('<STR_LIT>', returns_group1.distribution.mean /<EOL>returns_group1.distribution.variance**<NUM_LIT> *<EOL>np.sqrt(<NUM_LIT>))<EOL>pm.Deterministic('<STR_LIT>', returns_group2.distribution.mean /<EOL>returns_group2.distribution.variance**<NUM_LIT> *<EOL>np.sqrt(<NUM_LIT>))<EOL>trace = pm.sample(samples, progressbar=progressbar)<EOL><DEDENT>return model, trace<EOL>

Bayesian Estimation Supersedes the T-Test

This model runs a Bayesian hypothesis comparing if y1 and y2 come
from the same distribution. Returns are assumed to be T-distributed.

In addition, computes annual volatility and Sharpe of in and
out-of-sample periods.

This model replicates the example used in:
Kruschke, John. (2012) Bayesian estimation supersedes the t
test. Journal of Experimental Psychology: General.

Parameters
----------
y1 : array-like
    Array of returns (e.g. in-sample)
y2 : array-like
    Array of returns (e.g. out-of-sample)
samples : int, optional
    Number of posterior samples to draw.

Returns
-------
model : pymc.Model object
    PyMC3 model containing all random variables.
trace : pymc3.sampling.BaseTrace object
    A PyMC3 trace object that contains samples for each parameter
    of the posterior.

See Also
--------
plot_stoch_vol : plotting of tochastic volatility model


transactions = _groupby_consecutive(transactions)<EOL>roundtrips = []<EOL>for sym, trans_sym in transactions.groupby('<STR_LIT>'):<EOL><INDENT>trans_sym = trans_sym.sort_index()<EOL>price_stack = deque()<EOL>dt_stack = deque()<EOL>trans_sym['<STR_LIT>'] = trans_sym.price *np.sign(trans_sym.amount)<EOL>trans_sym['<STR_LIT>'] = trans_sym.amount.abs().astype(int)<EOL>for dt, t in trans_sym.iterrows():<EOL><INDENT>if t.price < <NUM_LIT:0>:<EOL><INDENT>warnings.warn('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>continue<EOL><DEDENT>indiv_prices = [t.signed_price] * t.abs_amount<EOL>if (len(price_stack) == <NUM_LIT:0>) or(copysign(<NUM_LIT:1>, price_stack[-<NUM_LIT:1>]) == copysign(<NUM_LIT:1>, t.amount)):<EOL><INDENT>price_stack.extend(indiv_prices)<EOL>dt_stack.extend([dt] * len(indiv_prices))<EOL><DEDENT>else:<EOL><INDENT>pnl = <NUM_LIT:0><EOL>invested = <NUM_LIT:0><EOL>cur_open_dts = []<EOL>for price in indiv_prices:<EOL><INDENT>if len(price_stack) != <NUM_LIT:0> and(copysign(<NUM_LIT:1>, price_stack[-<NUM_LIT:1>]) != copysign(<NUM_LIT:1>, price)):<EOL><INDENT>prev_price = price_stack.popleft()<EOL>prev_dt = dt_stack.popleft()<EOL>pnl += -(price + prev_price)<EOL>cur_open_dts.append(prev_dt)<EOL>invested += abs(prev_price)<EOL><DEDENT>else:<EOL><INDENT>price_stack.append(price)<EOL>dt_stack.append(dt)<EOL><DEDENT><DEDENT>roundtrips.append({'<STR_LIT>': pnl,<EOL>'<STR_LIT>': cur_open_dts[<NUM_LIT:0>],<EOL>'<STR_LIT>': dt,<EOL>'<STR_LIT>': price < <NUM_LIT:0>,<EOL>'<STR_LIT>': pnl / invested,<EOL>'<STR_LIT>': sym,<EOL>})<EOL><DEDENT><DEDENT><DEDENT>roundtrips = pd.DataFrame(roundtrips)<EOL>roundtrips['<STR_LIT>'] = roundtrips['<STR_LIT>'].sub(roundtrips['<STR_LIT>'])<EOL>if portfolio_value is not None:<EOL><INDENT>pv = pd.DataFrame(portfolio_value,<EOL>columns=['<STR_LIT>']).assign(date=portfolio_value.index)<EOL>roundtrips['<STR_LIT:date>'] = roundtrips.close_dt.apply(lambda x:<EOL>x.replace(hour=<NUM_LIT:0>,<EOL>minute=<NUM_LIT:0>,<EOL>second=<NUM_LIT:0>))<EOL>tmp = roundtrips.join(pv, on='<STR_LIT:date>', lsuffix='<STR_LIT:_>')<EOL>roundtrips['<STR_LIT>'] = tmp.pnl / tmp.portfolio_value<EOL>roundtrips = roundtrips.drop('<STR_LIT:date>', axis='<STR_LIT>')<EOL><DEDENT>return roundtrips<EOL>

Group transactions into "round trips". First, transactions are
    grouped by day and directionality. Then, long and short
    transactions are matched to create round-trip round_trips for which
    PnL, duration and returns are computed. Crossings where a position
    changes from long to short and vice-versa are handled correctly.

    Under the hood, we reconstruct the individual shares in a
    portfolio over time and match round_trips in a FIFO-order.

    For example, the following transactions would constitute one round trip:
    index                  amount   price    symbol
    2004-01-09 12:18:01    10       50      'AAPL'
    2004-01-09 15:12:53    10       100      'AAPL'
    2004-01-13 14:41:23    -10      100      'AAPL'
    2004-01-13 15:23:34    -10      200       'AAPL'

    First, the first two and last two round_trips will be merged into a two
    single transactions (computing the price via vwap). Then, during
    the portfolio reconstruction, the two resulting transactions will
    be merged and result in 1 round-trip trade with a PnL of
    (150 * 20) - (75 * 20) = 1500.

    Note, that round trips do not have to close out positions
    completely. For example, we could have removed the last
    transaction in the example above and still generated a round-trip
    over 10 shares with 10 shares left in the portfolio to be matched
    with a later transaction.

    Parameters
    ----------
    transactions : pd.DataFrame
        Prices and amounts of executed round_trips. One row per trade.
        - See full explanation in tears.create_full_tear_sheet

    portfolio_value : pd.Series (optional)
        Portfolio value (all net assets including cash) over time.
        Note that portfolio_value needs to beginning of day, so either
        use .shift() or positions.sum(axis='columns') / (1+returns).

    Returns
    -------
    round_trips : pd.DataFrame
        DataFrame with one row per round trip.  The returns column
        contains returns in respect to the portfolio value while
        rt_returns are the returns in regards to the invested capital
        into that partiulcar round-trip.


def vwap(transaction):<EOL><INDENT>if transaction.amount.sum() == <NUM_LIT:0>:<EOL><INDENT>warnings.warn('<STR_LIT>')<EOL>return np.nan<EOL><DEDENT>return (transaction.amount * transaction.price).sum() /transaction.amount.sum()<EOL><DEDENT>out = []<EOL>for sym, t in txn.groupby('<STR_LIT>'):<EOL><INDENT>t = t.sort_index()<EOL>t.index.name = '<STR_LIT>'<EOL>t = t.reset_index()<EOL>t['<STR_LIT>'] = t.amount > <NUM_LIT:0><EOL>t['<STR_LIT>'] = (t.order_sign.shift(<EOL><NUM_LIT:1>) != t.order_sign).astype(int).cumsum()<EOL>t['<STR_LIT>'] = ((t.dt.sub(t.dt.shift(<NUM_LIT:1>))) ><EOL>max_delta).astype(int).cumsum()<EOL>grouped_price = (t.groupby(('<STR_LIT>',<EOL>'<STR_LIT>'))<EOL>.apply(vwap))<EOL>grouped_price.name = '<STR_LIT>'<EOL>grouped_rest = t.groupby(('<STR_LIT>', '<STR_LIT>')).agg({<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>'})<EOL>grouped = grouped_rest.join(grouped_price)<EOL>out.append(grouped)<EOL><DEDENT>out = pd.concat(out)<EOL>out = out.set_index('<STR_LIT>')<EOL>return out<EOL>

Merge transactions of the same direction separated by less than
    max_delta time duration.

    Parameters
    ----------
    transactions : pd.DataFrame
        Prices and amounts of executed round_trips. One row per trade.
        - See full explanation in tears.create_full_tear_sheet

    max_delta : pandas.Timedelta (optional)
        Merge transactions in the same direction separated by less
        than max_delta time duration.

    Returns
    -------
    transactions : pd.DataFrame


missing_stocks = positions.columns.difference(<EOL>factor_loadings.index.get_level_values(<NUM_LIT:1>).unique()<EOL>)<EOL>num_stocks = len(positions.columns) - <NUM_LIT:1><EOL>missing_stocks = missing_stocks.drop('<STR_LIT>')<EOL>num_stocks_covered = num_stocks - len(missing_stocks)<EOL>missing_ratio = round(len(missing_stocks) / num_stocks, ndigits=<NUM_LIT:3>)<EOL>if num_stocks_covered == <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if len(missing_stocks) > <NUM_LIT:0>:<EOL><INDENT>if len(missing_stocks) > <NUM_LIT:5>:<EOL><INDENT>missing_stocks_displayed = (<EOL>"<STR_LIT>"<EOL>).format(len(missing_stocks),<EOL>'<STR_LIT:U+002CU+0020>'.join(missing_stocks[:<NUM_LIT:5>].map(str)),<EOL>missing_stocks[-<NUM_LIT:1>])<EOL>avg_allocation_msg = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>missing_stocks_displayed = (<EOL>"<STR_LIT>"<EOL>).format(list(missing_stocks))<EOL>avg_allocation_msg = "<STR_LIT>"<EOL><DEDENT>missing_stocks_warning_msg = (<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT:\n>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT:\n>"<EOL>"<STR_LIT>"<EOL>).format(<EOL>missing_stocks_displayed,<EOL>missing_ratio,<EOL>avg_allocation_msg,<EOL>positions[missing_stocks[:<NUM_LIT:5>].union(missing_stocks[[-<NUM_LIT:1>]])].mean(),<EOL>)<EOL>warnings.warn(missing_stocks_warning_msg)<EOL>positions = positions.drop(missing_stocks, axis='<STR_LIT>',<EOL>errors='<STR_LIT:ignore>')<EOL><DEDENT>missing_factor_loadings_index = positions.index.difference(<EOL>factor_loadings.index.get_level_values(<NUM_LIT:0>).unique()<EOL>)<EOL>missing_factor_loadings_index = positions.index.difference(<EOL>factor_loadings.index.get_level_values(<NUM_LIT:0>).unique()<EOL>)<EOL>if len(missing_factor_loadings_index) > <NUM_LIT:0>:<EOL><INDENT>if len(missing_factor_loadings_index) > <NUM_LIT:5>:<EOL><INDENT>missing_dates_displayed = (<EOL>"<STR_LIT>"<EOL>).format(<EOL>missing_factor_loadings_index[<NUM_LIT:0>],<EOL>missing_factor_loadings_index[-<NUM_LIT:1>]<EOL>)<EOL><DEDENT>else:<EOL><INDENT>missing_dates_displayed = list(missing_factor_loadings_index)<EOL><DEDENT>warning_msg = (<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>).format(len(missing_factor_loadings_index), missing_dates_displayed)<EOL>warnings.warn(warning_msg)<EOL>positions = positions.drop(missing_factor_loadings_index,<EOL>errors='<STR_LIT:ignore>')<EOL>returns = returns.drop(missing_factor_loadings_index, errors='<STR_LIT:ignore>')<EOL>factor_returns = factor_returns.drop(missing_factor_loadings_index,<EOL>errors='<STR_LIT:ignore>')<EOL><DEDENT>if transactions is not None and pos_in_dollars:<EOL><INDENT>turnover = get_turnover(positions, transactions).mean()<EOL>if turnover > PERF_ATTRIB_TURNOVER_THRESHOLD:<EOL><INDENT>warning_msg = (<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT:\n>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL>warnings.warn(warning_msg)<EOL><DEDENT><DEDENT>return (returns, positions, factor_returns, factor_loadings)<EOL>

Make sure that all inputs have matching dates and tickers,
and raise warnings if necessary.


if costs is None:<EOL><INDENT>return ep.cum_returns(returns)<EOL><DEDENT>return ep.cum_returns(returns - costs)<EOL>


summary = OrderedDict()<EOL>total_returns = perf_attrib['<STR_LIT>']<EOL>specific_returns = perf_attrib['<STR_LIT>']<EOL>common_returns = perf_attrib['<STR_LIT>']<EOL>summary['<STR_LIT>'] =ep.annual_return(specific_returns)<EOL>summary['<STR_LIT>'] =ep.annual_return(common_returns)<EOL>summary['<STR_LIT>'] =ep.annual_return(total_returns)<EOL>summary['<STR_LIT>'] =ep.sharpe_ratio(specific_returns)<EOL>summary['<STR_LIT>'] =ep.cum_returns_final(specific_returns)<EOL>summary['<STR_LIT>'] =ep.cum_returns_final(common_returns)<EOL>summary['<STR_LIT>'] =ep.cum_returns_final(total_returns)<EOL>summary = pd.Series(summary, name='<STR_LIT>')<EOL>annualized_returns_by_factor = [ep.annual_return(perf_attrib[c])<EOL>for c in risk_exposures.columns]<EOL>cumulative_returns_by_factor = [ep.cum_returns_final(perf_attrib[c])<EOL>for c in risk_exposures.columns]<EOL>risk_exposure_summary = pd.DataFrame(<EOL>data=OrderedDict([<EOL>(<EOL>'<STR_LIT>',<EOL>risk_exposures.mean(axis='<STR_LIT>')<EOL>),<EOL>('<STR_LIT>', annualized_returns_by_factor),<EOL>('<STR_LIT>', cumulative_returns_by_factor),<EOL>]),<EOL>index=risk_exposures.columns,<EOL>)<EOL>return summary, risk_exposure_summary<EOL>

Takes perf attribution data over a period of time and computes annualized
multifactor alpha, multifactor sharpe, risk exposures.


def plot_returns(perf_attrib_data, cost=None, ax=None):

if ax is None:<EOL><INDENT>ax = plt.gca()<EOL><DEDENT>returns = perf_attrib_data['<STR_LIT>']<EOL>total_returns_label = '<STR_LIT>'<EOL>cumulative_returns_less_costs = _cumulative_returns_less_costs(<EOL>returns,<EOL>cost<EOL>)<EOL>if cost is not None:<EOL><INDENT>total_returns_label += '<STR_LIT>'<EOL><DEDENT>specific_returns = perf_attrib_data['<STR_LIT>']<EOL>common_returns = perf_attrib_data['<STR_LIT>']<EOL>ax.plot(cumulative_returns_less_costs, color='<STR_LIT:b>',<EOL>label=total_returns_label)<EOL>ax.plot(ep.cum_returns(specific_returns), color='<STR_LIT:g>',<EOL>label='<STR_LIT>')<EOL>ax.plot(ep.cum_returns(common_returns), color='<STR_LIT:r>',<EOL>label='<STR_LIT>')<EOL>if cost is not None:<EOL><INDENT>ax.plot(-ep.cum_returns(cost), color='<STR_LIT:k>',<EOL>label='<STR_LIT>')<EOL><DEDENT>ax.set_title('<STR_LIT>')<EOL>ax.set_ylabel('<STR_LIT>')<EOL>configure_legend(ax)<EOL>return ax<EOL>

Plot total, specific, and common returns.

Parameters
----------
perf_attrib_data : pd.DataFrame
    df with factors, common returns, and specific returns as columns,
    and datetimes as index. Assumes the `total_returns` column is NOT
    cost adjusted.
    - Example:
                    momentum  reversal  common_returns  specific_returns
        dt
        2017-01-01  0.249087  0.935925        1.185012          1.185012
        2017-01-02 -0.003194 -0.400786       -0.403980         -0.403980

cost : pd.Series, optional
    if present, gets subtracted from `perf_attrib_data['total_returns']`,
    and gets plotted separately

ax :  matplotlib.axes.Axes
    axes on which plots are made. if None, current axes will be used

Returns
-------
ax :  matplotlib.axes.Axes


if ax is None:<EOL><INDENT>ax = plt.gca()<EOL><DEDENT>ax.hist(alpha_returns, color='<STR_LIT:g>', label='<STR_LIT>')<EOL>ax.set_title('<STR_LIT>')<EOL>ax.axvline(<NUM_LIT:0>, color='<STR_LIT:k>', linestyle='<STR_LIT>', label='<STR_LIT>')<EOL>avg = alpha_returns.mean()<EOL>ax.axvline(avg, color='<STR_LIT:b>', label='<STR_LIT>'.format(avg))<EOL>configure_legend(ax)<EOL>return ax<EOL>

Plot histogram of daily multi-factor alpha returns (specific returns).

Parameters
----------
alpha_returns : pd.Series
    series of daily alpha returns indexed by datetime

ax :  matplotlib.axes.Axes
    axes on which plots are made. if None, current axes will be used

Returns
-------
ax :  matplotlib.axes.Axes


if pos_in_dollars:<EOL><INDENT>positions = get_percent_alloc(positions)<EOL><DEDENT>positions = positions.drop('<STR_LIT>', axis='<STR_LIT>')<EOL>positions = positions.stack()<EOL>positions.index = positions.index.set_names(['<STR_LIT>', '<STR_LIT>'])<EOL>return positions<EOL>

Convert positions to percentages if necessary, and change them
to long format.

Parameters
----------
positions: pd.DataFrame
    Daily holdings (in dollars or percentages), indexed by date.
    Will be converted to percentages if positions are in dollars.
    Short positions show up as cash in the 'cash' column.

pos_in_dollars : bool
    Flag indicating whether `positions` are in dollars or percentages
    If True, positions are in dollars.


(returns,<EOL>positions,<EOL>factor_returns,<EOL>factor_loadings) = _align_and_warn(returns,<EOL>positions,<EOL>factor_returns,<EOL>factor_loadings,<EOL>transactions=transactions,<EOL>pos_in_dollars=pos_in_dollars)<EOL>positions = _stack_positions(positions, pos_in_dollars=pos_in_dollars)<EOL>return ep.perf_attrib(returns, positions, factor_returns, factor_loadings)<EOL>

Attributes the performance of a returns stream to a set of risk factors.

Preprocesses inputs, and then calls empyrical.perf_attrib. See
empyrical.perf_attrib for more info.

Performance attribution determines how much each risk factor, e.g.,
momentum, the technology sector, etc., contributed to total returns, as
well as the daily exposure to each of the risk factors. The returns that
can be attributed to one of the given risk factors are the
`common_returns`, and the returns that _cannot_ be attributed to a risk
factor are the `specific_returns`, or the alpha. The common_returns and
specific_returns summed together will always equal the total returns.

Parameters
----------
returns : pd.Series
    Returns for each day in the date range.
    - Example:
        2017-01-01   -0.017098
        2017-01-02    0.002683
        2017-01-03   -0.008669

positions: pd.DataFrame
    Daily holdings (in dollars or percentages), indexed by date.
    Will be converted to percentages if positions are in dollars.
    Short positions show up as cash in the 'cash' column.
    - Examples:
                    AAPL  TLT  XOM  cash
        2017-01-01    34   58   10     0
        2017-01-02    22   77   18     0
        2017-01-03   -15   27   30    15

                        AAPL       TLT       XOM  cash
        2017-01-01  0.333333  0.568627  0.098039   0.0
        2017-01-02  0.188034  0.658120  0.153846   0.0
        2017-01-03  0.208333  0.375000  0.416667   0.0

factor_returns : pd.DataFrame
    Returns by factor, with date as index and factors as columns
    - Example:
                    momentum  reversal
        2017-01-01  0.002779 -0.005453
        2017-01-02  0.001096  0.010290

factor_loadings : pd.DataFrame
    Factor loadings for all days in the date range, with date and ticker as
    index, and factors as columns.
    - Example:
                           momentum  reversal
        dt         ticker
        2017-01-01 AAPL   -1.592914  0.852830
                   TLT     0.184864  0.895534
                   XOM     0.993160  1.149353
        2017-01-02 AAPL   -0.140009 -0.524952
                   TLT    -1.066978  0.185435
                   XOM    -1.798401  0.761549

transactions : pd.DataFrame, optional
    Executed trade volumes and fill prices. Used to check the turnover of
    the algorithm. Default is None, in which case the turnover check is
    skipped.

    - One row per trade.
    - Trades on different names that occur at the
      same time will have identical indicies.
    - Example:
        index                  amount   price    symbol
        2004-01-09 12:18:01    483      324.12   'AAPL'
        2004-01-09 12:18:01    122      83.10    'MSFT'
        2004-01-13 14:12:23    -75      340.43   'AAPL'

pos_in_dollars : bool
    Flag indicating whether `positions` are in dollars or percentages
    If True, positions are in dollars.

Returns
-------
tuple of (risk_exposures_portfolio, perf_attribution)

risk_exposures_portfolio : pd.DataFrame
    df indexed by datetime, with factors as columns
    - Example:
                    momentum  reversal
        dt
        2017-01-01 -0.238655  0.077123
        2017-01-02  0.821872  1.520515

perf_attribution : pd.DataFrame
    df with factors, common returns, and specific returns as columns,
    and datetimes as index
    - Example:
                    momentum  reversal  common_returns  specific_returns
        dt
        2017-01-01  0.249087  0.935925        1.185012          1.185012
        2017-01-02 -0.003194 -0.400786       -0.403980         -0.403980


def configure_legend(ax, autofmt_xdate=True, change_colors=False,<EOL>rotation=<NUM_LIT:30>, ha='<STR_LIT:right>'):

chartBox = ax.get_position()<EOL>ax.set_position([chartBox.x0, chartBox.y0,<EOL>chartBox.width * <NUM_LIT>, chartBox.height])<EOL>handles, labels = ax.get_legend_handles_labels()<EOL>handles_and_labels_sorted = sorted(zip(handles, labels),<EOL>key=lambda x: x[<NUM_LIT:0>].get_ydata()[-<NUM_LIT:1>],<EOL>reverse=True)<EOL>handles_sorted = [h[<NUM_LIT:0>] for h in handles_and_labels_sorted]<EOL>labels_sorted = [h[<NUM_LIT:1>] for h in handles_and_labels_sorted]<EOL>if change_colors:<EOL><INDENT>for handle, color in zip(handles_sorted,<EOL>cycle(COLORS)):<EOL><INDENT>handle.set_color(color)<EOL><DEDENT><DEDENT>ax.legend(handles=handles_sorted,<EOL>labels=labels_sorted,<EOL>frameon=True,<EOL>framealpha=<NUM_LIT:0.5>,<EOL>loc='<STR_LIT>',<EOL>bbox_to_anchor=(<NUM_LIT>, <NUM_LIT:1>),<EOL>fontsize='<STR_LIT>')<EOL>if autofmt_xdate:<EOL><INDENT>for label in ax.get_xticklabels():<EOL><INDENT>label.set_ha(ha)<EOL>label.set_rotation(rotation)<EOL><DEDENT><DEDENT>

Format legend for perf attribution plots:
- put legend to the right of plot instead of overlapping with it
- make legend order match up with graph lines
- set colors according to colormap


def check_intraday(estimate, returns, positions, transactions):

if estimate == '<STR_LIT>':<EOL><INDENT>if positions is not None and transactions is not None:<EOL><INDENT>if detect_intraday(positions, transactions):<EOL><INDENT>warnings.warn('<STR_LIT>' +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT>')<EOL>return estimate_intraday(returns, positions, transactions)<EOL><DEDENT>else:<EOL><INDENT>return positions<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return positions<EOL><DEDENT><DEDENT>elif estimate:<EOL><INDENT>if positions is not None and transactions is not None:<EOL><INDENT>return estimate_intraday(returns, positions, transactions)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return positions<EOL><DEDENT>

Logic for checking if a strategy is intraday and processing it.

Parameters
----------
estimate: boolean or str, optional
    Approximate returns for intraday strategies.
    See description in tears.create_full_tear_sheet.
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in create_full_tear_sheet.
positions : pd.DataFrame
    Daily net position values.
     - See full explanation in create_full_tear_sheet.
transactions : pd.DataFrame
    Prices and amounts of executed trades. One row per trade.
     - See full explanation in create_full_tear_sheet.

Returns
-------
pd.DataFrame
    Daily net position values, adjusted for intraday movement.


return '<STR_LIT>' % x<EOL>


SETTINGS['<STR_LIT>'] = func<EOL>

Registers the 'returns_func' that will be called for
retrieving returns data.

Parameters
----------
func : function
    A function that returns a pandas Series of asset returns.
    The signature of the function must be as follows

    >>> func(symbol)

    Where symbol is an asset identifier

Returns
-------
None


try:<EOL><INDENT>df.index = df.index.tz_localize('<STR_LIT>')<EOL><DEDENT>except TypeError:<EOL><INDENT>df.index = df.index.tz_convert('<STR_LIT>')<EOL><DEDENT>return df<EOL>

For use in tests; applied UTC timestamp to DataFrame.


return df[df.columns[<NUM_LIT:0>]]<EOL>

For use in tests; converts DataFrame's first column to Series.


if isinstance(table, pd.Series):<EOL><INDENT>table = pd.DataFrame(table)<EOL><DEDENT>if name is not None:<EOL><INDENT>table.columns.name = name<EOL><DEDENT>html = table.to_html(float_format=float_format, formatters=formatters)<EOL>if header_rows is not None:<EOL><INDENT>n_cols = html.split('<STR_LIT>')[<NUM_LIT:1>].split('<STR_LIT>')[<NUM_LIT:0>].count('<STR_LIT>')<EOL>rows = '<STR_LIT>'<EOL>for name, value in header_rows.items():<EOL><INDENT>rows += ('<STR_LIT>' +<EOL>'<STR_LIT>') % (name, n_cols, value)<EOL><DEDENT>html = html.replace('<STR_LIT>', '<STR_LIT>' + rows)<EOL><DEDENT>display(HTML(html))<EOL>

Pretty print a pandas DataFrame.

Uses HTML output if running inside Jupyter Notebook, otherwise
formatted text output.

Parameters
----------
table : pandas.Series or pandas.DataFrame
    Table to pretty-print.
name : str, optional
    Table name to display in upper left corner.
float_format : function, optional
    Formatter to use for displaying table elements, passed as the
    `float_format` arg to pd.Dataframe.to_html.
    E.g. `'{0:.2%}'.format` for displaying 100 as '100.00%'.
formatters : list or dict, optional
    Formatters to use by column, passed as the `formatters` arg to
    pd.Dataframe.to_html.
header_rows : dict, optional
    Extra rows to display at the top of the table.


def wrapper(df, *args, **kwargs):<EOL><INDENT>if df.ndim == <NUM_LIT:1>:<EOL><INDENT>return func(df, *args, **kwargs)<EOL><DEDENT>elif df.ndim == <NUM_LIT:2>:<EOL><INDENT>return df.apply(func, *args, **kwargs)<EOL><DEDENT><DEDENT>return wrapper<EOL>

Decorator so that functions can be written to work on Series but
may still be called with DataFrames.


return returns.rolling(rolling_vol_window).std()* np.sqrt(APPROX_BDAYS_PER_YEAR)<EOL>

Determines the rolling volatility of a strategy.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in tears.create_full_tear_sheet.
rolling_vol_window : int
    Length of rolling window, in days, over which to compute.

Returns
-------
pd.Series
    Rolling volatility.


def perf_stats(returns, factor_returns=None, positions=None,<EOL>transactions=None, turnover_denom='<STR_LIT>'):

stats = pd.Series()<EOL>for stat_func in SIMPLE_STAT_FUNCS:<EOL><INDENT>stats[STAT_FUNC_NAMES[stat_func.__name__]] = stat_func(returns)<EOL><DEDENT>if positions is not None:<EOL><INDENT>stats['<STR_LIT>'] = gross_lev(positions).mean()<EOL>if transactions is not None:<EOL><INDENT>stats['<STR_LIT>'] = get_turnover(positions,<EOL>transactions,<EOL>turnover_denom).mean()<EOL><DEDENT><DEDENT>if factor_returns is not None:<EOL><INDENT>for stat_func in FACTOR_STAT_FUNCS:<EOL><INDENT>res = stat_func(returns, factor_returns)<EOL>stats[STAT_FUNC_NAMES[stat_func.__name__]] = res<EOL><DEDENT><DEDENT>return stats<EOL>

Calculates various performance metrics of a strategy, for use in
plotting.show_perf_stats.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in tears.create_full_tear_sheet.
factor_returns : pd.Series, optional
    Daily noncumulative returns of the benchmark factor to which betas are
    computed. Usually a benchmark such as market returns.
     - This is in the same style as returns.
     - If None, do not compute alpha, beta, and information ratio.
positions : pd.DataFrame
    Daily net position values.
     - See full explanation in tears.create_full_tear_sheet.
transactions : pd.DataFrame
    Prices and amounts of executed trades. One row per trade.
    - See full explanation in tears.create_full_tear_sheet.
turnover_denom : str
    Either AGB or portfolio_value, default AGB.
    - See full explanation in txn.get_turnover.

Returns
-------
pd.Series
    Performance metrics.


return ep.tail_ratio(returns) *(<NUM_LIT:1> + ep.annual_return(returns))<EOL>

Common sense ratio is the multiplication of the tail ratio and the
Gain-to-Pain-Ratio -- sum(profits) / sum(losses).

See http://bit.ly/1ORzGBk for more information on motivation of
this metric.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in tears.create_full_tear_sheet.

Returns
-------
float
    common sense ratio


valley = np.argmin(underwater)  <EOL>peak = underwater[:valley][underwater[:valley] == <NUM_LIT:0>].index[-<NUM_LIT:1>]<EOL>try:<EOL><INDENT>recovery = underwater[valley:][underwater[valley:] == <NUM_LIT:0>].index[<NUM_LIT:0>]<EOL><DEDENT>except IndexError:<EOL><INDENT>recovery = np.nan  <EOL><DEDENT>return peak, valley, recovery<EOL>

Determines peak, valley, and recovery dates given an 'underwater'
DataFrame.

An underwater DataFrame is a DataFrame that has precomputed
rolling drawdown.

Parameters
----------
underwater : pd.Series
   Underwater returns (rolling drawdown) of a strategy.

Returns
-------
peak : datetime
    The maximum drawdown's peak.
valley : datetime
    The maximum drawdown's valley.
recovery : datetime
    The maximum drawdown's recovery.


def rolling_regression(returns, factor_returns,<EOL>rolling_window=APPROX_BDAYS_PER_MONTH * <NUM_LIT:6>,<EOL>nan_threshold=<NUM_LIT:0.1>):

<EOL>ret_no_na = returns.dropna()<EOL>columns = ['<STR_LIT>'] + factor_returns.columns.tolist()<EOL>rolling_risk = pd.DataFrame(columns=columns,<EOL>index=ret_no_na.index)<EOL>rolling_risk.index.name = '<STR_LIT>'<EOL>for beg, end in zip(ret_no_na.index[:-rolling_window],<EOL>ret_no_na.index[rolling_window:]):<EOL><INDENT>returns_period = ret_no_na[beg:end]<EOL>factor_returns_period = factor_returns.loc[returns_period.index]<EOL>if np.all(factor_returns_period.isnull().mean()) < nan_threshold:<EOL><INDENT>factor_returns_period_dnan = factor_returns_period.dropna()<EOL>reg = linear_model.LinearRegression(fit_intercept=True).fit(<EOL>factor_returns_period_dnan,<EOL>returns_period.loc[factor_returns_period_dnan.index])<EOL>rolling_risk.loc[end, factor_returns.columns] = reg.coef_<EOL>rolling_risk.loc[end, '<STR_LIT>'] = reg.intercept_<EOL><DEDENT><DEDENT>return rolling_risk<EOL>

Computes rolling factor betas using a multivariate linear regression
(separate linear regressions is problematic because the factors may be
confounded).

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in tears.create_full_tear_sheet.
factor_returns : pd.DataFrame
    Daily noncumulative returns of the benchmark factor to which betas are
    computed. Usually a benchmark such as market returns.
     - Computes rolling beta for each column.
     - This is in the same style as returns.
rolling_window : int, optional
    The days window over which to compute the beta. Defaults to 6 months.
nan_threshold : float, optional
    If there are more than this fraction of NaNs, the rolling regression
    for the given date will be skipped.

Returns
-------
pandas.DataFrame
    DataFrame containing rolling beta coefficients to SMB, HML and UMD


returns = returns.copy()<EOL>df_cum = ep.cum_returns(returns, <NUM_LIT:1.0>)<EOL>running_max = np.maximum.accumulate(df_cum)<EOL>underwater = df_cum / running_max - <NUM_LIT:1><EOL>drawdowns = []<EOL>for t in range(top):<EOL><INDENT>peak, valley, recovery = get_max_drawdown_underwater(underwater)<EOL>if not pd.isnull(recovery):<EOL><INDENT>underwater.drop(underwater[peak: recovery].index[<NUM_LIT:1>:-<NUM_LIT:1>],<EOL>inplace=True)<EOL><DEDENT>else:<EOL><INDENT>underwater = underwater.loc[:peak]<EOL><DEDENT>drawdowns.append((peak, valley, recovery))<EOL>if (len(returns) == <NUM_LIT:0>) or (len(underwater) == <NUM_LIT:0>):<EOL><INDENT>break<EOL><DEDENT><DEDENT>return drawdowns<EOL>

Finds top drawdowns, sorted by drawdown amount.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in tears.create_full_tear_sheet.
top : int, optional
    The amount of top drawdowns to find (default 10).

Returns
-------
drawdowns : list
    List of drawdown peaks, valleys, and recoveries. See get_max_drawdown.


def summarize_paths(samples, cone_std=(<NUM_LIT:1.>, <NUM_LIT>, <NUM_LIT>), starting_value=<NUM_LIT:1.>):

cum_samples = ep.cum_returns(samples.T,<EOL>starting_value=starting_value).T<EOL>cum_mean = cum_samples.mean(axis=<NUM_LIT:0>)<EOL>cum_std = cum_samples.std(axis=<NUM_LIT:0>)<EOL>if isinstance(cone_std, (float, int)):<EOL><INDENT>cone_std = [cone_std]<EOL><DEDENT>cone_bounds = pd.DataFrame(columns=pd.Float64Index([]))<EOL>for num_std in cone_std:<EOL><INDENT>cone_bounds.loc[:, float(num_std)] = cum_mean + cum_std * num_std<EOL>cone_bounds.loc[:, float(-num_std)] = cum_mean - cum_std * num_std<EOL><DEDENT>return cone_bounds<EOL>

Gnerate the upper and lower bounds of an n standard deviation
cone of forecasted cumulative returns.

Parameters
----------
samples : numpy.ndarray
    Alternative paths, or series of possible outcomes.
cone_std : list of int/float
    Number of standard devations to use in the boundaries of
    the cone. If multiple values are passed, cone bounds will
    be generated for each value.

Returns
-------
samples : pandas.core.frame.DataFrame


return ep.beta(returns, factor_returns)<EOL>

Calculates beta.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
    - See full explanation in :func:`~pyfolio.timeseries.cum_returns`.
factor_returns : pd.Series
    Daily noncumulative returns of the benchmark factor to which betas are
    computed. Usually a benchmark such as market returns.
     - This is in the same style as returns.

Returns
-------
float
    Beta.


df_cum = ep.cum_returns(returns, <NUM_LIT:1.0>)<EOL>drawdown_periods = get_top_drawdowns(returns, top=top)<EOL>df_drawdowns = pd.DataFrame(index=list(range(top)),<EOL>columns=['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>'])<EOL>for i, (peak, valley, recovery) in enumerate(drawdown_periods):<EOL><INDENT>if pd.isnull(recovery):<EOL><INDENT>df_drawdowns.loc[i, '<STR_LIT>'] = np.nan<EOL><DEDENT>else:<EOL><INDENT>df_drawdowns.loc[i, '<STR_LIT>'] = len(pd.date_range(peak,<EOL>recovery,<EOL>freq='<STR_LIT:B>'))<EOL><DEDENT>df_drawdowns.loc[i, '<STR_LIT>'] = (peak.to_pydatetime()<EOL>.strftime('<STR_LIT>'))<EOL>df_drawdowns.loc[i, '<STR_LIT>'] = (valley.to_pydatetime()<EOL>.strftime('<STR_LIT>'))<EOL>if isinstance(recovery, float):<EOL><INDENT>df_drawdowns.loc[i, '<STR_LIT>'] = recovery<EOL><DEDENT>else:<EOL><INDENT>df_drawdowns.loc[i, '<STR_LIT>'] = (recovery.to_pydatetime()<EOL>.strftime('<STR_LIT>'))<EOL><DEDENT>df_drawdowns.loc[i, '<STR_LIT>'] = (<EOL>(df_cum.loc[peak] - df_cum.loc[valley]) / df_cum.loc[peak]) * <NUM_LIT:100><EOL><DEDENT>df_drawdowns['<STR_LIT>'] = pd.to_datetime(df_drawdowns['<STR_LIT>'])<EOL>df_drawdowns['<STR_LIT>'] = pd.to_datetime(df_drawdowns['<STR_LIT>'])<EOL>df_drawdowns['<STR_LIT>'] = pd.to_datetime(<EOL>df_drawdowns['<STR_LIT>'])<EOL>return df_drawdowns<EOL>

Places top drawdowns in a table.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in tears.create_full_tear_sheet.
top : int, optional
    The amount of top drawdowns to find (default 10).

Returns
-------
df_drawdowns : pd.DataFrame
    Information about top drawdowns.


@deprecated(msg=DEPRECATION_WARNING)<EOL>def sharpe_ratio(returns, risk_free=<NUM_LIT:0>, period=DAILY):

return ep.sharpe_ratio(returns, risk_free=risk_free, period=period)<EOL>

Determines the Sharpe ratio of a strategy.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
    - See full explanation in :func:`~pyfolio.timeseries.cum_returns`.
risk_free : int, float
    Constant risk-free return throughout the period.
period : str, optional
    Defines the periodicity of the 'returns' data for purposes of
    annualizing. Can be 'monthly', 'weekly', or 'daily'.
    - Defaults to 'daily'.

Returns
-------
float
    Sharpe ratio.
np.nan
    If insufficient length of returns or if if adjusted returns are 0.

Note
-----
See https://en.wikipedia.org/wiki/Sharpe_ratio for more details.


def var_cov_var_normal(P, c, mu=<NUM_LIT:0>, sigma=<NUM_LIT:1>):

alpha = sp.stats.norm.ppf(<NUM_LIT:1> - c, mu, sigma)<EOL>return P - P * (alpha + <NUM_LIT:1>)<EOL>

Variance-covariance calculation of daily Value-at-Risk in a
portfolio.

Parameters
----------
P : float
    Portfolio value.
c : float
    Confidence level.
mu : float, optional
    Mean.

Returns
-------
float
    Variance-covariance.


return ep.stability_of_timeseries(returns)<EOL>

Determines R-squared of a linear fit to the cumulative
log returns. Computes an ordinary least squares linear fit,
and returns R-squared.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
    - See full explanation in :func:`~pyfolio.timeseries.cum_returns`.

Returns
-------
float
    R-squared.


return starting_value * (returns / returns.iloc[<NUM_LIT:0>])<EOL>

Normalizes a returns timeseries based on the first value.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in tears.create_full_tear_sheet.
starting_value : float, optional
   The starting returns (default 1).

Returns
-------
pd.Series
    Normalized returns.


@deprecated(msg=DEPRECATION_WARNING)<EOL>def sortino_ratio(returns, required_return=<NUM_LIT:0>, period=DAILY):

return ep.sortino_ratio(returns, required_return=required_return)<EOL>

Determines the Sortino ratio of a strategy.

Parameters
----------
returns : pd.Series or pd.DataFrame
    Daily returns of the strategy, noncumulative.
    - See full explanation in :func:`~pyfolio.timeseries.cum_returns`.
required_return: float / series
    minimum acceptable return
period : str, optional
    Defines the periodicity of the 'returns' data for purposes of
    annualizing. Can be 'monthly', 'weekly', or 'daily'.
    - Defaults to 'daily'.

Returns
-------
depends on input type
series ==> float
DataFrame ==> np.array

    Annualized Sortino ratio.


exposure = positions.drop('<STR_LIT>', axis=<NUM_LIT:1>).abs().sum(axis=<NUM_LIT:1>)<EOL>return exposure / positions.sum(axis=<NUM_LIT:1>)<EOL>

Calculates the gross leverage of a strategy.

Parameters
----------
positions : pd.DataFrame
    Daily net position values.
     - See full explanation in tears.create_full_tear_sheet.

Returns
-------
pd.Series
    Gross leverage.


returns_dupe = returns.copy()<EOL>returns_dupe.index = returns_dupe.index.map(pd.Timestamp)<EOL>ranges = OrderedDict()<EOL>for name, (start, end) in PERIODS.items():<EOL><INDENT>try:<EOL><INDENT>period = returns_dupe.loc[start:end]<EOL>if len(period) == <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>ranges[name] = period<EOL><DEDENT>except BaseException:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>return ranges<EOL>

Extracts returns based on interesting events. See
gen_date_range_interesting.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in tears.create_full_tear_sheet.

Returns
-------
ranges : OrderedDict
    Date ranges, with returns, of all valid events.


return returns.rolling(rolling_sharpe_window).mean()/ returns.rolling(rolling_sharpe_window).std()* np.sqrt(APPROX_BDAYS_PER_YEAR)<EOL>

Determines the rolling Sharpe ratio of a strategy.

Parameters
----------
returns : pd.Series
    Daily returns of the strategy, noncumulative.
     - See full explanation in tears.create_full_tear_sheet.
rolling_sharpe_window : int
    Length of rolling window, in days, over which to compute.

Returns
-------
pd.Series
    Rolling Sharpe ratio.

Note
-----
See https://en.wikipedia.org/wiki/Sharpe_ratio for more details.


@deprecated(msg=DEPRECATION_WARNING)<EOL>def downside_risk(returns, required_return=<NUM_LIT:0>, period=DAILY):

return ep.downside_risk(returns,<EOL>required_return=required_return,<EOL>period=period)<EOL>

Determines the downside deviation below a threshold

Parameters
----------
returns : pd.Series or pd.DataFrame
    Daily returns of the strategy, noncumulative.
    - See full explanation in :func:`~pyfolio.timeseries.cum_returns`.
required_return: float / series
    minimum acceptable return
period : str, optional
    Defines the periodicity of the 'returns' data for purposes of
    annualizing. Can be 'monthly', 'weekly', or 'daily'.
    - Defaults to 'daily'.

Returns
-------
depends on input type
series ==> float
DataFrame ==> np.array

    Annualized downside deviation


txn_norm = transactions.copy()<EOL>txn_norm.index = txn_norm.index.normalize()<EOL>amounts = txn_norm.amount.abs()<EOL>prices = txn_norm.price<EOL>values = amounts * prices<EOL>daily_amounts = amounts.groupby(amounts.index).sum()<EOL>daily_values = values.groupby(values.index).sum()<EOL>daily_amounts.name = "<STR_LIT>"<EOL>daily_values.name = "<STR_LIT>"<EOL>return pd.concat([daily_values, daily_amounts], axis=<NUM_LIT:1>)<EOL>

Extract daily transaction data from set of transaction objects.

Parameters
----------
transactions : pd.DataFrame
    Time series containing one row per symbol (and potentially
    duplicate datetime indices) and columns for amount and
    price.

Returns
-------
pd.DataFrame
    Daily transaction volume and number of shares.
     - See full explanation in tears.create_full_tear_sheet.


transaction_list = []<EOL>for dt in transactions.index:<EOL><INDENT>txns = transactions.loc[dt]<EOL>if len(txns) == <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>for txn in txns:<EOL><INDENT>txn = map_transaction(txn)<EOL>transaction_list.append(txn)<EOL><DEDENT><DEDENT>df = pd.DataFrame(sorted(transaction_list, key=lambda x: x['<STR_LIT>']))<EOL>df['<STR_LIT>'] = -df['<STR_LIT>'] * df['<STR_LIT>']<EOL>df.index = list(map(pd.Timestamp, df.dt.values))<EOL>return df<EOL>

Formats a transaction DataFrame.

Parameters
----------
transactions : pd.DataFrame
    Contains improperly formatted transactional data.

Returns
-------
df : pd.DataFrame
    Daily transaction volume and dollar ammount.
     - See full explanation in tears.create_full_tear_sheet.


<EOL>response = make_get_request(session, '<STR_LIT>', params_data=query)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:result>']<EOL><DEDENT>else:<EOL><INDENT>raise UsersNotFoundException(<EOL>message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>'])<EOL><DEDENT>


jobs_data = {<EOL>'<STR_LIT>': job_ids<EOL>}<EOL>response = make_post_request(session, '<STR_LIT>', json_data=jobs_data)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:status>']<EOL><DEDENT>else:<EOL><INDENT>raise UserJobsNotAddedException(<EOL>message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>'])<EOL><DEDENT>

Add a list of jobs to the currently authenticated user


<EOL>if user_details:<EOL><INDENT>user_details['<STR_LIT>'] = True<EOL><DEDENT>response = make_get_request(session, '<STR_LIT>', params_data=user_details)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:result>']<EOL><DEDENT>else:<EOL><INDENT>raise SelfNotRetrievedException(<EOL>message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>']<EOL>)<EOL><DEDENT>

Get details about the currently authenticated user


response = make_get_request(session, '<STR_LIT>')<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return response.json()['<STR_LIT:result>']['<STR_LIT:id>']<EOL><DEDENT>else:<EOL><INDENT>raise UserIdNotRetrievedException(<EOL>'<STR_LIT>' % response.text, response.text)<EOL><DEDENT>


def get_user_by_id(session, user_id, user_details=None):

if user_details:<EOL><INDENT>user_details['<STR_LIT>'] = True<EOL><DEDENT>response = make_get_request(<EOL>session, '<STR_LIT>'.format(user_id), params_data=user_details)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:result>']<EOL><DEDENT>else:<EOL><INDENT>raise UserNotFoundException(<EOL>message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>']<EOL>)<EOL><DEDENT>


def create_project(session, title, description,<EOL>currency, budget, jobs):

project_data = {'<STR_LIT:title>': title,<EOL>'<STR_LIT:description>': description,<EOL>'<STR_LIT>': currency,<EOL>'<STR_LIT>': budget,<EOL>'<STR_LIT>': jobs<EOL>}<EOL>response = make_post_request(session, '<STR_LIT>', json_data=project_data)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>project_data = json_data['<STR_LIT:result>']<EOL>p = Project(project_data)<EOL>p.url = urljoin(session.url, '<STR_LIT>' % p.seo_url)<EOL>return p<EOL><DEDENT>else:<EOL><INDENT>raise ProjectNotCreatedException(message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>'],<EOL>)<EOL><DEDENT>


def get_jobs(session, job_ids, seo_details, lang):

get_jobs_data = {<EOL>'<STR_LIT>': job_ids,<EOL>'<STR_LIT>': seo_details,<EOL>'<STR_LIT>': lang,<EOL>}<EOL>response = make_get_request(session, '<STR_LIT>', params_data=get_jobs_data)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:result>']<EOL><DEDENT>else:<EOL><INDENT>raise JobsNotFoundException(<EOL>message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>'])<EOL><DEDENT>


def create_hourly_project(session, title, description,<EOL>currency, budget, jobs, hourly_project_info):

project_data = {'<STR_LIT:title>': title,<EOL>'<STR_LIT:description>': description,<EOL>'<STR_LIT>': currency,<EOL>'<STR_LIT>': budget,<EOL>'<STR_LIT>': jobs,<EOL>'<STR_LIT:type>': '<STR_LIT>',<EOL>'<STR_LIT>': hourly_project_info<EOL>}<EOL>response = make_post_request(session, '<STR_LIT>', json_data=project_data)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>project_data = json_data['<STR_LIT:result>']<EOL>p = Project(project_data)<EOL>p.url = urljoin(session.url, '<STR_LIT>' % p.seo_url)<EOL>return p<EOL><DEDENT>else:<EOL><INDENT>raise ProjectNotCreatedException(message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>'],<EOL>)<EOL><DEDENT>


def post_track(session, user_id, project_id, latitude, longitude):

tracking_data = {<EOL>'<STR_LIT>': user_id,<EOL>'<STR_LIT>': project_id,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': latitude,<EOL>'<STR_LIT>': longitude<EOL>}<EOL>}<EOL>response = make_post_request(session, '<STR_LIT>',<EOL>json_data=tracking_data)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:result>']<EOL><DEDENT>else:<EOL><INDENT>raise TrackNotCreatedException(message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>'])<EOL><DEDENT>

Start tracking a project by creating a track


def get_milestones(session, project_ids=[], milestone_ids=[], user_details=None, limit=<NUM_LIT:10>, offset=<NUM_LIT:0>):

get_milestones_data = {}<EOL>if milestone_ids:<EOL><INDENT>get_milestones_data['<STR_LIT>'] = milestone_ids<EOL><DEDENT>if project_ids:<EOL><INDENT>get_milestones_data['<STR_LIT>'] = project_ids<EOL><DEDENT>get_milestones_data['<STR_LIT>'] = limit<EOL>get_milestones_data['<STR_LIT>'] = offset<EOL>if user_details:<EOL><INDENT>get_milestones_data.update(user_details)<EOL><DEDENT>response = make_get_request(<EOL>session, '<STR_LIT>', params_data=get_milestones_data)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:result>']<EOL><DEDENT>else:<EOL><INDENT>raise MilestonesNotFoundException(<EOL>message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>']<EOL>)<EOL><DEDENT>


def place_project_bid(session, project_id, bidder_id, description, amount,<EOL>period, milestone_percentage):

bid_data = {<EOL>'<STR_LIT>': project_id,<EOL>'<STR_LIT>': bidder_id,<EOL>'<STR_LIT:description>': description,<EOL>'<STR_LIT>': amount,<EOL>'<STR_LIT>': period,<EOL>'<STR_LIT>': milestone_percentage,<EOL>}<EOL>response = make_post_request(session, '<STR_LIT>', json_data=bid_data)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>bid_data = json_data['<STR_LIT:result>']<EOL>return Bid(bid_data)<EOL><DEDENT>else:<EOL><INDENT>raise BidNotPlacedException(message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>'])<EOL><DEDENT>


def create_local_project(session, title, description,<EOL>currency, budget, jobs, location):

project_data = {'<STR_LIT:title>': title,<EOL>'<STR_LIT:description>': description,<EOL>'<STR_LIT>': currency,<EOL>'<STR_LIT>': budget,<EOL>'<STR_LIT>': jobs,<EOL>'<STR_LIT>': True,<EOL>'<STR_LIT:location>': location<EOL>}<EOL>response = make_post_request(session, '<STR_LIT>', json_data=project_data)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>project_data = json_data['<STR_LIT:result>']<EOL>p = Project(project_data)<EOL>p.url = urljoin(session.url, '<STR_LIT>' % p.seo_url)<EOL>return p<EOL><DEDENT>else:<EOL><INDENT>raise ProjectNotCreatedException(message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>'],<EOL>)<EOL><DEDENT>


headers = {<EOL>'<STR_LIT:Content-Type>': '<STR_LIT>'<EOL>}<EOL>bid_data = {<EOL>'<STR_LIT:action>': '<STR_LIT>'<EOL>}<EOL>endpoint = '<STR_LIT>'.format(bid_id)<EOL>response = make_put_request(session, endpoint, headers=headers,<EOL>params_data=bid_data)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:status>']<EOL><DEDENT>else:<EOL><INDENT>json_data = response.json()<EOL>raise BidNotAwardedException(<EOL>message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>']<EOL>)<EOL><DEDENT>


params_data = {<EOL>'<STR_LIT:action>': '<STR_LIT>',<EOL>}<EOL>endpoint = '<STR_LIT>'.format(milestone_request_id)<EOL>response = make_put_request(session, endpoint, params_data=params_data)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:status>']<EOL><DEDENT>else:<EOL><INDENT>raise MilestoneRequestNotAcceptedException(<EOL>message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>'])<EOL><DEDENT>


def get_milestone_by_id(session, milestone_id, user_details=None):

<EOL>endpoint = '<STR_LIT>'.format(milestone_id)<EOL>response = make_get_request(session, endpoint, params_data=user_details)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:result>']<EOL><DEDENT>else:<EOL><INDENT>raise MilestonesNotFoundException(<EOL>message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>']<EOL>)<EOL><DEDENT>


def get_track_by_id(session, track_id, track_point_limit=None, track_point_offset=None):

tracking_data = {}<EOL>if track_point_limit:<EOL><INDENT>tracking_data['<STR_LIT>'] = track_point_limit<EOL><DEDENT>if track_point_offset:<EOL><INDENT>tracking_data['<STR_LIT>'] = track_point_offset<EOL><DEDENT>response = make_get_request(session, '<STR_LIT>'.format(track_id),<EOL>params_data=tracking_data)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:result>']<EOL><DEDENT>else:<EOL><INDENT>raise TrackNotFoundException(message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>'])<EOL><DEDENT>


params_data = {<EOL>'<STR_LIT:action>': '<STR_LIT>',<EOL>}<EOL>endpoint = '<STR_LIT>'.format(milestone_request_id)<EOL>response = make_put_request(session, endpoint, params_data=params_data)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:status>']<EOL><DEDENT>else:<EOL><INDENT>raise MilestoneRequestNotRejectedException(<EOL>message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>'])<EOL><DEDENT>


def create_thread(session, member_ids, context_type, context, message):

headers = {<EOL>'<STR_LIT:Content-Type>': '<STR_LIT>'<EOL>}<EOL>thread_data = {<EOL>'<STR_LIT>': member_ids,<EOL>'<STR_LIT>': context_type,<EOL>'<STR_LIT>': context,<EOL>'<STR_LIT:message>': message,<EOL>}<EOL>response = make_post_request(session, '<STR_LIT>', headers,<EOL>form_data=thread_data)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return Thread(json_data['<STR_LIT:result>'])<EOL><DEDENT>else:<EOL><INDENT>raise ThreadNotCreatedException(message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>'])<EOL><DEDENT>


def get_messages(session, query, limit=<NUM_LIT:10>, offset=<NUM_LIT:0>):

query['<STR_LIT>'] = limit<EOL>query['<STR_LIT>'] = offset<EOL>response = make_get_request(session, '<STR_LIT>', params_data=query)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:result>']<EOL><DEDENT>else:<EOL><INDENT>raise MessagesNotFoundException(<EOL>message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>']<EOL>)<EOL><DEDENT>


def search_messages(session, thread_id, query, limit=<NUM_LIT:20>,<EOL>offset=<NUM_LIT:0>, message_context_details=None,<EOL>window_above=None, window_below=None):

query = {<EOL>'<STR_LIT>': thread_id,<EOL>'<STR_LIT>': query,<EOL>'<STR_LIT>': limit,<EOL>'<STR_LIT>': offset<EOL>}<EOL>if message_context_details:<EOL><INDENT>query['<STR_LIT>'] = message_context_details<EOL><DEDENT>if window_above:<EOL><INDENT>query['<STR_LIT>'] = window_above<EOL><DEDENT>if window_below:<EOL><INDENT>query['<STR_LIT>'] = window_below<EOL><DEDENT>response = make_get_request(session, '<STR_LIT>', params_data=query)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:result>']<EOL><DEDENT>else:<EOL><INDENT>raise MessagesNotFoundException(<EOL>message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>']<EOL>)<EOL><DEDENT>


<EOL>response = make_get_request(session, '<STR_LIT>', params_data=query)<EOL>json_data = response.json()<EOL>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>return json_data['<STR_LIT:result>']<EOL><DEDENT>else:<EOL><INDENT>raise ThreadsNotFoundException(<EOL>message=json_data['<STR_LIT:message>'],<EOL>error_code=json_data['<STR_LIT>'],<EOL>request_id=json_data['<STR_LIT>']<EOL>)<EOL><DEDENT>


def wrapper(func):<EOL><INDENT>@wraps(func)<EOL>def wrapped(self, *args, **kwargs):<EOL><INDENT>return func(self, *args, **kwargs)<EOL><DEDENT>wrapped.description = description<EOL>return wrapped<EOL><DEDENT>return wrapper<EOL>


self.cookie = CookieDict()<EOL>


def _connect(self, url, params, ok_codes, rtype, description,<EOL>redirect=False, consumer=None):

<EOL>if self.cookie:<EOL><INDENT>self.setHeader('<STR_LIT>', self.cookie.render_to_string())<EOL><DEDENT>response = super(TestCase, self)._connect(<EOL>url, params, ok_codes, rtype, description,<EOL>redirect=redirect, consumer=consumer)<EOL>for key, value in response.headers.items():<EOL><INDENT>if key.lower() == '<STR_LIT>':<EOL><INDENT>self.cookie.from_cookie_string(value)<EOL><DEDENT><DEDENT>content_type_header = response.headers.get('<STR_LIT>')<EOL>if content_type_header:<EOL><INDENT>if '<STR_LIT:;>' in content_type_header:<EOL><INDENT>content_type, charset_pair = content_type_header.split('<STR_LIT:;>', <NUM_LIT:1>)<EOL>if '<STR_LIT:=>' in charset_pair:<EOL><INDENT>_key, charset = charset_pair.split('<STR_LIT:=>', <NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>charset = None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>content_type = content_type_header<EOL>charset = None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>content_type = charset = None<EOL><DEDENT>response.content_type_header = content_type_header<EOL>if content_type:<EOL><INDENT>response.content_type = content_type.strip()<EOL><DEDENT>else:<EOL><INDENT>response.content_type = None<EOL><DEDENT>response.charset = charset<EOL>if response.content_type == '<STR_LIT:application/json>':<EOL><INDENT>response.data = json.loads(response.body)<EOL><DEDENT>else:<EOL><INDENT>response.data = None<EOL><DEDENT>return response<EOL>


return self.cookie.get(self.session_id_key)<EOL>


values = '<STR_LIT>'<EOL>for key, value in self.items():<EOL><INDENT>values += '<STR_LIT>'.format(key, value)<EOL><DEDENT>return values<EOL>


raise NotImplementedError<EOL>

Many DynamicArgs won't have a length that can be
precomputed. Most DynamicArgs objects will have an iteration
limit to guarantee eventual termination. If so, the maximum
possible number of arguments that could be generated should be
returned.


outputs_dir = os.path.join(info['<STR_LIT>'], '<STR_LIT>')<EOL>pattern = '<STR_LIT>' % info['<STR_LIT>']<EOL>flist = os.listdir(outputs_dir)<EOL>try:<EOL><INDENT>outputs = []<EOL>for tid in tids:<EOL><INDENT>matches = fnmatch.filter(flist, pattern.format(tid=tid))<EOL>if len(matches) != <NUM_LIT:1>:<EOL><INDENT>self.warning("<STR_LIT>" % tid)<EOL><DEDENT>contents = open(os.path.join(outputs_dir, matches[<NUM_LIT:0>]),'<STR_LIT:r>').read()<EOL>outputs.append(self.output_extractor(contents))<EOL><DEDENT>self._next_val = self._update_state(outputs)<EOL>self.trace.append((outputs, self._next_val))<EOL><DEDENT>except:<EOL><INDENT>self.warning("<STR_LIT>")<EOL>self._next_val = StopIteration<EOL><DEDENT>

Called to update the state of the iterator.  This methods
receives the set of task ids from the previous set of tasks
together with the launch information to allow the output
values to be parsed using the output_extractor. This data is then
used to determine the next desired point in the parameter
space by calling the _update_state method.


for (i, (val, args)) in enumerate(self.trace):<EOL><INDENT>if args is StopIteration:<EOL><INDENT>info = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>pprint = '<STR_LIT:U+002C>'.join('<STR_LIT:{>' + '<STR_LIT:U+002C>'.join('<STR_LIT>' % (k,v)<EOL>for (k,v) in arg.items()) + '<STR_LIT:}>' for arg in args)<EOL>info = ("<STR_LIT>" % pprint )<EOL><DEDENT>if i == <NUM_LIT:0>: print("<STR_LIT>" % (i, info))<EOL>else:      print("<STR_LIT>" % (i, info.capitalize(), val))<EOL><DEDENT>

Summarizes the trace of values used to update the DynamicArgs
and the arguments subsequently returned. May be used to
implement the summary method.


if not other: return self<EOL>dynamic = (isinstance(self, DynamicArgs),  isinstance(other, DynamicArgs))<EOL>if dynamic == (True, True):<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>elif (True in dynamic):<EOL><INDENT>return DynamicConcatenate(self,other)<EOL><DEDENT>else:<EOL><INDENT>return Concatenate(self,other)<EOL><DEDENT>

Concatenates two argument specifiers. See Concatenate and
DynamicConcatenate documentation respectively.


if len(launchers) == <NUM_LIT:0>: raise Exception('<STR_LIT>')<EOL>timestamps = [launcher.timestamp for launcher in launchers]<EOL>if not all(timestamps[<NUM_LIT:0>] == tstamp for tstamp in timestamps):<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>root_directories = []<EOL>for launcher in launchers:<EOL><INDENT>command = launcher.command<EOL>args = launcher.args<EOL>command.verify(args)<EOL>root_directory = launcher.get_root_directory()<EOL>if os.path.isdir(root_directory):<EOL><INDENT>raise Exception("<STR_LIT>" % root_directory)<EOL><DEDENT>if root_directory in root_directories:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>root_directories.append(root_directory)<EOL><DEDENT>

Performs consistency checks across all the launchers.


self._spec_log += specs <EOL>log_path = os.path.join(self.root_directory, ("<STR_LIT>" % self.batch_name))<EOL>core.Log.write_log(log_path, [spec for (_, spec) in specs], allow_append=True)<EOL>

The log contains the tids and corresponding specifications
used during launch with the specifications in JSON format.


def _launch_process_group(self, process_commands, streams_path):

processes = {}<EOL>def check_complete_processes(wait=False):<EOL><INDENT>"""<STR_LIT>"""<EOL>result = False<EOL>for proc in list(processes):<EOL><INDENT>if wait: proc.wait()<EOL>if proc.poll() is not None:<EOL><INDENT>self.debug("<STR_LIT>"<EOL>% (processes[proc]['<STR_LIT>'], proc.poll()))<EOL>processes[proc]['<STR_LIT>'].close()<EOL>processes[proc]['<STR_LIT>'].close()<EOL>del processes[proc]<EOL>result = True<EOL><DEDENT><DEDENT>return result<EOL><DEDENT>for cmd, tid in process_commands:<EOL><INDENT>self.debug("<STR_LIT>" % tid)<EOL>job_timestamp = time.strftime('<STR_LIT>')<EOL>basename = "<STR_LIT>" % (self.batch_name, job_timestamp, tid)<EOL>stdout_handle = open(os.path.join(streams_path, "<STR_LIT>"<EOL>% (basename, tid)), "<STR_LIT:wb>")<EOL>stderr_handle = open(os.path.join(streams_path, "<STR_LIT>"<EOL>% (basename, tid)), "<STR_LIT:wb>")<EOL>proc = subprocess.Popen(cmd, stdout=stdout_handle, stderr=stderr_handle)<EOL>processes[proc] = { '<STR_LIT>' : tid,<EOL>'<STR_LIT>' : stdout_handle,<EOL>'<STR_LIT>' : stderr_handle }<EOL>if self.max_concurrency:<EOL><INDENT>while len(processes) >= self.max_concurrency:<EOL><INDENT>if not check_complete_processes(len(processes)==<NUM_LIT:1>):<EOL><INDENT>time.sleep(<NUM_LIT:0.1>)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>while len(processes) > <NUM_LIT:0>:<EOL><INDENT>if not check_complete_processes(True):<EOL><INDENT>time.sleep(<NUM_LIT:0.1>)<EOL><DEDENT><DEDENT>

Launches processes defined by process_commands, but only
executes max_concurrency processes at a time; if a process
completes and there are still outstanding processes to be
executed, the next processes are run until max_concurrency is
reached again.


if timestamp is None: timestamp = self.timestamp<EOL>if self.timestamp_format is not None:<EOL><INDENT>root_name =  (time.strftime(self.timestamp_format, timestamp)<EOL>+ '<STR_LIT:->' + self.batch_name)<EOL><DEDENT>else:<EOL><INDENT>root_name = self.batch_name<EOL><DEDENT>path = os.path.join(self.output_directory,<EOL>*(self.subdir+[root_name]))<EOL>return os.path.abspath(path)<EOL>

A helper method that supplies the root directory name given a
timestamp.


def _qsub_block(self, output_dir, error_dir, tid_specs):

processes = []<EOL>job_names = []<EOL>for (tid, spec) in tid_specs:<EOL><INDENT>job_name = "<STR_LIT>" % (self.batch_name, self.job_timestamp, tid)<EOL>job_names.append(job_name)<EOL>cmd_args = self.command(<EOL>self.command._formatter(spec),<EOL>tid, self._launchinfo)<EOL>popen_args = self._qsub_args([("<STR_LIT>",error_dir), ('<STR_LIT>',job_name), ("<STR_LIT>",output_dir)],<EOL>cmd_args)<EOL>p = subprocess.Popen(popen_args, stdout=subprocess.PIPE)<EOL>(stdout, stderr) = p.communicate()<EOL>self.debug(stdout)<EOL>if p.poll() != <NUM_LIT:0>:<EOL><INDENT>raise EnvironmentError("<STR_LIT>" % p.poll())<EOL><DEDENT>processes.append(p)<EOL><DEDENT>self.message("<STR_LIT>" % len(processes))<EOL>if (self.reduction_fn is not None) or self.dynamic:<EOL><INDENT>self._qsub_collate_and_launch(output_dir, error_dir, job_names)<EOL><DEDENT>

This method handles static argument specifiers and cases where
the dynamic specifiers cannot be queued before the arguments
are known.


def _qsub_collate_and_launch(self, output_dir, error_dir, job_names):

job_name = "<STR_LIT>" % (self.batch_name,<EOL>self.job_timestamp,<EOL>self.collate_count)<EOL>overrides = [("<STR_LIT>",error_dir), ('<STR_LIT>',job_name), ("<STR_LIT>",output_dir),<EOL>('<STR_LIT>','<STR_LIT:U+002C>'.join(job_names))]<EOL>resume_cmds =["<STR_LIT>",<EOL>("<STR_LIT>"<EOL>% self.root_directory),<EOL>"<STR_LIT>",<EOL>"<STR_LIT>"]<EOL>cmd_args = [self.command.executable,<EOL>'<STR_LIT:-c>', '<STR_LIT:;>'.join(resume_cmds)]<EOL>popen_args = self._qsub_args(overrides, cmd_args)<EOL>p = subprocess.Popen(popen_args, stdout=subprocess.PIPE)<EOL>(stdout, stderr) = p.communicate()<EOL>self.debug(stdout)<EOL>if p.poll() != <NUM_LIT:0>:<EOL><INDENT>raise EnvironmentError("<STR_LIT>" % p.poll())<EOL><DEDENT>self.collate_count += <NUM_LIT:1><EOL>self.message("<STR_LIT>")<EOL>return job_name<EOL>

The method that actually runs qsub to invoke the python
process with the necessary commands to trigger the next
collation step and next block of jobs.


def __call__(self, spec, tid=None, info={}):

raise NotImplementedError<EOL>

Formats a single argument specification supplied as a
dictionary of argument name/value pairs. The info dictionary
contains launch information as defined in the _setup_launch
method of Launcher.


raise NotImplementedError<EOL>

A succinct summary of the Command configuration.  Unlike the
repr, a summary does not have to be complete but must supply
key information relevant to the user. Must begin by stating
the executable.


<EOL>if self.launch_args is not None:<EOL><INDENT>proceed = self.review_args(self.launch_args,<EOL>show_repr=True,<EOL>heading='<STR_LIT>')<EOL>if not proceed: return False<EOL><DEDENT>reviewers = [self.review_args,<EOL>self.review_command,<EOL>self.review_launcher]<EOL>for (count, launcher) in enumerate(launchers):<EOL><INDENT>if not all(reviewer(launcher) for reviewer in reviewers):<EOL><INDENT>print("<STR_LIT>")<EOL>return False<EOL><DEDENT>if len(launchers)!= <NUM_LIT:1> and count < len(launchers)-<NUM_LIT:1>:<EOL><INDENT>skip_remaining = self.input_options(['<STR_LIT:Y>', '<STR_LIT:n>','<STR_LIT>'],<EOL>'<STR_LIT>', default='<STR_LIT:y>')<EOL>if skip_remaining == '<STR_LIT:y>':          break<EOL>elif skip_remaining == '<STR_LIT>':     return False<EOL><DEDENT><DEDENT>if self.input_options(['<STR_LIT:y>','<STR_LIT:N>'], '<STR_LIT>', default='<STR_LIT:n>') != '<STR_LIT:y>':<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>return self._launch_all(launchers)<EOL><DEDENT>

Runs the review process for all the launchers.


launchinfo = self._setup_launch()<EOL>streams_path = self._setup_streams_path()<EOL>self.command.finalize(launchinfo)<EOL>self._record_info(launchinfo)<EOL>last_tid = <NUM_LIT:0><EOL>last_tids = []<EOL>for gid, groupspecs in enumerate(self.args):<EOL><INDENT>tids = list(range(last_tid, last_tid+len(groupspecs)))<EOL>last_tid += len(groupspecs)<EOL>allcommands = [self.command(<EOL>self.command._formatter(spec), tid, launchinfo)for (spec,tid) in zip(groupspecs,tids)]<EOL>self._append_log(list(zip(tids,groupspecs)))<EOL>self.message("<STR_LIT>" % (gid, len(allcommands)))<EOL>self._launch_process_group(zip(allcommands,tids), streams_path)<EOL>last_tids = tids[:]<EOL>if self.dynamic:<EOL><INDENT>self.args.update(last_tids, launchinfo)<EOL><DEDENT><DEDENT>self._record_info()<EOL>if self.reduction_fn is not None:<EOL><INDENT>self.reduction_fn(self._spec_log, self.root_directory)<EOL><DEDENT>

Call to start Launcher execution. Typically invoked by
review_and_launch but may be called directly by the user.


def review_args(self, obj, show_repr=False, heading='<STR_LIT>'):

args = obj.args if isinstance(obj, Launcher) else obj<EOL>print('<STR_LIT>' % self.summary_heading(heading))<EOL>args.summary()<EOL>if show_repr: print("<STR_LIT>" % args)<EOL>response = self.input_options(['<STR_LIT:y>', '<STR_LIT:N>','<STR_LIT>'],<EOL>'<STR_LIT>', default='<STR_LIT:n>')<EOL>if response == '<STR_LIT>': return False<EOL>if response == '<STR_LIT:y>':  args.show()<EOL>print('<STR_LIT>')<EOL>return True<EOL>

Reviews the given argument specification. Can review the
meta-arguments (launch_args) or the arguments themselves.


for launcher in launchers:<EOL><INDENT>print("<STR_LIT>" % launcher.batch_name)<EOL>launcher()<EOL><DEDENT>return True<EOL>


p = subprocess.Popen(['<STR_LIT>', '<STR_LIT>' % (self.batch_name,<EOL>self.job_timestamp)],<EOL>stdout=subprocess.PIPE)<EOL>(stdout, stderr) = p.communicate()<EOL>return p.poll()<EOL>

Runs qdel command to remove all remaining queued jobs using
the <batch_name>* pattern . Necessary when StopIteration is
raised with scheduled jobs left on the queue.
Returns exit-code of qdel.


self._launchinfo = self._setup_launch()<EOL>self.command.finalize(self._launchinfo)<EOL>self.job_timestamp = time.strftime('<STR_LIT>')<EOL>streams_path = self._setup_streams_path()<EOL>self.qsub_flag_options['<STR_LIT>'] = streams_path<EOL>self.qsub_flag_options['<STR_LIT>'] = streams_path<EOL>self.collate_and_launch()<EOL>self._record_info(self._launchinfo)<EOL>

Main entry point for the launcher. Collects the static
information about the launch and sets up the stdout and stderr
stream output directories. Generates the first call to
collate_and_launch().


def input_options(self, options, prompt='<STR_LIT>', default=None):

check_options = [x.lower() for x in options]<EOL>while True:<EOL><INDENT>response = input('<STR_LIT>' % (prompt, '<STR_LIT:U+002CU+0020>'.join(options))).lower()<EOL>if response in check_options: return response.strip()<EOL>elif response == '<STR_LIT>' and default is not None:<EOL><INDENT>return default.lower().strip()<EOL><DEDENT><DEDENT>

Helper to prompt the user for input on the commandline.


processes = []<EOL>for cmd, tid in process_commands:<EOL><INDENT>job_timestamp = time.strftime('<STR_LIT>')<EOL>basename = "<STR_LIT>" % (self.batch_name, job_timestamp, tid)<EOL>stdout_path = os.path.join(streams_path, "<STR_LIT>" % (basename, tid))<EOL>stderr_path = os.path.join(streams_path, "<STR_LIT>" % (basename, tid))<EOL>process = { '<STR_LIT>' : tid,<EOL>'<STR_LIT>' : cmd,<EOL>'<STR_LIT>' : stdout_path,<EOL>'<STR_LIT>' : stderr_path }<EOL>processes.append(process)<EOL><DEDENT>json_path = os.path.join(self.root_directory, self.json_name % (tid))<EOL>with open(json_path, '<STR_LIT:w>') as json_file:<EOL><INDENT>json.dump(processes, json_file, sort_keys=True, indent=<NUM_LIT:4>)<EOL><DEDENT>p = subprocess.Popen([self.script_path, json_path, self.batch_name,<EOL>str(len(processes)), str(self.max_concurrency)])<EOL>if p.wait() != <NUM_LIT:0>:<EOL><INDENT>raise EnvironmentError("<STR_LIT>" % p.poll())<EOL><DEDENT>

Aggregates all process_commands and the designated output files into a
list, and outputs it as JSON, after which the wrapper script is called.


return<EOL>

Optional, final check that ensures valid arguments have been
passed before launch. Allows the constant and varying_keys to
be be checked and can inspect the specs attribute if an
instance of Args. If invalid, raise an Exception with the
appropriate error message, otherwise return None.


(basename, ext) = os.path.splitext(filename)<EOL>basename = basename if (ext in self.extensions) else filename<EOL>ext = ext if (ext in self.extensions) else self.extensions[<NUM_LIT:0>]<EOL>savepath = os.path.abspath(os.path.join(self.directory,<EOL>'<STR_LIT>' % (basename, ext)))<EOL>return (tempfile.mkstemp(ext, basename + "<STR_LIT:_>", self.directory)[<NUM_LIT:1>]<EOL>if self.hash_suffix else savepath)<EOL>

Returns the full path for saving the file, adding an extension
and making the filename unique as necessary.


def save(self, filename,  metadata={}, **data):

intersection = set(metadata.keys()) & set(data.keys())<EOL>if intersection:<EOL><INDENT>msg = '<STR_LIT>'<EOL>raise Exception(msg  % '<STR_LIT:U+002C>'.join(intersection))<EOL><DEDENT>

The implementation in the base class simply checks there is no
clash between the metadata and data keys.


def save(self, filename, imdata, **data):

if isinstance(imdata, numpy.ndarray):<EOL><INDENT>imdata = Image.fromarray(numpy.uint8(imdata))<EOL><DEDENT>elif isinstance(imdata, Image.Image):<EOL><INDENT>imdata.save(self._savepath(filename))<EOL><DEDENT>

Data may be either a PIL Image object or a Numpy array.


raise NotImplementedError<EOL>


global _loaded<EOL>if not _loaded:<EOL><INDENT>_loaded = True<EOL>from lancet import launch<EOL>if sys.version_info[<NUM_LIT:0>] == <NUM_LIT:2>:<EOL><INDENT>launch.input = lambda *args, **kwargs: raw_input(*args, **kwargs)<EOL><DEDENT>plaintext_formatter = ip.display_formatter.formatters['<STR_LIT>']<EOL>plaintext_formatter.for_type(Args, repr_pretty_annotated)<EOL>plaintext_formatter.for_type(Command, repr_pretty_unannotated)<EOL>plaintext_formatter.for_type(Launcher, repr_pretty_unannotated)<EOL>plaintext_formatter.for_type(FileType, repr_pretty_unannotated)<EOL>plaintext_formatter.for_type(review_and_launch, repr_pretty_unannotated)<EOL><DEDENT>

IPython pretty printing support (optional). To load the extension
you may execute the following in IPython:

%load_ext lancet


def __call__(self, paths=[], **params_to_override):

p=param.ParamOverrides(self, dict(params_to_override, paths=paths))<EOL>if p.paths == []:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>paths = [p.paths] if isinstance(p.paths, str) else p.paths<EOL>def _desc(path, ind):<EOL><INDENT>for vcs in p.commands.keys():<EOL><INDENT>if os.path.exists(os.path.join(path, vcs)):<EOL><INDENT>proc = subprocess.Popen(p.commands[vcs][ind],<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE, cwd=path)<EOL>return str(proc.communicate()[<NUM_LIT:0>].decode()).strip()<EOL><DEDENT><DEDENT><DEDENT>abspaths = [os.path.abspath(path) for path in paths]<EOL>return {'<STR_LIT>' : dict((path, _desc(path,<NUM_LIT:0>)) for path in abspaths),<EOL>'<STR_LIT>':  dict((path, _desc(path,<NUM_LIT:1>)) for path in abspaths),<EOL>'<STR_LIT>':     dict((path, _desc(path,<NUM_LIT:2>)) for path in abspaths)}<EOL>

Takes a single path string or a list of path strings and
returns the corresponing version control information.


return type(spec)((k, str(v)) for (k,v) in spec.items())<EOL>

Formats the elements of an argument set appropriately


items,  data_keys = [], None<EOL>for key, filename in table.items():<EOL><INDENT>data_dict = self.filetype.data(filename[<NUM_LIT:0>])<EOL>current_keys = tuple(sorted(data_dict.keys()))<EOL>values = [data_dict[k] for k in current_keys]<EOL>if data_keys is None:<EOL><INDENT>data_keys = current_keys<EOL><DEDENT>elif data_keys != current_keys:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>items.append((key, values))<EOL><DEDENT>return Table(items, kdims=table.kdims, vdims=data_keys)<EOL>

Load the file contents into the supplied Table using the
specified key and filetype. The input table should have the
filenames as values which will be replaced by the loaded
data. If data_key is specified, this key will be used to index
the loaded data to retrive the specified item.


raise NotImplementedError<EOL>

Returns the list of parameter names whose values are constant
as the argument specifier is iterated.  Note that the union of
constant and varying_keys should partition the entire set of
keys in the case where there are no unsortable keys.


@classmethod<EOL><INDENT>def directory(cls, directory, root=None, extension=None, **kwargs):<DEDENT>

root = os.getcwd() if root is None else root<EOL>suffix = '<STR_LIT>' if extension is None else '<STR_LIT:.>' + extension.rsplit('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL>pattern = directory + os.sep + '<STR_LIT:*>' + suffix<EOL>key = os.path.join(root, directory,'<STR_LIT:*>').rsplit(os.sep)[-<NUM_LIT:2>]<EOL>format_parse = list(string.Formatter().parse(key))<EOL>if not all([el is None for el in zip(*format_parse)[<NUM_LIT:1>]]):<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>return cls(key, pattern, root, **kwargs)<EOL>

Load all the files in a given directory selecting only files
with the given extension if specified. The given kwargs are
passed through to the normal constructor.


def _info(self, source, key, filetype, ignore):

specs, mdata = [], {}<EOL>mdata_clashes  = set()<EOL>for spec in source.specs:<EOL><INDENT>if key not in spec:<EOL><INDENT>raise Exception("<STR_LIT>" % key)<EOL><DEDENT>mdata = dict((k,v) for (k,v) in filetype.metadata(spec[key]).items()<EOL>if k not in ignore)<EOL>mdata_spec = {}<EOL>mdata_spec.update(spec)<EOL>mdata_spec.update(mdata)<EOL>specs.append(mdata_spec)<EOL>mdata_clashes = mdata_clashes | (set(spec.keys()) & set(mdata.keys()))<EOL><DEDENT>if mdata_clashes:<EOL><INDENT>self.warning("<STR_LIT>")<EOL><DEDENT>return specs<EOL>

Generates the union of the source.specs and the metadata
dictionary loaded by the filetype object.


log_path = (log_path if os.path.isfile(log_path)<EOL>else os.path.join(os.getcwd(), log_path))<EOL>with open(log_path,'<STR_LIT:r>') as log:<EOL><INDENT>splits = (line.split() for line in log)<EOL>uzipped = ((int(split[<NUM_LIT:0>]), json.loads("<STR_LIT:U+0020>".join(split[<NUM_LIT:1>:]))) for split in splits)<EOL>szipped = [(i, dict((str(k),v) for (k,v) in d.items())) for (i,d) in uzipped]<EOL><DEDENT>return dict_type(szipped)<EOL>

Parses the log file generated by a launcher and returns
dictionary with tid keys and specification values.

Ordering can be maintained by setting dict_type to the
appropriate constructor (i.e. OrderedDict). Keys are converted
from unicode to strings for kwarg use.


def pprint_args(self, pos_args, keyword_args, infix_operator=None, extra_params={}):

if infix_operator and not (len(pos_args)==<NUM_LIT:2> and keyword_args==[]):<EOL><INDENT>raise Exception('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>(kwargs,_,_,_) = self._pprint_args<EOL>self._pprint_args = (keyword_args + kwargs, pos_args, infix_operator, extra_params)<EOL>

Method to define the positional arguments and keyword order
for pretty printing.


sep = '<STR_LIT>'<EOL>float_codes = ['<STR_LIT:e>','<STR_LIT:E>','<STR_LIT:f>', '<STR_LIT:F>','<STR_LIT:g>', '<STR_LIT>', '<STR_LIT:n>']<EOL>typecodes = dict([(k,float) for k in float_codes]<EOL>+ [('<STR_LIT:b>',bin), ('<STR_LIT:d>',int), ('<STR_LIT:o>',oct), ('<STR_LIT:x>',hex)])<EOL>parse = list(string.Formatter().parse(pattern))<EOL>text, fields, codes, _ = zip(*parse)<EOL>types = []<EOL>for (field, code) in zip(fields, codes):<EOL><INDENT>if code in ['<STR_LIT>', None]: continue<EOL>constructor =  typecodes.get(code[-<NUM_LIT:1>], None)<EOL>if constructor: types += [(field, constructor)]<EOL><DEDENT>stars =  ['<STR_LIT>' if not f else '<STR_LIT:*>' for f in fields]<EOL>globpat = '<STR_LIT>'.join(text+star for (text,star) in zip(text,stars))<EOL>refields = ['<STR_LIT>' if not f else sep+('<STR_LIT>'% f)+sep for f in fields]<EOL>parts = '<STR_LIT>'.join(text+group for (text,group) in zip(text, refields)).split(sep)<EOL>for i in range(<NUM_LIT:0>, len(parts), <NUM_LIT:2>): parts[i] = re.escape(parts[i])<EOL>regexp_pattern = '<STR_LIT>'.join(parts).replace('<STR_LIT>','<STR_LIT>')<EOL>fields = list(f for f in fields if f)<EOL>return globpat, regexp_pattern , fields, dict(types)<EOL>

Given a path pattern with format declaration, generates a
four-tuple (glob_pattern, regexp pattern, fields, type map)


@staticmethod<EOL><INDENT>def write_log(log_path, data, allow_append=True):<DEDENT>

append = os.path.isfile(log_path)<EOL>islist = isinstance(data, list)<EOL>if append and not allow_append:<EOL><INDENT>raise Exception('<STR_LIT>'<EOL>'<STR_LIT>' % log_path)<EOL><DEDENT>if not (islist or isinstance(data, Args)):<EOL><INDENT>raise Exception('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>specs = data if islist else data.specs<EOL>if not all(isinstance(el,dict) for el in specs):<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>log_file = open(log_path, '<STR_LIT>') if append else open(log_path, '<STR_LIT:w>')<EOL>start = int(log_file.readlines()[-<NUM_LIT:1>].split()[<NUM_LIT:0>])+<NUM_LIT:1> if append else <NUM_LIT:0><EOL>ascending_indices = range(start, start+len(data))<EOL>log_str = '<STR_LIT:\n>'.join(['<STR_LIT>' % (tid, json.dumps(el))<EOL>for (tid, el) in zip(ascending_indices,specs)])<EOL>log_file.write("<STR_LIT:\n>"+log_str if append else log_str)<EOL>log_file.close()<EOL>

Writes the supplied specifications to the log path. The data
may be supplied as either as a an Args or as a list of
dictionaries.

By default, specifications will be appropriately appended to
an existing log file. This can be disabled by setting
allow_append to False.


raise StopIteration<EOL>

Called to get a list of specifications: dictionaries with
parameter name keys and string values.


ordering = self.constant_keys + self.varying_keys<EOL>spec_lines = ['<STR_LIT:U+002CU+0020>'.join(['<STR_LIT>' % (k, s[k]) for k in ordering<EOL>if (k in s) and (k not in exclude)])<EOL>for s in self.specs]<EOL>print('<STR_LIT:\n>'.join(['<STR_LIT>' % (i,l) for (i,l) in enumerate(spec_lines)]))<EOL>

Convenience method to inspect the available argument values in
human-readable format. The ordering of keys is determined by
how quickly they vary.

The exclude list allows specific keys to be excluded for
readability (e.g. to hide long, absolute filenames).


def _build_specs(self, specs, kwargs, fp_precision):

if specs is None:<EOL><INDENT>overrides = param.ParamOverrides(self, kwargs,<EOL>allow_extra_keywords=True)<EOL>extra_kwargs = overrides.extra_keywords()<EOL>kwargs = dict([(k,v) for (k,v) in kwargs.items()<EOL>if k not in extra_kwargs])<EOL>rounded_specs = list(self.round_floats([extra_kwargs],<EOL>fp_precision))<EOL>if extra_kwargs=={}: return [], kwargs, True<EOL>else:                return rounded_specs, kwargs, False<EOL><DEDENT>return list(self.round_floats(specs, fp_precision)), kwargs, True<EOL>

Returns the specs, the remaining kwargs and whether or not the
constructor was called with kwarg or explicit specs.


ns = u'<STR_LIT>' % namespace<EOL>nsl = len(ns)<EOL>for elem in doc.getiterator():<EOL><INDENT>if elem.tag.startswith(ns):<EOL><INDENT>elem.tag = elem.tag[nsl:]<EOL>elem.attrib['<STR_LIT>'] = namespace<EOL><DEDENT><DEDENT>

Remove namespace in the passed document in place.


raise NotImplementedError()<EOL>


def mergeItems(self, localItem, remoteItem, changeSpec):

raise ConflictError('<STR_LIT>')<EOL>

[OPTIONAL] Merges the properties of `remoteItem`, which is an item
provided by a remote peer during a synchronization, into the
`localItem`, which is an item retrieved from this agent either via
:meth:`getItem` or :meth:`matchItem`. `changeSpec` will represent
the changes applied to `localItem` since `remoteItem` was last
synchronized, or will be ``None`` when called as a result of a
slow-sync :meth:`matchItem` call.

This method should return a new change-spec (see
:meth:`replaceItem` for details) that represents the changes
applied to `localItem` from `remoteItem`.

If the items cannot be merged, then a `pysyncml.ConflictError`
should be raised with more descriptive information on what failed
during the merge --- in which case pysyncml will revert to the
conflict resolution policy defined by `store.conflictPolicy` or
`adapter.conflictPolicy`.

IMPORTANT: if the merge fails, `localItem` and `remoteItem` must
stay untouched by this call; most importantly, if the merge fails
with a ConflictError, then `remoteItem` must be in the identical
state as when it entered the call.

This method by default raises a ConflictError, which means that if
any changes are made to the same item simultaneously by two
different peers, they will result in a conflict and will not be
auto-mergeable.

See :doc:`../merging` for details.


raise NotImplementedError()<EOL>

Deletes the local datastore item with ID `itemID`.


def dumpItem(self, item, stream, contentType=None, version=None):

raise NotImplementedError()<EOL>

Converts the specified `item` to serialized form (such that it can
transported over the wire) and writes it to the provided file-like
`stream` object. For agents that support multiple content-types,
the desired `contentType` and `version` will be specified as a
parameter. If `contentType` and `version` are None, appropriate
default values should be used. For agents that concurrently use
multiple content-types, the return value may be a two-element
tuple of (contentType, version), thus overriding or enhancing the
provided values.


def loadItem(self, stream, contentType=None, version=None):

raise NotImplementedError()<EOL>

Reverses the effects of the :meth:`dumpItem` method, and returns
the de-serialized Item from the file-like source `stream`.

Note: `version` will typically be ``None``, so it should either be
auto-determined, or not used. This is an issue in the SyncML
protocol, and is only here for symmetry with :meth:`dumpItem`
and as "future-proofing".