<EOL>self.log.debug("<STR_LIT>")<EOL>self.connected.clear()<EOL>self.reconnect_required.set()<EOL>if self.socket:<EOL><INDENT>self.socket.close()<EOL><DEDENT>

Issues a reconnection by setting the reconnect_required event.

        :return:


Creates a websocket connection.

        :return:


self.log.debug("<STR_LIT>")<EOL>self._connect()<EOL>

Main method of Thread.

        :return:


self._stop_timers()<EOL>raw, received_at = message, time.time()<EOL>self.log.debug("<STR_LIT>",<EOL>raw, received_at)<EOL>try:<EOL><INDENT>data = json.loads(raw)<EOL><DEDENT>except json.JSONDecodeError:<EOL><INDENT>return<EOL><DEDENT>if isinstance(data, dict):<EOL><INDENT>self._system_handler(data, received_at)<EOL><DEDENT>else:<EOL><INDENT>if data[<NUM_LIT:1>] == '<STR_LIT>':<EOL><INDENT>self._heartbeat_handler()<EOL><DEDENT>else:<EOL><INDENT>self._data_handler(data, received_at)<EOL><DEDENT><DEDENT>self._start_timers()<EOL>

Handles and passes received data to the appropriate handlers.

        :return:


Stops ping, pong and connection timers.

        :return:


self.log.debug("<STR_LIT>")<EOL>self._stop_timers()<EOL>self.ping_timer = Timer(self.ping_interval, self.send_ping)<EOL>self.ping_timer.start()<EOL>self.connection_timer = Timer(self.connection_timeout,<EOL>self._connection_timed_out)<EOL>self.connection_timer.start()<EOL>

Resets and starts timers for API data and connection.

        :return:


self.log.debug("<STR_LIT>")<EOL>self.socket.send(json.dumps({'<STR_LIT>': '<STR_LIT>'}))<EOL>self.pong_timer = Timer(self.pong_timeout, self._check_pong)<EOL>self.pong_timer.start()<EOL>

Sends a ping message to the API and starts pong timers.

        :return:


Checks if a Pong message was received.

        :return:


def send(self, api_key=None, secret=None, list_data=None, auth=False, **kwargs):

if auth:<EOL><INDENT>nonce = str(int(time.time() * <NUM_LIT>))<EOL>auth_string = '<STR_LIT>' + nonce<EOL>auth_sig = hmac.new(secret.encode(), auth_string.encode(),<EOL>hashlib.sha384).hexdigest()<EOL>payload = {'<STR_LIT>': '<STR_LIT>', '<STR_LIT>': api_key, '<STR_LIT>': auth_sig,<EOL>'<STR_LIT>': auth_string, '<STR_LIT>': nonce}<EOL>payload = json.dumps(payload)<EOL><DEDENT>elif list_data:<EOL><INDENT>payload = json.dumps(list_data)<EOL><DEDENT>else:<EOL><INDENT>payload = json.dumps(kwargs)<EOL><DEDENT>self.log.debug("<STR_LIT>", payload)<EOL>try:<EOL><INDENT>self.socket.send(payload)<EOL><DEDENT>except websocket.WebSocketConnectionClosedException:<EOL><INDENT>self.log.error("<STR_LIT>", kwargs)<EOL><DEDENT>

Sends the given Payload to the API via the websocket connection.

        :param kwargs: payload paarameters as key=value pairs
        :return:


def pass_to_client(self, event, data, *args):

self.q.put((event, data, *args))<EOL>

Passes data up to the client via a Queue().

        :param event:
        :param data:
        :param args:
        :return:


self.log.debug("<STR_LIT>")<EOL>self.reconnect()<EOL>

Issues a reconnection if the connection timed out.

        :return:


self.log.debug("<STR_LIT>")<EOL>self.paused.set()<EOL>

Pauses the connection.

        :return:


self.log.debug("<STR_LIT>")<EOL>self.paused.clear()<EOL>self.log.debug("<STR_LIT>")<EOL>self._resubscribe(soft=True)<EOL>

Unpauses the connection.

        Send a message up to client that he should re-subscribe to all
        channels.

        :return:


<EOL>self.log.debug("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>self._start_timers()<EOL>

Handles heartbeat messages.

        :return:


<EOL>self.log.debug("<STR_LIT>")<EOL>self.pong_received = True<EOL>

Handle a pong response.

        :return:


self.log.debug("<STR_LIT>", data)<EOL>event = data.pop('<STR_LIT>')<EOL>if event == '<STR_LIT>':<EOL><INDENT>self.log.debug("<STR_LIT>",<EOL>data)<EOL>self._pong_handler()<EOL><DEDENT>elif event == '<STR_LIT:info>':<EOL><INDENT>self.log.debug("<STR_LIT>",<EOL>data)<EOL>self._info_handler(data)<EOL><DEDENT>elif event == '<STR_LIT:error>':<EOL><INDENT>self.log.debug("<STR_LIT>",<EOL>data)<EOL>self._error_handler(data)<EOL><DEDENT>elif event in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>self.log.debug("<STR_LIT>"<EOL>"<STR_LIT>", data)<EOL>self._response_handler(event, data, ts)<EOL><DEDENT>else:<EOL><INDENT>self.log.error("<STR_LIT>", event, data)<EOL><DEDENT>

Distributes system messages to the appropriate handler.

        System messages include everything that arrives as a dict,
        or a list containing a heartbeat.

        :param data:
        :param ts:
        :return:


def _response_handler(self, event, data, ts):

self.log.debug("<STR_LIT>", data)<EOL>self.pass_to_client(event, data, ts)<EOL>

Handles responses to (un)subscribe and conf commands.

        Passes data up to client.

        :param data:
        :param ts:
        :return:


def raise_exception():<EOL><INDENT>"""<STR_LIT>"""<EOL>self.log.error("<STR_LIT>", data['<STR_LIT:code>'], info_message[data['<STR_LIT:code>']])<EOL>raise ValueError("<STR_LIT>" % (data['<STR_LIT:code>'], info_message[data['<STR_LIT:code>']]))<EOL><DEDENT>if '<STR_LIT:code>' not in data and '<STR_LIT:version>' in data:<EOL><INDENT>self.log.info('<STR_LIT>', data['<STR_LIT:version>'])<EOL>return<EOL><DEDENT>info_message = {<NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>'<EOL>'<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>'<EOL>'<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>'<EOL>'<STR_LIT>'}<EOL>codes = {<NUM_LIT>: self.reconnect, <NUM_LIT>: self._pause,<EOL><NUM_LIT>: self._unpause}<EOL>if '<STR_LIT:version>' in data:<EOL><INDENT>self.log.info("<STR_LIT>", data['<STR_LIT:version>'])<EOL>return<EOL><DEDENT>try:<EOL><INDENT>self.log.info(info_message[data['<STR_LIT:code>']])<EOL>codes[data['<STR_LIT:code>']]()<EOL><DEDENT>except KeyError as e:<EOL><INDENT>self.log.exception(e)<EOL>self.log.error("<STR_LIT>", data['<STR_LIT:code>'])<EOL>raise<EOL><DEDENT>

Handle INFO messages from the API and issues relevant actions.

:param data:
:param ts:


errors = {<NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>'<EOL>}<EOL>try:<EOL><INDENT>self.log.error(errors[data['<STR_LIT:code>']])<EOL><DEDENT>except KeyError:<EOL><INDENT>self.log.error("<STR_LIT>"<EOL>"<STR_LIT>", data)<EOL><DEDENT>

Handle Error messages and log them accordingly.

:param data:
:param ts:


<EOL>self.log.debug("<STR_LIT>",<EOL>data)<EOL>self.pass_to_client('<STR_LIT:data>', data, ts)<EOL>

Handles data messages by passing them up to the client.

        :param data:
        :param ts:
        :return:


<EOL>if self.bitfinex_config:<EOL><INDENT>self.send(**self.bitfinex_config)<EOL><DEDENT>q_list = []<EOL>while True:<EOL><INDENT>try:<EOL><INDENT>identifier, q = self.channel_configs.popitem(last=True if soft else False)<EOL><DEDENT>except KeyError:<EOL><INDENT>break<EOL><DEDENT>q_list.append((identifier, q.copy()))<EOL>if identifier == '<STR_LIT>':<EOL><INDENT>self.send(**q, auth=True)<EOL>continue<EOL><DEDENT>if soft:<EOL><INDENT>q['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>self.send(**q)<EOL><DEDENT>if soft:<EOL><INDENT>for identifier, q in reversed(q_list):<EOL><INDENT>self.channel_configs[identifier] = q<EOL>self.send(**q)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for identifier, q in q_list:<EOL><INDENT>self.channel_configs[identifier] = q<EOL><DEDENT><DEDENT>

Resubscribes to all channels found in self.channel_configs.

        :param soft: if True, unsubscribes first.
        :return: None


def __init__(self, key=None, secret=None, log_level=None, **wss_kwargs):

self.key = key if key else '<STR_LIT>'<EOL>self.secret = secret if secret else '<STR_LIT>'<EOL>self.conn = WebSocketConnection(log_level=log_level,<EOL>**wss_kwargs)<EOL>self.queue_processor = QueueProcessor(self.conn.q,<EOL>log_level=log_level)<EOL>

Initializes BtfxWss Instance.
:param key: Api Key as string
:param secret: Api secret as string
:param addr: Websocket API Address


self.conn.start()<EOL>self.queue_processor.start()<EOL>


self.conn.disconnect()<EOL>self.queue_processor.join()<EOL>


self.conn.reconnect()<EOL>while not self.conn.connected.is_set():<EOL><INDENT>log.info("<STR_LIT>")<EOL>time.sleep(<NUM_LIT:1>)<EOL><DEDENT>for key in self.channel_configs:<EOL><INDENT>self.conn.send(**self.channel_configs[key])<EOL><DEDENT>


key = ('<STR_LIT>', pair)<EOL>return self.queue_processor.tickers[key]<EOL>

Return a queue containing all received ticker data.

        :param pair:
        :return: Queue()


key = ('<STR_LIT>', pair)<EOL>return self.queue_processor.books[key]<EOL>

Return a queue containing all received book data.

        :param pair:
        :return: Queue()


key = ('<STR_LIT>', pair)<EOL>return self.queue_processor.raw_books[key]<EOL>

Return a queue containing all received raw book data.

        :param pair:
        :return: Queue()


key = ('<STR_LIT>', pair)<EOL>return self.queue_processor.trades[key]<EOL>

Return a queue containing all received trades data.

        :param pair:
        :return: Queue()


def candles(self, pair, timeframe=None):

timeframe = '<STR_LIT>' if not timeframe else timeframe<EOL>key = ('<STR_LIT>', pair, timeframe)<EOL>return self.queue_processor.candles[key]<EOL>

Return a queue containing all received candles data.

        :param pair: str, Symbol pair to request data for
        :param timeframe: str
        :return: Queue()


def config(self, decimals_as_strings=True, ts_as_dates=False,<EOL>sequencing=False, ts=False, **kwargs):

flags = <NUM_LIT:0><EOL>if decimals_as_strings:<EOL><INDENT>flags += <NUM_LIT:8><EOL><DEDENT>if ts_as_dates:<EOL><INDENT>flags += <NUM_LIT:32><EOL><DEDENT>if ts:<EOL><INDENT>flags += <NUM_LIT><EOL><DEDENT>if sequencing:<EOL><INDENT>flags += <NUM_LIT><EOL><DEDENT>q = {'<STR_LIT>': '<STR_LIT>', '<STR_LIT>': flags}<EOL>q.update(kwargs)<EOL>self.conn.bitfinex_config = q<EOL>self.conn.send(**q)<EOL>

Send configuration to websocket server

        :param decimals_as_strings: bool, turn on/off decimals as strings
        :param ts_as_dates: bool, decide to request timestamps as dates instead
        :param sequencing: bool, turn on sequencing
        :param ts: bool, request the timestamp to be appended to every array
               sent by the server
        :param kwargs:
        :return:


@is_connected<EOL><INDENT>def subscribe_to_ticker(self, pair, **kwargs):<DEDENT>

identifier = ('<STR_LIT>', pair)<EOL>self._subscribe('<STR_LIT>', identifier, symbol=pair, **kwargs)<EOL>

Subscribe to the passed pair's ticker channel.

        :param pair: str, Symbol pair to request data for
        :param kwargs:
        :return:


@is_connected<EOL><INDENT>def unsubscribe_from_ticker(self, pair, **kwargs):<DEDENT>

identifier = ('<STR_LIT>', pair)<EOL>self._unsubscribe('<STR_LIT>', identifier, symbol=pair, **kwargs)<EOL>

Unsubscribe to the passed pair's ticker channel.

        :param pair: str, Symbol pair to request data for
        :param kwargs:
        :return:


@is_connected<EOL><INDENT>def subscribe_to_order_book(self, pair, **kwargs):<DEDENT>

Subscribe to the passed pair's order book channel.

        :param pair: str, Symbol pair to request data for
        :param kwargs:
        :return:


@is_connected<EOL><INDENT>def unsubscribe_from_order_book(self, pair, **kwargs):<DEDENT>

Unsubscribe to the passed pair's order book channel.

        :param pair: str, Symbol pair to request data for
        :param kwargs:
        :return:


@is_connected<EOL><INDENT>def subscribe_to_raw_order_book(self, pair, prec=None, **kwargs):<DEDENT>

identifier = ('<STR_LIT>', pair)<EOL>prec = '<STR_LIT>' if prec is None else prec<EOL>self._subscribe('<STR_LIT>', identifier, pair=pair, prec=prec, **kwargs)<EOL>

Subscribe to the passed pair's raw order book channel.

        :param pair: str, Symbol pair to request data for
        :param prec:
        :param kwargs:
        :return:


@is_connected<EOL><INDENT>def unsubscribe_from_raw_order_book(self, pair, prec=None, **kwargs):<DEDENT>

identifier = ('<STR_LIT>', pair)<EOL>prec = '<STR_LIT>' if prec is None else prec<EOL>self._unsubscribe('<STR_LIT>', identifier, pair=pair, prec=prec, **kwargs)<EOL>

Unsubscribe to the passed pair's raw order book channel.

        :param pair: str, Symbol pair to request data for
        :param prec:
        :param kwargs:
        :return:


@is_connected<EOL><INDENT>def subscribe_to_trades(self, pair, **kwargs):<DEDENT>

Subscribe to the passed pair's trades channel.

        :param pair: str, Symbol pair to request data for
        :param kwargs:
        :return:


@is_connected<EOL><INDENT>def unsubscribe_from_trades(self, pair, **kwargs):<DEDENT>

Unsubscribe to the passed pair's trades channel.

        :param pair: str, Symbol pair to request data for
        :param kwargs:
        :return:


@is_connected<EOL><INDENT>def subscribe_to_candles(self, pair, timeframe=None, **kwargs):<DEDENT>

valid_tfs = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>if timeframe:<EOL><INDENT>if timeframe not in valid_tfs:<EOL><INDENT>raise ValueError("<STR_LIT>" % valid_tfs)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>timeframe = '<STR_LIT>'<EOL><DEDENT>identifier = ('<STR_LIT>', pair, timeframe)<EOL>pair = '<STR_LIT:t>' + pair if not pair.startswith('<STR_LIT:t>') else pair<EOL>key = '<STR_LIT>' + timeframe + '<STR_LIT::>' + pair<EOL>self._subscribe('<STR_LIT>', identifier, key=key, **kwargs)<EOL>

Subscribe to the passed pair's OHLC data channel.

        :param pair: str, Symbol pair to request data for
        :param timeframe: str, {1m, 5m, 15m, 30m, 1h, 3h, 6h, 12h,
                                1D, 7D, 14D, 1M}
        :param kwargs:
        :return:


@is_connected<EOL><INDENT>def unsubscribe_from_candles(self, pair, timeframe=None, **kwargs):<DEDENT>

valid_tfs = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>if timeframe:<EOL><INDENT>if timeframe not in valid_tfs:<EOL><INDENT>raise ValueError("<STR_LIT>" % valid_tfs)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>timeframe = '<STR_LIT>'<EOL><DEDENT>identifier = ('<STR_LIT>', pair, timeframe)<EOL>pair = '<STR_LIT:t>' + pair if not pair.startswith('<STR_LIT:t>') else pair<EOL>key = '<STR_LIT>' + timeframe + '<STR_LIT::>' + pair<EOL>self._unsubscribe('<STR_LIT>', identifier, key=key, **kwargs)<EOL>

Unsubscribe to the passed pair's OHLC data channel.

        :param timeframe: str, {1m, 5m, 15m, 30m, 1h, 3h, 6h, 12h,
                                1D, 7D, 14D, 1M}
        :param kwargs:
        :return:


if not self.key and not self.secret:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>self.channel_configs['<STR_LIT>'] = {'<STR_LIT>': self.key, '<STR_LIT>': self.secret}<EOL>self.conn.send(api_key=self.key, secret=self.secret, auth=True)<EOL>

Authenticate with the Bitfinex API.

        :return:


self._send_auth_command('<STR_LIT>', order_settings)<EOL>

Post a new Order via Websocket.

        :param kwargs:
        :return:


@is_connected<EOL><INDENT>def cancel_order(self, multi=False, **order_identifiers):<DEDENT>

if multi:<EOL><INDENT>self._send_auth_command('<STR_LIT>', order_identifiers)<EOL><DEDENT>else:<EOL><INDENT>self._send_auth_command('<STR_LIT>', order_identifiers)<EOL><DEDENT>

Cancel one or multiple orders via Websocket.

        :param multi: bool, whether order_settings contains settings for one, or
                      multiples orders
        :param order_identifiers: Identifiers for the order(s) you with to cancel
        :return:


self._send_auth_command('<STR_LIT>', operations)<EOL>

Execute multiple, order-related operations via Websocket.

        :param operations: operations to send to the websocket
        :return:


self._send_auth_command('<STR_LIT>', calculations)<EOL>

Request one or several operations via Websocket.

        :param calculations: calculations as strings to send to the websocket
        :return:


def __init__(self, data_q, log_level=None,<EOL>*args, **kwargs):

super(QueueProcessor, self).__init__(*args, **kwargs)<EOL>self.q = data_q<EOL>self._response_handlers = {'<STR_LIT>': self._handle_unsubscribed,<EOL>'<STR_LIT>': self._handle_subscribed,<EOL>'<STR_LIT>': self._handle_conf,<EOL>'<STR_LIT>': self._handle_auth,<EOL>'<STR_LIT>': self._handle_auth}<EOL>self._data_handlers = {'<STR_LIT>': self._handle_ticker,<EOL>'<STR_LIT>': self._handle_book,<EOL>'<STR_LIT>': self._handle_raw_book,<EOL>'<STR_LIT>': self._handle_candles,<EOL>'<STR_LIT>': self._handle_trades}<EOL>self._registry = {}<EOL>self.channel_directory = {}<EOL>self.channel_handlers = {}<EOL>self.last_update = {}<EOL>self.tickers = defaultdict(Queue)<EOL>self.books = defaultdict(Queue)<EOL>self.raw_books = defaultdict(Queue)<EOL>self.trades = defaultdict(Queue)<EOL>self.candles = defaultdict(Queue)<EOL>self.account = Queue()<EOL>self._stopped = Event()<EOL>self.log = logging.getLogger(self.__module__)<EOL>self.log.setLevel(level=logging.INFO if not log_level else log_level)<EOL>

Initialze a QueueProcessor instance.

        :param data_q: Queue()
        :param log_level: logging level
        :param args: Thread *args
        :param kwargs: Thread **kwargs


self._stopped.set()<EOL>super(QueueProcessor, self).join(timeout=timeout)<EOL>

Set sentinel for run() method and join thread.

        :param timeout:
        :return:


while not self._stopped.is_set():<EOL><INDENT>try:<EOL><INDENT>message = self.q.get(timeout=<NUM_LIT:0.1>)<EOL><DEDENT>except Empty:<EOL><INDENT>continue<EOL><DEDENT>dtype, data, ts = message<EOL>if dtype in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>self._response_handlers[dtype](dtype, data, ts)<EOL><DEDENT>except KeyError:<EOL><INDENT>self.log.error("<STR_LIT>"<EOL>"<STR_LIT>", dtype, message)<EOL><DEDENT><DEDENT>elif dtype == '<STR_LIT:data>':<EOL><INDENT>try:<EOL><INDENT>channel_id = data[<NUM_LIT:0>]<EOL>if channel_id != <NUM_LIT:0>:<EOL><INDENT>channel_type, *_ = self.channel_directory[channel_id]<EOL>self._data_handlers[channel_type](channel_type, data, ts)<EOL>self.update_timestamps(channel_id, ts)<EOL><DEDENT>else:<EOL><INDENT>self._handle_account(data=data, ts=ts)<EOL><DEDENT><DEDENT>except KeyError:<EOL><INDENT>self.log.error("<STR_LIT>",<EOL>message)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.log.error("<STR_LIT>", message)<EOL>continue<EOL><DEDENT><DEDENT>


def _handle_subscribed(self, dtype, data, ts,):

self.log.debug("<STR_LIT>", dtype, data, ts)<EOL>channel_name = data.pop('<STR_LIT>')<EOL>channel_id = data.pop('<STR_LIT>')<EOL>config = data<EOL>if '<STR_LIT>' in config:<EOL><INDENT>symbol = config['<STR_LIT>']<EOL>if symbol.startswith('<STR_LIT:t>'):<EOL><INDENT>symbol = symbol[<NUM_LIT:1>:]<EOL><DEDENT><DEDENT>elif '<STR_LIT>' in config:<EOL><INDENT>symbol = config['<STR_LIT>']<EOL>if symbol.startswith('<STR_LIT:t>'):<EOL><INDENT>symbol = symbol[<NUM_LIT:1>:]<EOL><DEDENT><DEDENT>elif '<STR_LIT:key>' in config:<EOL><INDENT>symbol = config['<STR_LIT:key>'].split('<STR_LIT::>')[<NUM_LIT:2>][<NUM_LIT:1>:]  <EOL><DEDENT>else:<EOL><INDENT>symbol = None<EOL><DEDENT>if '<STR_LIT>' in config and config['<STR_LIT>'].startswith('<STR_LIT:R>'):<EOL><INDENT>channel_name = '<STR_LIT>' + channel_name<EOL><DEDENT>self.channel_handlers[channel_id] = self._data_handlers[channel_name]<EOL>if '<STR_LIT:key>' in config:<EOL><INDENT>identifier = (channel_name, symbol, config['<STR_LIT:key>'].split('<STR_LIT::>')[<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>identifier = (channel_name, symbol)<EOL><DEDENT>self.channel_handlers[channel_id] = identifier<EOL>self.channel_directory[identifier] = channel_id<EOL>self.channel_directory[channel_id] = identifier<EOL>self.log.info("<STR_LIT>", identifier)<EOL>

Handles responses to subscribe() commands.

        Registers a channel id with the client and assigns a data handler to it.

        :param dtype:
        :param data:
        :param ts:
        :return:


def _handle_unsubscribed(self, dtype, data, ts):

self.log.debug("<STR_LIT>", dtype, data, ts)<EOL>channel_id = data.pop('<STR_LIT>')<EOL>chan_identifier = self.channel_directory.pop(channel_id)<EOL>self.channel_directory.pop(chan_identifier)<EOL>self.channel_handlers.pop(channel_id)<EOL>self.last_update.pop(channel_id)<EOL>self.log.info("<STR_LIT>", chan_identifier)<EOL>

Handles responses to unsubscribe() commands.

        Removes a channel id from the client.

        :param dtype:
        :param data:
        :param ts:
        :return:


def _handle_auth(self, dtype, data, ts):

<EOL>if dtype == '<STR_LIT>':<EOL><INDENT>raise NotImplementedError<EOL><DEDENT>channel_id = data.pop('<STR_LIT>')<EOL>user_id = data.pop('<STR_LIT>')<EOL>identifier = ('<STR_LIT>', user_id)<EOL>self.channel_handlers[identifier] = channel_id<EOL>self.channel_directory[identifier] = channel_id<EOL>self.channel_directory[channel_id] = identifier<EOL>

Handles authentication responses.

        :param dtype:
        :param data:
        :param ts:
        :return:


def _handle_conf(self, dtype, data, ts):

self.log.debug("<STR_LIT>", dtype, data, ts)<EOL>self.log.info("<STR_LIT>", dtype)<EOL>return<EOL>

Handles configuration messages.

        :param dtype:
        :param data:
        :param ts:
        :return:


def update_timestamps(self, chan_id, ts):

try:<EOL><INDENT>self.last_update[chan_id] = ts<EOL><DEDENT>except KeyError:<EOL><INDENT>self.log.warning("<STR_LIT>"<EOL>"<STR_LIT>",<EOL>self.channel_directory[chan_id])<EOL><DEDENT>

Updates the timestamp for the given channel id.

        :param chan_id:
        :param ts:
        :return:


<EOL>chan_id, channel_short_name, *data = data<EOL>entry = (channel_short_name, data, ts)<EOL>self.account.put(entry)<EOL>

Handles Account related data.

        translation table for channel names:
            Data Channels
            os      -   Orders
            hos     -   Historical Orders
            ps      -   Positions
            hts     -   Trades (snapshot)
            te      -   Trade Event
            tu      -   Trade Update
            ws      -   Wallets
            bu      -   Balance Info
            miu     -   Margin Info
            fiu     -   Funding Info
            fos     -   Offers
            hfos    -   Historical Offers
            fcs     -   Credits
            hfcs    -   Historical Credits
            fls     -   Loans
            hfls    -   Historical Loans
            htfs    -   Funding Trades
            n       -   Notifications (WIP)

        :param dtype:
        :param data:
        :param ts:
        :return:


def _handle_ticker(self, dtype, data, ts):

self.log.debug("<STR_LIT>", dtype, data, ts)<EOL>channel_id, *data = data<EOL>channel_identifier = self.channel_directory[channel_id]<EOL>entry = (data, ts)<EOL>self.tickers[channel_identifier].put(entry)<EOL>

Adds received ticker data to self.tickers dict, filed under its channel
        id.

        :param dtype:
        :param data:
        :param ts:
        :return:


def _handle_book(self, dtype, data, ts):

self.log.debug("<STR_LIT>", dtype, data, ts)<EOL>channel_id, *data = data<EOL>log.debug("<STR_LIT>", ts, channel_id, data)<EOL>channel_identifier = self.channel_directory[channel_id]<EOL>entry = (data, ts)<EOL>self.books[channel_identifier].put(entry)<EOL>

Updates the order book stored in self.books[chan_id].

        :param dtype:
        :param data:
        :param ts:
        :return:


def _handle_raw_book(self, dtype, data, ts):

self.log.debug("<STR_LIT>", dtype, data, ts)<EOL>channel_id, *data = data<EOL>channel_identifier = self.channel_directory[channel_id]<EOL>entry = (data, ts)<EOL>self.raw_books[channel_identifier].put(entry)<EOL>

Updates the raw order books stored in self.raw_books[chan_id].

        :param dtype:
        :param data:
        :param ts:
        :return:


def _handle_trades(self, dtype, data, ts):

self.log.debug("<STR_LIT>", dtype, data, ts)<EOL>channel_id, *data = data<EOL>channel_identifier = self.channel_directory[channel_id]<EOL>entry = (data, ts)<EOL>self.trades[channel_identifier].put(entry)<EOL>

Files trades in self._trades[chan_id].

        :param dtype:
        :param data:
        :param ts:
        :return:


def _handle_candles(self, dtype, data, ts):

self.log.debug("<STR_LIT>", dtype, data, ts)<EOL>channel_id, *data = data<EOL>channel_identifier = self.channel_directory[channel_id]<EOL>entry = (data, ts)<EOL>self.candles[channel_identifier].put(entry)<EOL>

Stores OHLC data received via wss in self.candles[chan_id].

        :param dtype:
        :param data:
        :param ts:
        :return:


if not isinstance(arg, bool):<EOL><INDENT>return arg.lower() in ("<STR_LIT:yes>", "<STR_LIT:true>", "<STR_LIT:t>", "<STR_LIT:1>", "<STR_LIT:y>")<EOL><DEDENT>return arg<EOL>

convert2boolean is used for environmental variables
       that must be returned as boolean

       Parameters
       ==========
       arg: the argument to convert to boolean. Must be a string given
       grabbed from environment.


def getenv(variable_key, default=None, required=False, silent=True):

variable = os.environ.get(variable_key, default)<EOL>if variable is None and required:<EOL><INDENT>bot.error("<STR_LIT>" %variable_key)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if not silent and variable is not None:<EOL><INDENT>bot.verbose("<STR_LIT>" %(variable_key,variable))<EOL><DEDENT>return variable<EOL>

getenv will attempt to get an environment variable. If the variable
       is not found, None is returned.

       Parameters
       ==========
       variable_key: the variable name
       required: exit with error if not found
        silent: Do not print debugging information for variable


from sregistry.main import get_client<EOL>cli = get_client(quiet=args.quiet)<EOL>for query in args.query:<EOL><INDENT>if query in ['<STR_LIT>','<STR_LIT:*>']:<EOL><INDENT>query = None<EOL><DEDENT>cli.ls(query=query)<EOL><DEDENT>

the list command corresponds with listing images for an external
       resource. This is different from listing images that are local to the
       database, which should be done with "images"


<EOL>if response['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>bucket = response['<STR_LIT>']['<STR_LIT>']['<STR_LIT:location>']<EOL>obj = response['<STR_LIT>']['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>]<EOL>bot.custom("<STR_LIT>", response['<STR_LIT>'], '<STR_LIT>')<EOL>bot.custom("<STR_LIT>", response['<STR_LIT:size>'], '<STR_LIT>')<EOL>bot.custom(response['<STR_LIT:status>'], bucket + obj , '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>bot.custom(response['<STR_LIT:status>'], '<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>bot.custom("<STR_LIT>", response['<STR_LIT>'], '<STR_LIT>')<EOL>if "<STR_LIT>" in response:<EOL><INDENT>bot.custom('<STR_LIT>', response['<STR_LIT>'], '<STR_LIT>')<EOL><DEDENT>if output_file != None:    <EOL><INDENT>with open(output_file, '<STR_LIT:w>') as filey:<EOL><INDENT>if response['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>filey.writelines('<STR_LIT>' % response['<STR_LIT>'])    <EOL>filey.writelines('<STR_LIT>' % response['<STR_LIT:size>'])    <EOL><DEDENT>filey.writelines('<STR_LIT>' % (response['<STR_LIT:status>'], bucket, obj))<EOL>filey.writelines('<STR_LIT>' % response['<STR_LIT>'])<EOL>if "<STR_LIT>" in response:<EOL><INDENT>filey.writelines('<STR_LIT>' % response['<STR_LIT>'])<EOL><DEDENT><DEDENT><DEDENT>

print the output to the console for the user. If the user wants the content
       also printed to an output file, do that.

       Parameters
       ==========
       response: the response from the builder, with metadata added
       output_file: if defined, write output also to file


from sregistry.main import Client as cli<EOL>if len(args.commands) > <NUM_LIT:0>:<EOL><INDENT>for name in args.commands:<EOL><INDENT>cli.destroy(name)<EOL><DEDENT><DEDENT>sys.exit(<NUM_LIT:0>)<EOL>

kill is a helper function to call the "kill" function of the client,
       meaning we bring down an instance.


from sregistry.main import Client as cli<EOL>cli.list_builders()<EOL>sys.exit(<NUM_LIT:0>)<EOL>

list running instances for a user, including all builders and report
       instance names and statuses.


from sregistry.main import get_client<EOL>cli = get_client(init=False)<EOL>if len(args.commands) > <NUM_LIT:0>:<EOL><INDENT>template_name = args.commands.pop(<NUM_LIT:0>)<EOL><DEDENT>cli.list_templates(template_name)<EOL>sys.exit(<NUM_LIT:0>)<EOL>

list a specific template (if a name is provided) or all templates
       available.

       Parameters
       ==========
       args: the argparse object to look for a template name
       template_name: if not set, show all


from sregistry.main import Client as cli    <EOL>if len(args.commands) > <NUM_LIT:0>:<EOL><INDENT>container_name = args.commands.pop(<NUM_LIT:0>)<EOL><DEDENT>cli.logs(container_name)<EOL>sys.exit(<NUM_LIT:0>)<EOL>

list a specific log for a builder, or the latest log if none provided

       Parameters
       ==========
       args: the argparse object to look for a container name
       container_name: a default container name set to be None (show latest log)


from sregistry.main import get_client<EOL>cli = get_client(quiet=args.quiet)<EOL>for query in args.query:<EOL><INDENT>if query in ['<STR_LIT>','<STR_LIT:*>']:<EOL><INDENT>query = None<EOL><DEDENT>cli.images(query=query)<EOL><DEDENT>

the images entrypoint is intended to list images locally in the user
       database, optionally taking one or more query string to subset the
       search


<EOL>from sregistry.main import get_client<EOL>client = get_client(args.endpoint)<EOL>client.announce(args.command)<EOL>from IPython import embed<EOL>embed()<EOL>

give the user an ipython shell, optionally with an endpoint of choice.


actions = [action for action in parser._actions <EOL>if isinstance(action, argparse._SubParsersAction)]<EOL>subparsers = dict()<EOL>for action in actions:<EOL><INDENT>for choice, subparser in action.choices.items():<EOL><INDENT>subparsers[choice] = subparser<EOL><DEDENT><DEDENT>return subparsers<EOL>

get_subparser will get a dictionary of subparsers, to help with printing help


from sregistry.main import Client as cli<EOL>parser = get_parser()<EOL>subparsers = get_subparsers(parser)<EOL>def help(return_code=<NUM_LIT:0>):<EOL><INDENT>'''<STR_LIT>'''<EOL>version = sregistry.__version__<EOL>name = cli.client_name<EOL>print("<STR_LIT>" %(version, name))<EOL>parser.print_help()<EOL>sys.exit(return_code)<EOL><DEDENT>if len(sys.argv) == <NUM_LIT:1>:<EOL><INDENT>help()<EOL><DEDENT>try:<EOL><INDENT>args = parser.parse_args()<EOL><DEDENT>except:<EOL><INDENT>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>if args.debug is False:<EOL><INDENT>os.environ['<STR_LIT>'] = "<STR_LIT>"<EOL><DEDENT>if args.command == "<STR_LIT:version>":<EOL><INDENT>print(sregistry.__version__)<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>from sregistry.logger import bot<EOL>if args.command == "<STR_LIT>": from .add import main<EOL>elif args.command == "<STR_LIT>": from .backend import main<EOL>elif args.command == "<STR_LIT>": from .build import main<EOL>elif args.command == "<STR_LIT>": from .get import main<EOL>elif args.command == "<STR_LIT>": from .delete import main<EOL>elif args.command == "<STR_LIT>": from .inspect import main<EOL>elif args.command == "<STR_LIT>": from .images import main<EOL>elif args.command == "<STR_LIT>": from .labels import main<EOL>elif args.command == "<STR_LIT>": from .mv import main<EOL>elif args.command == "<STR_LIT>": from .push import main<EOL>elif args.command == "<STR_LIT>": from .pull import main<EOL>elif args.command == "<STR_LIT>": from .rename import main<EOL>elif args.command == "<STR_LIT>": from .rm import main<EOL>elif args.command == "<STR_LIT>": from .rmi import main<EOL>elif args.command == "<STR_LIT>": from .search import main<EOL>elif args.command == "<STR_LIT>": from .share import main<EOL>elif args.command == "<STR_LIT>": from .shell import main<EOL>return_code = <NUM_LIT:0><EOL>try:<EOL><INDENT>main(args=args,<EOL>parser=parser,<EOL>subparser=subparsers[args.command])<EOL>sys.exit(return_code)<EOL><DEDENT>except UnboundLocalError:<EOL><INDENT>return_code = <NUM_LIT:1><EOL><DEDENT>help(return_code)<EOL>

main is the entrypoint to the sregistry client. The flow works to first
    to determine the subparser in use based on the command. The command then
    imports the correct main (files imported in this folder) associated with
    the action of choice. When the client is imported, it is actually importing
    a return of the function get_client() under sregistry/main, which plays
    the job of "sniffing" the environment to determine what flavor of client
    the user wants to activate. Installed within a singularity image, this
    start up style maps well to Standard Container Integration Format (SCIF)
    apps, where each client is a different entrypoint activated based on the
    environment variables.


from sregistry.main import get_client<EOL>images = args.image<EOL>if not isinstance(images,list):<EOL><INDENT>images = [images]<EOL><DEDENT>for image in images:<EOL><INDENT>print(image)<EOL>cli = get_client(image, quiet=args.quiet)<EOL>cli.announce(args.command)<EOL>cli.share(image, share_to=args.share_to)<EOL><DEDENT>

sharing an image means sending a remote share from an image you
       control to a contact, usually an email.


print('<STR_LIT>')<EOL>settings = read_client_secrets()<EOL>print('<STR_LIT>' %len(settings))<EOL>if '<STR_LIT>' in settings:<EOL><INDENT>print('<STR_LIT>' %settings['<STR_LIT>'])<EOL>update_secrets(settings)<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>

print the status for all or one of the backends.


def add(backend, variable, value, force=False):

print('<STR_LIT>')<EOL>settings = read_client_secrets()<EOL>prefix = '<STR_LIT>' %backend.upper()<EOL>if not variable.startswith(prefix):<EOL><INDENT>variable = '<STR_LIT>' %(prefix, variable)<EOL><DEDENT>variable = variable.upper()<EOL>bot.info("<STR_LIT>" %(variable, value))<EOL>if backend in settings:<EOL><INDENT>if variable in settings[backend] and force is False:<EOL><INDENT>previous = settings[backend][variable]<EOL>bot.error('<STR_LIT>' %(variable, previous))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>if backend not in settings:<EOL><INDENT>settings[backend] = {}<EOL><DEDENT>settings[backend][variable] = value<EOL>update_secrets(settings)<EOL>


print('<STR_LIT>')<EOL>settings = read_client_secrets()<EOL>prefixed = variable<EOL>prefix = '<STR_LIT>' %backend.upper()<EOL>if not variable.startswith(prefix):<EOL><INDENT>prefixed = '<STR_LIT>' %(prefix, variable)<EOL><DEDENT>variable = variable.upper()<EOL>bot.info(variable)<EOL>if backend in settings:<EOL><INDENT>if variable in settings[backend]:<EOL><INDENT>del settings[backend][variable]           <EOL><DEDENT>if prefixed in settings[backend]:<EOL><INDENT>del settings[backend][prefixed]           <EOL><DEDENT>update_secrets(settings)<EOL><DEDENT>

remove a variable from the config, if found.


settings = read_client_secrets()<EOL>if backend is not None:<EOL><INDENT>settings['<STR_LIT>'] = backend<EOL>update_secrets(settings)<EOL>print('<STR_LIT>' %backend)<EOL><DEDENT>

activate a backend by adding it to the .sregistry configuration file.


settings = read_client_secrets()<EOL>if backend in settings:<EOL><INDENT>del settings[backend]<EOL>if '<STR_LIT>' in settings:<EOL><INDENT>if settings['<STR_LIT>'] == backend:<EOL><INDENT>del settings['<STR_LIT>']<EOL><DEDENT><DEDENT>update_secrets(settings)<EOL>print('<STR_LIT>' %backend)<EOL><DEDENT>else:<EOL><INDENT>if backend is not None:<EOL><INDENT>print('<STR_LIT>' %backend)<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT><DEDENT>

delete a backend, and update the secrets file


settings = read_client_secrets()<EOL>if "<STR_LIT>" in settings:<EOL><INDENT>del settings['<STR_LIT>']<EOL>update_secrets(settings)<EOL>print('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>

deactivate any active backend by removing it from the .sregistry
       configuration file


settings = read_client_secrets()<EOL>backends = list(settings.keys())    <EOL>backends = [b for b in backends if b!='<STR_LIT>']<EOL>if backend in backends:<EOL><INDENT>bot.info(backend)<EOL>print(json.dumps(settings[backend], indent=<NUM_LIT:4>, sort_keys=True))<EOL><DEDENT>else:<EOL><INDENT>if backend is not None:<EOL><INDENT>print('<STR_LIT>' %backend)<EOL><DEDENT>bot.info("<STR_LIT>")<EOL>print('<STR_LIT:\n>'.join(backends))<EOL><DEDENT>

return a list of backends installed for the user, which is based on
       the config file keys found present

       Parameters
       ==========
       backend: a specific backend to list. If defined, just list parameters.


name = name.lower()<EOL>templates = dict()<EOL>templates['<STR_LIT>'] = {"<STR_LIT>": <NUM_LIT:0>,<EOL>"<STR_LIT>": <NUM_LIT:0>,<EOL>"<STR_LIT>": "<STR_LIT:root>",<EOL>"<STR_LIT>": "<STR_LIT:root>",<EOL>"<STR_LIT>": <NUM_LIT>}<EOL>if name in templates:<EOL><INDENT>bot.debug("<STR_LIT>" % (name))<EOL>return templates[name]<EOL><DEDENT>else:<EOL><INDENT>bot.warning("<STR_LIT>" % (name))<EOL><DEDENT>

return a default template for some function in sregistry
       If there is no template, None is returned.

       Parameters
       ==========
       name: the name of the template to retrieve


return os.path.realpath(path.strip("<STR_LIT:U+0020>"))<EOL>

clean_path will canonicalize the path
    :param path: the path to clean


if not isinstance(arg, bool):<EOL><INDENT>return arg.lower() in ("<STR_LIT:yes>", "<STR_LIT:true>", "<STR_LIT:t>", "<STR_LIT:1>", "<STR_LIT:y>")<EOL><DEDENT>return arg<EOL>

convert2boolean is used for environmental variables that must be
    returned as boolean


def getenv(variable_key, required=False, default=None, silent=False):

variable = os.environ.get(variable_key, default)<EOL>if variable is None and required:<EOL><INDENT>bot.error(<EOL>"<STR_LIT>" %<EOL>variable_key)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if silent:<EOL><INDENT>bot.verbose2("<STR_LIT>" % variable_key)<EOL><DEDENT>else:<EOL><INDENT>if variable is not None:<EOL><INDENT>bot.verbose2("<STR_LIT>" % (variable_key, variable))<EOL><DEDENT>else:<EOL><INDENT>bot.verbose2("<STR_LIT>" % variable_key)<EOL><DEDENT><DEDENT>return variable<EOL>

getenv will attempt to get an environment variable. If the variable
    is not found, None is returned.
    :param variable_key: the variable name
    :param required: exit with error if not found
    :param silent: Do not print debugging information for variable


DISABLE_CACHE = convert2boolean(getenv("<STR_LIT>",<EOL>default=False))<EOL>if DISABLE_CACHE:<EOL><INDENT>SINGULARITY_CACHE = tempfile.mkdtemp()<EOL><DEDENT>else:<EOL><INDENT>userhome = pwd.getpwuid(os.getuid())[<NUM_LIT:5>]<EOL>_cache = os.path.join(userhome, "<STR_LIT>")<EOL>SINGULARITY_CACHE = getenv("<STR_LIT>", default=_cache)<EOL><DEDENT>cache_base = clean_path(SINGULARITY_CACHE)<EOL>if subfolder is not None:<EOL><INDENT>cache_base = "<STR_LIT>" % (cache_base, subfolder)<EOL><DEDENT>mkdir_p(cache_base)<EOL>if not quiet:<EOL><INDENT>bot.debug("<STR_LIT>" % cache_base)<EOL><DEDENT>return cache_base<EOL>

get_cache will return the user's cache for singularity.
    :param subfolder: a subfolder in the cache base to retrieve, specifically


if singularity_version is None:        <EOL><INDENT>singularity_version = os.environ.get("<STR_LIT>")<EOL><DEDENT>if singularity_version is None:<EOL><INDENT>try:<EOL><INDENT>cmd = ['<STR_LIT>','<STR_LIT>']<EOL>output = run_command(cmd)<EOL>if isinstance(output['<STR_LIT:message>'],bytes):<EOL><INDENT>output['<STR_LIT:message>'] = output['<STR_LIT:message>'].decode('<STR_LIT:utf-8>')<EOL><DEDENT>singularity_version = output['<STR_LIT:message>'].strip('<STR_LIT:\n>')<EOL>bot.info("<STR_LIT>" % singularity_version)<EOL><DEDENT>except:<EOL><INDENT>singularity_version = None<EOL>bot.warning("<STR_LIT>")<EOL><DEDENT><DEDENT>return singularity_version<EOL>

get_singularity_version will determine the singularity version for a
       build first, an environmental variable is looked at, followed by
       using the system version.

       Parameters
       ==========
       singularity_version: if not defined, look for in environment. If still
       not find, try finding via executing --version to Singularity. Only return
       None if not set in environment or installed.


if software is None:<EOL><INDENT>software = "<STR_LIT>"<EOL><DEDENT>cmd = ['<STR_LIT>', software ]<EOL>try:<EOL><INDENT>result = run_command(cmd)<EOL>if strip_newline is True:<EOL><INDENT>result['<STR_LIT:message>'] = result['<STR_LIT:message>'].strip('<STR_LIT:\n>')<EOL><DEDENT>return result<EOL><DEDENT>except: <EOL><INDENT>return None<EOL><DEDENT>

get_install will return the path to where Singularity (or another
       executable) is installed.


if software is None:<EOL><INDENT>software = "<STR_LIT>"<EOL><DEDENT>cmd = [software, '<STR_LIT>']<EOL>try:<EOL><INDENT>version = run_command(cmd,software)<EOL><DEDENT>except: <EOL><INDENT>return False<EOL><DEDENT>if version is not None:<EOL><INDENT>if quiet is False and version['<STR_LIT>'] == <NUM_LIT:0>:<EOL><INDENT>version = version['<STR_LIT:message>']<EOL>bot.info("<STR_LIT>" % (software.upper(), version))<EOL><DEDENT>return True <EOL><DEDENT>return False<EOL>

check_install will attempt to run the singularity command, and
       return True if installed. The command line utils will not run
       without this check.

       Parameters
       ==========
       software: the software to check if installed
       quiet: should we be quiet? (default True)


return os.path.abspath(os.path.dirname(os.path.dirname(__file__)))<EOL>

get_installdir returns the installation directory of the application


from sregistry.defaults import SREGISTRY_THUMBNAIL<EOL>if SREGISTRY_THUMBNAIL is not None:<EOL><INDENT>if os.path.exists(SREGISTRY_THUMBNAIL):<EOL><INDENT>return SREGISTRY_THUMBNAIL<EOL><DEDENT><DEDENT>return "<STR_LIT>" %get_installdir()<EOL>

return the robot.png thumbnail from the database folder.
       if the user has exported a different image, use that instead.


if sudo is True:<EOL><INDENT>cmd = ['<STR_LIT>'] + cmd<EOL><DEDENT>try:<EOL><INDENT>output = Popen(cmd, stderr=STDOUT, stdout=PIPE)<EOL><DEDENT>except FileNotFoundError:<EOL><INDENT>cmd.pop(<NUM_LIT:0>)<EOL>output = Popen(cmd, stderr=STDOUT, stdout=PIPE)<EOL><DEDENT>t = output.communicate()[<NUM_LIT:0>],output.returncode<EOL>output = {'<STR_LIT:message>':t[<NUM_LIT:0>],<EOL>'<STR_LIT>':t[<NUM_LIT:1>]}<EOL>if isinstance(output['<STR_LIT:message>'], bytes):<EOL><INDENT>output['<STR_LIT:message>'] = output['<STR_LIT:message>'].decode('<STR_LIT:utf-8>')<EOL><DEDENT>return output<EOL>

run_command uses subprocess to send a command to the terminal.

    Parameters
    ==========
    cmd: the command to send, should be a list for subprocess
    error_message: the error message to give to user if fails,
    if none specified, will alert that command failed.


hasher = hashlib.md5()<EOL>with open(image_path, "<STR_LIT:rb>") as f:<EOL><INDENT>for chunk in iter(lambda: f.read(<NUM_LIT>), b"<STR_LIT>"):<EOL><INDENT>hasher.update(chunk)<EOL><DEDENT><DEDENT>return hasher.hexdigest()<EOL>

return an md5 hash of the file based on a criteria level. This
       is intended to give the file a reasonable version.

       Parameters
       ==========
       image_path: full path to the singularity image


def set_default(item, default, use_default):

if item == None and use_default:<EOL><INDENT>return default<EOL><DEDENT>return item<EOL>

if an item provided is None and boolean use_default is set to True,
       return the default. Otherwise, return the item.


if special_characters is None:<EOL><INDENT>special_characters = []<EOL><DEDENT>return '<STR_LIT>'.join(e.lower()<EOL>for e in name if e.isalnum() or e in special_characters)<EOL>

format_container_name will take a name supplied by the user,
    remove all special characters (except for those defined by "special-characters"
    and return the new image name.


<EOL>image = image or '<STR_LIT>'<EOL>regexp = re.compile('<STR_LIT>')<EOL>uri = regexp.match(image)<EOL>if uri is not None:<EOL><INDENT>uri = (uri.group().lower()<EOL>.replace('<STR_LIT:_>','<STR_LIT:->')<EOL>.replace('<STR_LIT>','<STR_LIT>'))<EOL>accepted_uris = ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT:http>', '<STR_LIT>', <EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>', <EOL>'<STR_LIT>',<EOL>'<STR_LIT>', <EOL>'<STR_LIT>']<EOL>if "<STR_LIT>" in uri: uri = "<STR_LIT>"<EOL>if uri not in accepted_uris:<EOL><INDENT>bot.warning('<STR_LIT>' % uri)<EOL>uri = None<EOL><DEDENT><DEDENT>return uri<EOL>

get the uri for an image, if within acceptable

       Parameters
       ==========
       image: the image uri, in the format <uri>://<registry>/<namespace>:<tag>


return re.sub('<STR_LIT>','<STR_LIT>', image)<EOL>

remove_uri will remove the uri from the image path, if provided.

       Parameters
       ==========
       image: the image uri, in the format <uri>://<registry>/<namespace>:<tag>


try:<EOL><INDENT>os.makedirs(path)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno == errno.EEXIST and os.path.isdir(path):<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>bot.error("<STR_LIT>" % path)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>

mkdir_p attempts to get the same functionality as mkdir -p
    :param path: the path to create.


<EOL>tmpdir = get_tmpdir(requested_tmpdir)<EOL>if tmpdir is not None:<EOL><INDENT>prefix = os.path.join(tmpdir, os.path.basename(prefix))<EOL><DEDENT>fd, tmp_file = tempfile.mkstemp(prefix=prefix) <EOL>os.close(fd)<EOL>return tmp_file<EOL>

get a temporary file with an optional prefix. By default will be
       created in /tmp unless SREGISTRY_TMPDIR is set. By default, the file
       is closed (and just a name returned).

       Parameters
       ==========
       requested_tmpdir: an optional requested temporary directory, first
       priority as is coming from calling function.
       prefix: Given a need for a sandbox (or similar), prefix the file
       with this string.


def get_tmpdir(requested_tmpdir=None, prefix="<STR_LIT>", create=True):

from sregistry.defaults import SREGISTRY_TMPDIR<EOL>tmpdir = requested_tmpdir or SREGISTRY_TMPDIR<EOL>prefix = prefix or "<STR_LIT>"<EOL>prefix = "<STR_LIT>" %(prefix, next(tempfile._get_candidate_names()))<EOL>tmpdir = os.path.join(tmpdir, prefix)<EOL>if not os.path.exists(tmpdir) and create is True:<EOL><INDENT>os.mkdir(tmpdir)<EOL><DEDENT>return tmpdir<EOL>

get a temporary directory for an operation. If SREGISTRY_TMPDIR
       is set, return that. Otherwise, return the output of tempfile.mkdtemp

       Parameters
       ==========
       requested_tmpdir: an optional requested temporary directory, first
       priority as is coming from calling function.
       prefix: Given a need for a sandbox (or similar), we will need to
       create a subfolder *within* the SREGISTRY_TMPDIR.
       create: boolean to determine if we should create folder (True)


return pwd.getpwuid(os.getuid())[<NUM_LIT:5>]<EOL>

get the user home based on the effective uid


def extract_tar(archive, output_folder, handle_whiteout=False):

from .terminal import run_command<EOL>if handle_whiteout is True:<EOL><INDENT>return _extract_tar(archive, output_folder)<EOL><DEDENT>args = '<STR_LIT>'<EOL>if archive.endswith("<STR_LIT>"):<EOL><INDENT>args = '<STR_LIT>'<EOL><DEDENT>command = ["<STR_LIT>", args, archive, "<STR_LIT>", output_folder, "<STR_LIT>"]<EOL>if not bot.is_quiet():<EOL><INDENT>print("<STR_LIT>" % archive)<EOL><DEDENT>return run_command(command)<EOL>

extract a tar archive to a specified output folder

        Parameters
        ==========
        archive: the archive file to extract
        output_folder: the output folder to extract to
        handle_whiteout: use docker2oci variation to handle whiteout files


from .terminal import ( run_command, which )<EOL>result = which('<STR_LIT>')<EOL>if result['<STR_LIT>'] != <NUM_LIT:0>:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>script = result['<STR_LIT:message>'] <EOL>command = ['<STR_LIT>' ,script, '<STR_LIT>', archive, '<STR_LIT>', output_folder]<EOL>if not bot.is_quiet():<EOL><INDENT>print("<STR_LIT>" % archive)<EOL><DEDENT>return run_command(command)<EOL>

use blob2oci to handle whiteout files for extraction. Credit for this
       script goes to docker2oci by Olivier Freyermouth, and see script
       folder for license.

       Parameters
       ==========
       archive: the archive to extract
       output_folder the output folder (sandbox) to extract to


if output_folder is None:<EOL><INDENT>output_folder = tempfile.mkdtemp()<EOL><DEDENT>finished_tar = None<EOL>additions = []<EOL>contents = []<EOL>for entity in files:<EOL><INDENT>info = tarfile.TarInfo(name=entity['<STR_LIT:name>'])<EOL>info.mode = entity['<STR_LIT>']<EOL>info.mtime = int(datetime.datetime.now().strftime('<STR_LIT:%s>'))<EOL>info.uid = entity["<STR_LIT>"]<EOL>info.gid = entity["<STR_LIT>"]<EOL>info.uname = entity["<STR_LIT>"]<EOL>info.gname = entity["<STR_LIT>"]<EOL>filey = io.StringIO()<EOL>content = None<EOL>try:  <EOL><INDENT>info.size = filey.write(entity['<STR_LIT:content>'])<EOL>content = io.BytesIO(entity['<STR_LIT:content>'].encode('<STR_LIT:utf8>'))<EOL><DEDENT>except Exception:  <EOL><INDENT>info.size = int(filey.write(entity['<STR_LIT:content>'].decode('<STR_LIT:utf-8>')))<EOL>content = io.BytesIO(entity['<STR_LIT:content>'].encode('<STR_LIT:utf8>'))<EOL><DEDENT>pass<EOL>if content is not None:<EOL><INDENT>addition = {'<STR_LIT:content>': content,<EOL>'<STR_LIT:info>': info}<EOL>additions.append(addition)<EOL>contents.append(content)<EOL><DEDENT><DEDENT>if len(additions) > <NUM_LIT:0>:<EOL><INDENT>hashy = get_content_hash(contents)<EOL>finished_tar = "<STR_LIT>" % (output_folder, hashy)<EOL>if os.path.exists(finished_tar):<EOL><INDENT>msg = "<STR_LIT>" % finished_tar<EOL>msg += "<STR_LIT>"<EOL>bot.debug(msg)<EOL><DEDENT>tar = tarfile.open(finished_tar, "<STR_LIT>")<EOL>for a in additions:<EOL><INDENT>tar.addfile(a["<STR_LIT:info>"], a["<STR_LIT:content>"])<EOL><DEDENT>tar.close()<EOL><DEDENT>else:<EOL><INDENT>msg = "<STR_LIT>"<EOL>msg += "<STR_LIT>"<EOL>bot.debug(msg)<EOL><DEDENT>return finished_tar<EOL>

create_memory_tar will take a list of files (each a dictionary
        with name, permission, and content) and write the tarfile
        (a sha256 sum name is used) to the output_folder.
        If there is no output folde specified, the
        tar is written to a temporary folder.


hasher = hashlib.sha256()<EOL>for content in contents:<EOL><INDENT>if isinstance(content, io.BytesIO):<EOL><INDENT>content = content.getvalue()<EOL><DEDENT>if not isinstance(content, bytes):<EOL><INDENT>content = bytes(content)<EOL><DEDENT>hasher.update(content)<EOL><DEDENT>return hasher.hexdigest()<EOL>

get_content_hash will return a hash for a list of content (bytes/other)


hasher = hashlib.sha256()<EOL>with open(filename, "<STR_LIT:rb>") as f:<EOL><INDENT>for chunk in iter(lambda: f.read(<NUM_LIT>), b"<STR_LIT>"):<EOL><INDENT>hasher.update(chunk)<EOL><DEDENT><DEDENT>return hasher.hexdigest()<EOL>


def copyfile(source, destination, force=True):

<EOL>if source == destination and force is False:<EOL><INDENT>return destination<EOL><DEDENT>if os.path.exists(destination) and force is True:<EOL><INDENT>os.remove(destination)<EOL><DEDENT>shutil.copyfile(source, destination)<EOL>return destination<EOL>

copy a file from a source to its destination.


def write_file(filename, content, mode="<STR_LIT:w>"):

with open(filename, mode) as filey:<EOL><INDENT>filey.writelines(content)<EOL><DEDENT>return filename<EOL>

write_file will open a file, "filename" and write content, "content"
       and properly close the file


def write_json(json_obj, filename, mode="<STR_LIT:w>", print_pretty=True):

with open(filename, mode) as filey:<EOL><INDENT>if print_pretty:<EOL><INDENT>filey.writelines(print_json(json_obj))<EOL><DEDENT>else:<EOL><INDENT>filey.writelines(json.dumps(json_obj))<EOL><DEDENT><DEDENT>return filename<EOL>

write_json will (optionally,pretty print) a json object to file

       Parameters
       ==========
       json_obj: the dict to print to json
       filename: the output file to write to
       pretty_print: if True, will use nicer formatting


return json.dumps(<EOL>json_obj,<EOL>indent=<NUM_LIT:4>,<EOL>separators=(<EOL>'<STR_LIT:U+002C>',<EOL>'<STR_LIT>'))<EOL>

just dump the json in a "pretty print" format


def read_file(filename, mode="<STR_LIT:r>", readlines=True):

with open(filename, mode) as filey:<EOL><INDENT>if readlines is True:<EOL><INDENT>content = filey.readlines()<EOL><DEDENT>else:<EOL><INDENT>content = filey.read()<EOL><DEDENT><DEDENT>return content<EOL>


with open(filename, mode) as filey:<EOL><INDENT>data = json.load(filey)<EOL><DEDENT>return data<EOL>

read_json reads in a json file and returns
       the data structure as dict.


if not isinstance(files, list):<EOL><INDENT>files = [files]<EOL><DEDENT>for f in files:<EOL><INDENT>if os.path.exists(f):<EOL><INDENT>bot.verbose3("<STR_LIT>" % f)<EOL>os.remove(f)<EOL><DEDENT><DEDENT>

clean up will delete a list of files, only if they exist


tag = re.sub('<STR_LIT>','<STR_LIT>', path)<EOL>if tag in ['<STR_LIT>', None]:<EOL><INDENT>tag = '<STR_LIT>'<EOL><DEDENT>return tag<EOL>

get a recipe tag (the extension of a Singularity file). The extension
       determines the tag. If no extension is found, latest is used.

       Parameters
       ==========
       path: the path to the recipe file (e.g. /opt/Singularity.tag)


def parse_header(recipe, header="<STR_LIT>", remove_header=True):

parsed_header = None<EOL>fromline = [x for x in recipe.split('<STR_LIT:\n>') if "<STR_LIT>" %header in x.lower()]<EOL>if len(fromline) == <NUM_LIT:0>:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>if len(fromline) > <NUM_LIT:0>:<EOL><INDENT>fromline = fromline[<NUM_LIT:0>]<EOL>parsed_header = fromline.strip()<EOL><DEDENT>if remove_header is True:<EOL><INDENT>parsed_header = fromline.split('<STR_LIT::>', <NUM_LIT:1>)[-<NUM_LIT:1>].strip()<EOL><DEDENT>return parsed_header<EOL>

take a recipe, and return the complete header, line. If
       remove_header is True, only return the value.

       Parameters
       ==========
       recipe: the recipe file
       headers: the header key to find and parse
       remove_header: if true, remove the key


def find_recipes(folders, pattern=None, base=None):

<EOL>if folders is None:<EOL><INDENT>folders = os.getcwd()<EOL><DEDENT>if not isinstance(folders,list):<EOL><INDENT>folders = [folders]<EOL><DEDENT>manifest = dict()<EOL>for base_folder in folders:<EOL><INDENT>custom_pattern = None<EOL>if os.path.isfile(base_folder):  <EOL><INDENT>manifest = find_single_recipe(filename=base_folder,<EOL>pattern=pattern,<EOL>manifest=manifest)<EOL>continue<EOL><DEDENT>elif not os.path.isdir(base_folder):<EOL><INDENT>custom_pattern = base_folder.split('<STR_LIT:/>')[-<NUM_LIT:1>:][<NUM_LIT:0>]<EOL>base_folder = "<STR_LIT:/>".join(base_folder.split('<STR_LIT:/>')[<NUM_LIT:0>:-<NUM_LIT:1>])<EOL><DEDENT>manifest = find_folder_recipes(base_folder=base_folder,<EOL>pattern=custom_pattern or pattern,<EOL>manifest=manifest,<EOL>base=base)<EOL><DEDENT>return manifest<EOL>

find recipes will use a list of base folders, files,
       or patterns over a subset of content to find recipe files
       (indicated by Starting with Singularity

       Parameters
       ==========
        base: if defined, consider folders recursively below this level.


<EOL>if manifest is None:<EOL><INDENT>manifest = dict()<EOL><DEDENT>for root, dirnames, filenames in os.walk(base_folder):<EOL><INDENT>for filename in fnmatch.filter(filenames, pattern):<EOL><INDENT>container_path = os.path.join(root, filename)<EOL>if base is not None:<EOL><INDENT>container_base = container_path.replace(base,'<STR_LIT>').strip('<STR_LIT:/>')<EOL>collection = container_base.split('<STR_LIT:/>')[<NUM_LIT:0>]<EOL>recipe = os.path.basename(container_base)<EOL>container_uri = "<STR_LIT>" %(collection,recipe)<EOL><DEDENT>else:<EOL><INDENT>container_uri = '<STR_LIT:/>'.join(container_path.strip('<STR_LIT:/>').split('<STR_LIT:/>')[-<NUM_LIT:2>:])<EOL><DEDENT>add_container = True<EOL>if container_uri in manifest:<EOL><INDENT>if manifest[container_uri]['<STR_LIT>'] > os.path.getmtime(container_path):<EOL><INDENT>add_container = False<EOL><DEDENT><DEDENT>if add_container:<EOL><INDENT>manifest[container_uri] = {'<STR_LIT:path>': os.path.abspath(container_path),<EOL>'<STR_LIT>':os.path.getmtime(container_path)}<EOL><DEDENT><DEDENT><DEDENT>return manifest<EOL>

find folder recipes will find recipes based on a particular pattern.

       Parameters
       ==========
       base_folder: the base folder to recursively walk
       pattern: a default pattern to search for
       manifest: an already started manifest
       base: if defined, consider folders under this level recursively.


def find_single_recipe(filename, pattern="<STR_LIT>", manifest=None):

if pattern is None:<EOL><INDENT>pattern = "<STR_LIT>"<EOL><DEDENT>recipe = None<EOL>file_basename = os.path.basename(filename)<EOL>if fnmatch.fnmatch(file_basename, pattern):<EOL><INDENT>recipe = {'<STR_LIT:path>': os.path.abspath(filename),<EOL>'<STR_LIT>':os.path.getmtime(filename)}<EOL><DEDENT>if manifest is not None and recipe is not None:<EOL><INDENT>container_uri = '<STR_LIT:/>'.join(filename.split('<STR_LIT:/>')[-<NUM_LIT:2>:])<EOL>if container_uri in manifest:<EOL><INDENT>if manifest[container_uri]['<STR_LIT>'] < os.path.getmtime(filename):<EOL><INDENT>manifest[container_uri] = recipe<EOL><DEDENT><DEDENT>else:<EOL><INDENT>manifest[container_uri] = recipe<EOL><DEDENT>return manifest<EOL><DEDENT>return recipe<EOL>

find_single_recipe will parse a single file, and if valid,
       return an updated manifest

       Parameters
       ==========
       filename: the filename to assess for a recipe
       pattern: a default pattern to search for
       manifest: an already started manifest


from sregistry.database.models import Collection<EOL>collection = self.get_collection(name)<EOL>if collection is None:<EOL><INDENT>collection = Collection(name=name)<EOL>self.session.add(collection)<EOL>self.session.commit()<EOL><DEDENT>return collection<EOL>

get a collection if it exists. If it doesn't exist, create it first.

    Parameters
    ==========
    name: the collection name, usually parsed from get_image_names()['name']


from sregistry.database.models import Collection<EOL>return Collection.query.filter(Collection.name == name).first()<EOL>

get a collection, if it exists, otherwise return None.


from sregistry.database.models import Collection, Container<EOL>names = parse_image_name( remove_uri (name) )<EOL>collection = self.get_collection(name=names['<STR_LIT>'])<EOL>container = None<EOL>if collection is not None:<EOL><INDENT>container = self.get_container(collection_id=collection.id,<EOL>name=names['<STR_LIT:image>'], <EOL>tag=names['<STR_LIT>'],<EOL>version=names['<STR_LIT:version>'])<EOL>if container is not None and quiet is False:<EOL><INDENT>if container.image is not None:<EOL><INDENT>print(container.image)<EOL><DEDENT>elif container.url is not None:<EOL><INDENT>print(container.url)<EOL><DEDENT>else:<EOL><INDENT>bot.info('<STR_LIT>' %name)<EOL><DEDENT><DEDENT><DEDENT>return container<EOL>

Do a get for a container, and then a collection, and then return None
       if no result is found.
    Parameters
    ==========
    name: should coincide with either the collection name, or the container
          name with the collection. A query is done first for the collection,
          and then the container, and the path to the image file returned.


from sregistry.database.models import Collection, Container<EOL>rows = []<EOL>if query is not None:   <EOL><INDENT>like = "<STR_LIT:%>" + query + "<STR_LIT:%>"<EOL>containers = Container.query.filter(or_(Container.name == query,<EOL>Container.tag.like(like),<EOL>Container.uri.like(like),<EOL>Container.name.like(like))).all() <EOL><DEDENT>else:<EOL><INDENT>containers = Container.query.all()<EOL><DEDENT>if len(containers) > <NUM_LIT:0>:<EOL><INDENT>message = "<STR_LIT>"<EOL>bot.custom(prefix='<STR_LIT>', message=message, color="<STR_LIT>")<EOL>for c in containers:<EOL><INDENT>uri = c.get_uri()<EOL>created_at = c.created_at.strftime('<STR_LIT>')<EOL>rows.append([created_at, "<STR_LIT>" %c.client, uri])<EOL><DEDENT>bot.table(rows) <EOL><DEDENT>return containers<EOL>

List local images in the database, optionally with a query.

       Paramters
       =========
       query: a string to search for in the container or collection name|tag|uri


print(name)<EOL>container = self.get(name)<EOL>if container is not None:<EOL><INDENT>collection = container.collection.name<EOL>fields = container.__dict__.copy()<EOL>fields['<STR_LIT>'] = collection        <EOL>fields['<STR_LIT>'] = json.loads(fields['<STR_LIT>'])<EOL>del fields['<STR_LIT>']<EOL>fields['<STR_LIT>'] = str(fields['<STR_LIT>'])<EOL>print(json.dumps(fields, indent=<NUM_LIT:4>, sort_keys=True))<EOL>return fields<EOL><DEDENT>

Inspect a local image in the database, which typically includes the
       basic fields in the model.


container = self.get(image_name, quiet=True)<EOL>if container is not None:<EOL><INDENT>if container.image is not None:<EOL><INDENT>dirname = os.path.dirname(container.image)<EOL>names = parse_image_name( remove_uri (path) )<EOL>storage = os.path.join( self.storage,<EOL>os.path.dirname(names['<STR_LIT>']) )<EOL>if not os.path.exists(storage):<EOL><INDENT>os.mkdir(storage)<EOL><DEDENT>fullpath = os.path.abspath(os.path.join(dirname, names['<STR_LIT>']))<EOL>container = self.cp(move_to=fullpath,<EOL>container=container,<EOL>command="<STR_LIT>")<EOL>if container is not None:<EOL><INDENT>container.uri = names['<STR_LIT>']<EOL>self.session.commit()<EOL>return container<EOL><DEDENT><DEDENT><DEDENT>bot.warning('<STR_LIT>' %(image_name))<EOL>

rename performs a move, but ensures the path is maintained in storage

       Parameters
       ==========
       image_name: the image name (uri) to rename to.
       path: the name to rename (basename is taken)


container = self.get(image_name, quiet=True)<EOL>if container is not None:<EOL><INDENT>name = container.uri or container.get_uri()<EOL>image = container.image or '<STR_LIT>'<EOL>if os.path.exists(image):<EOL><INDENT>filename = os.path.basename(image)<EOL>filedir = os.path.abspath(path)<EOL>if not os.path.isdir(path):<EOL><INDENT>filename = os.path.basename(path)<EOL>filedir = os.path.dirname(path)<EOL><DEDENT>if filedir == '<STR_LIT>':<EOL><INDENT>filedir = os.getcwd()<EOL><DEDENT>fullpath = os.path.abspath(os.path.join(filedir,filename))<EOL>return self.cp(move_to=fullpath, <EOL>container=container,<EOL>command="<STR_LIT>")<EOL><DEDENT><DEDENT>bot.warning('<STR_LIT>' %(image_name))<EOL>

Move an image from it's current location to a new path.
       Removing the image from organized storage is not the recommended approach
       however is still a function wanted by some.

       Parameters
       ==========
       image_name: the parsed image name.
       path: the location to move the image to


def cp(self, move_to, image_name=None, container=None, command="<STR_LIT>"):

if container is None and image_name is None:<EOL><INDENT>bot.error('<STR_LIT>' %command)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if container is None:<EOL><INDENT>container = self.get(image_name, quiet=True)<EOL><DEDENT>image = container.image or '<STR_LIT>'<EOL>if os.path.exists(image):<EOL><INDENT>filedir = os.path.dirname(move_to)<EOL>if move_to == image:<EOL><INDENT>bot.warning('<STR_LIT>' %image)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if not os.path.exists(filedir):<EOL><INDENT>bot.error('<STR_LIT>' %filedir)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if not os.access(filedir, os.W_OK):<EOL><INDENT>bot.error('<STR_LIT>' %filedir)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>original = os.path.basename(image)<EOL>try:<EOL><INDENT>shutil.move(image, move_to)<EOL>container.image = move_to<EOL>self.session.commit()<EOL>bot.info('<STR_LIT>' %(command, original, move_to))<EOL>return container<EOL><DEDENT>except:<EOL><INDENT>bot.error('<STR_LIT>' %(command, original, move_to))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>bot.warning('''<STR_LIT>''' %(container.uri, command))<EOL>

_cp is the shared function between mv (move) and rename, and performs
       the move, and returns the updated container

       Parameters
       ==========
       image_name: an image_uri to look up a container in the database
       container: the container object to move (must have a container.image
       move_to: the full path to move it to


container = self.rm(image_name, delete=True)<EOL>if container is not None:<EOL><INDENT>bot.info("<STR_LIT>" % container)<EOL><DEDENT>

Remove an image from the database and filesystem.


container = self.get(image_name)<EOL>if container is not None:<EOL><INDENT>name = container.uri or container.get_uri()<EOL>image = container.image<EOL>self.session.delete(container)<EOL>self.session.commit()<EOL>if image is not None:<EOL><INDENT>if os.path.exists(image) and delete is True:<EOL><INDENT>os.remove(container.image)<EOL><DEDENT>return image<EOL><DEDENT>bot.info("<STR_LIT>" % name)<EOL><DEDENT>

Remove an image from the database, akin to untagging the image. This
    does not delete the file from the cache, unless delete is set to True
    (as called by rmi).


from sregistry.database.models import (<EOL>Container,<EOL>Collection<EOL>)<EOL>if image_path is not None:<EOL><INDENT>if not os.path.exists(image_path) and save is True:<EOL><INDENT>bot.error('<STR_LIT>' %image_path)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>if image_uri is None:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>names = parse_image_name( remove_uri(image_uri) )<EOL>bot.debug('<STR_LIT>' % names['<STR_LIT>'])    <EOL>metadata = self.get_metadata(image_path, names=names)<EOL>collection = self.get_or_create_collection(names['<STR_LIT>'])<EOL>version = names.get('<STR_LIT:version>')<EOL>if version == None:<EOL><INDENT>if image_path != None:<EOL><INDENT>version = get_image_hash(image_path)<EOL><DEDENT>else:<EOL><INDENT>version = '<STR_LIT>'  <EOL><DEDENT>names = parse_image_name( remove_uri(image_uri), version=version )<EOL><DEDENT>if save is True and image_path is not None:<EOL><INDENT>if image_name is None:      <EOL><INDENT>image_name = self._get_storage_name(names)<EOL><DEDENT>if copy is True:<EOL><INDENT>copyfile(image_path, image_name)<EOL><DEDENT>else:<EOL><INDENT>shutil.move(image_path, image_name)<EOL><DEDENT>image_path = image_name<EOL><DEDENT>if url is None and "<STR_LIT:url>" in metadata:<EOL><INDENT>url = metadata['<STR_LIT:url>']<EOL><DEDENT>container = self.get_container(name=names['<STR_LIT:image>'],<EOL>collection_id=collection.id, <EOL>tag=names['<STR_LIT>'],<EOL>version=version)<EOL>if container is None:<EOL><INDENT>action = "<STR_LIT>"<EOL>container = Container(metrics=json.dumps(metadata),<EOL>name=names['<STR_LIT:image>'],<EOL>image=image_path,<EOL>client=self.client_name,<EOL>tag=names['<STR_LIT>'],<EOL>version=version,<EOL>url=url,<EOL>uri=names['<STR_LIT>'],<EOL>collection_id=collection.id)<EOL>self.session.add(container)<EOL>collection.containers.append(container)<EOL><DEDENT>else:<EOL><INDENT>action="<STR_LIT>"<EOL>metrics=json.loads(container.metrics)<EOL>metrics.update(metadata)<EOL>container.url= url<EOL>container.client=self.client_name<EOL>if image_path is not None:<EOL><INDENT>container.image=image_path<EOL><DEDENT>container.metrics=json.dumps(metrics)<EOL><DEDENT>self.session.commit()<EOL>bot.info("<STR_LIT>" % (action,names['<STR_LIT>']))<EOL>return container<EOL>

get or create a container, including the collection to add it to.
    This function can be used from a file on the local system, or via a URL
    that has been downloaded. Either way, if one of url, version, or image_file
    is not provided, the model is created without it. If a version is not
    provided but a file path is, then the file hash is used.

    Parameters
    ==========
    image_path: full path to image file
    image_name: if defined, the user wants a custom name (and not based on uri)
    metadata: any extra metadata to keep for the image (dict)
    save: if True, move the image to the cache if it's not there
    copy: If True, copy the image instead of moving it.

    image_name: a uri that gets parsed into a names object that looks like:

    {'collection': 'vsoch',
     'image': 'hello-world',
     'storage': 'vsoch/hello-world-latest.img',
     'tag': 'latest',
     'version': '12345'
     'uri': 'vsoch/hello-world:latest@12345'}

    After running add, the user will take some image in a working
    directory, add it to the database, and have it available for search
    and use under SREGISTRY_STORAGE/<collection>/<container>

    If the container was retrieved from a webby place, it should have version
    If no version is found, the file hash is used.


<EOL>if image_path is not None:<EOL><INDENT>if not os.path.exists(image_path):<EOL><INDENT>bot.error('<STR_LIT>' %image_path)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>if image_uri is None:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>names = parse_image_name( remove_uri(image_uri) )<EOL>bot.debug('<STR_LIT>' % names['<STR_LIT>'])    <EOL>class DummyContainer:<EOL><INDENT>def __init__(self, image_path, client_name, url, names):<EOL><INDENT>self.image=image_path<EOL>self.client=client_name<EOL>self.url=url<EOL>self.name=names['<STR_LIT:image>']<EOL>self.tag=names['<STR_LIT>']<EOL>self.uri=names['<STR_LIT>']<EOL><DEDENT><DEDENT>container = DummyContainer(image_path, self.client_name, url, names)<EOL>bot.info("<STR_LIT>" % (action,names['<STR_LIT>']))<EOL>return container<EOL>

dummy add simple returns an object that mimics a database entry, so the
       calling function (in push or pull) can interact with it equally. Most
       variables (other than image_path) are not used.


<EOL>self.database = '<STR_LIT>'<EOL>

initialize the database, meaning we just set the name to be dummy


<EOL>self.database = '<STR_LIT>' % db_path<EOL>self.storage = SREGISTRY_STORAGE<EOL>bot.debug("<STR_LIT>" % self.database)<EOL>self.engine = create_engine(self.database, convert_unicode=True)<EOL>self.session = scoped_session(sessionmaker(autocommit=False,<EOL>autoflush=False,<EOL>bind=self.engine))<EOL>Base.query = self.session.query_property()<EOL>Base.metadata.create_all(bind=self.engine)<EOL>self.Base = Base<EOL>

initialize the database, with the default database path or custom of

       the format sqlite:////scif/data/expfactory.db

    The custom path can be set with the environment variable SREGISTRY_DATABASE
    when a user creates the client, we must initialize this db
    the database should use the .singularity cache folder to cache
    layers and images, and .singularity/sregistry.db as a database


return "<STR_LIT>"<EOL>


uri = "<STR_LIT>" %(self.collection.name, self.name, self.tag)<EOL>if self.version not in [None,'<STR_LIT>']:<EOL><INDENT>uri = "<STR_LIT>" %(uri, self.version)<EOL><DEDENT>return uri<EOL>

generate a uri on the fly from database parameters if one is not
        saved with the initial model (it should be, but might not be possible)


count = len(it) if expected_size is None else expected_size<EOL>with Bar(label=label, <EOL>width=width,<EOL>hide=hide,<EOL>empty_char=BAR_EMPTY_CHAR,<EOL>filled_char=BAR_FILLED_CHAR,<EOL>expected_size=count, <EOL>every=every)as bar:<EOL><INDENT>for i, item in enumerate(it):<EOL><INDENT>yield item<EOL>bar.show(i + <NUM_LIT:1>)<EOL><DEDENT><DEDENT>

Progress iterator. Wrap your iterables with it.


level = os.environ.get("<STR_LIT>", INFO)<EOL>if isinstance(level, int):<EOL><INDENT>return level<EOL><DEDENT>if level == "<STR_LIT>":<EOL><INDENT>return CRITICAL<EOL><DEDENT>elif level == "<STR_LIT>":<EOL><INDENT>return ABORT<EOL><DEDENT>elif level == "<STR_LIT>":<EOL><INDENT>return ERROR<EOL><DEDENT>elif level == "<STR_LIT>":<EOL><INDENT>return WARNING<EOL><DEDENT>elif level == "<STR_LIT>":<EOL><INDENT>return LOG<EOL><DEDENT>elif level == "<STR_LIT>":<EOL><INDENT>return INFO<EOL><DEDENT>elif level == "<STR_LIT>":<EOL><INDENT>return QUIET<EOL><DEDENT>elif level.startswith("<STR_LIT>"):<EOL><INDENT>return VERBOSE3<EOL><DEDENT>elif level == "<STR_LIT>":<EOL><INDENT>return LOG<EOL><DEDENT>elif level == "<STR_LIT>":<EOL><INDENT>return DEBUG<EOL><DEDENT>return level<EOL>

get_logging_level will configure a logging to standard out based on the user's
    selected level, which should be in an environment variable called
    MESSAGELEVEL. if MESSAGELEVEL is not set, the maximum level
    (5) is assumed (all messages).


if not isinstance(arg, bool):<EOL><INDENT>return arg.lower() in ("<STR_LIT:yes>", "<STR_LIT:true>", "<STR_LIT:t>", "<STR_LIT:1>", "<STR_LIT:y>")<EOL><DEDENT>return arg<EOL>

convert2boolean is used for environmental variables that must be
    returned as boolean


COLORIZE = get_user_color_preference()<EOL>if COLORIZE is not None:<EOL><INDENT>return COLORIZE<EOL><DEDENT>streams = [self.errorStream, self.outputStream]<EOL>for stream in streams:<EOL><INDENT>if not hasattr(stream, '<STR_LIT>'):<EOL><INDENT>return False<EOL><DEDENT>if not stream.isatty():<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

useColor will determine if color should be added
        to a print. Will check if being run in a terminal, and
        if has support for asci


if self.colorize:<EOL><INDENT>if level in self.colors:<EOL><INDENT>text = "<STR_LIT>" % (self.colors[level],<EOL>text,<EOL>self.colors["<STR_LIT>"])<EOL><DEDENT><DEDENT>return text<EOL>

addColor to the prompt (usually prefix) if terminal
        supports, and specified to do so


if level in [ABORT,<EOL>ERROR,<EOL>WARNING,<EOL>VERBOSE,<EOL>VERBOSE1,<EOL>VERBOSE2,<EOL>VERBOSE3,<EOL>DEBUG]:<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>

determine if a level should print to
        stderr, includes all levels but INFO and QUIET


if level in [LOG,<EOL>INFO]:<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>

determine if a level should print to stdout
        only includes INFO


if messageLevel <= self.level:<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>

check if a messageLevel is enabled to emit a level


def emit(self, level, message, prefix=None, color=None):

if color is None:<EOL><INDENT>color = level<EOL><DEDENT>if prefix is not None:<EOL><INDENT>prefix = self.addColor(color, "<STR_LIT>" % (prefix))<EOL><DEDENT>else:<EOL><INDENT>prefix = "<STR_LIT>"<EOL>message = self.addColor(color, message)<EOL><DEDENT>message = "<STR_LIT>" % (prefix, message)<EOL>if not message.endswith('<STR_LIT:\n>'):<EOL><INDENT>message = "<STR_LIT>" % message<EOL><DEDENT>if self.level == QUIET:<EOL><INDENT>pass<EOL><DEDENT>elif self.isEnabledFor(level):<EOL><INDENT>if self.emitError(level):<EOL><INDENT>self.write(self.errorStream, message)<EOL><DEDENT>else:<EOL><INDENT>self.write(self.outputStream, message)<EOL><DEDENT><DEDENT>self.history.append(message)<EOL>

emit is the main function to print the message
        optionally with a prefix
        :param level: the level of the message
        :param message: the message to print
        :param prefix: a prefix for the message


if isinstance(message, bytes):<EOL><INDENT>message = message.decode('<STR_LIT:utf-8>')<EOL><DEDENT>stream.write(message)<EOL>

write will write a message to a stream,
        first checking the encoding


if join_newline:<EOL><INDENT>return '<STR_LIT:\n>'.join(self.history)<EOL><DEDENT>return self.history<EOL>

get_logs will return the complete history, joined by newline
        (default) or as is.


if not self.level == QUIET:<EOL><INDENT>percent = <NUM_LIT:100> * (iteration / float(total))<EOL>progress = int(length * iteration // total)<EOL>if suffix is None:<EOL><INDENT>suffix = '<STR_LIT>'<EOL><DEDENT>if prefix is None:<EOL><INDENT>prefix = '<STR_LIT>'<EOL><DEDENT>if percent >= <NUM_LIT:100>:<EOL><INDENT>percent = <NUM_LIT:100><EOL>progress = length<EOL><DEDENT>if symbol is None:<EOL><INDENT>symbol = "<STR_LIT:=>"<EOL><DEDENT>if progress < length:<EOL><INDENT>bar = symbol * progress + '<STR_LIT:|>' + '<STR_LIT:->' * (length - progress - <NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>bar = symbol * progress + '<STR_LIT:->' * (length - progress)<EOL><DEDENT>if self.level > min_level:<EOL><INDENT>percent = "<STR_LIT>" % ("<STR_LIT>").format(percent)<EOL>output = '<STR_LIT:\r>' + prefix +"<STR_LIT>" % (bar, percent, '<STR_LIT:%>', suffix)<EOL>sys.stdout.write(output),<EOL>if iteration == total and carriage_return:<EOL><INDENT>sys.stdout.write('<STR_LIT:\n>')<EOL><DEDENT>sys.stdout.flush()<EOL><DEDENT><DEDENT>

create a terminal progress bar, default bar shows for verbose+

           Parameters
           ==========
           iteration: current iteration (Int)
           total: total iterations (Int)
           length: character length of bar (Int)


if self.level < <NUM_LIT:1>:<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

is_quiet returns true if the level is under 1


def table(self, rows, col_width=<NUM_LIT:2>):

labels = [str(x) for x in range(<NUM_LIT:1>,len(rows)+<NUM_LIT:1>)]<EOL>if isinstance(rows, dict):<EOL><INDENT>labels = list(rows.keys())<EOL>rows = list(rows.values())<EOL><DEDENT>for row in rows: <EOL><INDENT>label = labels.pop(<NUM_LIT:0>)<EOL>label = label.ljust(col_width)<EOL>message = "<STR_LIT:\t>".join(row)<EOL>self.custom(prefix=label,<EOL>message=message)<EOL><DEDENT>

table will print a table of entries. If the rows is
        a dictionary, the keys are interpreted as column names. if
        not, a numbered list is used.


def generate(self, delim='<STR_LIT:->', length=<NUM_LIT:4>, chars='<STR_LIT>'):

descriptor = self._select(self._descriptors)<EOL>noun = self._select(self._nouns)<EOL>numbers = '<STR_LIT>'.join((self._select(chars) for _ in range(length)))<EOL>return delim.join([descriptor, noun, numbers])<EOL>

Generate a robot name. Inspiration from Haikunator, but much more
         poorly implemented ;)

Parameters
==========
delim: Delimiter
length: TokenLength
chars: TokenChars


if len(select_from) <= <NUM_LIT:0>:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>return choice(select_from)<EOL>

select an element from a list using random.choice

            Parameters
            ==========
            should be a list of things to select from


path = os.path.abspath(path)<EOL>bot.debug("<STR_LIT>" % path)<EOL>if not os.path.exists(path):<EOL><INDENT>bot.error('<STR_LIT>' %path)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>names = parse_image_name(remove_uri(name),tag=tag)<EOL>if names['<STR_LIT:version>'] is None:<EOL><INDENT>version = get_image_hash(path)<EOL>names = parse_image_name(remove_uri(name), tag=tag, version=version)    <EOL><DEDENT>metadata = self.get_metadata(path, names=names)<EOL>if "<STR_LIT:data>" in metadata:<EOL><INDENT>metadata = metadata['<STR_LIT:data>']<EOL><DEDENT>metadata.update(names)<EOL>manifest = self._upload(source=path, <EOL>destination=names['<STR_LIT>'],<EOL>metadata=metadata)<EOL>print(manifest['<STR_LIT>'])<EOL>

push an image to Google Cloud Storage, meaning uploading it

    path: should correspond to an absolte image path (or derive it)
    name: should be the complete uri that the user has requested to push.
    tag: should correspond with an image tag. This is provided to mirror Docker


@retry(wait_exponential_multiplier=<NUM_LIT:1000>, wait_exponential_max=<NUM_LIT>)<EOL>def upload(self, source, <EOL>destination, <EOL>bucket,<EOL>chunk_size = <NUM_LIT:2> * <NUM_LIT> * <NUM_LIT>, <EOL>metadata=None,<EOL>keep_private=True):

env = '<STR_LIT>'<EOL>keep_private = self._get_and_update_setting(env) or keep_private<EOL>media = MediaFileUpload(source, chunksize=chunk_size, resumable=True)<EOL>request = self._storage_service.objects().insert(bucket=bucket.name, <EOL>name=destination,<EOL>media_body=media)<EOL>response = None<EOL>total = request.resumable._size / (<NUM_LIT>*<NUM_LIT>)<EOL>bar = ProgressBar(expected_size=total, filled_char='<STR_LIT:=>', hide=self.quiet)<EOL>while response is None:<EOL><INDENT>error = None<EOL>try:<EOL><INDENT>progress, response = request.next_chunk()<EOL>if progress:<EOL><INDENT>bar.show(progress.resumable_progress / (<NUM_LIT>*<NUM_LIT>))<EOL><DEDENT><DEDENT>except:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>blob = bucket.blob(destination)<EOL>if blob.exists():<EOL><INDENT>if not keep_private:<EOL><INDENT>blob.make_public()<EOL><DEDENT>if metadata is not None:<EOL><INDENT>body = prepare_metadata(metadata)<EOL>blob.metadata = metadata   <EOL>blob._properties['<STR_LIT>'] = metadata<EOL>blob.patch()<EOL><DEDENT><DEDENT>return response<EOL>

upload a file from a source to a destination. The client is expected
       to have a bucket (self._bucket) that is created when instantiated.

       This would be the method to do the same using the storage client,
       but not easily done for resumable

       blob = self._bucket.blob(destination)
       blob.upload_from_filename(filename=source,
                                 content_type="application/zip",
                                 client=self._service)

       url = blob.public_url
       if isinstance(url, six.binary_type):
           url = url.decode('utf-8')

       return url


def pull(self, images, file_name=None, save=True, **kwargs):

if not isinstance(images,list):<EOL><INDENT>images = [images]<EOL><DEDENT>bot.debug('<STR_LIT>' %len(images))<EOL>finished = []<EOL>for image in images:<EOL><INDENT>q = parse_image_name(remove_uri(image))<EOL>bot.info('<STR_LIT>' %(q['<STR_LIT>'],self._bucket_name))<EOL>matches = self._container_query(q['<STR_LIT>'], quiet=True)<EOL>if len(matches) == <NUM_LIT:0>:<EOL><INDENT>bot.info('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>if file_name is None:<EOL><INDENT>file_name = q['<STR_LIT>'].replace('<STR_LIT:/>','<STR_LIT:->')<EOL><DEDENT>image = matches[<NUM_LIT:0>]<EOL>image_file = self.download(url=image.media_link,<EOL>file_name=file_name,<EOL>show_progress=True)<EOL>if save is True:<EOL><INDENT>image_uri = q['<STR_LIT>']<EOL>metadata = image.metadata<EOL>if "<STR_LIT>" in metadata:<EOL><INDENT>metadata['<STR_LIT:url>'] = metadata['<STR_LIT>']<EOL><DEDENT>metadata['<STR_LIT>'] = image.self_link<EOL>container = self.add(image_path=image_file,<EOL>image_uri=image_uri,<EOL>metadata=metadata,<EOL>url=image.media_link)<EOL>image_file = container.image<EOL><DEDENT>if os.path.exists(image_file):<EOL><INDENT>bot.debug('<STR_LIT>' % image_file)<EOL>bot.custom(prefix="<STR_LIT>", message=image_file)<EOL>finished.append(image_file)<EOL><DEDENT><DEDENT>if len(finished) == <NUM_LIT:1>:<EOL><INDENT>finished = finished[<NUM_LIT:0>]<EOL><DEDENT>return finished<EOL>

pull an image from google storage, based on the identifier

       Parameters
       ==========
       images: refers to the uri given by the user to pull in the format
               <collection>/<namespace>. You should have an API that is able to
               retrieve a container based on parsing this uri.
       file_name: the user's requested name for the file. It can
               optionally be None if the user wants a default.
       save: if True, you should save the container to the database
             using self.add()

       Returns
       =======
       finished: a single container path, or list of paths


def build(self, name,<EOL>recipe="<STR_LIT>",<EOL>context=None, <EOL>preview=False):

bot.debug("<STR_LIT>" % recipe)<EOL>names = parse_image_name(remove_uri(name))<EOL>config = self._load_build_config(name=names['<STR_LIT>'], recipe=recipe)<EOL>build_package = [recipe]<EOL>if context not in [None, '<STR_LIT>', []]:<EOL><INDENT>if '<STR_LIT:.>' in context:<EOL><INDENT>context = glob(os.getcwd() + '<STR_LIT>', recursive=True)<EOL><DEDENT>build_package = build_package + context<EOL><DEDENT>package = create_build_package(build_package)<EOL>destination='<STR_LIT>' % os.path.basename(package)<EOL>blob = self._build_bucket.blob(destination)<EOL>if not blob.exists() and preview is False:<EOL><INDENT>bot.log('<STR_LIT>')<EOL>manifest = self._upload(source=package, <EOL>bucket=self._build_bucket,<EOL>destination=destination)<EOL><DEDENT>else:<EOL><INDENT>bot.log('<STR_LIT>' % self._build_bucket.name)<EOL><DEDENT>config["<STR_LIT:source>"]["<STR_LIT>"]['<STR_LIT>'] = self._build_bucket.name<EOL>config["<STR_LIT:source>"]["<STR_LIT>"]['<STR_LIT:object>'] = destination<EOL>if preview is False:<EOL><INDENT>config = self._run_build(config, self._bucket, names)<EOL><DEDENT>if not self._get_and_update_setting('<STR_LIT>'):<EOL><INDENT>blob.delete()<EOL><DEDENT>shutil.rmtree(os.path.dirname(package))<EOL>return config<EOL>

trigger a build on Google Cloud (builder then storage) given a name
       recipe, and Github URI where the recipe can be found.

       Parameters
       ==========
       recipe: the local recipe to build.
       name: should be the complete uri that the user has requested to push.
       context: the dependency files needed for the build. If not defined, only
                the recipe is uploaded.
       preview: if True, preview but don't run the build

       Environment
       ===========
       SREGISTRY_GOOGLE_BUILD_SINGULARITY_VERSION: the version of Singularity
           to use, defaults to 3.0.2-slim
       SREGISTRY_GOOGLE_BUILD_CLEANUP: after build, delete intermediate
           dependencies in cloudbuild bucket.


<EOL>for package_file in package_files:<EOL><INDENT>if not os.path.exists(package_file):<EOL><INDENT>bot.exit('<STR_LIT>' % package_file)<EOL><DEDENT><DEDENT>bot.log('<STR_LIT>' % len(package_files))<EOL>build_dir = get_tmpdir(prefix="<STR_LIT>")<EOL>build_tar = '<STR_LIT>' % build_dir<EOL>tar = tarfile.open(build_tar, "<STR_LIT>")<EOL>for package_file in package_files:<EOL><INDENT>tar.add(package_file)<EOL><DEDENT>tar.close()<EOL>sha256 = get_file_hash(build_tar)<EOL>hash_tar =  "<STR_LIT>" %(build_dir, sha256)<EOL>shutil.move(build_tar, hash_tar)<EOL>return hash_tar<EOL>

given a list of files, copy them to a temporary folder,
       compress into a .tar.gz, and rename based on the file hash.
       Return the full path to the .tar.gz in the temporary folder.

       Parameters
       ==========
       package_files: a list of files to include in the tar.gz


def load_build_config(self, name, recipe):

version_envar = '<STR_LIT>'<EOL>version = self._get_and_update_setting(version_envar, '<STR_LIT>')<EOL>config = get_build_template()<EOL>container_name = '<STR_LIT>' % name.replace('<STR_LIT:/>','<STR_LIT:->', <NUM_LIT:1>)<EOL>config['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT:name>'] = '<STR_LIT>' % version<EOL>config['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT:args>'] = ['<STR_LIT>', container_name, recipe]<EOL>config["<STR_LIT>"]["<STR_LIT>"]["<STR_LIT:location>"] = "<STR_LIT>" % self._bucket_name<EOL>config["<STR_LIT>"]["<STR_LIT>"]["<STR_LIT>"] = [container_name]<EOL>return config<EOL>

load a google compute config, meaning that we start with a template,
       and mimic the following example cloudbuild.yaml:

        steps:
        - name: "singularityware/singularity:${_SINGULARITY_VERSION}"
          args: ['build', 'julia-centos-another.sif', 'julia.def']
        artifacts:
          objects:
            location: 'gs://sregistry-gcloud-build-vanessa'
            paths: ['julia-centos-another.sif']

        Parameters
        ==========
        recipe: the local recipe file for the builder.
        name: the name of the container, based on the uri


def run_build(self, config, bucket, names):

project = self._get_project()<EOL>bot.custom('<STR_LIT>', project, "<STR_LIT>")<EOL>bot.custom('<STR_LIT>', config['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT:name>'], "<STR_LIT>")<EOL>response = self._build_service.projects().builds().create(body=config, <EOL>projectId=project).execute()<EOL>build_id = response['<STR_LIT>']['<STR_LIT>']['<STR_LIT:id>']<EOL>status = response['<STR_LIT>']['<STR_LIT>']['<STR_LIT:status>']<EOL>bot.log("<STR_LIT>" % (build_id, status))<EOL>start = time.time()<EOL>while status not in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>time.sleep(<NUM_LIT:15>)<EOL>response = self._build_service.projects().builds().get(id=build_id, <EOL>projectId=project).execute()<EOL>build_id = response['<STR_LIT:id>']<EOL>status = response['<STR_LIT:status>']<EOL>bot.log("<STR_LIT>" % (build_id, status))<EOL><DEDENT>end = time.time()<EOL>bot.log('<STR_LIT>' % (round(end - start, <NUM_LIT:2>)))<EOL>if status == '<STR_LIT>':<EOL><INDENT>env = '<STR_LIT>'<EOL>blob = bucket.blob(response['<STR_LIT>']['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>])<EOL>if self._get_and_update_setting(env) == None:<EOL><INDENT>blob.make_public()<EOL>response['<STR_LIT>'] = blob.public_url<EOL><DEDENT>update_blob_metadata(blob, response, config, bucket, names)<EOL>response['<STR_LIT>'] = blob.media_link<EOL>response['<STR_LIT:size>'] = blob.size<EOL>response['<STR_LIT>'] = blob.md5_hash<EOL><DEDENT>return response<EOL>

run a build, meaning creating a build. Retry if there is failure


def update_blob_metadata(blob, response, config, bucket, names):

manifest = os.path.basename(response['<STR_LIT>']['<STR_LIT>'])<EOL>manifest = json.loads(bucket.blob(manifest).download_as_string())<EOL>metadata = {'<STR_LIT>': manifest['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT:value>'],<EOL>'<STR_LIT>': response['<STR_LIT>']['<STR_LIT>'],<EOL>'<STR_LIT:location>': manifest['<STR_LIT:location>'],<EOL>'<STR_LIT>': config['<STR_LIT:source>']['<STR_LIT>']['<STR_LIT>'],<EOL>'<STR_LIT>': config['<STR_LIT:source>']['<STR_LIT>']['<STR_LIT:object>'],<EOL>'<STR_LIT>': '<STR_LIT:U+0020>'.join(config['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT:args>']),<EOL>'<STR_LIT>': config['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT:name>'],<EOL>'<STR_LIT>': blob.media_link,<EOL>'<STR_LIT>': blob.self_link,<EOL>'<STR_LIT:size>': blob.size,<EOL>'<STR_LIT:name>': names['<STR_LIT>'],<EOL>'<STR_LIT:type>': "<STR_LIT>"} <EOL>blob.metadata = metadata   <EOL>blob._properties['<STR_LIT>'] = metadata<EOL>blob.patch()<EOL>

a specific function to take a blob, along with a SUCCESS response
       from Google build, the original config, and update the blob
       metadata with the artifact file name, dependencies, and image hash.


def search(self, query=None, args=None):

if query is not None:<EOL><INDENT>return self._container_query(query)<EOL><DEDENT>return self._search_all()<EOL>

query a bucket for images that match a particular pattern. If no
       query is provided, all images in the bucket are listed that have type
       "container" in the metadata and client "sregistry"


results = []<EOL>for image in self._bucket.list_blobs():<EOL><INDENT>if image.metadata is not None:<EOL><INDENT>if "<STR_LIT:type>" in image.metadata:<EOL><INDENT>if image.metadata['<STR_LIT:type>'] == "<STR_LIT>":<EOL><INDENT>results.append(image)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if len(results) == <NUM_LIT:0>:<EOL><INDENT>bot.info("<STR_LIT>")<EOL><DEDENT>return results<EOL>

return a list of containers, determined by finding the metadata field
       "type" with value "container." We alert the user to no containers
       if results is empty, and exit

       {'metadata': {'items':
                              [
                               {'key': 'type', 'value': 'container'}, ...
                              ]
                    }
       }


results = self._list_containers()<EOL>bot.info("<STR_LIT>" %self._bucket_name)<EOL>rows = []<EOL>for i in results:<EOL><INDENT>size = round(i.size / (<NUM_LIT>*<NUM_LIT>))<EOL>size = ("<STR_LIT>" %size).rjust(<NUM_LIT:10>)<EOL>rows.append([size, i.metadata['<STR_LIT:name>']])<EOL><DEDENT>bot.table(rows)<EOL>return rows<EOL>

a "list all" search that doesn't require a query. Here we return to
       the user all objects that have custom metadata value of "container"

       IMPORTANT: the upload function adds this metadata. For a container to
       be found by the client, it must have the type as container in metadata.


def container_query(self, query, quiet=False):

results = self._list_containers()<EOL>matches = []<EOL>for result in results:<EOL><INDENT>for key,val in result.metadata.items():<EOL><INDENT>if query in val and result not in matches:<EOL><INDENT>matches.append(result)<EOL><DEDENT><DEDENT><DEDENT>if not quiet:<EOL><INDENT>bot.info("<STR_LIT>" %(self._bucket_name,len(matches)))<EOL>for image in matches:<EOL><INDENT>size = round(image.size / (<NUM_LIT>*<NUM_LIT>))<EOL>bot.custom(prefix=image.name, color="<STR_LIT>")<EOL>bot.custom(prefix='<STR_LIT>', message=image.id)<EOL>bot.custom(prefix='<STR_LIT>', message=image.metadata['<STR_LIT:name>'])<EOL>bot.custom(prefix='<STR_LIT>', message=image.updated)<EOL>bot.custom(prefix='<STR_LIT>',  message='<STR_LIT>' %(size))<EOL>bot.custom(prefix='<STR_LIT>', message=image.md5_hash)<EOL>if "<STR_LIT>" in image.metadata:<EOL><INDENT>public_url = image.metadata['<STR_LIT>']<EOL>bot.custom(prefix='<STR_LIT>', message=public_url)<EOL><DEDENT>bot.newline()<EOL><DEDENT><DEDENT>return matches<EOL>

search for a specific container.
       This function would likely be similar to the above, but have different
       filter criteria from the user (based on the query)


bot.info('<STR_LIT>' % self._bucket_name)<EOL>

add the bucket name to be printed to the user at appropriate times


env = '<STR_LIT>'<EOL>self._secrets = self._get_and_update_setting(env)<EOL>if self._secrets is None:<EOL><INDENT>bot.error('<STR_LIT>' %env)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>

The user is required to have an application secrets file in his
           or her environment. The information isn't saved to the secrets
           file, but the client exists with error if the variable isn't found.


<EOL>self._get_services()<EOL>env = '<STR_LIT>'<EOL>self._bucket_name = self._get_and_update_setting(env)<EOL>if self._bucket_name is None:<EOL><INDENT>self._bucket_name = '<STR_LIT>' %os.environ['<STR_LIT>']<EOL><DEDENT>self._build_bucket_name = "<STR_LIT>" % self._bucket_name<EOL>self._bucket = self._get_bucket(self._bucket_name)<EOL>self._build_bucket = self._get_bucket(self._build_bucket_name)<EOL>

init client will check if the user has defined a bucket that
           differs from the default, use the application credentials to
           get the bucket, and then instantiate the client.


self._bucket_service = storage.Client()<EOL>creds = GoogleCredentials.get_application_default()<EOL>self._storage_service = discovery_build('<STR_LIT>', version, credentials=creds)<EOL>self._build_service = discovery_build('<STR_LIT>', version, credentials=creds)<EOL>

get version 1 of the google compute and storage service

            Parameters
            ==========
            version: version to use (default is v1)


<EOL>try:<EOL><INDENT>bucket = self._bucket_service.get_bucket(bucket_name)<EOL><DEDENT>except google.cloud.exceptions.NotFound:<EOL><INDENT>bucket = self._bucket_service.create_bucket(bucket_name)<EOL><DEDENT>except:<EOL><INDENT>bot.error('<STR_LIT>' % bucket_name)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return bucket<EOL>

get a bucket based on a bucket name. If it doesn't exist, create it.

           Parameters
           ==========
           bucket_name: the name of the bucket to get (or create). It should
                        not contain google, and should be all lowercase with -
                        or underscores.


return self._required_get_and_update('<STR_LIT>', project)<EOL>

get project returns the active project, and exists if not found.

           Parameters
           ==========
           project: a project to default to, if not found in the environment
           zone: a default zone, will be us-west1-a by default


return self._get_and_update_setting('<STR_LIT>', zone)<EOL>

get zone returns the zone set in the environment, or the default

           Parameters
           ==========
           zone: a default zone, will be us-west1-a by default


content = None<EOL>results = self._list_logs()<EOL>print(results)<EOL>if name is not None:<EOL><INDENT>for result in results:<EOL><INDENT>matches = False<EOL>if name in result.name:<EOL><INDENT>matches=True<EOL><DEDENT>for key,val in result.metadata.items():<EOL><INDENT>if name in val:<EOL><INDENT>matches = True<EOL><DEDENT><DEDENT>if matches is True:<EOL><INDENT>content = self._print_log(result.name)  <EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if len(results) > <NUM_LIT:0>:<EOL><INDENT>latest = results[<NUM_LIT:0>]<EOL>for result in results:<EOL><INDENT>if result.time_created >= latest.time_created:<EOL><INDENT>latest = result                   <EOL><DEDENT><DEDENT>content = self._print_log(result.name)  <EOL><DEDENT><DEDENT>return content<EOL>

return logs for a particular container. The logs file is equivalent to
       the name, but with extension .log. If there is no name, the most recent
       log is returned.

       Parameters
       ==========
       name: the container name to print logs for.


results = []<EOL>for image in self._bucket.list_blobs():<EOL><INDENT>if image.name.endswith('<STR_LIT>'):<EOL><INDENT>results.append(image)<EOL><DEDENT><DEDENT>if len(results) == <NUM_LIT:0>:<EOL><INDENT>bot.info("<STR_LIT>")<EOL><DEDENT>return results<EOL>

return a list of logs. We return any file that ends in .log


content = None<EOL>logfile = self._bucket.get_blob(logname)<EOL>print(logname)<EOL>if logfile:<EOL><INDENT>bot.info('<STR_LIT>' %logname)<EOL>content = requests.get(logfile.media_link).text<EOL>print(content)<EOL><DEDENT>return content<EOL>

helper function to retrieve a particular log, and print.

       Parameters
       ==========
       name: the name of the log to retrieve


try:<EOL><INDENT>operation = service.objects().delete(bucket=bucket_name,<EOL>object=object_name).execute()<EOL><DEDENT>except HttpError as e:<EOL><INDENT>pass<EOL>operation = e<EOL><DEDENT>return operation<EOL>

delete object will delete a file from a bucket

       Parameters
       ==========
       storage_service: the service obtained with get_storage_service
       bucket_name: the name of the bucket
       object_name: the "name" parameter of the object.


bot.debug("<STR_LIT>" % name)<EOL>for file_object in files:<EOL><INDENT>if isinstance(file_object, dict):<EOL><INDENT>if "<STR_LIT>" in file_object:<EOL><INDENT>if file_object['<STR_LIT>'] == "<STR_LIT>":<EOL><INDENT>object_name = "<STR_LIT:/>".join(file_object['<STR_LIT:id>'].split('<STR_LIT:/>')[:-<NUM_LIT:1>])<EOL>object_name = re.sub('<STR_LIT>' %self._bucket['<STR_LIT:name>'],'<STR_LIT>', object_name,<NUM_LIT:1>)<EOL>delete_object(service=self._bucket_service,<EOL>bucket_name=bucket['<STR_LIT:name>'],<EOL>object_name=object_name)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

delete an image from Google Storage.

       Parameters
       ==========
       name: the name of the file (or image) to delete