if not os.path.splitext(filename)[<NUM_LIT:1>][<NUM_LIT:1>:] in self.suffixes_to_compress:<EOL><INDENT>return False<EOL><DEDENT>file_stats = None<EOL>compressed_stats = None<EOL>compressed_filename = '<STR_LIT>'.format(filename, self.suffix)<EOL>try:<EOL><INDENT>file_stats = os.stat(filename)<EOL>compressed_stats = os.stat(compressed_filename)<EOL><DEDENT>except OSError:  <EOL><INDENT>pass<EOL><DEDENT>if file_stats and compressed_stats:<EOL><INDENT>return (compressed_filename<EOL>if file_stats.st_mtime > compressed_stats.st_mtime<EOL>else False)<EOL><DEDENT>else:<EOL><INDENT>return compressed_filename<EOL><DEDENT>

If the given filename should be compressed, returns the
        compressed filename.

        A file can be compressed if:

        - It is a whitelisted extension
        - The compressed file does not exist
        - The compressed file exists by is older than the file itself

        Otherwise, it returns False.


assert opacity >= <NUM_LIT:0> and opacity <= <NUM_LIT:1><EOL>if im.mode != '<STR_LIT>':<EOL><INDENT>im = im.convert('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>im = im.copy()<EOL><DEDENT>alpha = im.split()[<NUM_LIT:3>]<EOL>alpha = ImageEnhance.Brightness(alpha).enhance(opacity)<EOL>im.putalpha(alpha)<EOL>return im<EOL>


def watermark(im, mark, position, opacity=<NUM_LIT:1>):

if opacity < <NUM_LIT:1>:<EOL><INDENT>mark = reduce_opacity(mark, opacity)<EOL><DEDENT>if im.mode != '<STR_LIT>':<EOL><INDENT>im = im.convert('<STR_LIT>')<EOL><DEDENT>layer = Image.new('<STR_LIT>', im.size, (<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>))<EOL>if position == '<STR_LIT>':<EOL><INDENT>for y in range(<NUM_LIT:0>, im.size[<NUM_LIT:1>], mark.size[<NUM_LIT:1>]):<EOL><INDENT>for x in range(<NUM_LIT:0>, im.size[<NUM_LIT:0>], mark.size[<NUM_LIT:0>]):<EOL><INDENT>layer.paste(mark, (x, y))<EOL><DEDENT><DEDENT><DEDENT>elif position == '<STR_LIT>':<EOL><INDENT>ratio = min(<EOL>float(im.size[<NUM_LIT:0>]) / mark.size[<NUM_LIT:0>], float(im.size[<NUM_LIT:1>]) / mark.size[<NUM_LIT:1>])<EOL>w = int(mark.size[<NUM_LIT:0>] * ratio)<EOL>h = int(mark.size[<NUM_LIT:1>] * ratio)<EOL>mark = mark.resize((w, h))<EOL>layer.paste(mark, (int((im.size[<NUM_LIT:0>] - w) / <NUM_LIT:2>),<EOL>int((im.size[<NUM_LIT:1>] - h) / <NUM_LIT:2>)))<EOL><DEDENT>else:<EOL><INDENT>layer.paste(mark, position)<EOL><DEDENT>return Image.composite(layer, im, layer)<EOL>


writer = PageWriter(gallery.settings, index_title=gallery.title)<EOL>for album in gallery.albums.values():<EOL><INDENT>medias = album.medias<EOL>next_medias = medias[<NUM_LIT:1>:] + [None]<EOL>previous_medias = [None] + medias[:-<NUM_LIT:1>]<EOL>media_groups = zip(medias, next_medias, previous_medias)<EOL>for media_group in media_groups:<EOL><INDENT>writer.write(album, media_group)<EOL><DEDENT><DEDENT>

Generates and writes the media pages for all media in the gallery


from sigal import __url__ as sigal_link<EOL>file_path = os.path.join(album.dst_path, media_group[<NUM_LIT:0>].filename)<EOL>page = self.template.render({<EOL>'<STR_LIT>': album,<EOL>'<STR_LIT>': media_group[<NUM_LIT:0>],<EOL>'<STR_LIT>': media_group[-<NUM_LIT:1>],<EOL>'<STR_LIT>': media_group[<NUM_LIT:1>],<EOL>'<STR_LIT>': self.index_title,<EOL>'<STR_LIT>': self.settings,<EOL>'<STR_LIT>': sigal_link,<EOL>'<STR_LIT>': {'<STR_LIT:name>': os.path.basename(self.theme),<EOL>'<STR_LIT:url>': url_from_path(os.path.relpath(self.theme_path,<EOL>album.dst_path))},<EOL>})<EOL>output_file = "<STR_LIT>" % file_path<EOL>with open(output_file, '<STR_LIT:w>', encoding='<STR_LIT:utf-8>') as f:<EOL><INDENT>f.write(page)<EOL><DEDENT>


pass<EOL>

Sigal - Simple Static Gallery Generator.

    Sigal is yet another python script to prepare a static gallery of images:
    resize images, create thumbnails with some options, generate html pages.


if os.path.isfile(path):<EOL><INDENT>print("<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>from pkg_resources import resource_string<EOL>conf = resource_string(__name__, '<STR_LIT>')<EOL>with open(path, '<STR_LIT:w>', encoding='<STR_LIT:utf-8>') as f:<EOL><INDENT>f.write(conf.decode('<STR_LIT:utf8>'))<EOL><DEDENT>print("<STR_LIT>".format(path))<EOL>

Copy a sample config file in the current directory (default to
    'sigal.conf.py'), or use the provided 'path'.


@main.command()<EOL>@argument('<STR_LIT:source>', required=False)<EOL>@argument('<STR_LIT>', required=False)<EOL>@option('<STR_LIT>', '<STR_LIT>', is_flag=True,<EOL>help="<STR_LIT>")<EOL>@option('<STR_LIT>', '<STR_LIT>', is_flag=True, help="<STR_LIT>")<EOL>@option('<STR_LIT>', '<STR_LIT>', is_flag=True,<EOL>help="<STR_LIT>")<EOL>@option('<STR_LIT:-c>', '<STR_LIT>', default=_DEFAULT_CONFIG_FILE, show_default=True,<EOL>help="<STR_LIT>")<EOL>@option('<STR_LIT>', '<STR_LIT>', help="<STR_LIT>"<EOL>"<STR_LIT>")<EOL>@option('<STR_LIT>', help="<STR_LIT>")<EOL>@option('<STR_LIT>', '<STR_LIT>', help="<STR_LIT>")<EOL>def build(source, destination, debug, verbose, force, config, theme, title,<EOL>ncpu):

level = ((debug and logging.DEBUG) or (verbose and logging.INFO) or<EOL>logging.WARNING)<EOL>init_logging(__name__, level=level)<EOL>logger = logging.getLogger(__name__)<EOL>if not os.path.isfile(config):<EOL><INDENT>logger.error("<STR_LIT>", config)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>start_time = time.time()<EOL>settings = read_settings(config)<EOL>for key in ('<STR_LIT:source>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>arg = locals()[key]<EOL>if arg is not None:<EOL><INDENT>settings[key] = os.path.abspath(arg)<EOL><DEDENT>logger.info("<STR_LIT>", key.capitalize(), settings[key])<EOL><DEDENT>if not settings['<STR_LIT:source>'] or not os.path.isdir(settings['<STR_LIT:source>']):<EOL><INDENT>logger.error("<STR_LIT>", settings['<STR_LIT:source>'])<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>relative_check = True<EOL>try:<EOL><INDENT>relative_check = os.path.relpath(settings['<STR_LIT>'],<EOL>settings['<STR_LIT:source>']).startswith('<STR_LIT:..>')<EOL><DEDENT>except ValueError:<EOL><INDENT>pass<EOL><DEDENT>if not relative_check:<EOL><INDENT>logger.error("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if title:<EOL><INDENT>settings['<STR_LIT:title>'] = title<EOL><DEDENT>locale.setlocale(locale.LC_ALL, settings['<STR_LIT>'])<EOL>init_plugins(settings)<EOL>gal = Gallery(settings, ncpu=ncpu)<EOL>gal.build(force=force)<EOL>for src, dst in settings['<STR_LIT>']:<EOL><INDENT>src = os.path.join(settings['<STR_LIT:source>'], src)<EOL>dst = os.path.join(settings['<STR_LIT>'], dst)<EOL>logger.debug('<STR_LIT>', src, dst)<EOL>copy(src, dst, symlink=settings['<STR_LIT>'], rellink=settings['<STR_LIT>'])<EOL><DEDENT>stats = gal.stats<EOL>def format_stats(_type):<EOL><INDENT>opt = ["<STR_LIT>".format(stats[_type + '<STR_LIT:_>' + subtype], subtype)<EOL>for subtype in ('<STR_LIT>', '<STR_LIT>')<EOL>if stats[_type + '<STR_LIT:_>' + subtype] > <NUM_LIT:0>]<EOL>opt = '<STR_LIT>'.format('<STR_LIT:U+002CU+0020>'.join(opt)) if opt else '<STR_LIT>'<EOL>return '<STR_LIT>'.format(stats[_type], _type, opt)<EOL><DEDENT>print('<STR_LIT>'<EOL>.format(format_stats('<STR_LIT:image>'), format_stats('<STR_LIT>'),<EOL>time.time() - start_time))<EOL>

Run sigal to process a directory.

    If provided, 'source', 'destination' and 'theme' will override the
    corresponding values from the settings file.


logger = logging.getLogger(__name__)<EOL>logger.debug('<STR_LIT>', settings['<STR_LIT>'])<EOL>for path in settings['<STR_LIT>']:<EOL><INDENT>sys.path.insert(<NUM_LIT:0>, path)<EOL><DEDENT>for plugin in settings['<STR_LIT>']:<EOL><INDENT>try:<EOL><INDENT>if isinstance(plugin, str):<EOL><INDENT>mod = importlib.import_module(plugin)<EOL>mod.register(settings)<EOL><DEDENT>else:<EOL><INDENT>plugin.register(settings)<EOL><DEDENT>logger.debug('<STR_LIT>', plugin)<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.error('<STR_LIT>', plugin, e)<EOL><DEDENT><DEDENT>for path in settings['<STR_LIT>']:<EOL><INDENT>sys.path.remove(path)<EOL><DEDENT>


@main.command()<EOL>@argument('<STR_LIT>', default='<STR_LIT>')<EOL>@option('<STR_LIT>', '<STR_LIT>', help="<STR_LIT>", default=<NUM_LIT>)<EOL>@option('<STR_LIT:-c>', '<STR_LIT>', default=_DEFAULT_CONFIG_FILE,<EOL>show_default=True, help='<STR_LIT>')<EOL>def serve(destination, port, config):

if os.path.exists(destination):<EOL><INDENT>pass<EOL><DEDENT>elif os.path.exists(config):<EOL><INDENT>settings = read_settings(config)<EOL>destination = settings.get('<STR_LIT>')<EOL>if not os.path.exists(destination):<EOL><INDENT>sys.stderr.write("<STR_LIT>"<EOL>"<STR_LIT>".format(destination))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>sys.stderr.write("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>.format(destination=destination, config=config))<EOL>sys.exit(<NUM_LIT:2>)<EOL><DEDENT>print('<STR_LIT>'.format(destination))<EOL>os.chdir(destination)<EOL>Handler = server.SimpleHTTPRequestHandler<EOL>httpd = socketserver.TCPServer(("<STR_LIT>", port), Handler, False)<EOL>print("<STR_LIT>".format(port))<EOL>try:<EOL><INDENT>httpd.allow_reuse_address = True<EOL>httpd.server_bind()<EOL>httpd.server_activate()<EOL>httpd.serve_forever()<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>


@main.command()<EOL>@argument('<STR_LIT:target>')<EOL>@argument('<STR_LIT>', nargs=-<NUM_LIT:1>)<EOL>@option('<STR_LIT>', '<STR_LIT>', default=False, is_flag=True,<EOL>help='<STR_LIT>')<EOL>def set_meta(target, keys, overwrite=False):

if not os.path.exists(target):<EOL><INDENT>sys.stderr.write("<STR_LIT>".format(target))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if len(keys) < <NUM_LIT:2> or len(keys) % <NUM_LIT:2> > <NUM_LIT:0>:<EOL><INDENT>sys.stderr.write("<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if os.path.isdir(target):<EOL><INDENT>descfile = os.path.join(target, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>descfile = os.path.splitext(target)[<NUM_LIT:0>] + '<STR_LIT>'<EOL><DEDENT>if os.path.exists(descfile) and not overwrite:<EOL><INDENT>sys.stderr.write("<STR_LIT>"<EOL>"<STR_LIT>".format(descfile))<EOL>sys.exit(<NUM_LIT:2>)<EOL><DEDENT>with open(descfile, "<STR_LIT:w>") as fp:<EOL><INDENT>for i in range(len(keys) // <NUM_LIT:2>):<EOL><INDENT>k, v = keys[i * <NUM_LIT:2>:(i + <NUM_LIT:1>) * <NUM_LIT:2>]<EOL>fp.write("<STR_LIT>".format(k.capitalize(), v))<EOL><DEDENT><DEDENT>print("<STR_LIT>".format(len(keys) // <NUM_LIT:2>, descfile))<EOL>

Write metadata keys to .md file.

    TARGET can be a media file or an album directory. KEYS are key/value pairs.

    Ex, to set the title of test.jpg to "My test image":

    sigal set_meta test.jpg title "My test image"


path, filen = os.path.split(filename)<EOL>name, ext = os.path.splitext(filen)<EOL>if ext.lower() in settings['<STR_LIT>']:<EOL><INDENT>ext = '<STR_LIT>'<EOL><DEDENT>return join(path, settings['<STR_LIT>'], settings['<STR_LIT>'] +<EOL>name + settings['<STR_LIT>'] + ext)<EOL>

Return the path to the thumb.

    examples:
    >>> default_settings = create_settings()
    >>> get_thumb(default_settings, "bar/foo.jpg")
    "bar/thumbnails/foo.jpg"
    >>> get_thumb(default_settings, "bar/foo.png")
    "bar/thumbnails/foo.png"

    for videos, it returns a jpg file:
    >>> get_thumb(default_settings, "bar/foo.webm")
    "bar/thumbnails/foo.jpg"


logger = logging.getLogger(__name__)<EOL>logger.info("<STR_LIT>")<EOL>settings = _DEFAULT_CONFIG.copy()<EOL>if filename:<EOL><INDENT>logger.debug("<STR_LIT>", filename)<EOL>settings_path = os.path.dirname(filename)<EOL>tempdict = {}<EOL>with open(filename) as f:<EOL><INDENT>code = compile(f.read(), filename, '<STR_LIT>')<EOL>exec(code, tempdict)<EOL><DEDENT>settings.update((k, v) for k, v in tempdict.items()<EOL>if k not in ['<STR_LIT>'])<EOL>paths = ['<STR_LIT:source>', '<STR_LIT>', '<STR_LIT>']<EOL>if os.path.isdir(join(settings_path, settings['<STR_LIT>'])) andos.path.isdir(join(settings_path, settings['<STR_LIT>'],<EOL>'<STR_LIT>')):<EOL><INDENT>paths.append('<STR_LIT>')<EOL><DEDENT>for p in paths:<EOL><INDENT>path = settings[p]<EOL>if path and not isabs(path):<EOL><INDENT>settings[p] = abspath(normpath(join(settings_path, path)))<EOL>logger.debug("<STR_LIT>", p, path, settings[p])<EOL><DEDENT><DEDENT><DEDENT>for key in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>w, h = settings[key]<EOL>if h > w:<EOL><INDENT>settings[key] = (h, w)<EOL>logger.warning("<STR_LIT>"<EOL>"<STR_LIT>", key)<EOL><DEDENT><DEDENT>if not settings['<STR_LIT>']:<EOL><INDENT>logger.info('<STR_LIT>')<EOL><DEDENT>logger.debug('<STR_LIT>', pformat(settings, width=<NUM_LIT>))<EOL>return settings<EOL>

Read settings from a config file in the source_dir root.


settings = _DEFAULT_CONFIG.copy()<EOL>settings.update(kwargs)<EOL>return settings<EOL>

Create a new default setting copy and initialize it with kwargs.


from . import __url__ as sigal_link<EOL>self.logger.info("<STR_LIT>", album)<EOL>return {<EOL>'<STR_LIT>': album,<EOL>'<STR_LIT>': self.index_title,<EOL>'<STR_LIT>': self.settings,<EOL>'<STR_LIT>': sigal_link,<EOL>'<STR_LIT>': {'<STR_LIT:name>': os.path.basename(self.theme),<EOL>'<STR_LIT:url>': url_from_path(os.path.relpath(self.theme_path,<EOL>album.dst_path))},<EOL>}<EOL>

Generate the context dict for the given path.


page = self.template.render(**self.generate_context(album))<EOL>output_file = os.path.join(album.dst_path, album.output_file)<EOL>with open(output_file, '<STR_LIT:w>', encoding='<STR_LIT:utf-8>') as f:<EOL><INDENT>f.write(page)<EOL><DEDENT>


def copy(src, dst, symlink=False, rellink=False):

func = os.symlink if symlink else shutil.copy2<EOL>if symlink and os.path.lexists(dst):<EOL><INDENT>os.remove(dst)<EOL><DEDENT>if rellink:  <EOL><INDENT>func(os.path.relpath(src, os.path.dirname(dst)), dst)<EOL><DEDENT>else:<EOL><INDENT>func(src, dst)<EOL><DEDENT>


if not os.path.isdir(path):<EOL><INDENT>os.makedirs(path)<EOL><DEDENT>

Create the directory if it does not exist


if os.sep != '<STR_LIT:/>':<EOL><INDENT>path = '<STR_LIT:/>'.join(path.split(os.sep))<EOL><DEDENT>return quote(path)<EOL>

Transform path to url, converting backslashes to slashes if needed.


global MD<EOL>with open(filename, '<STR_LIT:r>', encoding='<STR_LIT>') as f:<EOL><INDENT>text = f.read()<EOL><DEDENT>if MD is None:<EOL><INDENT>MD = Markdown(extensions=['<STR_LIT>',<EOL>'<STR_LIT>'],<EOL>output_format='<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>MD.reset()<EOL>MD.Meta = {}<EOL><DEDENT>output = {'<STR_LIT:description>': Markup(MD.convert(text))}<EOL>try:<EOL><INDENT>meta = MD.Meta.copy()<EOL><DEDENT>except AttributeError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>output['<STR_LIT>'] = meta<EOL>try:<EOL><INDENT>output['<STR_LIT:title>'] = MD.Meta['<STR_LIT:title>'][<NUM_LIT:0>]<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return output<EOL>

Reads markdown file, converts output and fetches title and meta-data for
    further processing.


return ext in VIDEO_MIMES.keys()<EOL>

Checks if ext is a supported HTML5 video.


return VIDEO_MIMES[ext]<EOL>


def check_subprocess(cmd, source, outname):

logger = logging.getLogger(__name__)<EOL>try:<EOL><INDENT>res = subprocess.run(cmd, stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE)<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>logger.debug('<STR_LIT>', outname)<EOL>if os.path.isfile(outname):<EOL><INDENT>os.remove(outname)<EOL><DEDENT>raise<EOL><DEDENT>if res.returncode:<EOL><INDENT>logger.debug('<STR_LIT>', res.stdout.decode('<STR_LIT:utf8>'))<EOL>logger.debug('<STR_LIT>', res.stderr.decode('<STR_LIT:utf8>'))<EOL>if os.path.isfile(outname):<EOL><INDENT>logger.debug('<STR_LIT>', outname)<EOL>os.remove(outname)<EOL><DEDENT>raise SubprocessException('<STR_LIT>' + source)<EOL><DEDENT>

Run the command to resize the video and remove the output file if the
    processing fails.


res = subprocess.run([converter, '<STR_LIT>', source], stderr=subprocess.PIPE)<EOL>stderr = res.stderr.decode('<STR_LIT:utf8>')<EOL>pattern = re.compile(r'<STR_LIT>')<EOL>match = pattern.search(stderr)<EOL>rot_pattern = re.compile(r'<STR_LIT>')<EOL>rot_match = rot_pattern.search(stderr)<EOL>if match:<EOL><INDENT>x, y = int(match.groups()[<NUM_LIT:0>]), int(match.groups()[<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>x = y = <NUM_LIT:0><EOL><DEDENT>if rot_match:<EOL><INDENT>x, y = y, x<EOL><DEDENT>return x, y<EOL>


def generate_video(source, outname, settings, options=None):

logger = logging.getLogger(__name__)<EOL>converter = settings['<STR_LIT>']<EOL>w_src, h_src = video_size(source, converter=converter)<EOL>w_dst, h_dst = settings['<STR_LIT>']<EOL>logger.debug('<STR_LIT>', w_src, h_src, w_dst, h_dst)<EOL>base, src_ext = splitext(source)<EOL>base, dst_ext = splitext(outname)<EOL>if dst_ext == src_ext and w_src <= w_dst and h_src <= h_dst:<EOL><INDENT>logger.debug('<STR_LIT>')<EOL>shutil.copy(source, outname)<EOL>return<EOL><DEDENT>if h_dst * w_src < h_src * w_dst:<EOL><INDENT>resize_opt = ['<STR_LIT>', "<STR_LIT>" % h_dst]<EOL><DEDENT>else:<EOL><INDENT>resize_opt = ['<STR_LIT>', "<STR_LIT>" % w_dst]<EOL><DEDENT>if w_src <= w_dst and h_src <= h_dst:<EOL><INDENT>resize_opt = []<EOL><DEDENT>cmd = [converter, '<STR_LIT>', source, '<STR_LIT>']  <EOL>if options is not None:<EOL><INDENT>cmd += options<EOL><DEDENT>cmd += resize_opt + [outname]<EOL>logger.debug('<STR_LIT>', '<STR_LIT:U+0020>'.join(cmd))<EOL>check_subprocess(cmd, source, outname)<EOL>

Video processor.

    :param source: path to a video
    :param outname: path to the generated video
    :param settings: settings dict
    :param options: array of options passed to ffmpeg


def generate_thumbnail(source, outname, box, delay, fit=True, options=None,<EOL>converter='<STR_LIT>'):

logger = logging.getLogger(__name__)<EOL>tmpfile = outname + "<STR_LIT>"<EOL>cmd = [converter, '<STR_LIT>', source, '<STR_LIT>', '<STR_LIT>', '<STR_LIT:1>',<EOL>'<STR_LIT>', delay, '<STR_LIT>', '<STR_LIT:1>', '<STR_LIT>', tmpfile]<EOL>logger.debug('<STR_LIT>', '<STR_LIT:U+0020>'.join(cmd))<EOL>check_subprocess(cmd, source, outname)<EOL>image.generate_thumbnail(tmpfile, outname, box, fit=fit, options=options)<EOL>os.unlink(tmpfile)<EOL>

Create a thumbnail image for the video source, based on ffmpeg.


def process_video(filepath, outpath, settings):

logger = logging.getLogger(__name__)<EOL>filename = os.path.split(filepath)[<NUM_LIT:1>]<EOL>basename, ext = splitext(filename)<EOL>try:<EOL><INDENT>if settings['<STR_LIT>'] and is_valid_html5_video(ext):<EOL><INDENT>outname = os.path.join(outpath, filename)<EOL>utils.copy(filepath, outname, symlink=settings['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>valid_formats = ['<STR_LIT>', '<STR_LIT>']<EOL>video_format = settings['<STR_LIT>']<EOL>if video_format not in valid_formats:<EOL><INDENT>logger.error('<STR_LIT>',<EOL>valid_formats)<EOL>raise ValueError<EOL><DEDENT>outname = os.path.join(outpath, basename + '<STR_LIT:.>' + video_format)<EOL>generate_video(filepath, outname, settings,<EOL>options=settings.get(video_format + '<STR_LIT>'))<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>if logger.getEffectiveLevel() == logging.DEBUG:<EOL><INDENT>raise<EOL><DEDENT>else:<EOL><INDENT>return Status.FAILURE<EOL><DEDENT><DEDENT>if settings['<STR_LIT>']:<EOL><INDENT>thumb_name = os.path.join(outpath, get_thumb(settings, filename))<EOL>try:<EOL><INDENT>generate_thumbnail(<EOL>outname, thumb_name, settings['<STR_LIT>'],<EOL>settings['<STR_LIT>'], fit=settings['<STR_LIT>'],<EOL>options=settings['<STR_LIT>'],<EOL>converter=settings['<STR_LIT>'])<EOL><DEDENT>except Exception:<EOL><INDENT>if logger.getEffectiveLevel() == logging.DEBUG:<EOL><INDENT>raise<EOL><DEDENT>else:<EOL><INDENT>return Status.FAILURE<EOL><DEDENT><DEDENT><DEDENT>return Status.SUCCESS<EOL>

Process a video: resize, create thumbnail.


return url_from_path(self.filename)<EOL>


if self.settings['<STR_LIT>']:<EOL><INDENT>s = self.settings<EOL>if s['<STR_LIT>']:<EOL><INDENT>return self.filename<EOL><DEDENT>orig_path = join(s['<STR_LIT>'], self.path, s['<STR_LIT>'])<EOL>check_or_create_dir(orig_path)<EOL>big_path = join(orig_path, self.src_filename)<EOL>if not isfile(big_path):<EOL><INDENT>copy(self.src_path, big_path, symlink=s['<STR_LIT>'],<EOL>rellink=self.settings['<STR_LIT>'])<EOL><DEDENT>return join(s['<STR_LIT>'], self.src_filename)<EOL><DEDENT>

Path to the original image, if ``keep_orig`` is set (relative to the
        album directory). Copy the file if needed.


if self.big is not None:<EOL><INDENT>return url_from_path(self.big)<EOL><DEDENT>


if not isfile(self.thumb_path):<EOL><INDENT>self.logger.debug('<STR_LIT>', self)<EOL>path = (self.dst_path if os.path.exists(self.dst_path)<EOL>else self.src_path)<EOL>try:<EOL><INDENT>s = self.settings<EOL>if self.type == '<STR_LIT:image>':<EOL><INDENT>image.generate_thumbnail(<EOL>path, self.thumb_path, s['<STR_LIT>'],<EOL>fit=s['<STR_LIT>'])<EOL><DEDENT>elif self.type == '<STR_LIT>':<EOL><INDENT>video.generate_thumbnail(<EOL>path, self.thumb_path, s['<STR_LIT>'],<EOL>s['<STR_LIT>'], fit=s['<STR_LIT>'],<EOL>converter=s['<STR_LIT>'])<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>self.logger.error('<STR_LIT>', e)<EOL>return<EOL><DEDENT><DEDENT>return url_from_path(self.thumb_name)<EOL>

Path to the thumbnail image (relative to the album directory).


self.description = '<STR_LIT>'<EOL>self.meta = {}<EOL>self.title = '<STR_LIT>'<EOL>descfile = splitext(self.src_path)[<NUM_LIT:0>] + '<STR_LIT>'<EOL>if isfile(descfile):<EOL><INDENT>meta = read_markdown(descfile)<EOL>for key, val in meta.items():<EOL><INDENT>setattr(self, key, val)<EOL><DEDENT><DEDENT>

Get image metadata from filename.md: title, description, meta.


descfile = join(self.src_path, self.description_file)<EOL>self.description = '<STR_LIT>'<EOL>self.meta = {}<EOL>self.title = os.path.basename(self.path if self.path != '<STR_LIT:.>'<EOL>else self.src_path)<EOL>if isfile(descfile):<EOL><INDENT>meta = read_markdown(descfile)<EOL>for key, val in meta.items():<EOL><INDENT>setattr(self, key, val)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>self.author = self.meta['<STR_LIT>'][<NUM_LIT:0>]<EOL><DEDENT>except KeyError:<EOL><INDENT>self.author = self.settings.get('<STR_LIT>')<EOL><DEDENT>

Get album metadata from `description_file` (`index.md`):

        -> title, thumbnail image, description


check_or_create_dir(self.dst_path)<EOL>if self.medias:<EOL><INDENT>check_or_create_dir(join(self.dst_path,<EOL>self.settings['<STR_LIT>']))<EOL><DEDENT>if self.medias and self.settings['<STR_LIT>']:<EOL><INDENT>self.orig_path = join(self.dst_path, self.settings['<STR_LIT>'])<EOL>check_or_create_dir(self.orig_path)<EOL><DEDENT>

Create output directories for thumbnails and original images.


for media in self.medias:<EOL><INDENT>if media.type == '<STR_LIT:image>':<EOL><INDENT>yield media<EOL><DEDENT><DEDENT>

List of images (:class:`~sigal.gallery.Image`).


for media in self.medias:<EOL><INDENT>if media.type == '<STR_LIT>':<EOL><INDENT>yield media<EOL><DEDENT><DEDENT>

List of videos (:class:`~sigal.gallery.Video`).


root_path = self.path if self.path != '<STR_LIT:.>' else '<STR_LIT>'<EOL>return [self.gallery.albums[join(root_path, path)]<EOL>for path in self.subdirs]<EOL>

List of :class:`~sigal.gallery.Album` objects for each
        sub-directory.


url = self.name.encode('<STR_LIT:utf-8>')<EOL>return url_quote(url) + '<STR_LIT:/>' + self.url_ext<EOL>

URL of the album, relative to its parent.


if self._thumbnail:<EOL><INDENT>return self._thumbnail<EOL><DEDENT>thumbnail = self.meta.get('<STR_LIT>', ['<STR_LIT>'])[<NUM_LIT:0>]<EOL>if thumbnail and isfile(join(self.src_path, thumbnail)):<EOL><INDENT>self._thumbnail = url_from_path(join(<EOL>self.name, get_thumb(self.settings, thumbnail)))<EOL>self.logger.debug("<STR_LIT>", self, self._thumbnail)<EOL>return self._thumbnail<EOL><DEDENT>else:<EOL><INDENT>for f in self.medias:<EOL><INDENT>ext = splitext(f.filename)[<NUM_LIT:1>]<EOL>if ext.lower() in self.settings['<STR_LIT>']:<EOL><INDENT>size = f.size<EOL>if size is None:<EOL><INDENT>size = get_size(f.src_path)<EOL><DEDENT>if size['<STR_LIT:width>'] > size['<STR_LIT>']:<EOL><INDENT>self._thumbnail = (url_quote(self.name) + '<STR_LIT:/>' +<EOL>f.thumbnail)<EOL>self.logger.debug(<EOL>"<STR_LIT>",<EOL>self, self._thumbnail)<EOL>return self._thumbnail<EOL><DEDENT><DEDENT><DEDENT>if not self._thumbnail and self.medias:<EOL><INDENT>for media in self.medias:<EOL><INDENT>if media.thumbnail is not None:<EOL><INDENT>self._thumbnail = (url_quote(self.name) + '<STR_LIT:/>' +<EOL>media.thumbnail)<EOL>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.logger.warning("<STR_LIT>", self)<EOL>return None<EOL><DEDENT>self.logger.debug("<STR_LIT>",<EOL>self, self._thumbnail)<EOL>return self._thumbnail<EOL><DEDENT>if not self._thumbnail:<EOL><INDENT>for path, album in self.gallery.get_albums(self.path):<EOL><INDENT>if album.thumbnail:<EOL><INDENT>self._thumbnail = (url_quote(self.name) + '<STR_LIT:/>' +<EOL>album.thumbnail)<EOL>self.logger.debug(<EOL>"<STR_LIT>",<EOL>self, self._thumbnail)<EOL>return self._thumbnail<EOL><DEDENT><DEDENT><DEDENT><DEDENT>self.logger.error('<STR_LIT>', self)<EOL>return None<EOL>


if self.path == '<STR_LIT:.>':<EOL><INDENT>return []<EOL><DEDENT>path = self.path<EOL>breadcrumb = [((self.url_ext or '<STR_LIT:.>'), self.title)]<EOL>while True:<EOL><INDENT>path = os.path.normpath(os.path.join(path, '<STR_LIT:..>'))<EOL>if path == '<STR_LIT:.>':<EOL><INDENT>break<EOL><DEDENT>url = (url_from_path(os.path.relpath(path, self.path)) + '<STR_LIT:/>' +<EOL>self.url_ext)<EOL>breadcrumb.append((url, self.gallery.albums[path].title))<EOL><DEDENT>breadcrumb.reverse()<EOL>return breadcrumb<EOL>

List of ``(url, title)`` tuples defining the current breadcrumb
        path.


return any(image.has_location() for image in self.images)<EOL>

Check if we have at least one photo with GPS location in the album


zip_gallery = self.settings['<STR_LIT>']<EOL>if zip_gallery and len(self) > <NUM_LIT:0>:<EOL><INDENT>zip_gallery = zip_gallery.format(album=self)<EOL>archive_path = join(self.dst_path, zip_gallery)<EOL>if (self.settings.get('<STR_LIT>', False) and<EOL>isfile(archive_path)):<EOL><INDENT>self.logger.debug("<STR_LIT>",<EOL>archive_path)<EOL>return zip_gallery<EOL><DEDENT>archive = zipfile.ZipFile(archive_path, '<STR_LIT:w>', allowZip64=True)<EOL>attr = ('<STR_LIT>' if self.settings['<STR_LIT>'] == '<STR_LIT>'<EOL>else '<STR_LIT>')<EOL>for p in self:<EOL><INDENT>path = getattr(p, attr)<EOL>try:<EOL><INDENT>archive.write(path, os.path.split(path)[<NUM_LIT:1>])<EOL><DEDENT>except OSError as e:<EOL><INDENT>self.logger.warn('<STR_LIT>', p, e)<EOL><DEDENT><DEDENT>archive.close()<EOL>self.logger.debug('<STR_LIT>', archive_path)<EOL>return zip_gallery<EOL><DEDENT>

Make a ZIP archive with all media files and return its path.

        If the ``zip_gallery`` setting is set,it contains the location of a zip
        archive with all original images of the corresponding directory.


return self.settings['<STR_LIT:title>'] or self.albums['<STR_LIT:.>'].title<EOL>


for name in self.albums[path].subdirs:<EOL><INDENT>subdir = os.path.normpath(join(path, name))<EOL>yield subdir, self.albums[subdir]<EOL>for subname, album in self.get_albums(subdir):<EOL><INDENT>yield subname, self.albums[subdir]<EOL><DEDENT><DEDENT>

Return the list of all sub-directories of path.


if not self.albums:<EOL><INDENT>self.logger.warning("<STR_LIT>")<EOL>return<EOL><DEDENT>def log_func(x):<EOL><INDENT>available_length = get_terminal_size()[<NUM_LIT:0>] - <NUM_LIT:64><EOL>if x and available_length > <NUM_LIT:10>:<EOL><INDENT>return x.name[:available_length]<EOL><DEDENT>else:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT><DEDENT>try:<EOL><INDENT>with progressbar(self.albums.values(), label="<STR_LIT>",<EOL>item_show_func=log_func, show_eta=False,<EOL>file=self.progressbar_target) as albums:<EOL><INDENT>media_list = [f for album in albums<EOL>for f in self.process_dir(album, force=force)]<EOL><DEDENT><DEDENT>except KeyboardInterrupt:<EOL><INDENT>sys.exit('<STR_LIT>')<EOL><DEDENT>bar_opt = {'<STR_LIT:label>': "<STR_LIT>",<EOL>'<STR_LIT>': True,<EOL>'<STR_LIT:file>': self.progressbar_target}<EOL>failed_files = []<EOL>if self.pool:<EOL><INDENT>try:<EOL><INDENT>with progressbar(length=len(media_list), **bar_opt) as bar:<EOL><INDENT>for res in self.pool.imap_unordered(worker, media_list):<EOL><INDENT>if res:<EOL><INDENT>failed_files.append(res)<EOL><DEDENT>bar.update(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>self.pool.close()<EOL>self.pool.join()<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>self.pool.terminate()<EOL>sys.exit('<STR_LIT>')<EOL><DEDENT>except pickle.PicklingError:<EOL><INDENT>self.logger.critical(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>exc_info=True)<EOL>sys.exit('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>with progressbar(media_list, **bar_opt) as medias:<EOL><INDENT>for media_item in medias:<EOL><INDENT>res = process_file(media_item)<EOL>if res:<EOL><INDENT>failed_files.append(res)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if failed_files:<EOL><INDENT>self.remove_files(failed_files)<EOL><DEDENT>if self.settings['<STR_LIT>']:<EOL><INDENT>album_writer = AlbumPageWriter(self.settings,<EOL>index_title=self.title)<EOL>album_list_writer = AlbumListPageWriter(self.settings,<EOL>index_title=self.title)<EOL>with progressbar(self.albums.values(),<EOL>label="<STR_LIT>" % "<STR_LIT>",<EOL>item_show_func=log_func, show_eta=False,<EOL>file=self.progressbar_target) as albums:<EOL><INDENT>for album in albums:<EOL><INDENT>if album.albums:<EOL><INDENT>if album.medias:<EOL><INDENT>self.logger.warning(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>album.title, album.title<EOL>)<EOL><DEDENT>album_list_writer.write(album)<EOL><DEDENT>else:<EOL><INDENT>album_writer.write(album)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>print('<STR_LIT>')<EOL>signals.gallery_build.send(self)<EOL>


def process_dir(self, album, force=False):

for f in album:<EOL><INDENT>if isfile(f.dst_path) and not force:<EOL><INDENT>self.logger.info("<STR_LIT>", f.filename)<EOL>self.stats[f.type + '<STR_LIT>'] += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>self.stats[f.type] += <NUM_LIT:1><EOL>yield (f.type, f.path, f.filename, f.src_path, album.dst_path,<EOL>self.settings)<EOL><DEDENT><DEDENT>

Process a list of images in a directory.


build_status = generate_status = install_status = <NUM_LIT:0><EOL>if not hasattr(args, '<STR_LIT>'):<EOL><INDENT>vars(args)['<STR_LIT>'] = []<EOL><DEDENT>if '<STR_LIT:test>' in args.build_targets:<EOL><INDENT>logging.error('<STR_LIT>')<EOL>return {'<STR_LIT:status>':<NUM_LIT:1>}<EOL><DEDENT>cwd = os.getcwd()<EOL>c = validate.currentDirectoryModule()<EOL>if not c:<EOL><INDENT>return {'<STR_LIT:status>':<NUM_LIT:1>}<EOL><DEDENT>try:<EOL><INDENT>target, errors = c.satisfyTarget(args.target, additional_config=args.config)<EOL><DEDENT>except access_common.AccessException as e:<EOL><INDENT>logging.error(e)<EOL>return {'<STR_LIT:status>':<NUM_LIT:1>}<EOL><DEDENT>if errors:<EOL><INDENT>for error in errors:<EOL><INDENT>logging.error(error)<EOL><DEDENT>return {'<STR_LIT:status>':<NUM_LIT:1>}<EOL><DEDENT>vars(args)['<STR_LIT>'] = None<EOL>vars(args)['<STR_LIT>'] = False<EOL>if not hasattr(args, '<STR_LIT>'):<EOL><INDENT>if '<STR_LIT>' in args.build_targets:<EOL><INDENT>vars(args)['<STR_LIT>'] = '<STR_LIT:all>'<EOL><DEDENT>elif not len(args.build_targets):<EOL><INDENT>vars(args)['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>vars(args)['<STR_LIT>'] = '<STR_LIT:all>'<EOL><DEDENT><DEDENT>install_status = install.execCommand(args, [])<EOL>builddir = os.path.join(cwd, '<STR_LIT>', target.getName())<EOL>all_deps = c.getDependenciesRecursive(<EOL>target = target,<EOL>available_components = [(c.getName(), c)],<EOL>test = True<EOL>)<EOL>missing = <NUM_LIT:0><EOL>for d in all_deps.values():<EOL><INDENT>if not d and not (d.isTestDependency() and args.install_test_deps != '<STR_LIT:all>'):<EOL><INDENT>logging.error('<STR_LIT>' % os.path.split(d.path)[<NUM_LIT:1>])<EOL>missing += <NUM_LIT:1><EOL><DEDENT><DEDENT>if missing:<EOL><INDENT>logging.error('<STR_LIT>')<EOL>return {'<STR_LIT:status>': <NUM_LIT:1>, '<STR_LIT>':install_status, '<STR_LIT>':missing}<EOL><DEDENT>generator = cmakegen.CMakeGen(builddir, target)<EOL>config = generator.configure(c, all_deps)<EOL>logging.debug("<STR_LIT>", config['<STR_LIT>'])<EOL>script_environment = {<EOL>'<STR_LIT>': config['<STR_LIT>']<EOL>}<EOL>runScriptWithModules(c, all_deps.values(), '<STR_LIT>', script_environment)<EOL>app = c if len(c.getBinaries()) else None<EOL>for error in generator.generateRecursive(c, all_deps, builddir, application=app):<EOL><INDENT>logging.error(error)<EOL>generate_status = <NUM_LIT:1><EOL><DEDENT>logging.debug("<STR_LIT>")<EOL>runScriptWithModules(c, all_deps.values(), '<STR_LIT>', script_environment)<EOL>if (not hasattr(args, '<STR_LIT>')) or (not args.generate_only):<EOL><INDENT>error = target.build(<EOL>builddir, c, args, release_build=args.release_build,<EOL>build_args=following_args, targets=args.build_targets,<EOL>release_no_debug_info_build=args.release_no_debug_info_build<EOL>)<EOL>if error:<EOL><INDENT>logging.error(error)<EOL>build_status = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>runScriptWithModules(c, all_deps.values(), '<STR_LIT>', script_environment)<EOL><DEDENT>if install_status:<EOL><INDENT>logging.warning(<EOL>"<STR_LIT>"+<EOL>"<STR_LIT>"+<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT><DEDENT>return {<EOL>'<STR_LIT:status>': build_status or generate_status or install_status,<EOL>'<STR_LIT>': missing,<EOL>'<STR_LIT>': build_status,<EOL>'<STR_LIT>': generate_status,<EOL>'<STR_LIT>': install_status<EOL>}<EOL>

Perform the build command, but provide detailed error information.
        Returns {status:0, build_status:0, generate_status:0, install_status:0} on success.
        If status: is nonzero there was some sort of error. Other properties
        are optional, and may not be set if that step was not attempted.


def displayOutdated(modules, dependency_specs, use_colours):

if use_colours:<EOL><INDENT>DIM    = colorama.Style.DIM       <EOL>NORMAL = colorama.Style.NORMAL    <EOL>BRIGHT = colorama.Style.BRIGHT    <EOL>YELLOW = colorama.Fore.YELLOW     <EOL>RED    = colorama.Fore.RED        <EOL>GREEN  = colorama.Fore.GREEN      <EOL>RESET  = colorama.Style.RESET_ALL <EOL><DEDENT>else:<EOL><INDENT>DIM = BRIGHT = YELLOW = RED = GREEN = RESET = u'<STR_LIT>'<EOL><DEDENT>status = <NUM_LIT:0><EOL>from yotta.lib import access<EOL>from yotta.lib import access_common<EOL>from yotta.lib import sourceparse<EOL>for name, m in modules.items():<EOL><INDENT>if m.isTestDependency():<EOL><INDENT>continue<EOL><DEDENT>try:<EOL><INDENT>latest_v = access.latestSuitableVersion(name, '<STR_LIT:*>', registry='<STR_LIT>', quiet=True)<EOL><DEDENT>except access_common.Unavailable as e:<EOL><INDENT>latest_v = None<EOL><DEDENT>if not m:<EOL><INDENT>m_version = u'<STR_LIT:U+0020>' + RESET + BRIGHT + RED + u"<STR_LIT>" + RESET<EOL><DEDENT>else:<EOL><INDENT>m_version = DIM + u'<STR_LIT>' % (m.version)<EOL><DEDENT>if not latest_v:<EOL><INDENT>print(u'<STR_LIT>' % (RED, name, m_version, NORMAL, RESET))<EOL>status = <NUM_LIT:2><EOL>continue<EOL><DEDENT>elif not m or m.version < latest_v:<EOL><INDENT>update_prevented_by = '<STR_LIT>'<EOL>if m:<EOL><INDENT>specs_preventing_update = [<EOL>x for x in dependency_specs<EOL>if x.name == name and not<EOL>sourceparse.parseSourceURL(x.nonShrinkwrappedVersionReq()).semanticSpecMatches(latest_v)<EOL>]<EOL>shrinkwrap_prevents_update = [<EOL>x for x in dependency_specs<EOL>if x.name == name and x.isShrinkwrapped() and not<EOL>sourceparse.parseSourceURL(x.versionReq()).semanticSpecMatches(latest_v)<EOL>]<EOL>if len(specs_preventing_update):<EOL><INDENT>update_prevented_by = '<STR_LIT>' % (<EOL>'<STR_LIT:U+002CU+0020>'.join(['<STR_LIT>' % (x.version_req, x.specifying_module) for x in specs_preventing_update])<EOL>)<EOL><DEDENT>if len(shrinkwrap_prevents_update):<EOL><INDENT>update_prevented_by += '<STR_LIT>'<EOL><DEDENT>if m.version.major() < latest_v.major():<EOL><INDENT>colour = GREEN<EOL><DEDENT>elif m.version.minor() < latest_v.minor():<EOL><INDENT>colour = YELLOW<EOL><DEDENT>else:<EOL><INDENT>colour = RED<EOL><DEDENT><DEDENT>else:<EOL><INDENT>colour = RED<EOL><DEDENT>print(u'<STR_LIT>' % (name, m_version, RESET, colour, latest_v.version, update_prevented_by, RESET))<EOL>if not status:<EOL><INDENT>status = <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>return status<EOL>

print information about outdated modules,
        return 0 if there is nothing to be done and nonzero otherwise


<EOL>tests = []<EOL>add_test_re = re.compile('<STR_LIT>', flags=re.IGNORECASE)<EOL>for root, dirs, files in os.walk(builddir, topdown=True):<EOL><INDENT>if not recurse_yotta_modules:<EOL><INDENT>dirs = [d for d in dirs if d != '<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in files:<EOL><INDENT>with open(os.path.join(root, '<STR_LIT>'), '<STR_LIT:r>') as ctestf:<EOL><INDENT>dir_tests = []<EOL>for line in ctestf:<EOL><INDENT>if line.lower().startswith('<STR_LIT>'):<EOL><INDENT>match = add_test_re.search(line)<EOL>if match:<EOL><INDENT>dir_tests.append((match.group(<NUM_LIT:1>), match.group(<NUM_LIT:2>)))<EOL><DEDENT>else:<EOL><INDENT>logging.error(<EOL>"<STR_LIT>" %<EOL>line.rstrip('<STR_LIT:\n>')<EOL>)<EOL><DEDENT><DEDENT><DEDENT>if len(dir_tests):<EOL><INDENT>tests.append((root, dir_tests))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return tests<EOL>

returns a list of (directory_path, [list of tuples of (test name, test command)])


name, spec = _getNonRegistryRef(source_url)<EOL>if spec:<EOL><INDENT>return spec<EOL><DEDENT>try:<EOL><INDENT>url_is_spec = version.Spec(source_url)<EOL><DEDENT>except ValueError:<EOL><INDENT>url_is_spec = None<EOL><DEDENT>if url_is_spec is not None:<EOL><INDENT>return VersionSource('<STR_LIT>', '<STR_LIT>', source_url)<EOL><DEDENT>raise InvalidVersionSpec("<STR_LIT>" % (source_url))<EOL>

Parse the specified version source URL (or version spec), and return an
        instance of VersionSource


try:<EOL><INDENT>parseSourceURL(spec_or_source_url)<EOL>return True<EOL><DEDENT>except InvalidVersionSpec:<EOL><INDENT>return False<EOL><DEDENT>

Check if the specified version source URL (or version spec), can be
        parsed successfully.


import re<EOL>name, spec = _getNonRegistryRef(target_name_and_spec)<EOL>if name:<EOL><INDENT>return name, target_name_and_spec<EOL><DEDENT>split_at = '<STR_LIT:@>'<EOL>if target_name_and_spec.find('<STR_LIT:@>') > target_name_and_spec.find('<STR_LIT:U+002C>') and'<STR_LIT:U+002C>' in target_name_and_spec:<EOL><INDENT>split_at = '<STR_LIT:U+002C>'<EOL><DEDENT>name = target_name_and_spec.split(split_at)[<NUM_LIT:0>]<EOL>spec = target_name_and_spec[len(name)+<NUM_LIT:1>:]<EOL>name = name.strip()<EOL>if not spec:<EOL><INDENT>spec = '<STR_LIT:*>'<EOL><DEDENT>return name, spec<EOL>

Parse targetname[@versionspec] and return a tuple
        (target_name_string, version_spec_string).

        targetname[,versionspec] is also supported (this is how target names
        and specifications are stored internally, and was the documented way of
        setting the spec on the commandline)

        Also accepts raw github version specs (Owner/reponame#whatever), as the
        name can be deduced from these.

        Note that the specification split from the name is not validated. If
        there is no specification (just a target name) passed in, then '*' will
        be returned as the specification.


import re<EOL>name, spec = _getNonRegistryRef(module_name_and_spec)<EOL>if name:<EOL><INDENT>return name, module_name_and_spec<EOL><DEDENT>name = module_name_and_spec.split('<STR_LIT:@>')[<NUM_LIT:0>]<EOL>spec = module_name_and_spec[len(name)+<NUM_LIT:1>:]<EOL>name = name.strip()<EOL>if not spec:<EOL><INDENT>spec = '<STR_LIT:*>'<EOL><DEDENT>return name, spec<EOL>

Parse modulename[@versionspec] and return a tuple
        (module_name_string, version_spec_string).

        Also accepts raw github version specs (Owner/reponame#whatever), as the
        name can be deduced from these.

        Note that the specification split from the name is not validated. If
        there is no specification (just a module name) passed in, then '*' will
        be returned as the specification.


def __init__(self, version_string, url=None):

super(Version, self).__init__()<EOL>self.url = url<EOL>version_string = str(version_string.strip())<EOL>self.version = None<EOL>if version_string.startswith('<STR_LIT:v>') or version_string.startswith('<STR_LIT:=>'):<EOL><INDENT>self.version = semantic_version.Version(version_string[<NUM_LIT:1>:], partial=False)<EOL><DEDENT>elif not version_string:<EOL><INDENT>self.version = TipVersion()<EOL><DEDENT>else:<EOL><INDENT>self.version = semantic_version.Version(version_string, partial=False)<EOL><DEDENT>self.url = url<EOL>

Wrap the semantic_version Version class so that we can represent
            'tip' versions as well as specific versions, and store an optional
            URL that can represent the location from which we can retrieve this
            version.

            Also add some useful methods for manipulating versions.


def wrapped_fn(*args, **kwargs):<EOL><INDENT>q = multiprocessing.Queue()<EOL>p = multiprocessing.Process(target=_dropPrivsReturnViaQueue, args=(q, fn, args, kwargs))<EOL>p.start()<EOL>r = None<EOL>e = None<EOL>while True:<EOL><INDENT>msg = q.get()<EOL>if msg[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>r = msg[<NUM_LIT:1>]<EOL><DEDENT>if msg[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>e = msg[<NUM_LIT:1>](msg[<NUM_LIT:2>])<EOL><DEDENT>if msg[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>if e is not None:<EOL><INDENT>raise e <EOL><DEDENT>return r<EOL><DEDENT><DEDENT><DEDENT>return wrapped_fn<EOL>

decorator to drop su/sudo privilages before running a function on
        unix/linux.
        The *real* uid is modified, so privileges are permanently dropped for
        the process. (i.e. make sure you don't need to do

        If there is a SUDO_UID environment variable, then we drop to that,
        otherwise we drop to nobody.


<EOL>if os.path.split(program)[<NUM_LIT:0>]:<EOL><INDENT>if os.path.exists(program) and os.access(program, os.X_OK):<EOL><INDENT>return program<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for path in os.environ['<STR_LIT>'].split(os.pathsep):<EOL><INDENT>progpath = os.path.join(path, program)<EOL>if os.path.exists(progpath) and os.access(progpath, os.X_OK):<EOL><INDENT>return progpath<EOL><DEDENT><DEDENT><DEDENT>return None<EOL>

look for "program" in PATH, and return the path to it, or None if it
        was not found


def remoteComponentFor(name, version_required, registry='<STR_LIT>'):

try:<EOL><INDENT>vs = sourceparse.parseSourceURL(version_required)<EOL><DEDENT>except ValueError as e:<EOL><INDENT>raise access_common.Unavailable(<EOL>'<STR_LIT:%s>' % (e)<EOL>)<EOL><DEDENT>if vs.source_type == '<STR_LIT>':<EOL><INDENT>if registry not in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>raise Exception('<STR_LIT>' % registry)<EOL><DEDENT>return registry_access.RegistryThing.createFromSource(<EOL>vs, name, registry=registry<EOL>)<EOL><DEDENT>elif vs.source_type == '<STR_LIT>':<EOL><INDENT>return github_access.GithubComponent.createFromSource(vs, name)<EOL><DEDENT>elif vs.source_type == '<STR_LIT>':<EOL><INDENT>return git_access.GitComponent.createFromSource(vs, name)<EOL><DEDENT>elif vs.source_type == '<STR_LIT>':<EOL><INDENT>return hg_access.HGComponent.createFromSource(vs, name)<EOL><DEDENT>else:<EOL><INDENT>raise Exception('<STR_LIT>' % vs.source_type)<EOL><DEDENT>

Return a RemoteComponent sublclass for the specified component name and
        source url (or version specification)
        Raises an exception if any arguments are invalid.


def satisfyVersionFromSearchPaths(name, version_required, search_paths, update=False, type='<STR_LIT>', inherit_shrinkwrap=None):

<EOL>from yotta.lib import pack<EOL>v = None<EOL>try:<EOL><INDENT>sv = sourceparse.parseSourceURL(version_required)<EOL><DEDENT>except ValueError as e:<EOL><INDENT>logging.error(e)<EOL>return None<EOL><DEDENT>try:<EOL><INDENT>local_version = searchPathsFor(<EOL>name,<EOL>sv.semanticSpec(),<EOL>search_paths,<EOL>type,<EOL>inherit_shrinkwrap = inherit_shrinkwrap<EOL>)<EOL><DEDENT>except pack.InvalidDescription as e:<EOL><INDENT>logger.error(e)<EOL>return None<EOL><DEDENT>logger.debug("<STR_LIT>" % (('<STR_LIT>', '<STR_LIT>')[not local_version], name))<EOL>if local_version:<EOL><INDENT>if update and not local_version.installedLinked():<EOL><INDENT>v = latestSuitableVersion(name, version_required, registry=_registryNamespaceForType(type))<EOL>if local_version:<EOL><INDENT>local_version.setLatestAvailable(v)<EOL><DEDENT><DEDENT>if local_version.installedLinked() or not local_version.outdated():<EOL><INDENT>logger.debug("<STR_LIT>" % local_version.path)<EOL>if name != local_version.getName():<EOL><INDENT>raise Exception('<STR_LIT>' % (<EOL>local_version.getName(), name, local_version.path<EOL>))<EOL><DEDENT>return local_version<EOL><DEDENT>logger.info('<STR_LIT>' % (<EOL>name,<EOL>local_version.getVersion(),<EOL>v<EOL>))<EOL>fsutils.rmRf(local_version.path)<EOL>return _satisfyVersionByInstallingVersion(<EOL>name, version_required, local_version.path, v, type=type, inherit_shrinkwrap=inherit_shrinkwrap<EOL>)<EOL><DEDENT>return None<EOL>

returns a Component/Target for the specified version, if found in the
        list of search paths. If `update' is True, then also check for newer
        versions of the found component, and update it in-place (unless it was
        installed via a symlink).


def satisfyVersionByInstalling(name, version_required, working_directory, type='<STR_LIT>', inherit_shrinkwrap=None):

v = latestSuitableVersion(name, version_required, _registryNamespaceForType(type))<EOL>install_into = os.path.join(working_directory, name)<EOL>return _satisfyVersionByInstallingVersion(<EOL>name, version_required, install_into, v, type=type, inherit_shrinkwrap = inherit_shrinkwrap<EOL>)<EOL>

installs and returns a Component/Target for the specified name+version
        requirement, into a subdirectory of `working_directory'


def _satisfyVersionByInstallingVersion(name, version_required, working_directory, version, type='<STR_LIT>', inherit_shrinkwrap=None):

assert(version)<EOL>logger.info('<STR_LIT>', version)<EOL>version.unpackInto(working_directory)<EOL>r = _clsForType(type)(working_directory, inherit_shrinkwrap = inherit_shrinkwrap)<EOL>if not r:<EOL><INDENT>raise Exception(<EOL>'<STR_LIT>' % (name, version_required, type)<EOL>)<EOL><DEDENT>if name != r.getName():<EOL><INDENT>raise Exception('<STR_LIT>' % (<EOL>type, name, version_required, r.getName()<EOL>))<EOL><DEDENT>r.runScript('<STR_LIT>')<EOL>return r<EOL>

installs and returns a Component/Target for the specified version requirement into
        'working_directory' using the provided remote version object.
        This function is not normally called via `satisfyVersionByInstalling',
        which looks up a suitable remote version object.


r = satisfyFromAvailable(name, available, type=type)<EOL>if r is not None:<EOL><INDENT>if not sourceparse.parseSourceURL(version_required).semanticSpecMatches(r.getVersion()):<EOL><INDENT>raise access_common.SpecificationNotMet(<EOL>"<STR_LIT>" % (type, name, version_required)<EOL>)<EOL><DEDENT>return r<EOL><DEDENT>r = satisfyVersionFromSearchPaths(<EOL>name,<EOL>version_required,<EOL>search_paths,<EOL>(update_installed == '<STR_LIT>'),<EOL>type = type,<EOL>inherit_shrinkwrap = inherit_shrinkwrap<EOL>)<EOL>if r is not None:<EOL><INDENT>return r<EOL><DEDENT>return satisfyVersionByInstalling(<EOL>name, version_required, working_directory, type=type, inherit_shrinkwrap = inherit_shrinkwrap<EOL>)<EOL>

returns a Component/Target for the specified version (either to an already
        installed copy (from the available list, or from disk), or to a newly
        downloaded one), or None if the version could not be satisfied.

        update_installed = None / 'Update'
            None:   prevent any attempt to look for new versions if the
                    component/target already exists
            Update: replace any existing version with the newest available, if
                    the newest available has a higher version


@functools.wraps(fn)<EOL>def wrapped(*args, **kwargs):<EOL><INDENT>try:<EOL><INDENT>return fn(*args, **kwargs)<EOL><DEDENT>except requests.exceptions.RequestException as e:<EOL><INDENT>return "<STR_LIT>" % (e.response.status_code, e.message)<EOL><DEDENT><DEDENT>return wrapped<EOL>

Decorator that captures requests.exceptions.RequestException errors
        and returns them as an error message. If no error occurs the reture
        value of the wrapped function is returned (normally None).


@functools.wraps(fn)<EOL>def wrapped(*args, **kwargs):<EOL><INDENT>from yotta.lib import auth<EOL>interactive = globalconf.get('<STR_LIT>')<EOL>try:<EOL><INDENT>return fn(*args, **kwargs)<EOL><DEDENT>except requests.exceptions.HTTPError as e:<EOL><INDENT>if e.response.status_code == requests.codes.unauthorized: <EOL><INDENT>logger.debug('<STR_LIT>', fn)<EOL>auth.authorizeUser(provider=None, interactive=interactive)<EOL>if interactive:<EOL><INDENT>logger.debug('<STR_LIT>')<EOL>return fn(*args, **kwargs)<EOL><DEDENT><DEDENT>raise<EOL><DEDENT><DEDENT>return wrapped<EOL>

Decorator to re-try API calls after asking the user for authentication.


@functools.wraps(fn)<EOL>def wrapped(*args, **kwargs):<EOL><INDENT>try:<EOL><INDENT>return fn(*args, **kwargs)<EOL><DEDENT>except requests.exceptions.HTTPError as e:<EOL><INDENT>if e.response.status_code == requests.codes.unauthorized: <EOL><INDENT>logger.error('<STR_LIT>')<EOL><DEDENT>elif e.response.status_code == requests.codes.bad and '<STR_LIT>' in e.response.text.lower(): <EOL><INDENT>logger.error('<STR_LIT>', e.response.status_code, e.response.text)<EOL>logger.error('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>logger.error('<STR_LIT>', e.response.status_code, e.response.text)<EOL><DEDENT>raise<EOL><DEDENT><DEDENT>return wrapped<EOL>

Decorator to print a friendly you-are-not-authorised message. Use
        **outside** the _handleAuth decorator to only print the message after
        the user has been given a chance to login.


def __raiseUnavailableFor401(fn):<EOL><INDENT>def wrapped(*args, **kwargs):<EOL><INDENT>try:<EOL><INDENT>return fn(*args, **kwargs)<EOL><DEDENT>except requests.exceptions.HTTPError as e:<EOL><INDENT>if e.response.status_code == requests.codes.unauthorized:<EOL><INDENT>raise access_common.Unavailable(message)<EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT><DEDENT>return wrapped<EOL><DEDENT>return __raiseUnavailableFor401<EOL>

Returns a decorator to swallow a requests exception for modules that
        are not accessible without logging in, and turn it into an Unavailable
        exception.


@_swallowRequestExceptions(fail_return="<STR_LIT>")<EOL>@_retryConnectionErrors<EOL>@_friendlyAuthError<EOL>@_handleAuth<EOL>def publish(namespace, name, version, description_file, tar_file, readme_file,<EOL>readme_file_ext, registry=None):

registry = registry or Registry_Base_URL<EOL>url = '<STR_LIT>' % (<EOL>registry,<EOL>namespace,<EOL>name,<EOL>version<EOL>)<EOL>if readme_file_ext == '<STR_LIT>':<EOL><INDENT>readme_section_name = '<STR_LIT>'<EOL><DEDENT>elif readme_file_ext == '<STR_LIT>':<EOL><INDENT>readme_section_name = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' % readme_file_ext)<EOL><DEDENT>body = OrderedDict([('<STR_LIT>', (None, description_file.read(),'<STR_LIT:application/json>')),<EOL>('<STR_LIT>',('<STR_LIT>', tar_file)),<EOL>(readme_section_name, (readme_section_name, readme_file))])<EOL>headers = _headersForRegistry(registry)<EOL>response = requests.put(url, headers=headers, files=body)<EOL>response.raise_for_status()<EOL>return None<EOL>

Publish a tarblob to the registry, if the request fails, an exception
        is raised, which either triggers re-authentication, or is turned into a
        return value by the decorators. (If successful, the decorated function
        returns None)


registry = registry or Registry_Base_URL<EOL>url = '<STR_LIT>' % (<EOL>registry,<EOL>namespace,<EOL>name,<EOL>version<EOL>)<EOL>headers = _headersForRegistry(registry)<EOL>response = requests.delete(url, headers=headers)<EOL>response.raise_for_status()<EOL>return None<EOL>

Try to unpublish a recently published version. Return any errors that
        occur.


registry = registry or Registry_Base_URL<EOL>url = '<STR_LIT>' % (<EOL>registry,<EOL>namespace,<EOL>name<EOL>)<EOL>request_headers = _headersForRegistry(registry)<EOL>response = requests.get(url, headers=request_headers)<EOL>if response.status_code == <NUM_LIT>:<EOL><INDENT>logger.error('<STR_LIT>' % (namespace[:-<NUM_LIT:1>], name))<EOL>return None<EOL><DEDENT>response.raise_for_status()<EOL>return ordered_json.loads(response.text)<EOL>

List the owners of a module or target (owners are the people with
        permission to publish versions and add/remove the owners).


registry = registry or Registry_Base_URL<EOL>url = '<STR_LIT>' % (<EOL>registry,<EOL>namespace,<EOL>name,<EOL>owner<EOL>)<EOL>request_headers = _headersForRegistry(registry)<EOL>response = requests.put(url, headers=request_headers)<EOL>if response.status_code == <NUM_LIT>:<EOL><INDENT>logger.error('<STR_LIT>' % (namespace[:-<NUM_LIT:1>], name))<EOL>return<EOL><DEDENT>response.raise_for_status()<EOL>return True<EOL>

Add an owner for a module or target (owners are the people with
        permission to publish versions and add/remove the owners).


registry = registry or Registry_Base_URL<EOL>url = '<STR_LIT>' % (<EOL>registry,<EOL>namespace,<EOL>name,<EOL>owner<EOL>)<EOL>request_headers = _headersForRegistry(registry)<EOL>response = requests.delete(url, headers=request_headers)<EOL>if response.status_code == <NUM_LIT>:<EOL><INDENT>logger.error('<STR_LIT>' % (namespace[:-<NUM_LIT:1>], name))<EOL>return<EOL><DEDENT>response.raise_for_status()<EOL>return True<EOL>

Remove an owner for a module or target (owners are the people with
        permission to publish versions and add/remove the owners).


@_retryConnectionErrors<EOL>def search(query='<STR_LIT>', keywords=[], registry=None):

registry = registry or Registry_Base_URL<EOL>url = '<STR_LIT>' % registry<EOL>headers = _headersForRegistry(registry)<EOL>params = {<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:50><EOL>}<EOL>if len(query):<EOL><INDENT>params['<STR_LIT>'] = query<EOL><DEDENT>if len(keywords):<EOL><INDENT>params['<STR_LIT>'] = keywords<EOL><DEDENT>while True:<EOL><INDENT>response = requests.get(url, headers=headers, params=params)<EOL>response.raise_for_status()<EOL>objects = ordered_json.loads(response.text)<EOL>if len(objects):<EOL><INDENT>for o in objects:<EOL><INDENT>yield o<EOL><DEDENT>params['<STR_LIT>'] += params['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>

generator of objects returned by the search endpoint (both modules and
        targets).

        Query is a full-text search (description, name, keywords), keywords
        search only the module/target description keywords lists.

        If both parameters are specified the search is the intersection of the
        two queries.


if (registry is None) or (registry == Registry_Base_URL):<EOL><INDENT>return<EOL><DEDENT>sources = _getSources()<EOL>source = None<EOL>for s in sources:<EOL><INDENT>if _sourceMatches(s, registry):<EOL><INDENT>source = s<EOL><DEDENT><DEDENT>if source is None:<EOL><INDENT>source = {<EOL>'<STR_LIT:type>':'<STR_LIT>',<EOL>'<STR_LIT:url>':registry,<EOL>}<EOL>sources.append(source)<EOL><DEDENT>source['<STR_LIT>'] = api_key<EOL>settings.set('<STR_LIT>', sources)<EOL>

Set the api key for accessing a registry. This is only necessary for
        development/test registries.


registry = registry or Registry_Base_URL<EOL>pubkey_pem = None<EOL>if _isPublicRegistry(registry):<EOL><INDENT>pubkey_pem = settings.getProperty('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>for s in _getSources():<EOL><INDENT>if _sourceMatches(s, registry):<EOL><INDENT>if '<STR_LIT>' in s and s['<STR_LIT>'] and '<STR_LIT>' in s['<STR_LIT>']:<EOL><INDENT>pubkey_pem = s['<STR_LIT>']['<STR_LIT>']<EOL>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if not pubkey_pem:<EOL><INDENT>pubkey_pem, privatekey_pem = _generateAndSaveKeys()<EOL><DEDENT>else:<EOL><INDENT>pubkey_pem = pubkey_pem.encode('<STR_LIT:ascii>')<EOL><DEDENT>if b'<STR_LIT>' in pubkey_pem:<EOL><INDENT>pubkey = serialization.load_pem_public_key(pubkey_pem, default_backend())<EOL><DEDENT>else:<EOL><INDENT>pubkey_der = binascii.unhexlify(pubkey_pem)<EOL>pubkey = serialization.load_der_public_key(pubkey_der, default_backend())<EOL><DEDENT>return _pubkeyWireFormat(pubkey)<EOL>

Return the user's public key (generating and saving a new key pair if necessary)


registry = registry or Registry_Base_URL<EOL>url = '<STR_LIT>' % (<EOL>registry<EOL>)<EOL>request_headers = _headersForRegistry(registry)<EOL>logger.debug('<STR_LIT>', request_headers)<EOL>try:<EOL><INDENT>response = requests.get(url, headers=request_headers)<EOL><DEDENT>except requests.RequestException as e:<EOL><INDENT>logger.debug(str(e))<EOL>return None<EOL><DEDENT>if response.status_code == requests.codes.unauthorized: <EOL><INDENT>logger.debug('<STR_LIT>')<EOL>return None<EOL><DEDENT>elif response.status_code == requests.codes.not_found: <EOL><INDENT>logger.debug('<STR_LIT>')<EOL>return None<EOL><DEDENT>body = response.text<EOL>logger.debug('<STR_LIT>' % body);<EOL>r = {}<EOL>parsed_response = ordered_json.loads(body)<EOL>if '<STR_LIT:error>' in parsed_response:<EOL><INDENT>raise AuthError(parsed_response['<STR_LIT:error>'])<EOL><DEDENT>for token in parsed_response:<EOL><INDENT>if '<STR_LIT>' in token and token['<STR_LIT>'] and '<STR_LIT>' in token:<EOL><INDENT>r[token['<STR_LIT>']] = token['<STR_LIT>']<EOL>break<EOL><DEDENT><DEDENT>logger.debug('<STR_LIT>' % r);<EOL>return r<EOL>

Poll the registry to get the result of a completed authentication
        (which, depending on the authentication the user chose or was directed
        to, will include a github or other access token)


@classmethod<EOL><INDENT>def createFromSource(cls, vs, name, registry):<DEDENT>

<EOL>if registry == '<STR_LIT>':<EOL><INDENT>name_match = re.match('<STR_LIT>', name)<EOL>if not name_match:<EOL><INDENT>raise access_common.AccessException(<EOL>'<STR_LIT>' % name<EOL>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>name_match = re.match('<STR_LIT>', name)<EOL>if not name_match:<EOL><INDENT>raise access_common.AccessException(<EOL>'<STR_LIT>' % name<EOL>)<EOL><DEDENT><DEDENT>assert(vs.semantic_spec)<EOL>return RegistryThing(name, vs.semantic_spec, registry)<EOL>

returns a registry component for anything that's a valid package
            name (this does not guarantee that the component actually exists in
            the registry: use availableVersions() for that).


return _listVersions(self.namespace, self.name)<EOL>

return a list of Version objects, each able to retrieve a tarball


<EOL>result = type(args[<NUM_LIT:0>])()<EOL>for k, v in itertools.chain(*[x.items() for x in args]):<EOL><INDENT>if not k in result:<EOL><INDENT>result[k] = v<EOL><DEDENT>elif isinstance(result[k], dict) and isinstance(v, dict):<EOL><INDENT>result[k] = _mergeDictionaries(result[k], v)<EOL><DEDENT><DEDENT>return result<EOL>

merge dictionaries of dictionaries recursively, with elements from
        dictionaries earlier in the argument sequence taking precedence


result = type(dictionary)()<EOL>for k in dictionary.keys():<EOL><INDENT>if isinstance(dictionary[k], dict):<EOL><INDENT>result[k] = _mirrorStructure(dictionary[k], value)<EOL><DEDENT>else:<EOL><INDENT>result[k] = value<EOL><DEDENT><DEDENT>return result<EOL>

create a new nested dictionary object with the same structure as
        'dictionary', but with all scalar values replaced with 'value'


error = None<EOL>config = {}<EOL>if not config_path:<EOL><INDENT>return (error, config)<EOL><DEDENT>if os.path.isfile(config_path):<EOL><INDENT>try:<EOL><INDENT>config = ordered_json.load(config_path)<EOL><DEDENT>except Exception as e:<EOL><INDENT>error = "<STR_LIT>" % (config_path, e)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>config = ordered_json.loads(config_path)<EOL><DEDENT>except Exception as e:<EOL><INDENT>if '<STR_LIT:{>' in config_path or '<STR_LIT:}>' in config_path:<EOL><INDENT>error = "<STR_LIT>" % e<EOL><DEDENT>else:<EOL><INDENT>error = "<STR_LIT>" % config_path<EOL><DEDENT><DEDENT><DEDENT>logger.debug('<STR_LIT>', config)<EOL>return (error, config)<EOL>


inherits = self.description.get('<STR_LIT>', {})<EOL>if len(inherits) == <NUM_LIT:1>:<EOL><INDENT>name, version_req = list(inherits.items())[<NUM_LIT:0>]<EOL>shrinkwrap_version_req = self.getShrinkwrapMapping('<STR_LIT>').get(name, None)<EOL>if shrinkwrap_version_req is not None:<EOL><INDENT>logger.debug(<EOL>'<STR_LIT>', shrinkwrap_version_req, name<EOL>)<EOL><DEDENT>return pack.DependencySpec(<EOL>name,<EOL>version_req,<EOL>shrinkwrap_version_req = shrinkwrap_version_req<EOL>)<EOL><DEDENT>elif len(inherits) > <NUM_LIT:1>:<EOL><INDENT>logger.error('<STR_LIT>', self.getName())<EOL><DEDENT>return None<EOL>

returns pack.DependencySpec for the base target of this target (or
            None if this target does not inherit from another target.


def __init__(self, leaf_target, base_targets, app_config, additional_config):

<EOL>super(DerivedTarget, self).__init__(<EOL>path = leaf_target.path,<EOL>installed_linked = leaf_target.installed_linked,<EOL>latest_suitable_version = leaf_target.latest_suitable_version<EOL>)<EOL>self.hierarchy = [leaf_target] + base_targets[:]<EOL>self.config = None<EOL>self.config_blame = None<EOL>self.app_config = app_config<EOL>self.additional_config = additional_config or {}<EOL>

Initialise a DerivedTarget (representing an inheritance hierarchy of
            Targets.), given the most-derived Target description, and a set of
            available Targets to compose the rest of the lineage from.

            DerivedTarget provides build & debug commands, and access to the
            derived target config info (merged with the application config
            info from config.json, if any).

            It's possible to update the application config for an existing
            DerivedTarget instance.

            DerivedTarget can also be used as a stand-in for the most-derived
            (leaf) target in the inheritance hierarchy.


for t in self.hierarchy:<EOL><INDENT>s = t.getScript(scriptname)<EOL>if s:<EOL><INDENT>return s<EOL><DEDENT><DEDENT>return None<EOL>

return the specified script if one exists (possibly inherited from
            a base target)


config_dicts = [self.additional_config, self.app_config] + [t.getConfig() for t in self.hierarchy]<EOL>config_blame = [<EOL>_mirrorStructure(self.additional_config, '<STR_LIT>'),<EOL>_mirrorStructure(self.app_config, '<STR_LIT>'),<EOL>] + [<EOL>_mirrorStructure(t.getConfig(), t.getName()) for t in self.hierarchy<EOL>]<EOL>self.config = _mergeDictionaries(*config_dicts)<EOL>self.config_blame = _mergeDictionaries(*config_blame)<EOL>

load the configuration information from the target hierarchy


return reversed([<EOL>os.path.join(x.path, x.description['<STR_LIT>']) for x in self.hierarchy if '<STR_LIT>' in x.description<EOL>])<EOL>

return a list of toolchain file paths in override order (starting
            at the bottom/leaf of the hierarchy and ending at the base).
            The list is returned in the order they should be included
            (most-derived last).


return reversed([<EOL>os.path.join(t.path, include_file)<EOL>for t in self.hierarchy<EOL>for include_file in t.description.get('<STR_LIT>', [])<EOL>])<EOL>

Return the list of cmake files which are to be included by yotta in
            every module built. The list is returned in the order they should
            be included (most-derived last).


for t in self.hierarchy:<EOL><INDENT>if t and t.getName() == target_name or target_name in t.description.get('<STR_LIT>', {}):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Return true if this target inherits from the named target (directly
            or indirectly. Also returns true if this target is the named
            target. Otherwise return false.


try:<EOL><INDENT>child = subprocess.Popen(cmd, cwd=builddir)<EOL>child.wait()<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno == errno.ENOENT:<EOL><INDENT>if cmd[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>' % (cmd[<NUM_LIT:0>])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return '<STR_LIT>' % (cmd)<EOL><DEDENT><DEDENT>if child.returncode:<EOL><INDENT>return '<STR_LIT>' % (cmd)<EOL><DEDENT>

Execute the given command, returning an error message if an error occured
            or None if the command was succesful.


@fsutils.dropRootPrivs<EOL><INDENT>def build(self, builddir, component, args, release_build=False, build_args=None, targets=None,<EOL>release_no_debug_info_build=False):<DEDENT>

if build_args is None:<EOL><INDENT>build_args = []<EOL><DEDENT>if targets is None:<EOL><INDENT>targets = []<EOL><DEDENT>if release_no_debug_info_build:<EOL><INDENT>build_type = '<STR_LIT>'<EOL><DEDENT>elif release_build:<EOL><INDENT>build_type = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>build_type = '<STR_LIT>'<EOL><DEDENT>cmd = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>' % build_type, '<STR_LIT>', args.cmake_generator, '<STR_LIT:.>']<EOL>res = self.exec_helper(cmd, builddir)<EOL>if res is not None:<EOL><INDENT>return res<EOL><DEDENT>from yotta.lib import cmake_fixups<EOL>cmake_fixups.applyFixupsForFenerator(args.cmake_generator, builddir, component)<EOL>build_command = self.overrideBuildCommand(args.cmake_generator, targets=targets)<EOL>if build_command:<EOL><INDENT>cmd = build_command + build_args<EOL><DEDENT>else:<EOL><INDENT>cmd = ['<STR_LIT>', '<STR_LIT>', builddir]<EOL>if len(targets):<EOL><INDENT>cmd += ['<STR_LIT>', targets[<NUM_LIT:0>]]<EOL><DEDENT>cmd += build_args<EOL><DEDENT>res = self.exec_helper(cmd, builddir)<EOL>if res is not None:<EOL><INDENT>return res<EOL><DEDENT>hint = self.hintForCMakeGenerator(args.cmake_generator, component)<EOL>if hint:<EOL><INDENT>logger.info(hint)<EOL><DEDENT>

Execute the commands necessary to build this component, and all of
            its dependencies.


def findProgram(self, builddir, program):

<EOL>if os.path.isfile(os.path.join(builddir, program)):<EOL><INDENT>logging.info('<STR_LIT>' % program)<EOL>return program<EOL><DEDENT>exact_matches = []<EOL>insensitive_matches = []<EOL>approx_matches = []<EOL>for path, dirs, files in os.walk(builddir):<EOL><INDENT>if program in files:<EOL><INDENT>exact_matches.append(os.path.relpath(os.path.join(path, program), builddir))<EOL>continue<EOL><DEDENT>files_lower = [f.lower() for f in files]<EOL>if program.lower() in files_lower:<EOL><INDENT>insensitive_matches.append(<EOL>os.path.relpath(<EOL>os.path.join(path, files[files_lower.index(program.lower())]),<EOL>builddir<EOL>)<EOL>)<EOL>continue<EOL><DEDENT>pg_basen_lower_noext = os.path.splitext(os.path.basename(program).lower())[<NUM_LIT:0>]<EOL>for f in files_lower:<EOL><INDENT>if pg_basen_lower_noext in f:<EOL><INDENT>approx_matches.append(<EOL>os.path.relpath(<EOL>os.path.join(path, files[files_lower.index(f)]),<EOL>builddir<EOL>)<EOL>)<EOL><DEDENT><DEDENT><DEDENT>if len(exact_matches) == <NUM_LIT:1>:<EOL><INDENT>logging.info('<STR_LIT>', program, exact_matches[<NUM_LIT:0>])<EOL>return exact_matches[<NUM_LIT:0>]<EOL><DEDENT>elif len(exact_matches) > <NUM_LIT:1>:<EOL><INDENT>logging.error(<EOL>'<STR_LIT>' % (<EOL>program,<EOL>'<STR_LIT>'.join(['<STR_LIT:">'+os.path.join(m, program)+'<STR_LIT:">' for m in exact_matches])<EOL>)<EOL>)<EOL>return None<EOL><DEDENT>reduced_approx_matches = []<EOL>for m in approx_matches:<EOL><INDENT>root = os.path.splitext(m)[<NUM_LIT:0>]<EOL>if (m == root) or (root not in approx_matches):<EOL><INDENT>reduced_approx_matches.append(m)<EOL><DEDENT><DEDENT>approx_matches = reduced_approx_matches<EOL>for matches in (insensitive_matches, approx_matches):<EOL><INDENT>if len(matches) == <NUM_LIT:1>:<EOL><INDENT>logging.info('<STR_LIT>' % (<EOL>program, matches[<NUM_LIT:0>]<EOL>))<EOL>return matches[<NUM_LIT:0>]<EOL><DEDENT>elif len(matches) > <NUM_LIT:1>:<EOL><INDENT>logging.error(<EOL>'<STR_LIT>' % (<EOL>program,<EOL>'<STR_LIT:\n>'.join(matches)<EOL>)<EOL>)<EOL>return None<EOL><DEDENT><DEDENT>logging.error('<STR_LIT>' %  program)<EOL>return None<EOL>

Return the builddir-relative path of program, if only a partial
            path is specified. Returns None and logs an error message if the
            program is ambiguous or not found


@fsutils.dropRootPrivs<EOL><INDENT>def start(self, builddir, program, forward_args):<DEDENT>

child = None<EOL>try:<EOL><INDENT>prog_path = self.findProgram(builddir, program)<EOL>if prog_path is None:<EOL><INDENT>return<EOL><DEDENT>start_env, start_vars = self.buildProgEnvAndVars(prog_path, builddir)<EOL>if self.getScript('<STR_LIT:start>'):<EOL><INDENT>cmd = [<EOL>os.path.expandvars(string.Template(x).safe_substitute(**start_vars))<EOL>for x in self.getScript('<STR_LIT:start>')<EOL>] + forward_args<EOL><DEDENT>else:<EOL><INDENT>cmd = shlex.split('<STR_LIT>' + prog_path) + forward_args<EOL><DEDENT>logger.debug('<STR_LIT>', cmd)<EOL>child = subprocess.Popen(<EOL>cmd, cwd = builddir, env = start_env<EOL>)<EOL>child.wait()<EOL>if child.returncode:<EOL><INDENT>return "<STR_LIT>" % child.returncode<EOL><DEDENT>child = None<EOL><DEDENT>except OSError as e:<EOL><INDENT>import errno<EOL>if e.errno == errno.ENOEXEC:<EOL><INDENT>return ("<STR_LIT>"+<EOL>"<STR_LIT>"<EOL>"<STR_LIT>") % prog_path<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>if child is not None:<EOL><INDENT>_tryTerminate(child)<EOL><DEDENT><DEDENT>

Launch the specified program. Uses the `start` script if specified
            by the target, attempts to run it natively if that script is not
            defined.


try:<EOL><INDENT>signal.signal(signal.SIGINT, _ignoreSignal);<EOL>if self.getScript('<STR_LIT>') is not None:<EOL><INDENT>return self._debugWithScript(builddir, program)<EOL><DEDENT>elif '<STR_LIT>' in self.description:<EOL><INDENT>logger.warning(<EOL>'<STR_LIT>'+<EOL>'<STR_LIT>', self.getName()<EOL>)<EOL>return self._debugDeprecated(builddir, program)<EOL><DEDENT>else:<EOL><INDENT>return "<STR_LIT>" % self<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>signal.signal(signal.SIGINT, signal.SIG_DFL);<EOL><DEDENT>

Launch a debugger for the specified program. Uses the `debug`
            script if specified by the target, falls back to the `debug` and
            `debugServer` commands if not. `program` is inserted into the
            $program variable in commands.


@functools.wraps(fn)<EOL>def wrapped(*args, **kwargs):<EOL><INDENT>interactive = globalconf.get('<STR_LIT>')<EOL>def retryWithAuthOrRaise(original_exception):<EOL><INDENT>auth.authorizeUser(provider='<STR_LIT>', interactive=interactive)<EOL>if not interactive:<EOL><INDENT>raise original_exception<EOL><DEDENT>else:<EOL><INDENT>logger.debug('<STR_LIT>', settings.getProperty('<STR_LIT>', '<STR_LIT>'))<EOL>return fn(*args, **kwargs)<EOL><DEDENT><DEDENT>def handleRateLimitExceeded(original_exception):<EOL><INDENT>if not _userAuthedWithGithub():<EOL><INDENT>logger.warning('<STR_LIT>')<EOL>return retryWithAuthOrRaise(original_exception)<EOL><DEDENT>else:<EOL><INDENT>raise original_exception<EOL><DEDENT><DEDENT>try:<EOL><INDENT>return fn(*args, **kwargs)<EOL><DEDENT>except requests.exceptions.HTTPError as e:<EOL><INDENT>if e.response.status_code == <NUM_LIT>:<EOL><INDENT>return handleRateLimitExceeded(e)<EOL><DEDENT>if e.response.status_code == <NUM_LIT>:<EOL><INDENT>return retryWithAuthOrRaise(e)<EOL><DEDENT>raise<EOL><DEDENT>except github.BadCredentialsException as e:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return retryWithAuthOrRaise(e)<EOL><DEDENT>except github.UnknownObjectException as e:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>if not _userAuthedWithGithub():<EOL><INDENT>logger.info('<STR_LIT>')<EOL>return retryWithAuthOrRaise(e)<EOL><DEDENT>raise<EOL><DEDENT>except github.RateLimitExceededException as e:<EOL><INDENT>return handleRateLimitExceeded(e)<EOL><DEDENT>except github.GithubException as e:<EOL><INDENT>if e.status == <NUM_LIT>:<EOL><INDENT>return handleRateLimitExceeded(e)<EOL><DEDENT>raise<EOL><DEDENT><DEDENT>return wrapped<EOL>

Decorator to re-try API calls after asking the user for authentication.


logger.debug('<STR_LIT>', repo)<EOL>g = Github(settings.getProperty('<STR_LIT>', '<STR_LIT>'))<EOL>repo = g.get_repo(repo)<EOL>tags = repo.get_tags()<EOL>logger.debug('<STR_LIT>', repo, [t.name for t in tags])<EOL>return {t.name: _ensureDomainPrefixed(t.tarball_url) for t in tags}<EOL>

return a dictionary of {tag: tarball_url}


g = Github(settings.getProperty('<STR_LIT>', '<STR_LIT>'))<EOL>repo = g.get_repo(repo)<EOL>return repo.get_archive_link('<STR_LIT>')<EOL>

return a string containing a tarball url


g = Github(settings.getProperty('<STR_LIT>', '<STR_LIT>'))<EOL>repo = g.get_repo(repo)<EOL>return repo.get_archive_link('<STR_LIT>', commit)<EOL>


@_handleAuth<EOL>def _getTarball(url, into_directory, cache_key, origin_info=None):

try:<EOL><INDENT>access_common.unpackFromCache(cache_key, into_directory)<EOL><DEDENT>except KeyError as e:<EOL><INDENT>tok = settings.getProperty('<STR_LIT>', '<STR_LIT>')<EOL>headers = {}<EOL>if tok is not None:<EOL><INDENT>headers['<STR_LIT>'] = '<STR_LIT>' + str(tok)<EOL><DEDENT>logger.debug('<STR_LIT>', url)<EOL>response = requests.get(url, allow_redirects=True, stream=True, headers=headers)<EOL>response.raise_for_status()<EOL>logger.debug('<STR_LIT>', url)<EOL>logger.debug('<STR_LIT>', response.headers)<EOL>response.raise_for_status()<EOL>access_common.unpackTarballStream(<EOL>stream = response,<EOL>into_directory = into_directory,<EOL>hash = {},<EOL>cache_key = cache_key,<EOL>origin_info = origin_info<EOL>)<EOL><DEDENT>

unpack the specified tarball url into the specified directory


@classmethod<EOL><INDENT>def createFromSource(cls, vs, name=None):<DEDENT>

return GithubComponent(vs.location, vs.spec, vs.semantic_spec, name)<EOL>

returns a github component for any github url (including
            git+ssh:// git+http:// etc. or None if this is not a Github URL.
            For all of these we use the github api to grab a tarball, because
            that's faster.

            Normally version will be empty, unless the original url was of the
            form: 'owner/repo @version' or 'url://...#version', which can be used
            to grab a particular tagged version.

            (Note that for github components we ignore the component name - it
             doesn't have to match the github module name)


r = []<EOL>for t in self._getTags():<EOL><INDENT>logger.debug("<STR_LIT>", t)<EOL>if not len(t[<NUM_LIT:0>].strip()):<EOL><INDENT>continue<EOL><DEDENT>try:<EOL><INDENT>r.append(GithubComponentVersion(t[<NUM_LIT:0>], t[<NUM_LIT:0>], url=t[<NUM_LIT:1>], name=self.name, cache_key=None))<EOL><DEDENT>except ValueError:<EOL><INDENT>logger.debug('<STR_LIT>', t)<EOL><DEDENT><DEDENT>return r<EOL>

return a list of Version objects, each with a tarball URL set


return [<EOL>GithubComponentVersion(<EOL>'<STR_LIT>', t[<NUM_LIT:0>], t[<NUM_LIT:1>], self.name, cache_key=_createCacheKey('<STR_LIT>', t[<NUM_LIT:0>], t[<NUM_LIT:1>], self.name)<EOL>) for t in self._getTags()<EOL>]<EOL>

return a list of GithubComponentVersion objects for all tags


return [<EOL>GithubComponentVersion(<EOL>'<STR_LIT>', b[<NUM_LIT:0>], b[<NUM_LIT:1>], self.name, cache_key=None<EOL>) for b in _getBranchHeads(self.repo).items()<EOL>]<EOL>

return a list of GithubComponentVersion objects for the tip of each branch


import re<EOL>commit_match = re.match('<STR_LIT>', self.tagOrBranchSpec(), re.I)<EOL>if commit_match:<EOL><INDENT>return GithubComponentVersion(<EOL>'<STR_LIT>', '<STR_LIT>', _getCommitArchiveURL(self.repo, self.tagOrBranchSpec()), self.name, cache_key=None<EOL>)<EOL><DEDENT>return None<EOL>

return a GithubComponentVersion object for a specific commit if valid


<EOL>if vs.location.startswith('<STR_LIT>'):<EOL><INDENT>location = vs.location[<NUM_LIT:3>:]<EOL><DEDENT>else:<EOL><INDENT>location = vs.location<EOL><DEDENT>return HGComponent(location, vs.spec)<EOL>

returns a hg component for any hg:// url, or None if this is not
            a hg component.

            Normally version will be empty, unless the original url was of the
            form 'hg+ssh://...#version', which can be used to grab a particular
            tagged version.


if v is False:<EOL><INDENT>return False<EOL><DEDENT>elif v is None:<EOL><INDENT>return False<EOL><DEDENT>elif v == <NUM_LIT:0>:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>return True<EOL><DEDENT>

Determine yotta-config truthiness. In yotta config land truthiness is
        different to python or json truthiness (in order to map nicely only
        preprocessor and CMake definediness):

          json      -> python -> truthy/falsey
          false     -> False  -> Falsey
          null      -> None   -> Falsey
          undefined -> None   -> Falsey
          0         -> 0      -> Falsey
          ""        -> ""     -> Truthy (different from python)
          "0"       -> "0"    -> Truthy
          {}        -> {}     -> Truthy (different from python)
          []        -> []     -> Truthy (different from python)
          everything else is truthy


deps = []<EOL>def specForDependency(name, version_spec, istest):<EOL><INDENT>shrinkwrap = self.getShrinkwrapMapping()<EOL>shrinkwrap_version_req = None<EOL>if name in shrinkwrap:<EOL><INDENT>shrinkwrap_version_req = shrinkwrap[name]<EOL>logger.debug(<EOL>'<STR_LIT>', self.getName(), shrinkwrap_version_req, name<EOL>)<EOL><DEDENT>return pack.DependencySpec(<EOL>name,<EOL>version_spec,<EOL>istest,<EOL>shrinkwrap_version_req = shrinkwrap_version_req,<EOL>specifying_module = self.getName()<EOL>)<EOL><DEDENT>deps += [specForDependency(x[<NUM_LIT:0>], x[<NUM_LIT:1>], False) for x in self.description.get('<STR_LIT>', {}).items()]<EOL>target_deps = self.description.get('<STR_LIT>', {})<EOL>if target is not None:<EOL><INDENT>for conf_key, target_conf_deps in target_deps.items():<EOL><INDENT>if _truthyConfValue(target.getConfigValue(conf_key)) or conf_key in target.getSimilarTo_Deprecated():<EOL><INDENT>logger.debug(<EOL>'<STR_LIT>' %<EOL>(conf_key, self.getName())<EOL>)<EOL>deps += [specForDependency(x[<NUM_LIT:0>], x[<NUM_LIT:1>], False) for x in target_conf_deps.items()]<EOL><DEDENT><DEDENT><DEDENT>deps += [specForDependency(x[<NUM_LIT:0>], x[<NUM_LIT:1>], True) for x in self.description.get('<STR_LIT>', {}).items()]<EOL>target_deps = self.description.get('<STR_LIT>', {})<EOL>if target is not None:<EOL><INDENT>for conf_key, target_conf_deps in target_deps.items():<EOL><INDENT>if _truthyConfValue(target.getConfigValue(conf_key)) or conf_key in target.getSimilarTo_Deprecated():<EOL><INDENT>logger.debug(<EOL>'<STR_LIT>' %<EOL>(conf_key, self.getName())<EOL>)<EOL>deps += [specForDependency(x[<NUM_LIT:0>], x[<NUM_LIT:1>], True) for x in target_conf_deps.items()]<EOL><DEDENT><DEDENT><DEDENT>seen = set()<EOL>r = []<EOL>for dep in deps:<EOL><INDENT>if not dep.name in seen:<EOL><INDENT>r.append(dep)<EOL>seen.add(dep.name)<EOL><DEDENT><DEDENT>return r<EOL>

Returns [DependencySpec]

            These are returned in the order that they are listed in the
            component description file: this is so that dependency resolution
            proceeds in a predictable way.


def satisfyTarget(self, target_name_and_version, update_installed=False, additional_config=None, install_missing=True):

<EOL>from yotta.lib import target<EOL>application_dir = None<EOL>if self.isApplication():<EOL><INDENT>application_dir = self.path<EOL><DEDENT>return target.getDerivedTarget(<EOL>target_name_and_version,<EOL>self.targetsPath(),<EOL>install_missing = install_missing,<EOL>application_dir = application_dir,<EOL>update_installed = update_installed,<EOL>additional_config = additional_config,<EOL>shrinkwrap = self.getShrinkwrap()<EOL>)<EOL>

Ensure that the specified target name (and optionally version,
            github ref or URL) is installed in the targets directory of the
            current component

            returns (derived_target, errors)


def getTarget(self, target_name_and_version, additional_config=None):

derived_target, errors = self.satisfyTarget(<EOL>target_name_and_version,<EOL>additional_config = additional_config,<EOL>install_missing = False<EOL>)<EOL>if len(errors):<EOL><INDENT>return None<EOL><DEDENT>else:<EOL><INDENT>return derived_target<EOL><DEDENT>

Return a derived target object representing the selected target: if
            the target is not installed, or is invalid then the returned object
            will test false in a boolean context.

            Returns derived_target

            Errors are not displayed.


return self.installed_dependencies<EOL>

Return true if satisfyDependencies has been called.

            Note that this is slightly different to when all of the
            dependencies are actually satisfied, but can be used as if it means
            that.


return bool(len(self.getBinaries()))<EOL>

Return true if this module is an application instead of a reusable
            library


<EOL>if '<STR_LIT>' in self.description:<EOL><INDENT>return {os.path.normpath(self.description['<STR_LIT>']): self.getName()}<EOL><DEDENT>else:<EOL><INDENT>return {}<EOL><DEDENT>

Return a dictionary of binaries to compile: {"dirname":"exename"},
            this is used when automatically generating CMakeLists

            Note that currently modules may define only a single executable
            binary or library to be built by the automatic build system, by
            specifying `"bin": "dir-to-be-built-into-binary"`, or `"lib":
            "dir-to-be-built-into-library"`, and the bin/lib will always have
            the same name as the module. The default behaviour if nothing is
            specified is for the 'source' directory to be built into a library.

            The module.json syntax may allow for other combinations in the
            future (and callers of this function should not rely on it
            returning only a single item). For example, a "bin": {"dirname":
            "exename"} syntax might be supported, however currently more
            complex builds must be controlled by custom CMakeLists.


if '<STR_LIT>' in self.description:<EOL><INDENT>return {os.path.normpath(self.description['<STR_LIT>']): self.getName()}<EOL><DEDENT>elif '<STR_LIT>' not in self.description and not explicit_only:<EOL><INDENT>return {'<STR_LIT:source>': self.getName()}<EOL><DEDENT>else:<EOL><INDENT>return {}<EOL><DEDENT>

Return a dictionary of libraries to compile: {"dirname":"libname"},
            this is used when automatically generating CMakeLists.

            If explicit_only is not set, then in the absence of both 'lib' and
            'bin' sections in the module.json file, the "source" directory
            will be returned.

            Note that currently modules may define only a single executable
            binary or library to be built by the automatic build system, by
            specifying `"bin": "dir-to-be-built-into-binary"`, or `"lib":
            "dir-to-be-built-into-library"`, and the bin/lib will always have
            the same name as the module. The default behaviour if nothing is
            specified is for the 'source' directory to be built into a library.

            The module.json syntax may allow for other combinations in the
            future (and callers of this function should not rely on it
            returning only a single item). For example, a "bin": {"dirname":
            "exename"} syntax might be supported, however currently more
            complex builds must be controlled by custom CMakeLists.


if '<STR_LIT>' in self.description:<EOL><INDENT>return [self.description['<STR_LIT>']]<EOL><DEDENT>else:<EOL><INDENT>return [x['<STR_LIT:type>'] for x in self.description['<STR_LIT>']]<EOL><DEDENT>

Return a list of licenses that apply to this module. (Strings,
            which may be SPDX identifiers)


if '<STR_LIT>' in self.description:<EOL><INDENT>return [os.path.normpath(x) for x in self.description['<STR_LIT>']]<EOL><DEDENT>else:<EOL><INDENT>return []<EOL><DEDENT>

Some components must export whole directories full of headers into
            the search path. This is really really bad, and they shouldn't do
            it, but support is provided as a concession to compatibility.


Some components (e.g. libc) must export directories of header files
            into the system include search path. They do this by adding a
            'extraSysIncludes' : [ array of directories ] field in their
            package description. This function returns the list of directories
            (or an empty list), if it doesn't exist.


if dirname == component_name:<EOL><INDENT>return '<STR_LIT>' % (component_name, dirname)<EOL><DEDENT>elif dirname.lower() in ('<STR_LIT:source>', '<STR_LIT:src>') and dirname != '<STR_LIT:source>':<EOL><INDENT>return '<STR_LIT>' % (component_name, dirname)<EOL><DEDENT>elif isPotentialTestDir(dirname) and dirname != '<STR_LIT:test>':<EOL><INDENT>return '<STR_LIT>' % (component_name, dirname)<EOL><DEDENT>elif not Source_Dir_Regex.match(dirname):<EOL><INDENT>corrected = Source_Dir_Invalid_Regex.sub('<STR_LIT>', dirname.lower())<EOL>if not corrected:<EOL><INDENT>corrected = '<STR_LIT:source>'<EOL><DEDENT>return '<STR_LIT>' % (component_name, dirname, corrected)<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

validate source directory names in components


for fn in filenames:<EOL><INDENT>try:<EOL><INDENT>self.configs[fn] = ordered_json.load(fn)<EOL><DEDENT>except IOError:<EOL><INDENT>self.configs[fn] = OrderedDict()<EOL><DEDENT>except Exception as e:<EOL><INDENT>self.configs[fn] = OrderedDict()<EOL>logging.warning(<EOL>"<STR_LIT>",<EOL>fn, e<EOL>)<EOL><DEDENT><DEDENT>

Read a list of files. Their configuration values are merged, with
             preference to values from files earlier in the list.


path = _splitPath(path)<EOL>for config in self.configs.values():<EOL><INDENT>cur = config<EOL>for el in path:<EOL><INDENT>if el in cur:<EOL><INDENT>cur = cur[el]<EOL><DEDENT>else:<EOL><INDENT>cur = None<EOL>break<EOL><DEDENT><DEDENT>if cur is not None:<EOL><INDENT>return cur<EOL><DEDENT><DEDENT>return None<EOL>

return a configuration value

            usage:
                get('section.property')

            Note that currently array indexes are not supported. You must
            get the whole array.

            returns None if any path element or the property is missing


def set(self, path, value=None, filename=None):

if filename is None:<EOL><INDENT>config = self._firstConfig()[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>config = self.configs[filename]<EOL><DEDENT>path = _splitPath(path)<EOL>for el in path[:-<NUM_LIT:1>]:<EOL><INDENT>if el in config:<EOL><INDENT>config = config[el]<EOL><DEDENT>else:<EOL><INDENT>config[el] = OrderedDict()<EOL>config = config[el]<EOL><DEDENT><DEDENT>config[path[-<NUM_LIT:1>]] = value<EOL>

Set a configuration value. If no filename is specified, the
            property is set in the first configuration file. Note that if a
            filename is specified and the property path is present in an
            earlier filename then set property will be hidden.

            usage:
                set('section.property', value='somevalue')

            Note that currently array indexes are not supported. You must
            set the whole array.


return self.version_req<EOL>

return the dependency specification ignoring any shrinkwrap


return self.shrinkwrap_version_req or self.version_req<EOL>

return the dependency specification, which may be from a shrinkwrap file


if self.origin_info is None:<EOL><INDENT>self.origin_info = {}<EOL>try:<EOL><INDENT>self.origin_info = ordered_json.load(os.path.join(self.path, Origin_Info_Fname))<EOL><DEDENT>except IOError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return self.origin_info.get('<STR_LIT:url>', None)<EOL>

Read the .yotta_origin.json file (if present), and return the value
            of the 'url' property


return self.error<EOL>

If this isn't a valid component/target, return some sort of
            explanation about why that is.


self.error = error<EOL>

Set an error: note that setting an error does not make the module
            invalid if it would otherwise be valid.


if self.latest_suitable_version and self.latest_suitable_version > self.version:<EOL><INDENT>return self.latest_suitable_version<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

Return a truthy object if a newer suitable version is available,
            otherwise return None.
            (in fact the object returned is a ComponentVersion that can be used
             to get the newer version)


if not self.vcs:<EOL><INDENT>return True<EOL><DEDENT>return self.vcs.isClean()<EOL>

Return true if the directory is not version controlled, or if it is
            version controlled with a supported system and is in a clean state


if not self.vcs:<EOL><INDENT>return<EOL><DEDENT>self.vcs.commit(message='<STR_LIT>' % tag, tag=tag)<EOL>

Commit the current working directory state (or do nothing if the
            working directory is not version controlled)


return self.version<EOL>

Return the version as specified by the package file.
            This will always be a real version: 1.2.3, not a hash or a URL.

            Note that a component installed through a URL still provides a real
            version - so if the first component to depend on some component C
            depends on it via a URI, and a second component depends on a
            specific version 1.2.3, dependency resolution will only succeed if
            the version of C obtained from the URL happens to be 1.2.3


test_path = PurePath('<STR_LIT:/>', path)<EOL>test_paths = tuple([test_path] + list(test_path.parents))<EOL>for exp in self.ignore_patterns:<EOL><INDENT>for tp in test_paths:<EOL><INDENT>if tp.match(exp):<EOL><INDENT>logger.debug('<STR_LIT>', path, tp, exp)<EOL>return True<EOL><DEDENT><DEDENT><DEDENT>return False<EOL>

Test if this module ignores the file at "path", which must be a
            path relative to the root of the module.

            If a file is within a directory that is ignored, the file is also
            ignored.


ordered_json.dump(os.path.join(self.path, self.description_filename), self.description)<EOL>if self.vcs:<EOL><INDENT>self.vcs.markForCommit(self.description_filename)<EOL><DEDENT>

Write the current (possibly modified) component description to a
            package description file in the component directory.


archive_name = '<STR_LIT>' % (self.getName(), self.getVersion())<EOL>def filterArchive(tarinfo):<EOL><INDENT>if tarinfo.name.find(archive_name) == <NUM_LIT:0> :<EOL><INDENT>unprefixed_name = tarinfo.name[len(archive_name)+<NUM_LIT:1>:]<EOL>tarinfo.mode &= <NUM_LIT><EOL><DEDENT>else:<EOL><INDENT>unprefixed_name = tarinfo.name<EOL><DEDENT>if self.ignores(unprefixed_name):<EOL><INDENT>return None<EOL><DEDENT>else:<EOL><INDENT>return tarinfo<EOL><DEDENT><DEDENT>with tarfile.open(fileobj=file_object, mode='<STR_LIT>') as tf:<EOL><INDENT>logger.info('<STR_LIT>' % archive_name)<EOL>tf.add(self.path, arcname=archive_name, filter=filterArchive)<EOL><DEDENT>

Write a tarball of the current component/target to the file object
            "file_object", which must already be open for writing at position 0


if (registry is None) or (registry == registry_access.Registry_Base_URL):<EOL><INDENT>if '<STR_LIT>' in self.description and self.description['<STR_LIT>']:<EOL><INDENT>return "<STR_LIT>" % (self.description_filename.split('<STR_LIT:.>')[<NUM_LIT:0>])<EOL><DEDENT><DEDENT>upload_archive = os.path.join(self.path, '<STR_LIT>')<EOL>fsutils.rmF(upload_archive)<EOL>fd = os.open(upload_archive, os.O_CREAT | os.O_EXCL | os.O_RDWR | getattr(os, "<STR_LIT>", <NUM_LIT:0>))<EOL>with os.fdopen(fd, '<STR_LIT>') as tar_file:<EOL><INDENT>tar_file.truncate()<EOL>self.generateTarball(tar_file)<EOL>logger.debug('<STR_LIT>', tar_file.tell())<EOL>tar_file.seek(<NUM_LIT:0>)<EOL>shasum = hashlib.sha256()<EOL>while True:<EOL><INDENT>chunk = tar_file.read(<NUM_LIT:1000>)<EOL>if not chunk:<EOL><INDENT>break<EOL><DEDENT>shasum.update(chunk)<EOL><DEDENT>logger.debug('<STR_LIT>', shasum.hexdigest())<EOL>tar_file.seek(<NUM_LIT:0>)<EOL>with self.findAndOpenReadme() as readme_file_wrapper:<EOL><INDENT>if not readme_file_wrapper:<EOL><INDENT>logger.warning("<STR_LIT>")<EOL><DEDENT>with open(self.getDescriptionFile(), '<STR_LIT:r>') as description_file:<EOL><INDENT>return registry_access.publish(<EOL>self.getRegistryNamespace(),<EOL>self.getName(),<EOL>self.getVersion(),<EOL>description_file,<EOL>tar_file,<EOL>readme_file_wrapper.file,<EOL>readme_file_wrapper.extension().lower(),<EOL>registry=registry<EOL>)<EOL><DEDENT><DEDENT><DEDENT>

Publish to the appropriate registry, return a description of any
            errors that occured, or None if successful.
            No VCS tagging is performed.


return registry_access.unpublish(<EOL>self.getRegistryNamespace(),<EOL>self.getName(),<EOL>self.getVersion(),<EOL>registry=registry<EOL>)<EOL>

Try to un-publish the current version. Return a description of any
            errors that occured, or None if successful.


script = self.description.get('<STR_LIT>', {}).get(scriptname, None)<EOL>if script is not None:<EOL><INDENT>if isinstance(script, str) or isinstance(script, type(u'<STR_LIT>')):<EOL><INDENT>import shlex<EOL>script = shlex.split(script)<EOL><DEDENT>if len(script) and script[<NUM_LIT:0>].lower().endswith('<STR_LIT>'):<EOL><INDENT>if not os.path.isabs(script[<NUM_LIT:0>]):<EOL><INDENT>absscript = os.path.abspath(os.path.join(self.path, script[<NUM_LIT:0>]))<EOL>logger.debug('<STR_LIT>', script[<NUM_LIT:0>], absscript)<EOL>script[<NUM_LIT:0>] = absscript<EOL><DEDENT>import sys<EOL>script = [sys.executable] + script<EOL><DEDENT><DEDENT>return script<EOL>

Return the specified script command. If the first part of the
            command is a .py file, then the current python interpreter is
            prepended.

            If the script is a single string, rather than an array, it is
            shlex-split.


@fsutils.dropRootPrivs<EOL><INDENT>def runScript(self, scriptname, additional_environment=None):<DEDENT>

import subprocess<EOL>import shlex<EOL>command = self.getScript(scriptname)<EOL>if command is None:<EOL><INDENT>logger.debug('<STR_LIT>', self, scriptname)<EOL>return <NUM_LIT:0><EOL><DEDENT>if not len(command):<EOL><INDENT>logger.error("<STR_LIT>", scriptname, self.getName())<EOL>return <NUM_LIT:1><EOL><DEDENT>env = os.environ.copy()<EOL>if additional_environment is not None:<EOL><INDENT>env.update(additional_environment)<EOL><DEDENT>errcode = <NUM_LIT:0><EOL>child = None<EOL>try:<EOL><INDENT>logger.debug('<STR_LIT>', command)<EOL>child = subprocess.Popen(<EOL>command, cwd = self.path, env = env<EOL>)<EOL>child.wait()<EOL>if child.returncode:<EOL><INDENT>logger.error(<EOL>"<STR_LIT>",<EOL>scriptname,<EOL>self.getName(),<EOL>child.returncode<EOL>)<EOL>errcode = child.returncode<EOL><DEDENT>child = None<EOL><DEDENT>finally:<EOL><INDENT>if child is not None:<EOL><INDENT>tryTerminate(child)<EOL><DEDENT><DEDENT>return errcode<EOL>

Run the specified script from the scripts section of the
            module.json file in the directory of this module.


def wrapper(*args, **kwargs):<EOL><INDENT>return fn(*args, **kwargs)<EOL><DEDENT>return wrapper<EOL>

decorator to drop su/sudo privilages before running a function on
        unix/linux.

        ** on windows this function does nothing **


<EOL>if os.path.exists(program) and os.access(program, os.X_OK):<EOL><INDENT>return program<EOL><DEDENT>for path in os.environ['<STR_LIT>'].split(os.pathsep):<EOL><INDENT>path = path.strip('<STR_LIT:">')<EOL>for ext in os.environ.get('<STR_LIT>', '<STR_LIT>').split(os.pathsep):<EOL><INDENT>progpath = os.path.join(path, program + ext)<EOL>if os.path.exists(progpath) and os.access(progpath, os.X_OK):<EOL><INDENT>return progpath<EOL><DEDENT><DEDENT><DEDENT>return None<EOL>

look for "program" in PATH (respecting PATHEXT), and return the path to
        it, or None if it was not found


next_x = None<EOL>first = True<EOL>for x in generator:<EOL><INDENT>if not first:<EOL><INDENT>yield (next_x, False)<EOL><DEDENT>next_x = x<EOL>first = False<EOL><DEDENT>if not first:<EOL><INDENT>yield (next_x, True)<EOL><DEDENT>

indicate whether the current item is the last one in a generator


def configure(self, component, all_dependencies):

r = {}<EOL>builddir = self.buildroot<EOL>available_dependencies = OrderedDict((k, v) for k, v in all_dependencies.items() if v)<EOL>self.set_toplevel_definitions = '<STR_LIT>'<EOL>if self.build_info_include_file is None:<EOL><INDENT>self.build_info_include_file, build_info_definitions = self.getBuildInfo(component.path, builddir)<EOL>self.set_toplevel_definitions += build_info_definitions<EOL><DEDENT>if self.config_include_file is None:<EOL><INDENT>self.config_include_file, config_definitions, self.config_json_file = self._getConfigData(available_dependencies, component, builddir, self.build_info_include_file)<EOL>self.set_toplevel_definitions += config_definitions<EOL><DEDENT>self.configured = True<EOL>return {<EOL>'<STR_LIT>': self.config_include_file,<EOL>'<STR_LIT>': self.config_json_file,<EOL>'<STR_LIT>': self.build_info_include_file<EOL>}<EOL>

Ensure all config-time files have been generated. Return a
            dictionary of generated items.